# /media/bill/SWAPPER/Qnial/MY_NDFS/email - ThunderBlog.ndf # www.BillHowell.ca Oct2018 based on "/email - split Thunderbird email folder.ndf" but haven't started coding!!! # Processes Thunderbird emails and posts to blog # motivation : simple email-based-blogging #************************************* # ToDos # 14Oct2018 make finds case-insensitive # 15Oct2018 need to fix problem of apos in keyStrsEachFldr # 16Oct2018 when email extracted to (link d_QNial_temp '181015 emSingle temp.txt') : # - convert mime with base64 # - chop off original email (useless and long) # #******************************************************************* f_emailSplit := 'email - split Thunderbird email folder.ndf' ; write 'loading email - split Thunderbird email folder.ndf' ; lq_emailSplit IS loaddefs link d_QNial_mine f_emailSplit # loaddefs link d_QNial_mine 'email - split Thunderbird email folder.ndf' #**************************** # Contents - email - extract, sort, cull addresses from text.ndf # unicode_to_ascii IS OP fin_name fas_name - Convert file from unicode to ascii # email_clean IS OP t_place line - Extract emails from text files #******************************************* # Setup (kind of like header info) flag_break := l ; flag_debug := l ; # Reminder of debug setup... # IF flag_break = l THEN Break ; ENDIF ; write_debug IS OP AAA { NONLOCAL flag_debug ; IF flag_debug THEN write AAA ; ENDIF ; } # Defined in Qnial/profile.ndf # d_QNial_mine := '/media/bill/SWAPPER/Qnial/MY_NDFS/' ; IF not in "TOSTRING_SEP (EACH first symbols 0) THEN loaddefs (link d_QNial_mine 'strings.ndf') 0 ; ENDIF; # *regexp_substitute* R S T [O] replace one or more substrings in string T that match the regular expression pattern R with string S # loaddefs link d_QNial_mine 'email - split Thunderbird email folder.ndf' #****************** # Global variables etc # see header file "/media/bill/SWAPPER/Qnial/MY_NDFS/email - split Thunderbird email folder, header.ndf" # # this header file is used to make it easier to follow the code! loaddefs link d_QNial_mine 'email - split Thunderbird email folder, header.ndf' ; #IF flag_break = l THEN Break ; ENDIF ; # loaddefs link d_QNial_mine 'email - split Thunderbird email folder.ndf' #*********************************************** # email_count IS OP pname_inn - count the number of emails in a text file # stays in "email - split Thunderbird email folder.ndf" write_debug 'loading email_count' ; email_count IS OP pname_inn { LOCAL emlCount finn line_current line_front line_last emlNew_lines ; % ; finn := open pname_inn "r ; % ; % a new email starts with : 2 empty lines, "From - " ; emlNew_lines := '' 'From - ' ; line_last := '' ; % This will catch a "From" at the start of a file! ; line_current := '' ; emlCount := 0 ; % ; WHILE (~= ??eof line_current) DO line_last := line_current ; line_current := readfile finn ; line_front := 7 take line_current ; IF (= (line_last line_front) emlNew_lines) THEN emlCount := emlCount + 1 ; ENDIF ; ENDWHILE ; % ; close finn ; emlCount } # tests - also see email_remove_massEmail email_count (link d_archive '181015 emSplit bad address1') EACH email_count (link d_archive '181015 emSplit bad address1') (link d_QNial_temp '181015 emSplit bad address1') email_count link d_QNial_temp 'emSplit leftover' email_count link d_QNial_temp 'emSplit good address' #IF flag_break = l THEN Break ; ENDIF ; # loaddefs link d_QNial_mine 'email - split Thunderbird email folder.ndf' #*********************************** # emailFilter_check IS OP i_filter_sequence - return 1st filter encountered # write_debug 'loading emailFilter_check' ; emailFilter_check IS OP i_filter_sequence { LOCAL em_chk filter_list filterStr i ; NONLOCAL keyStrsEachFldr p_hostCmdRslt p_emlFinished ; % ; filter_list := i_filter_sequence pick keyStrsEachFldr ; em_chk := o ; filterStr := '' ; FOR i WITH tell shape filter_list DO em_chk := string_in_file filter_list@i p_emlFinished p_hostCmdRslt ; IF em_chk THEN filterStr := filter_list@i ; EXIT '' ; ENDIF ; ENDFOR ; % later - use this to track [effectiveness, problems] with filter_lists ; em_chk filterStr } # emailFilter_check i_filter_sequence #IF flag_break = l THEN Break ; ENDIF ; # loaddefs link d_QNial_mine 'email - split Thunderbird email folder.ndf' #*********************************************** # email_remove_massEmail IS OP pname_inn pname_out - remove the mass email if included in reply # stays in "email - split Thunderbird email folder.ndf" # 16Oct2018 - this misses the first 4 or so line of html # can I code the lines before and after of the mass email to make this easier? will it stay in server replies? # # write_debug 'loading email_remove_massEmail' ; email_remove_massEmail IS OP pname_inn pname_out { LOCAL finn fout in_massEmail line ; % ; finn := open pname_inn "r ; fout := open pname_out "w ; in_massEmail := o ; WHILE (~= ??eof (line := readfile finn)) DO IF (OR (' ' EACHLEFT in_string line)) THEN in_massEmail := l ; ENDIF ; IF in_massEmail THEN IF (OR ('' '----- Message truncated -----' EACHLEFT in_string line)) THEN in_massEmail := o ; ENDIF ; ELSE writefile fout line ; ENDIF ; ENDWHILE ; % ; EACH close finn fout ; } # test email_remove_massEmail (link d_archive '181015 emSplit bad address1') (link d_QNial_temp '181015 emSplit bad address1') >> problem with : ----- Message truncated ----- # email_remove_massEmail (d_QNial_temp EACHRIGHT link 'emSplit bad address1' 'emSplit bad address1 2nd remove') # email_remove_massEmail (d_QNial_temp EACHRIGHT link 'emSplit bad address2' 'emSplit bad address2 2nd remove') #IF flag_break = l THEN Break ; ENDIF ; # loaddefs link d_QNial_mine 'email - split Thunderbird email folder.ndf' #*********************************************** # mime_types IS OP pname_inn pname_out - extract a list of mime_types from file # for setup & run, see "/media/bill/SWAPPER/Qnial/code develop_test/email - split- develop.txt" write_debug 'loading mime_types' ; mime_types IS OP plist_inn pname_out { LOCAL finn in_mime line line_up p_list pname_inn ; NONLOCAL mime_types_ALL ; % ; IF (= 'stdout' pname_out) THEN fout := 2 ; ELSE fout := open pname_out "w ; ENDIF ; p_list := pName_read_list plist_inn ; % ; FOR pname_inn WITH p_list DO fout EACHRIGHT writefile ' ' '**************************************' pname_inn ' ' ; finn := open pname_inn "r ; in_mime := o ; % ; WHILE (~= ??eof (line := readfile finn)) DO IF (= '--' (2 take line)) THEN IF (= '-->' (3 take line)) THEN in_mime := o ; %ELSEIF (= '----' (4 takeright line)) THEN in_mime := o ; ELSEIF (= '--' (2 takeright line)) THEN in_mime := o ; ELSE in_mime := l ; ENDIF ; writefile fout line ; IF in_mime THEN line := readfile finn ; IF (~= '' line) THEN writefile fout line ; line_up := toupper line ; IF (AND ('CONTENT-TYPE: MULTIPART/RELATED; BOUNDARY=' ~= (42 take line_up)) ('CONTENT-TYPE: MULTIPART/ALTERNATIVE; BOUNDARY=' ~= (46 take line_up))) THEN IF (NOT in line_up mime_types_ALL) THEN write line_up ; ENDIF ; ENDIF ; WHILE (AND (~= ??eof (line := readfile finn)) (~= '' line)) DO writefile fout line ; ENDWHILE ; ELSE in_mime := o ; ENDIF ; ENDIF ; ENDIF ; ENDWHILE ; % ; close finn ; ENDFOR ; % ; close fout ; } # qnial> mime_types '/media/bill/SWAPPER/Qnial/code develop_test/email - split emSplitFolderList.txt' '/media/bill/SWAPPER/Qnial/code develop_test/email - split mine_types list.txt' #IF flag_break = l THEN Break ; ENDIF ; # loaddefs link d_QNial_mine 'email - split Thunderbird email folder.ndf' #*********************************************** # mime_IMAGE IS OP finn fout - dump the image in an email folder write_debug 'loading mime_IMAGE' ; mime_IMAGE IS OP finn fout { LOCAL line ; WHILE (AND (~= ??eof (line := readfile finn)) (~= '' line)) DO writefile fout line ; ENDWHILE ; writefile fout line ; WHILE (AND (~= ??eof (line := readfile finn)) (~= '--' (2 take line))) DO null ; ENDWHILE ; writefile fout line ; } #IF flag_break = l THEN Break ; ENDIF ; # loaddefs link d_QNial_mine 'email - split Thunderbird email folder.ndf' #*********************************************** # mime_base64 IS OP finn fout - extract from email folder MIME coding (base64) and expand to unicode # 20Oct2018 base64 - 2 empty lines instead of one [before, after] mime decode # 20Oct2018 base64 - Chinese characters now corrupted (21Oct2018 - may be OK?) # 21Oct2018 base64 - I should catenate sucessive lines when 1st ends in "=" # 21Oct2018 base64 - [IronPort, Microsoft, etc] - excessive junk could be cut!! write_debug 'loading mime_base64' ; mime_base64 IS OP finn fout { LOCAL f_mimeCode f_mimeText line line_end ; NONLOCAL p_mimeCode p_mimeText ; % ; WHILE (AND (~= ??eof (line := readfile finn)) (~= '' line)) DO writefile fout line ; ENDWHILE ; writefile fout chr_newline ; % ; f_mimeCode := open p_mimeCode "w ; WHILE (AND (~= ??eof (line := readfile finn)) (~= '' line) (~= '--' (2 take line))) DO writefile f_mimeCode line ; ENDWHILE ; close f_mimeCode ; line_end := line ; % ; host link 'base64 --decode "' p_mimeCode '" >"' p_mimeText '" ' ; % ; f_mimeText := open p_mimeText "r ; WHILE (~= ??eof (line := readfile f_mimeText)) DO writefile fout line ; ENDWHILE ; close f_mimeText ; writefileWithBlanks fout line_end ; } #IF flag_break = l THEN Break ; ENDIF ; # loaddefs link d_QNial_mine 'email - split Thunderbird email folder.ndf' #*********************************************** # mime_from IS OP finn fout - extract MIME coding (base64) in email "From - " and expand to unicode write_debug 'loading mime_from' ; mime_from IS OP finn fout { LOCAL f_mimeCode f_mimeText line ; NONLOCAL p_mimeCode p_mimeText ; % ; in_from := o ; % ; WHILE (AND (~= ??eof (line := readfile finn)) (~= '' line)) DO writefile fout line ; IF (= 'Content-Transfer-Encoding: base64' (33 take line)) THEN in_from := l ; ENDIF ; ENDWHILE ; writefile fout line ; % ; IF in_from THEN f_mimeCode := open p_mimeCode "w ; WHILE (AND (~= ??eof (line := readfile finn)) (~= '' line)) DO writefile f_mimeCode line ; ENDWHILE ; close f_mimeCode ; % ; host link 'base64 --decode "' p_mimeCode '" >"' p_mimeText '" ' ; % ; f_mimeText := open p_mimeText "r ; WHILE (~= ??eof (line := readfile f_mimeText)) DO writefile fout line ; ENDWHILE ; close f_mimeText ; writefile fout chr_newline ; ENDIF ; } #IF flag_break = l THEN Break ; ENDIF ; # loaddefs link d_QNial_mine 'email - split Thunderbird email folder.ndf' #*********************************************** # mime_expand IS OP pname_inn pname_out - expand certain types of MIME coding (base64,image/png) within an email folder # goes in separate mime file in d_QNial_mine, or maybe fileops.ndf? # 16Oct2018 how am I going to do this? - mix of [text, html, binary, video, image, audio, etc] - or base64 always binary # 16Oct2018 also RSA signatures etc # sometimes indented spaces? # are mime recursive? - NO? - but they are nested write_debug 'loading mime_expand' ; mime_expand IS OP pname_inn pname_out { LOCAL finn fout line line_up mimeCount ; % ; finn := open pname_inn "r ; fout := open pname_out "w ; mimeCount := 0 ; % ; WHILE (~= ??eof (line := readfile finn)) DO writefile fout line ; IF (= '--' (2 take line)) THEN IF (= '-->' (3 take line)) THEN null ; ELSEIF (= '--' (2 takeright line)) THEN mimeCount := mimeCount - 1 ; ELSE mimeCount := mimeCount + 1 ; ENDIF ; ELSEIF (= 'From - ' (7 take line)) THEN mime_from finn fout ; ELSEIF (mimeCount > 0) THEN line_up := toupper line ; IF (in_string 'IMAGE/PNG' line_up) THEN mime_IMAGE finn fout ; ELSEIF (in_string 'Content-Transfer-Encoding: base64' line ) THEN mime_base64 finn fout ; ENDIF ; ENDIF ; ENDWHILE ; % ; EACH close finn fout ; } # qnial> mime_expand (link d_archive '181015 emSplit bad address1') (link d_QNial_temp '181015 emSplit bad address1') # qnial> mime_expand (link d_archive '181015 emSplit leftover' ) (link d_QNial_temp '181015 emSplit leftover' ) # actionable 'CONTENT-TYPE: IMAGE/PNG; NAME="ICON.PNG"' 'CONTENT-TYPE: MESSAGE/RFC822' 'CONTENT-TYPE: TEXT/RFC822-HEADERS' # discard images & old mass email # treat all the rest of mime_types_ALL as normal text (no processing)? # Comments 'CONTENT-DESCRIPTION: NOTIFICATION' all I have are ASCII --Apple-Mail=_6DAB1984-154F-4217-8E90-CF874AC4C4D1 Apple 'CONTENT-TYPE: MULTIPART/ALTERNATIVE; DIFFERENCES=CONTENT-TYPE;' boundary="dcabc192-4276-4442-b3df-bc6c9eb1f058" # two types of "Content-Transfer-Encoding: base64" - within (example) : --_000_ab713109b4234648afc18c233df09697MARCIANOautuniautacnz_ Content-Type: text/plain; charset="utf-8" Content-Transfer-Encoding: base64 - as part of "From - " contiguous lines, with empty line, then code #IF flag_break = l THEN Break ; ENDIF ; # loaddefs link d_QNial_mine 'email - split Thunderbird email folder.ndf' #*********************************************** # emailRead_one IS OP finn oldEmail_3lines - read one email from input folder, expand mime, remove mass email # - result is saved in p_emlNoMass # QNial "writefile" is necessary to avoid character problems with unix commands # - especially character "'" # 21Oct2018 results for which came first (mass,mime,mass) emSplit bad address1 (Mbytes), 1.1,5.2,1.1 To: ijcnn-bounce-, 16,3,16, away, 17,19,17, on leave, 1,3,1, confirming that your e-mail was received, 0,13,0, Total email count for all keyStrsEachFldr, 306,310,306, emSplit bad address2, 44,31,44, emSplit good address, 0,13,0, emSplit out of office1, 46,50,46, # 21Oct2018 base64 - mime-expand first, then chop IJCNN2019 email sent # create another condition for taking out remaining mass emails # delay didn't help : mime_expand p_emlRawInn p_emlNoMime ; waitSecs 1 ; email_remove_massEmail p_emlNoMime p_emlNoMass ; write_debug 'loading emailRead_one' ; emailRead_one IS OP finn oldEmail_3lines { LOCAL em_chk fraw filterStr i_counter i_filter_sequence last_3lines line_current emlNew_lines p_splitFolder test_3lines ; NONLOCAL d_QNial_temp filter_sequence i_leftover keyStrsALL keyCountsALL leftoverCount p_emlRawInn p_emlNoMime p_emlNoMass p_emlFinished p_folders p_log ; % ; % a new email starts with : 2 empty lines, "From - " ; emlNew_lines := '' '' 'From - ' ; line_current := last oldEmail_3lines ; last_3lines := (front oldEmail_3lines) append (7 take line_current) ; % ; fraw := open p_emlRawInn "w ; WHILE (AND (~= ??eof line_current) (~= test_3lines emlNew_lines)) DO writefile fraw line_current ; line_current := readfile finn ; test_3lines := (rest last_3lines) append (7 take line_current) ; last_3lines := (rest last_3lines) append line_current ; ENDWHILE ; close fraw ; % ; mime_expand p_emlRawInn p_emlNoMime ; p_emlFinished := p_emlNoMime ; % initially set in "email - split Thunderbird email folder, header.ndf" ; % ; % sort out result ; em_chk := o ; FOR i_filter_sequence WITH tell gage shape filter_sequence DO em_chk filterStr := emailFilter_check filter_sequence@i_filter_sequence ; IF em_chk THEN i_counter := find filterStr keyStrsALL ; keyCountsALL@i_counter := keyCountsALL@i_counter + 1 ; EXIT 'done for' ; ENDIF ; ENDFOR ; IF em_chk THEN p_splitFolder := link (i_filter_sequence pick filter_sequence pick p_folders) ; ELSE p_splitFolder := link d_QNial_temp 'emSplit leftover' ; leftoverCount := leftoverCount + 1 ; ENDIF ; % ; p_splitFolder last_3lines } # dunno how to filter IF (NOT em_chk) THEN em_chk p_splitFolder := o 'dunno' ; IF (NOT em_chk) THEN em_chk p_splitFolder := o 'inquiries' ; #IF flag_break = l THEN Break ; ENDIF ; # loaddefs link d_QNial_mine 'email - split Thunderbird email folder.ndf' #*********************************************** # emailSplit_folder IS - automatically split an email folder according to key phrases in emails # manual re-allocation follows. the original folder is NOT modified! Results are stored in d_QNial_temp # among split folders !! # Same code as emailSplit_folder up to "" # Code repeated as its likely this will sometimes be done after # computer shut off # timestamp_YYMMDD - from "/media/bill/SWAPPER/Qnial/setup.ndf" write_debug 'loading emailSplit_folder' ; emailSplit_folder IS { LOCAL finn fout i_count i_folder i_keyStrsALL last_3lines line_current p_splitFolder ; NONLOCAL p_fldrTmp d_QNial_temp folderCountsALL folderCountsTotal folders p_leftover leftoverCount keyCountsALL keyCountsTotal keyStrsEachFldr keyStrsALL p_emlNoMass p_emlNoMime p_fldrSrc p_log p_folders ; % ; %If this file is loaddefed - must rerun emailSplit_stats & emailSplit_folder ; host link 'echo >>"' p_log '" "*************************************"' ; host link 'echo >>"' p_log '" "emailSplit_folder start : ' timestamp_DDMMMYYYY_HMS '"' ; host link 'echo >>"' p_log '" " "' ; % ; % clear d_QNial_temp ; host 'find "' d_QNial_temp '" -maxdepth 1 -name "emSplit *" | tr \\n \\0 | xargs -0 rm' ; folderCountsALL := (gage shape folders ) reshape 0 ; keyCountsALL := (gage shape keyStrsALL) reshape 0 ; leftoverCount := 0 ; last_3lines := '' '' '' ; % This will catch a "From" at the start of a file! ; % ; % email_remove_massEmail - do in emailSplit_folder BEFORE & AFTER all emails sorted&filed! ; email_remove_massEmail p_fldrSrc p_fldrTmp ; % ; % process the target email folder ; finn := open p_fldrTmp "r ; REPEAT p_splitFolder last_3lines := emailRead_one finn last_3lines ; host link 'cat >>"' p_splitFolder '" "' p_emlNoMime '"' ; line_current := (last last_3lines) ; last_3lines := 'n/a' 'n/a' line_current ; UNTIL (= line_current ??eof) ENDREPEAT ; close finn ; % ; % sull ALL counts ; %waitSecs 2 ; keyCountsTotal := leftoverCount + sum keyCountsALL ; % ; % calculate email counts for all folders (leftover already tallied as "KeyStrs special" ; FOR i_folder WITH tell gage shape folders DO i_keyStrsALL := keyStrsEachFldr@i_folder EACHLEFT find keyStrsALL ; folderCountsALL@i_folder := sum (i_keyStrsALL EACHLEFT pick keyCountsALL) ; ENDFOR ; folderCountsTotal := leftoverCount + sum folderCountsALL ; % ; % email_remove_massEmail - do in emailSplit_folder BEFORE & AFTER all emails sorted&filed! ; FOR p_splitFolder WITH p_folders DO IF (file_exists '-f' p_splitFolder) THEN email_remove_massEmail p_splitFolder p_emlNoMass ; host link 'cp "' p_emlNoMass '" "' p_splitFolder '" ' ; ELSE host link 'echo "' (link p_splitFolder ' - no emails filtered here by emailSplit_folder.') '" >"' p_splitFolder '" ' ; ENDIF ; ENDFOR ; % special case of "emSplit leftover" ; email_remove_massEmail p_leftover p_emlNoMass ; host link 'cp "' p_emlNoMass '" "' p_leftover '" ' ; % ; % log summary info ; host link 'ls >>"' p_log '" -lh "' d_QNial_temp '" ' ; host link 'echo >>"' p_log '" " "' ; host link 'echo >>"' p_log '" "emailSplit_folder end : ' timestamp_DDMMMYYYY_HMS '"' ; host link 'echo >>"' p_log '" " "' ; } # Tests host link 'mv "/media/ramdisk/QNial host response.txt" "/home/bill/temp/"' find "/media/bill/ramdisk" -maxdepth 1 -name "emSplit *" | tr \\n \\0 | xargs -0 rm #IF flag_break = l THEN Break ; ENDIF ; # loaddefs link d_QNial_mine 'email - split Thunderbird email folder.ndf' #*********************************************** # emailSplit_stats IS - tally [keyCountsALL, folderCountsALL] in p_statsOut1_temp # runs emailSplit_folder and uses its [keyStrs, folders] counts # If this file is loaddefed - must rerun emailSplit_stats & emailSplit_folder write_debug 'loading emailSplit_stats' ; emailSplit_stats IS { LOCAL i_folder i_keyStrs ; NONLOCAL d_QNial_temp filters_manual keyStr_noApo leftoverCount folders folderCountsALL folderCountsTotal keyStrsALL keyCountsALL keyCountsTotal p_stats p_statsOut1_temp p_statsOut2_temp ; % ; host link 'cp --force "' p_stats '" "' p_statsOut1_temp '" ' ; % ; % list the date ; host link 'cat "' p_statsOut1_temp '" | sed ' chr_quotation 's/\(^Dates run.*$\)/\1' timestamp_DDMMMYYYY ',/' chr_quotation ' >"' p_statsOut2_temp '" ' ; host link 'cp "' p_statsOut2_temp '" "' p_statsOut1_temp '" ' ; % ; FOR i_keyStrs WITH tell gage shape keyStrsALL DO % must take care of apostrophes in keyStrsALL@i_keyStrs! ; keyStr_noApo := string_sub '''' (link chr_apo '\' chr_apo chr_apo) keyStrsALL@i_keyStrs ; host link 'cat "' p_statsOut1_temp '" | sed ' chr_quotation 's/\(^' keyStr_noApo '.*$\)/\1' (string keyCountsALL@i_keyStrs) ',/' chr_quotation ' >"' p_statsOut2_temp '" ' ; host link 'cp "' p_statsOut2_temp '" "' p_statsOut1_temp '" ' ; ENDFOR ; % ; % special case of "emSplit leftover" ; host link 'cat "' p_statsOut1_temp '" | sed ' chr_quotation 's/\(^leftover.*$\)/\1' (string leftoverCount) ',/' chr_quotation ' >"' p_statsOut2_temp '" ' ; host link 'cp "' p_statsOut2_temp '" "' p_statsOut1_temp '" ' ; % totals ; host link 'cat "' p_statsOut1_temp '" | sed ' chr_quotation 's/\(^Total email count for all keyStrs.*$\)/\1' (string keyCountsTotal) ',/' chr_quotation ' >"' p_statsOut2_temp '" ' ; host link 'cp "' p_statsOut2_temp '" "' p_statsOut1_temp '" ' ; % ; FOR i_folder WITH tell gage shape folders DO % must take care of apostrophes in folders@i_folder! ; folder_noApo := string_sub '''' (link chr_apo '\' chr_apo chr_apo) folders@i_folder ; host link 'cat "' p_statsOut1_temp '" | sed ' chr_quotation 's/\(^' folder_noApo '.*$\)/\1' (string folderCountsALL@i_folder) ',/' chr_quotation ' >"' p_statsOut2_temp '" ' ; host link 'cp "' p_statsOut2_temp '" "' p_statsOut1_temp '" ' ; ENDFOR ; % special case of "emSplit leftover" ; host link 'cat "' p_statsOut1_temp '" | sed ' chr_quotation 's/\(^emSplit leftover.*$\)/\1' (string leftoverCount) ',/' chr_quotation ' >"' p_statsOut2_temp '" ' ; host link 'cp "' p_statsOut2_temp '" "' p_statsOut1_temp '" ' ; % totals ; host link 'cat "' p_statsOut1_temp '" | sed ' chr_quotation 's/\(^Total email count for all folders.*$\)/\1' (string folderCountsTotal) ',/' chr_quotation ' >"' p_statsOut2_temp '" ' ; host link 'cp "' p_statsOut2_temp '" "' p_statsOut1_temp '" ' ; EACH write 'Manually transfer emails to : ' filters_manual ; } emailSplit_and_stats IS { emailSplit_folder ; emailSplit_stats ; } #IF flag_break = l THEN Break ; ENDIF ; # loaddefs link d_QNial_mine 'email - split Thunderbird email folder.ndf' #*********************************************** # emailSplit_statsManual IS - post folderCountsALL@i (but NOT keyCountsALL@i) in p_statsOut1_temp # MANUALLY done AFTER using emailSplit_folder and after manually re-allocating emails among split folders !! # BUT - emailSplit_and_stats updates to p_statsOut1_temp must still be in d_QNial_temp # does email count in each folder (doesn't use emailSplit_folder counts) write_debug 'loading emailSplit_statsManual' ; emailSplit_statsManual IS { LOCAL emCount emCountTotal i_folder ; NONLOCAL folders p_folders p_leftover p_statsOut1_temp p_statsOut2_temp filters_manual p_filters_manual ; % ; emCountTotal := 0 ; % do NOT list the date after manual ; % ; % counts of "normal" folders ; FOR i_folder WITH tell gage shape folders DO IF (file_exists '-f' p_folders@i_folder) THEN emCount := email_count p_folders@i_folder ; emCountTotal := emCountTotal + emCount ; host link 'cat "' p_statsOut1_temp '" | sed ' chr_quotation 's/\(^' folders@i_folder '.*$\)/\1' (string emCount) ';/' chr_quotation ' >"' p_statsOut2_temp '" ' ; host link 'cp "' p_statsOut2_temp '" "' p_statsOut1_temp '" ' ; ELSE host link 'cat "' p_statsOut1_temp '" | sed ' chr_quotation 's/\(^' folders@i_folder '.*$\)/\10;/' chr_quotation ' >"' p_statsOut2_temp '" ' ; host link 'cp "' p_statsOut2_temp '" "' p_statsOut1_temp '" ' ; ENDIF ; ENDFOR ; % ; % counts of filters_manual ; FOR i_folder WITH tell gage shape filters_manual DO IF (file_exists '-f' p_filters_manual@i_folder) THEN emCount := email_count p_filters_manual@i_folder ; emCountTotal := emCountTotal + emCount ; host link 'cat "' p_statsOut1_temp '" | sed ' chr_quotation 's/\(^' filters_manual@i_folder '.*$\)/\1' (string emCount) ';/' chr_quotation ' >"' p_statsOut2_temp '" ' ; host link 'cp "' p_statsOut2_temp '" "' p_statsOut1_temp '" ' ; ELSE host link 'cat "' p_statsOut1_temp '" | sed ' chr_quotation 's/\(^' filters_manual@i_folder '.*$\)/\10;/' chr_quotation ' >"' p_statsOut2_temp '" ' ; host link 'cp "' p_statsOut2_temp '" "' p_statsOut1_temp '" ' ; ENDIF ; ENDFOR ; % totals ; host link 'cat "' p_statsOut1_temp '" | sed ' chr_quotation 's/\(^Total email count for all folders.*$\)/\1' (string emCountTotal) ';/' chr_quotation ' >"' p_statsOut2_temp '" ' ; host link 'cp "' p_statsOut2_temp '" "' p_statsOut1_temp '" ' ; } #*********************************************** # emailSplit_archive IS - archive results for QNial "email - extract, sort, cull addresses from text.ndf" # # 15Oct2018 I need to add step to delete Thunderbird folders emailSplit_archive IS { NONLOCAL f_Src d_QNial_temp p_fldrTmp p_stats p_statsOut1_temp p_statsOut2_temp ; % ; %If this file is loaddefed - must rerun "emailSplit_folder p_fldrSrc" ; emailSplit_stats ; % ; %archive files ; host link 'mv "' p_fldrTmp '" "' d_archive timestamp_YYMMDD ' ' f_Src '" ' ; host link 'cp "' p_statsOut1_temp '" "' d_archive timestamp_YYMMDD ' ' f_stats '" ' ; host link 'mv "' p_statsOut1_temp '" "' p_stats '" ' ; host link 'rm "' p_statsOut2_temp '" ' ; host link 'find "' d_QNial_temp '" -maxdepth 1 -name "emSplit *" -printf "%f\n" | tr \\n \\0 | xargs -0 -IFILE mv "' d_QNial_temp 'FILE" "' d_archive timestamp_YYMMDD ' FILE" ' ; } # string_sub '''' (link chr_apo '\' chr_apo chr_apo) 'e-mail address you entered couldn''t be found' find "/media/bill/ramdisk" -maxdepth 1 -name "emSplit *" | tr \\n \\0 | xargs -0 ??????' find "/media/bill/ramdisk" -maxdepth 1 -name "emSplit *" -printf "%f\n" | tr \\n \\0 | xargs -0 -IFILE echo "/media/bill/ramdisk/FILE" "/media/bill/SWAPPER/a_INNS Lexicom email server/z_Archive/181015 FILE" # Tests cat "/media/bill/SWAPPER/a_INNS Lexicom email server/a_raw responses" | sed 's/\(user unknown.*$\)/\1999\,/' >"/media/bill/ramdisk/a_raw responses" #*********************************************** ## 12Aug2017 06:55 thoughts move_results AFTER manual inspection append to current folders? don't forget log file create log files for each filter phrase : date, number of hits (biased to most frequent, as others are "shut out" build SVM or Deep Learning to pick - keep ALL results on Midas to train grab old results from backups get rid of stderr handling - I don't get much now #********************************* # Procedure # 1. move '/media/bill/SWAPPER/Thunderbird/n4caryuo.default/Mail/Local Folders/0_Lexicom INNS email server.sbd/a_raw responses' to : d_archive (= '/media/bill/SWAPPER/a_INNS Lexicom email server/d_Archive/') and create an empty Thunderbird file 'a_raw responses' and delete corresponding '.msf' file to force regeneration Thunderbird menu -> to regenerate .msf # EITHER individually : # 1. qnial> emailSplit_folder - this splits "a_raw responses" into folders # qnial> emailSplit_stats - provides email counts in each folder # OR combined : # 1. qnial> emailSplit_and_stats - provides email stats BEFORE manually fixing folders # THEN : # 2. Manually check the results, and move emails from one folder to the next # filters_manual ('emSplit dunno','emSplit inquiries','emSplit leftover') are a particular focus # look for mis-classifications # 3. qnial> emailSplit_statsManual - provides email stats AFTER manually fixing folders # 4. Check "" for status # 5. When everything OK, : # qnial> emailSplit_archive # loaddefs link d_QNial_mine 'email - split Thunderbird email folder.ndf' # enddoc