#] #] ********************* #] loaddefs link d_Qndfs 'email - mass response processing/email - split Thunderbird email folder.ndf' # www.BillHowell.ca # 10Aug2017 based on previous manual filtering within Thunderbird (BIG improvement!) # 12Jan2020 modified to accommodate WCCI2020 HELP web-page # Splits Thunderbird email folder according to key phrases in emails # motivation : Thunderbird change, "permanent fatal" phrase doesn't work in "Quick Filter" # followed by "email - extract, sort, cull addresses from text.ndf" f_emailSplit := 'email - split Thunderbird email folder.ndf' ; loaddefs_start f_emailSplit ; #************ # List of operators, generated with : # $ grep "^#]" "$d_Qndfs""email - mass response processing/email - split Thunderbird email folder.ndf" | sed 's/^#\]/ /' # ********************* loaddefs link d_Qndfs 'email - mass response processing/email - split Thunderbird email folder.ndf' email_count IS OP pname_inn - count the number of emails in a text file email_count IS OP pname_inn - email_remove_massEmail IS OP pname_inn pname_out - remove the mass email if included in reply email_remove_massEmail IS OP pname_inn pname_out - preprocess_folder IS OP p_folder - [expand mime, remove mass mail] if flagged preprocess_folder IS OP p_fldrSrc p_fldrTwix p_fldrTmp - pstprocess_folder IS OP finn oldEmail_2lines - [expand mime, remove mass mail] if flagged pstprocess_folder IS OP finn oldEmail_2lines - remove_1stWhiteLines IS OP p_inn p_out - empty lines don't give proper email file! remove_1stwhiteLines IS OP p_inn p_out - emailCounts IS - emailCounts IS - clean_emlAddress IS OP line - remove ['"] and extra_whitespace from email address list clean_emlAddress IS OP line - find_emailHdr_items IS OP finn item - get an emailHdr_item (eg (From, Subject, CC]) from p_emlOne find_emailHdr_item IS OP finn item - email_filter IS - splits single email tempFile email_filter IS - emailRead_one IS OP finn oldEmail_2lines - read one email from input folder, expand mime, remove mass email emailRead_one IS OP finn oldEmail_2lines - emailSplit_folder IS - automatically split an email folder according to key phrases in emails emailSplit_folder IS - emailSplit_stats IS - tally [keyCountsALL, folderCountsALL] in p_stats1 emailSplit_stats IS - emailSplit_and_stats IS { emailSplit_folder ; emailSplit_stats ; } - emailSplit_statsManual IS - post folderCountsALL@i (but NOT keyCountsALL@i) in p_stats1 emailSplit_statsManual IS - emailSplit_archive IS - archive results for QNial "email - extract, sort, cull addresses from text.ndf" emailSplit_archive IS OP p_fldrSrc - #*********** # Setup (kind of like header info) flag_break := l ; flag_debug := o ; # Reminder of debug setup... # IF flag_break = l THEN Break ; ENDIF ; # this has : send_Thunderbird IS OP winName - find winID of email, send, wait loaddefs link d_Qndfs 'windows [open, close, ID, title], [get,set,move] cart [posn,size].ndf' ; loaddefs link d_Qndfs 'signals applications non-window.ndf' ; loaddefs link d_Qndfs 'email - mime.ndf' ; # *regexp_substitute* R S T [O] replace one or more substrings in string T that match the regular expression pattern R with string S # activate only ONE of : #loaddefs link d_Qndfs 'email - split massEmails.ndf' ; #loaddefs link d_Qndfs 'email - split HELPER_WCCI2020.ndf' ; # common to all : loaddefs link d_Qndfs 'email - split Thunderbird email folder, header.ndf' ; #IF flag_break = l THEN Break ; ENDIF ; # loaddefs link d_Qndfs 'email - mass response processing/email - split Thunderbird email folder.ndf' #*********************************************** #] email_count IS OP pname_inn - count the number of emails in a text file # stays in "email - mass response processing/email - split Thunderbird email folder.ndf" # 03May2020 - with a_Howell cool emails' it is obvious that the lastline thing doesn't work # This kills me, as 'From - ' could easuly arise in normal email text # they should have coded, eg : '#] From - ' or something! (shitty job) IF flag_debug THEN write 'loading email_count' ; ENDIF ; #] email_count IS OP pname_inn - email_count IS OP pname_inn { LOCAL emlCount finn line_current line_front line_last emlNew_lines ; % ; finn := open pname_inn "r ; emlNew_lines := 'From - ' ; % a new email starts with : "From - " ; emlCount := 0 ; WHILE (~= ??eof line_current) DO line_current := readfile finn ; IF (= (7 take line_current) emlNew_lines) THEN emlCount := emlCount + 1 ; ENDIF ; ENDWHILE ; close finn ; emlCount } # 03May2020 Old code : % a new email starts with : 2 empty lines, "From - " ; emlNew_lines := '' 'From - ' ; line_last := '' ; % This will catch a "From" at the start of a file! ; line_current := '' ; emlCount := 0 ; % ; WHILE (~= ??eof line_current) DO line_last := line_current ; line_current := readfile finn ; line_front := 7 take line_current ; IF (= (line_last line_front) emlNew_lines) THEN emlCount := emlCount + 1 ; ENDIF ; ENDWHILE ; % ; close finn ; emlCount # tests - also see email_remove_massEmail email_count (link d_archive '181015 emSplit bad address1') EACH email_count (link d_archive '181015 emSplit bad address1') (link d_temp '181015 emSplit bad address1') email_count link d_temp 'emSplit leftover' email_count link d_temp 'emSplit good address' email_count '/home/bill/Thunderbird/n4caryuo.default/Mail/Local Folders/a_Howell cool emails' #*********************************************** #] email_remove_massEmail IS OP pname_inn pname_out - remove the mass email if included in reply # stays in "email - mass response processing/email - split Thunderbird email folder.ndf" # 16Oct2018 - this misses the first 4 or so line of html # can I code the lines before and after of the mass email to make this easier? will it stay in server replies? # # IF flag_debug THEN write 'loading email_remove_massEmail' ; ENDIF ; #] email_remove_massEmail IS OP pname_inn pname_out - email_remove_massEmail IS OP pname_inn pname_out { LOCAL finn fout in_massEmail line ; % ; finn := open pname_inn "r ; fout := open pname_out "w ; in_massEmail := o ; WHILE (~= ??eof (line := readfile finn)) DO IF (OR (' ' EACHLEFT in_string line)) THEN in_massEmail := l ; ENDIF ; IF in_massEmail THEN IF (OR ('' ' - - - - - Message truncated - - - - -' EACHLEFT in_string line)) THEN in_massEmail := o ; ENDIF ; ELSE writefile fout line ; ENDIF ; ENDWHILE ; % ; EACH close finn fout ; } # test email_remove_massEmail (link d_archive '181015 emSplit bad address1') (link d_temp '181015 emSplit bad address1') >> problem with : - - - - - Message truncated - - - - - # email_remove_massEmail (d_temp EACHRIGHT link 'emSplit bad address1' 'emSplit bad address1 2nd remove') # email_remove_massEmail (d_temp EACHRIGHT link 'emSplit bad address2' 'emSplit bad address2 2nd remove') #*********************************************** #] preprocess_folder IS OP p_folder - [expand mime, remove mass mail] if flagged # doesn't remove images... IF flag_debug THEN write 'loading preprocess_folder' ; ENDIF ; #] preprocess_folder IS OP p_fldrSrc p_fldrTwix p_fldrTmp - preprocess_folder IS OP p_fldrSrc p_fldrTwix p_fldrTmp { NONLOCAL massEmailRemove_flag mime_expand_flag ; % ; IF mime_expand_flag THEN mime_expand p_fldrSrc p_fldrTwix ; % remove massEmail - ONLY if mime has been expanded (otherwise will often miss!) ; IF massEmailRemove_flag THEN email_remove_massEmail p_fldrTwix p_fldrTmp ; ELSE cmd := link 'mv "' p_fldrTwix '" "' p_fldrTmp '" ' ; host cmd ; ENDIF ; ELSE cmd := link 'cp "' p_fldrSrc '" "' p_fldrTmp '" ' ; host cmd ; ENDIF ; } #*********************************************** #] pstprocess_folder IS OP finn oldEmail_2lines - [expand mime, remove mass mail] if flagged # doesn't remove images... # 14Jan2020 I'm not currently using this - but I thought I did this for the massEmail system? # IF flag_debug THEN write 'loading pstprocess_folder' ; ENDIF ; #] pstprocess_folder IS OP finn oldEmail_2lines - pstprocess_folder IS OP finn oldEmail_2lines { LOCAL test ; NONLOCAL p_fldrTmp folderCountsALL keyCountsALL keyCountsTotal keyStrsALL email_filter_phrase p_emlOne p_log FAQFolderList ; % ; FOR p_splitFolder WITH FAQFolderList DO IF (file_exists ' -f' p_splitFolder) THEN %email_remove_massEmail p_splitFolder p_emlNoMass ; host link 'cp "' p_emlNoMass '" "' p_splitFolder '" ' ; ELSE host link 'echo "' (link p_splitFolder ' - no emails filtered here by emailSplit_folder.') '" >"' p_splitFolder '" ' ; ENDIF ; ENDFOR ; } #*********************************************** #] remove_1stWhiteLines IS OP p_inn p_out - empty lines don't give proper email file! # doesn't remove images... # 22Jan2020 move this to 'file_ops.ndf' when de-bugged IF flag_debug THEN write 'loading remove_1stwhiteLines' ; ENDIF ; #] remove_1stwhiteLines IS OP p_inn p_out - remove_1stwhiteLines IS OP p_inn p_out { NONLOCAL massEmailRemove_flag mime_expand_flag ; % ; finn := open p_inn "r ; fout := open p_out "w ; WHILE (is_whiteLine (line := readfile finn)) DO null ; ENDWHILE ; writefile fout line ; WHILE (~= ??eof (line := readfile finn)) DO writefile fout line ; ENDWHILE ; EACH close finn fout ; } # qnial> remove_1stwhiteLines (link d_temp 'single email.txt') (link d_temp 'single email.eml') #*********************************************** #] emailCounts IS - IF flag_debug THEN write 'loading emailCounts' ; ENDIF ; #] emailCounts IS - emailCounts IS { NONLOCAL keyCountsALL keyCountsTotal folderCountsALL folderCountsTotal ; % ; keyCountsTotal := sum keyCountsALL ; folderCountsTotal := sum folderCountsALL ; } # #*********************************************** #] clean_emlAddress IS OP line - remove ['"] and extra_whitespace from email address list # 16Jan2020 initial IF flag_debug THEN write 'loading clean_emlAddress' ; ENDIF ; #] clean_emlAddress IS OP line - clean_emlAddress IS OP line { strip_extra_whitespace (strip_chrs `, (strip_chrs `" (strip_chrs `' line))) } #*********************************************** #] find_emailHdr_items IS OP finn item - get an emailHdr_item (eg (From, Subject, CC]) from p_emlOne # 16Jan2020 Adapted from : # "$d_Qndfs""email - send many individualised.ndf" # "$d_bin""email - create auto.sh" # 16Jan2020 if a Hdr_item can;t be found, or is out of order, an error occurs IF flag_debug THEN write 'loading find_emailHdr_item' ; ENDIF ; #] find_emailHdr_item IS OP finn item - find_emailHdr_item IS OP finn item { LOCAL flag len_item line result ; len_item := gage shape item ; flag := l ; WHILE (AND flag (~= ??eof line)) DO line := readfile finn ; %writescreen line ; IF (= item (len_item take line)) THEN result := len_item drop line ; flag := o ; ENDIF ; ENDWHILE ; result } # $ loaddefs link d_Qndfs 'email - mass response processing/email - split Thunderbird email folder.ndf' # send_Thunderbird #*********************************************** #] email_filter IS - splits single email tempFile # 12Jan2020 - second part of the original emailRead_one # This operator is specific to each email system. Currently available as per Setup above. # dunno how to filter -> have to be done manually in leftover folder IF (NOT em_chk) THEN em_chk p_splitFolder := o 'dunno' ; IF (NOT em_chk) THEN em_chk p_splitFolder := o 'inquiries' ; IF flag_debug THEN write 'loading email_filter' ; ENDIF ; #] email_filter IS - email_filter IS { LOCAL key keyStr ; NONLOCAL daemon keyStrsALL keyCountsALL p_emlOne p_hostCmdRslt ; % ; keyStr := link daemon 'leftover' ; FOR key WITH (front keyStrsALL) DO IF (string_in_file key p_emlOne p_hostCmdRslt) THEN keyStr := key ; EXIT 'done for' ; ENDIF ; ENDFOR ; % ; % increment keyStr count ; i := find keyStr keyStrsALL ; keyCountsALL@i := keyCountsALL@i + 1 ; % ; keyStr } # # $ grep --ignore-case --fixed-strings 'IEEE WCCI 2020 HELP : can I submit a paper that has already appeared in [arXiv, TechRxiv.org, other]?' "/media/bill/ramdisk/test work temp.txt" # qnial> testus := link chr_apo 'IEEE WCCI 2020 HELP : can I submit a paper that has already appeared in [arXiv, TechRxiv.org, other]?' chr_apo # qnial> host link 'grep --ignore-case --fixed-strings ' testus ' "/media/bill/ramdisk/test work temp.txt" ' see strings.ndf : # qnial> string_in_file 'IEEE WCCI 2020 HELP : can I submit a paper that has already appeared in [arXiv, TechRxiv.org, other]?' (link d_temp 'test work temp.txt') (link d_temp 'QNial host response.txt') # for HELP-spcific IF (string_in_file (link 'Subject: ' key) p_emlOne p_hostCmdRslt) #*********************************************** #] emailRead_one IS OP finn oldEmail_2lines - read one email from input folder, expand mime, remove mass email # a new email starts with : 2 empty lines, 'From - ' ; #IF flag_break = l THEN Break ; ENDIF ; # loaddefs link d_Qndfs 'email - mass response processing/email - split Thunderbird email folder.ndf' # testSplit_daemonInbox IF flag_debug THEN write 'loading emailRead_one' ; ENDIF ; #] emailRead_one IS OP finn oldEmail_2lines - emailRead_one IS OP finn oldEmail_2lines { LOCAL fout line last1 ; NONLOCAL p_emlOne p_last_FROM_processed ; % ; % DON'T [open, close] finn!! emailSplit[...] does that! ; fout := open p_emlOne "w ; fout EACHRIGHT writefile oldEmail_2lines ; last1 := rest oldEmail_2lines ; write last1 ; WHILE (NOT OR ( (= ??eof (line := readfile finn)) (= 'From - ' (7 take line)) ) ) DO writefile fout line ; last1 := line ; ENDWHILE ; close fout ; IF (= 'From - ' (7 take line)) THEN host link 'echo "' line '" >"' p_last_FROM_processed '" ' ; ENDIF ; % remove leading whiteLines ; remove_1stwhiteLines p_emlOne p_emlOner ; %write last1 line ; last1 line } # may need this later? # 03May2020 - with a_Howell cool emails' it is obbvious that the lastline thing doesn't work # This kills me, as 'From - ' could easuly arise in normal email text # they should have coded, eg : '#] From - ' or something! (shitty job) (AND (= 'From - ' (7 take line)) (= '' last1) ) #*********************************************** #] emailSplit_folder IS - automatically split an email folder according to key phrases in emails # 21Oct2018 base64 - mime -expand first, then chop IJCNN2019 email sent # create another condition for taking out remaining mass emails # manual re -allocation follows. the original folder is NOT modified! # Results are stored in d_temp among split folderList !! # Same code as emailSplit_folder up to "" # Code repeated as its likely this will sometimes be done after computer shut off # timestamp_YYMMDD - from "/media/bill/SWAPPER/Qnial/setup.ndf" IF flag_debug THEN write 'loading emailSplit_folder' ; ENDIF ; #] emailSplit_folder IS - emailSplit_folder IS { LOCAL finn keyStr last_3lines p_fldrSrc p_splitFolder ; NONLOCAL email_filter_phrase FAQFolderList folderCountsALL keyCountsALL keyStrsALL d_log p_log p_emlOner p_fldrTwix p_fldrTmp ; % ; %If this file is loaddefed - must rerun emailSplit_stats & emailSplit_folder ; host link 'echo >>"' p_log '" "*************************************"' ; host link 'echo >>"' p_log '" "emailSplit_folder start : ' timestamp_DDMMMYYYY_HMS '"' ; host link 'echo >>"' p_log '" " "' ; % ; % zero the email counters ; p_fldrSrc := link d_log timestamp_YYMMDD_HMS ' ' f_Src ; folderCountsALL := (gage shape FAQFolderList ) reshape 0 ; keyCountsALL := (gage shape keyStrsALL) reshape 0 ; % depending on options, remove [massEmail, images], mime-to-text, ; preprocess_folder p_fldrSrc p_fldrTwix p_fldrTmp ; % ; % process the target email folder ; finn := open p_fldrTmp "r ; last_2lines := '' 'number 999' ; last_2lines := emailRead_one finn last_2lines ; WHILE (~= ??eof (last (last_2lines := emailRead_one finn last_2lines))) DO keyStr := execute email_filter_phrase ; p_splitFolder := link d_temp ((find keyStr keyStrsALL) pick FAQFolderList) ; host link 'cat "' p_emlOne '" >>"' p_splitFolder '" ' ; ENDWHILE ; close finn ; % ; %emailCounts ; % tally counts etc ; % ; % log summary info ; host link 'ls >>"' p_log '" -lh "' d_temp '" ' ; host link 'echo >>"' p_log '" " "' ; host link 'echo >>"' p_log '" "emailSplit_folder end : ' timestamp_DDMMMYYYY_HMS '"' ; host link 'echo >>"' p_log '" " "' ; } # Tests host link 'mv "/media/ramdisk/QNial host response.txt" "/home/bill/temp/"' find "/media/bill/ramdisk" -maxdepth 1 -name "emSplit *" | tr \\n \\0 | xargs -0 rm #*********************************************** #] emailSplit_stats IS - tally [keyCountsALL, folderCountsALL] in p_stats1 # runs emailSplit_folder and uses its [keyStrs, folderList] counts # If this file is loaddefed - must rerun emailSplit_stats & emailSplit_folder IF flag_debug THEN write 'loading emailSplit_stats' ; ENDIF ; #] emailSplit_stats IS - emailSplit_stats IS { LOCAL i_folder i_keyStrs ; NONLOCAL d_temp filters_manual keyStr_noApo leftoverCount folderList folderCountsALL folderCountsTotal keyStrsALL keyCountsALL keyCountsTotal p_stats p_stats1 p_stats2 ; % ; host link 'cp - -force "' p_stats '" "' p_stats1 '" ' ; % ; % list the date ; host link 'cat "' p_stats1 '" | sed ' chr_quotation 's/\(^Dates run.*$\)/\1' timestamp_DDMMMYYYY ',/' chr_quotation ' >"' p_stats2 '" ' ; host link 'cp "' p_stats2 '" "' p_stats1 '" ' ; % ; FOR i_keyStrs WITH tell gage shape keyStrsALL DO % must take care of apostrophes in keyStrsALL@i_keyStrs! ; keyStr_noApo := str_replace_subStr keyStrsALL@i_keyStrs '''' (link chr_apo '\' chr_apo chr_apo) ; host link 'cat "' p_stats1 '" | sed ' chr_quotation 's/\(^' keyStr_noApo '.*$\)/\1' (string keyCountsALL@i_keyStrs) ',/' chr_quotation ' >"' p_stats2 '" ' ; host link 'cp "' p_stats2 '" "' p_stats1 '" ' ; ENDFOR ; % ; % special case of "emSplit leftover" ; host link 'cat "' p_stats1 '" | sed ' chr_quotation 's/\(^leftover.*$\)/\1' (string leftoverCount) ',/' chr_quotation ' >"' p_stats2 '" ' ; host link 'cp "' p_stats2 '" "' p_stats1 '" ' ; % totals ; host link 'cat "' p_stats1 '" | sed ' chr_quotation 's/\(^Total email count for all keyStrs.*$\)/\1' (string keyCountsTotal) ',/' chr_quotation ' >"' p_stats2 '" ' ; host link 'cp "' p_stats2 '" "' p_stats1 '" ' ; % ; FOR i_folder WITH tell gage shape folderList DO % must take care of apostrophes in folderList@i_folder! ; folder_noApo := str_replace_subStr folderList@i_folder '''' (link chr_apo '\' chr_apo chr_apo) ; host link 'cat "' p_stats1 '" | sed ' chr_quotation 's/\(^' folder_noApo '.*$\)/\1' (string folderCountsALL@i_folder) ',/' chr_quotation ' >"' p_stats2 '" ' ; host link 'cp "' p_stats2 '" "' p_stats1 '" ' ; ENDFOR ; % special case of "emSplit leftover" ; host link 'cat "' p_stats1 '" | sed ' chr_quotation 's/\(^emSplit leftover.*$\)/\1' (string leftoverCount) ',/' chr_quotation ' >"' p_stats2 '" ' ; host link 'cp "' p_stats2 '" "' p_stats1 '" ' ; % totals ; host link 'cat "' p_stats1 '" | sed ' chr_quotation 's/\(^Total email count for all folderList.*$\)/\1' (string folderCountsTotal) ',/' chr_quotation ' >"' p_stats2 '" ' ; host link 'cp "' p_stats2 '" "' p_stats1 '" ' ; EACH write 'Manually transfer emails to : ' filters_manual ; } IF flag_debug THEN write 'loading emailSplit_and_stats' ; ENDIF ; #] emailSplit_and_stats IS { emailSplit_folder ; emailSplit_stats ; } - emailSplit_and_stats IS { emailSplit_folder ; emailSplit_stats ; } #*********************************************** #] emailSplit_statsManual IS - post folderCountsALL@i (but NOT keyCountsALL@i) in p_stats1 # MANUALLY done AFTER using emailSplit_folder and after manually re -allocating emails among split folderList !! # BUT - emailSplit_and_stats updates to p_stats1 must still be in d_temp # does email count in each folder (doesn't use emailSplit_folder counts) IF flag_debug THEN write 'loading emailSplit_statsManual' ; ENDIF ; #] emailSplit_statsManual IS - emailSplit_statsManual IS { LOCAL emCount emCountTotal i_folder filters_manual filters_manualCounts p_filters_manual ; NONLOCAL folderList FAQFolderList p_stats1 p_stats2 ; % ; emCountTotal := 0 ; % ; p_filters_manual := d_temp EACHRIGHT link filters_manual ; filters_manualCounts := (gage shape filters_manual) reshape 0 ; filters_manual := 'emSplit dunno' 'emSplit inquiries' 'emSplit leftover' ; % do NOT list the date after manual ; % ; % counts of "normal" folderList ; FOR i_folder WITH tell gage shape folderList DO IF (file_exists ' -f' FAQFolderList@i_folder) THEN emCount := email_count FAQFolderList@i_folder ; emCountTotal := emCountTotal + emCount ; host link 'cat "' p_stats1 '" | sed ' chr_quotation 's/\(^' folderList@i_folder '.*$\)/\1' (string emCount) ';/' chr_quotation ' >"' p_stats2 '" ' ; host link 'cp "' p_stats2 '" "' p_stats1 '" ' ; ELSE host link 'cat "' p_stats1 '" | sed ' chr_quotation 's/\(^' folderList@i_folder '.*$\)/\10;/' chr_quotation ' >"' p_stats2 '" ' ; host link 'cp "' p_stats2 '" "' p_stats1 '" ' ; ENDIF ; ENDFOR ; % ; % counts of filters_manual ; FOR i_folder WITH tell gage shape filters_manual DO IF (file_exists ' -f' p_filters_manual@i_folder) THEN emCount := email_count p_filters_manual@i_folder ; emCountTotal := emCountTotal + emCount ; host link 'cat "' p_stats1 '" | sed ' chr_quotation 's/\(^' filters_manual@i_folder '.*$\)/\1' (string emCount) ';/' chr_quotation ' >"' p_stats2 '" ' ; host link 'cp "' p_stats2 '" "' p_stats1 '" ' ; ELSE host link 'cat "' p_stats1 '" | sed ' chr_quotation 's/\(^' filters_manual@i_folder '.*$\)/\10;/' chr_quotation ' >"' p_stats2 '" ' ; host link 'cp "' p_stats2 '" "' p_stats1 '" ' ; ENDIF ; ENDFOR ; % totals ; host link 'cat "' p_stats1 '" | sed ' chr_quotation 's/\(^Total email count for all folderList.*$\)/\1' (string emCountTotal) ';/' chr_quotation ' >"' p_stats2 '" ' ; host link 'cp "' p_stats2 '" "' p_stats1 '" ' ; } #*********************************************** #] emailSplit_archive IS - archive results for QNial "email - extract, sort, cull addresses from text.ndf" # # 15Oct2018 I need to add step to delete Thunderbird folderList IF flag_debug THEN write 'loading emailSplit_archive' ; ENDIF ; #] emailSplit_archive IS OP p_fldrSrc - emailSplit_archive IS OP p_fldrSrc { NONLOCAL f_Src f_stats d_temp p_fldrTmp p_stats p_stats1 p_stats2 ; % ; %If this file is loaddefed - must rerun "emailSplit_folder p_fldrSrc" ; emailSplit_stats ; % ; %archive files ; host link 'mv "' p_fldrTmp '" "' d_archive timestamp_YYMMDD ' ' f_Src ' no massemails" ' ; host link 'cp "' p_stats1 '" "' d_archive timestamp_YYMMDD ' ' f_stats '" ' ; host link 'mv "' p_stats1 '" "' p_stats '" ' ; host link 'rm "' p_stats2 '" ' ; host link 'find "' d_temp '" -maxdepth 1 -name "emSplit *" -printf "%f\n" | tr \\n \\0 | xargs -0 -IFILE mv "' d_temp 'FILE" "' d_archive timestamp_YYMMDD ' FILE" ' ; host link 'mv "' p_fldrSrc '" "' d_archive timestamp_YYMMDD ' ' f_Src '" ' ; host link 'echo " " >"' (link p_fldrSrc '.msf') '" ' ; } # str_replace_subStr 'e -mail address you entered couldn''t be found' '''' (link chr_apo '\' chr_apo chr_apo) find "/media/bill/ramdisk" -maxdepth 1 -name "emSplit *" | tr \\n \\0 | xargs -0 ??????' find "/media/bill/ramdisk" -maxdepth 1 -name "emSplit *" -printf "%f\n" | tr \\n \\0 | xargs -0 -IFILE echo "/media/bill/ramdisk/FILE" "/media/bill/SWAPPER/a_INNS Lexicom email server/z_Archive/181015 FILE" # Tests cat "/media/bill/SWAPPER/a_INNS Lexicom email server/a_raw responses" | sed 's/\(user unknown.*$\)/\1999\,/' >"/media/bill/ramdisk/a_raw responses" #*********************************************** ## 12Aug2017 06:55 thoughts move_results AFTER manual inspection append to current folderList? don't forget log file create log files for each filter phrase : date, number of hits (biased to most frequent, as others are "shut out" build SVM or Deep Learning to pick - keep ALL results on Midas to train grab old results from backups get rid of stderr handling - I don't get much now #********************************* # Procedure for mass email processing : # 1. compact Thunderbird email folderList # run ONE of : # qnial> massEmails (from 'email - split massEmails.ndf') # +-----+ # qnial> HELPER_WCCI2020 (from 'email - split HELPER_WCCI2020.ndf') # first run after loaddefs : # 2. qnial> emailSplit_folder % this splits "a_raw responses" into folders ; # Repeats after first run, EITHER individually : # 2. qnial> emailSplit_folder - this splits "a_raw responses" into folders # qnial> emailSplit_stats - provides email counts in each folder # OR combined : # 2. qnial> emailSplit_and_stats - provides email stats BEFORE manually fixing folders # THEN : # 3. Manually check the results, and move emails from one folder to the next # filters_manual ('emSplit dunno','emSplit inquiries','emSplit leftover') are a particular focus # look for mis -classifications # 4. qnial> emailSplit_statsManual - provides email stats AFTER manually fixing folders # 5. Check "/media/bill/ramdisk/a_raw responses stats.txt1" for status # 6. When everything OK, : # qnial> emailSplit_archive # Next process - extract email addresses from folders # use "email - extract, sort, cull addresses from text.ndf" #********************************* # run # +-----+ # select one of : # emailSplit_folder # send_Thunderbird # loaddefs link d_Qndfs 'email - mass response processing/email - split Thunderbird email folder.ndf' loaddefs_ended f_emailSplit ; # enddoc