#] #] ********************* #] loaddefs link d_Qndfs 'webSite/webSite header.ndf' # used by 'Website updates.ndf' # www.BillHowell.ca 14Apr2020 initial, adapted from 'conference guides - header.ndf' # view this file in a text editor, with [constant width font, tab = 3 spaces], no line-wrap f_webHeaders := 'webSite header.ndf' ; loaddefs_start f_webHeaders ; #**************************** # List of operators, generated with : # $ cat "$d_Qndfs""webSite header.ndf" | grep "^#]" | sed 's/^#\]/ /' # ********************* loaddefs link d_Qndfs 'webSite/webSite header.ndf' +-----+ Global variables +-----+ Find all d_webRawe html files related to webSite [URLs, convert, update] webSite_extract_pathsSubDirsFnames IS - all relevant webSite paths to p_[all, html]FileList sorted [fname, all] lists webSite_readpathsSubDirsFnames IS - read stable [path, dir] lists #************************************************************* #] +-----+ #] Global variables IF flag_debug THEN write '+-----+' ; ENDIF ; IF flag_debug THEN write 'Global variables' ; ENDIF ; # d_webRawe d_webSite d_webWork - already defined in setup.ndf d_htmlBackup := link d_webRawe (link 'z_Archive/' timestamp_YYMMDD ' backups/') ; IF (NOT path_exists "w d_htmlBackup) THEN host link 'mkdir "' d_htmlBackup '" ' ; ENDIF ; # Note : webSite_convert creates a new backup directory every time it is used, eg % create a new backup directory for every use of webSite_convert, as damage can be VERY time-costly ; d_htmlBackup := link d_webRawe 'z_Archive/' timestamp_YYMMDD_HMS ' backups/' ; host link 'mkdir "' d_htmlBackup '" ' ; # Therefore, there is a conflict between the two, which will see-saw between them #+-----+ # The following globals # are primarily for link d_Qndfs 'website urls.ndf' (eg p_webPageList) # but also used by link d_Qndfs 'Website updates.ndf' p_log := link d_webWork 'webSite urls log.txt' ; p_allFileList := link d_webWork 'webSite allFileList.txt' ; p_webPageList := link d_webWork 'webSite webPageList.txt' ; p_allLinks := link d_webWork 'webURLs_extract allLinks.txt' ; p_webSiteURLlist := link d_webWork 'webSite urlList.txt' ; p_summary := link d_webWork 'webSite summary of [fail, unknown, OK,total] links.txt' ; p_linkTypeL := link d_webWork 'webSite linkType pnames.txt' ; p_html_lines_pct20 := link d_webWork 'webSite html lines pct20.txt' ; p_html_files_pct20 := link d_webWork 'webSite html files pct20.txt' ; p_errorsURLs := link d_webWork 'urls errors list.txt' ; p_howellURLs := link d_webWork 'urls howell list.txt' ; p_mailtoURLs := link d_webWork 'urls mailto list.txt' ; p_externURLs := link d_temp 'urls extern list.txt' ; p_externURL_fails := link d_webWork 'urls extern fails.txt' ; p_externURL_OK := link d_webWork 'urls extern OK.txt' ; p_internURLs := link d_temp 'urls intern list.txt' ; p_internURL_fails := link d_webWork 'urls intern fails.txt' ; p_internURL_OK := link d_webWork 'urls intern OK.txt' ; p_pgPosnURLs := link d_webWork 'urls pgPosn list.txt' ; p_pgPosnURL_fails := link d_webWork 'urls pgPosn fails.txt' ; p_pgPosnURL_OK := link d_webWork 'urls pgPosn OK.txt' ; # These globals arise from webSite_sortCullGradeupOn1st_[all, html]PathsAndFnames below : paths allFnames allPaths allSubDirs htmlPathsList htmlFnamesGradeupList htmlFnamesGradeupList htmlSubDirsList # old stuff p_html_url_clean := link d_temp 'html urls clean.txt' ; p_externURL_clean := link d_temp 'urls extern clean.txt' ; p_externURL_sort := link d_temp 'urls extern sorted.txt' ; p_internURL_clean := link d_temp 'urls intern clean.txt' ; p_internURL_sort := link d_temp 'urls intern sorted.txt' ; p_mailtoURL_clean := link d_temp 'urls mailto clean.txt' ; p_mailtoURL_sort := link d_temp 'urls mailto sorted.txt' ; p_pgPosnURL_clean := link d_temp 'urls pgPosn clean.txt' ; p_pgPosnURL_sort := link d_temp 'urls pgPosn sorted.txt' ; #************************************************************* #] +-----+ #] Find all d_webRawe html files related to webSite [URLs, convert, update] IF flag_debug THEN write '+-----+' ; ENDIF ; IF flag_debug THEN write 'Find all d_webRawe html files related to webSite [URLs, convert, update]' ; ENDIF ; # loaddefs link d_Qndfs 'webSite/webSite header.ndf' IF flag_debug THEN write 'loading countPathsDirs' ; ENDIF ; #] countPathsDirs IS - read stable [path, dir] lists # 11Nov2020?? initial countPathsDirs IS { NONLOCAL allPathsSortedByFname allSubDirsSortedByFname allFnamesSortedByFname allPathsSortedByPath allSubDirsSortedBySubdir htmlPathsSortedByFname htmlSubDirsSortedByFname htmlFnamesSortedByFname htmlPathsSortedByPath htmlSubDirsSortedBySubdir ; write table_addRowColTitles ( 'pathDir[pass,sortBy]' 'PathsSortedByFname' 'SubDirsSortedByFname' 'FnamesSortedByFname' 'PathsSortedByPath' 'SubDirsSortedBySubdir' ) ( 'pathDir[pass,sortBy]' 'all' 'html') (5 2 reshape (EACH (gage shape) allPathsSortedByFname allSubDirsSortedByFname allFnamesSortedByFname allPathsSortedByPath allSubDirsSortedBySubdir htmlPathsSortedByFname htmlSubDirsSortedByFname htmlFnamesSortedByFname htmlPathsSortedByPath htmlSubDirsSortedBySubdir ) ) ; } IF flag_debug THEN write 'loading webSite_extract_pathsSubDirsFnames' ; ENDIF ; #] webSite_extract_pathsSubDirsFnames IS - all relevant webSite paths to p_[all, html]FileList #] sorted [fname, all] lists # use this only after [create, move] cart [path, dir]s, but archive all backups first!!! # 28Oct2020 initial, lists_sortupCullOn1st is in setup.ndf # 28Oct2020 "Conference guides" are not included as I don't want to risk corrupting them & I won't change them anyways # 29Oct2020 - there are 2729 relevant files (except 15 ^[.]files, 1 ^[#] files I can't easily grep-out) excluding "Conference guides" # There must be something more [simple, efficient] than 'rows transpose mix'? ; #11Nov2020 the "seds" need a sed condition to reject timed backups eg /media/bill/SWAPPER/Website - raw/Wickson website/201111 15h32m25s Steven Wickson.html \|[1-9]\4 [1-9]\2h[1-9]2m[1-9]\2s - I'm not sure of the format cannot easily remove yymmdd files because I sometimes use that for legit files! IF flag_break THEN BREAK ; ENDIF ; webSite_extract_pathsSubDirsFnames IS { LOCAL fnames fnamePosns paths p_nonuniqueFileList subDirs ; NONLOCAL d_webRawe p_allFileList p_webPageList flag_webSite_extract_pathsSubDirsFnames allPathsSortedByFname allSubDirsSortedByFname allFnamesSortedByFname allPathsSortedByPath allSubDirsSortedBySubdir allMulplicateFnames htmlPathsSortedByFname htmlSubDirsSortedByFname htmlFnamesSortedByFname htmlPathsSortedByPath htmlSubDirsSortedBySubdir ; % ; IF (NOT in ((phrase (str_toUpperCase 'flag_webSite_extract_pathsSubDirsFnames')) "var) (symbols 0)) THEN flag_webSite_extract_pathsSubDirsFnames := l ; EACH write ' executing webSite_extract_pathsSubDirsFnames' ' This is done only on initial loaddefs, or when manually invoked after changes' ; p_nonuniqueFileList := link d_temp 'webSite_extract_pathsSubDirsFnames nonuniqueFileList.txt' ; host link 'find "' d_webRawe '" -type f -name "*" | grep --invert-match "z_Old\|z_Archive\|code develop_test\|System_maintenance\|Qnial_bag\|Cool emails/\|Electric Universe/References/\|Electric Universe/References/\|Yoonsuck Choe - conf program book/\|fin Head\|Menu\|fin [F,f]ooter\|fin organisations\|i9018xtp.default/extensions/\|[0-9]\{6\} [0-9]\{2\}h[0-9]\{2\}m[0-9]\{2\}s " | sort >"' p_nonuniqueFileList '" ' ; allMulplicateFnames := aryList_extractMulplicate_subArys "path_extract_fname (strList_readFrom_path p_nonuniqueFileList) ; host link 'sort -u "' p_nonuniqueFileList '" >"' p_allFileList '" ' ; allPathsSortedByPath := strList_readFrom_path p_allFileList ; fnamePosns := `/ EACHRIGHT (1 + last findall) allPathsSortedByPath ; subDirs fnames := rows transpose mix (fnamePosns EACHBOTH [take, drop] allPathsSortedByPath) ; allFnamesSortedByFname allSubDirsSortedByFname allPathsSortedByFname := lists_sortupCullOn1st ( fnames subDirs allPathsSortedByPath) ; allSubDirsSortedByFname := allSubDirsSortedByFname EACHLEFT str_extractPast_strFront d_webRawe ; allSubDirsSortedBySubdir := cull sortup allSubDirsSortedByFname ; % ; host link 'find "' d_webRawe '" -type f -name "*.html" | grep --invert-match "z_Old\|z_Archive\|System_maintenance\|Qnial_bag\|Cool emails/\|Electric Universe/References/\|Electric Universe/References/\|Yoonsuck Choe - conf program book/\|fin Head\|Menu\|fin [F,f]ooter\|fin organisations|i9018xtp.default/extensions/" | sort -u >"' p_webPageList '" ' ; htmlPathsSortedByPath := strList_readFrom_path p_webPageList ; fnamePosns := `/ EACHRIGHT (1 + last findall) htmlPathsSortedByPath ; subDirs fnames := rows transpose mix (fnamePosns EACHBOTH [take, drop] htmlPathsSortedByPath) ; htmlFnamesSortedByFname htmlSubDirsSortedByFname htmlPathsSortedByFname := lists_sortupCullOn1st ( fnames subDirs htmlPathsSortedByPath) ; htmlSubDirsSortedByFname := htmlSubDirsSortedByFname EACHLEFT str_extractPast_strFront d_webRawe ; htmlSubDirsSortedBySubdir := cull sortup htmlSubDirsSortedByFname ; write countPathsDirs ; ENDIF ; } webSite_sortCullGradeupOn1st_allPathsAndFnames IS webSite_extract_pathsSubDirsFnames webSite_extractAll_pathsSubDirsFnames IS webSite_extract_pathsSubDirsFnames webSite_extract_pathsSubDirsFnames ; # EACH (gage shape) allPathsSortedByFname allSubDirsSortedByFname allFnamesSortedByFname allPathsSortedByPath allSubDirsSortedBySubdir # EACH (gage shape) htmlPathsSortedByFname htmlSubDirsSortedByFname htmlFnamesSortedByFname htmlPathsSortedByPath htmlSubDirsSortedBySubdir # olde code p_htmlPathsSiteList htmlPathsSiteSortedByPath ; host link 'find "' d_webSite '" -maxdepth 4 -type f -name "*.html" | grep --invert-match "Conference guides\|z_Old\|z_Archive\|System_maintenance\|Qnial_bag\|Cool emails/\|Electric Universe/References/\|Electric Universe/References/\|Yoonsuck Choe - conf program book/\|fin Head\|Menu\|fin [F,f]ooter\|fin organisations|i9018xtp.default/extensions/" | sort -u >"' p_htmlPathsSiteList '" ' ; htmlPathsSiteSortedByPath := strList_readFrom_path p_htmlPathsSiteList ; IF flag_debug THEN write 'loading webSite_readpathsSubDirsFnames' ; ENDIF ; #] webSite_readpathsSubDirsFnames IS - read stable [path, dir] lists # 11Nov2020 initial # do this only after [create, move] cart [path, dir]s webSite_readpathsSubDirsFnames IS { NONLOCAL d_webRawe p_webPageList allPathsSortedByFname allSubDirsSortedByFname allFnamesSortedByFname allPathsSortedByPath allSubDirsSortedBySubdir htmlPathsSortedByFname htmlSubDirsSortedByFname htmlFnamesSortedByFname htmlPathsSortedByPath htmlSubDirsSortedBySubdir htmlNormalPages htmlConfGuidePages ; % ; % There must be something more [simple, efficient] than 'rows transpose mix'? ; allPathsSortedByPath := strList_readFrom_path p_allFileList ; fnamePosns := (`/ EACHRIGHT (1 + last findall) allPathsSortedByPath) ; subDirs fnames := rows transpose mix (fnamePosns EACHBOTH [take, drop] allPathsSortedByPath) ; allFnamesSortedByFname allSubDirsSortedByFname allPathsSortedByFname := lists_sortupCullOn1st ( fnames subDirs allPathsSortedByPath) ; allSubDirsSortedByFname := allSubDirsSortedByFname EACHLEFT str_extractPast_strFront d_webRawe ; allSubDirsSortedBySubdir := cull sortup allSubDirsSortedByFname ; % ; % There must be something more [simple, efficient] than 'rows transpose mix'? ; htmlPathsSortedByPath := strList_readFrom_path p_webPageList ; fnamePosns := `/ EACHRIGHT (1 + last findall) htmlPathsSortedByPath ; subDirs fnames := rows transpose mix (fnamePosns EACHBOTH [take, drop] htmlPathsSortedByPath) ; htmlFnamesSortedByFname htmlSubDirsSortedByFname htmlPathsSortedByFname := lists_sortupCullOn1st ( fnames subDirs htmlPathsSortedByPath) ; htmlSubDirsSortedByFname := htmlSubDirsSortedByFname EACHLEFT str_extractPast_strFront d_webRawe ; htmlSubDirsSortedBySubdir := cull sortup htmlSubDirsSortedByFname ; htmlConfGuidePageFlags := 'Neural nets/Conference guides/' EACHRIGHT subStr_in_str htmlPathsSortedByPath ; htmlConfGuidePages := htmlConfGuidePageFlags sublist htmlPathsSortedByPath ; htmlNormalPages := (NOT htmlConfGuidePageFlags) sublist htmlPathsSortedByPath ; } webSite_readpathsSubDirsFnames ; #********************* # Instructions # webSite_extract_pathsSubDirsFnames - manually do this # webSite_readpathsSubDirsFnames - this is done automatically loaddefs_ended f_webHeaders ; # enddoc