#] #] ********************* #] "$d_Qndfs""webSite/1_webSite instructions.txt" - fixes for webSite link problems www.BillHowell.ca 01Jun2021 initial gathered mostly from "$d_Qndfs"'0_website notes.txt' # for geany text editor instructions, especially regular expression search-replace, see "$d_SysMaint""text processors/geany notes.txt" # for examples, see "$d_Qndfs"'0_website updates notes.txt' # uploads with "$d_PROJECTS"'Website secure/lftp update www-BillHowell-ca.sh' #**************************** # Table of , generated with : # $ grep "^#] " "$d_Qndfs/webSite/""1_webSite instructions.txt" | sed 's/^#\]/ /' # ********************* "$d_Qnd""webSite/1_webSite instructions.txt" +-----+ Commandments to avoid time-costly problems!!! 1. NEVER put [apo, quote, &]s in [subDir, fname]s - instructions below for easy removal (impractical by hand) 2. +-----+ Fixes for [apo, quote, &]s in [subDir, fname, link]s backup files!! - important safety measure during code development restore files from backups - important safety measure during code development update a webPageRawe after editing - webPageRawe_update & webPageSite_update [find, grep] specific str Detailed instructions - [search, replace] '!!linkError!!s' due to an apostrophe in the [subDir, fname] 0. Setup - define variables to make processing easier, reduce errors that be catastrophic 1. change the affected [subDir, fname]s in "$d_webRawe" (not $d_webSite") 2. find pList - files with fname 3. prepare QNial "pList" variable assignment expression use htmlList of last step, regular expression search-replace with txt-editor geany 4. replace erroneous [subDir, fname] in pList with QNial QNial is slow, but much safer with [apo, quote]s than [find, sed, grep], although the latter do work 5. Re-list to temp2 file, but this time for d_webRawe where changes were made : +-----+ other instructions search allFnamesSortedByFname for an fname search allFnamesSortedByFname for part of an fname [find, grep, sed] to replace a specific str, example : !!linkError!! be careful - can screw up many files if str is not unique!!! change strOld to strNew in pathList, for strPattern, automatic path backups to d_backup test for path existance, example failure +-----+ for an example, see "$d_Qndfs"'0_website notes.txt' : 08********08 #] +-----+ #] Commandments to avoid time-costly problems!!! #] NEVER put [apo, quote, &??]s in [subDir, fname]s - instructions below for easy removal (impractical by hand) >> usually happens absent-mindedly when creating [subDir, fname]s #] backup files!! - important safety measure during code development # pathList_backupTo_dir htmlPathsSortedByPath (link d_webRawe 'z_Archive/') #] restore files from backups - important safety measure during code development # dirBackup_restoreTo_paths l (link d_webRawe 'z_Archive/201117 17h00m21s backups/') # htmlPathsSortedByPath # flag_fnamesDated = l means that fnames are dated, for [one, several] files at a time # flag_fnamesDated = o means that fnames are NOT dated, for [pathList, dir] processing, many paths 08********08 #] +-----+ #] Overall process #] 1. [add, change, delete, restructure] d_webRawe [page, file, subDir] #] 2. Update menues and hand-check link in menus - these are NOT modified by my software!! #] 3. 'webSite maintain [menu, header, footer, body] links, TableOfContents.ndf' It's easiest to follow the code of webRaweSite_doAll, shown below with "^>> ". Handy steps for each of the clusterOps below : +-----+ Re-try : qnial> bye $ qnial qnial> loaddefs link d_Qndfs 'webSite/webSite maintain [menu, header, footer, body] links, TableOfContents.ndf' qnial> select one : clusterOps_Rsync clusterOps_WebRawe clusterOps_WebSite clusterOps_urlExternCheck qnial> webRaweSite_doAll #] 4. individual clusterOps when things aren't working clusterOps_Rsync IS { NONLOCAL d_webRawe d_webSite ; % ; EACH write '' 'clusterOps_Rsync : ' ; % rsync (ensure up-to-date-copies) from d_webRawe -> d_webSite ; % only non-html files are transferred with this command ; % html files are handled by 'webAllRawOrSite_update "webPageSite_update' below ; write 'stepin : rsync website.sh' ; host 'bash "$d_bin""rsync website.sh"' ; } clusterOps_WebRawe IS { LOCAL d_backup ; NONLOCAL d_webRawe d_webSite ; % ; EACH write '' 'clusterOps_WebRawe : ' ; % ; % force 1st update of [subDir, fname] lists. ; write 'stepin : webRawe_extract_pathsSubDirsFnames' ; webRawe_extract_pathsSubDirsFnames ; % ; % d_webRawe - update lists of [path, link, url] links as [OK, bad, unknown] ; write 'stepin : webPageRawe_update' ; d_backup := webAllRawOrSite_update "webPageRawe_update ; write 'stepin : urls_check "intern"' ; urls_check 'intern' d_backup ; write 'stepin : urls_check "bkmkEx"' ; urls_check 'bkmkEx' d_backup ; EACH write '' '' ; d_backup } clusterOps_WebSite IS { NONLOCAL d_webRawe d_webSite ; % ; EACH write '' 'clusterOps_WebSite : ' ; % work on d_webSite ; write 'Any "z_Archive|z_Old" subDirs in d_webSite will be listed below their respective commands below' ; write 'stepin : find z_Archive' ; host 'find "$d_webSite" -type d -name "z_Archive"' ; write 'stepin : find z_Old' ; host 'find "$d_webSite" -type d -name "z_Old"' ; % execute_embeds to fill out each webPage with calculated relative addressing ; write 'stepin : webPageSite_update' ; webAllRawOrSite_update "webPageSite_update ; % ??circular updates [webRawe, webURLs_extract] ; EACH write '' '' ; } clusterOps_urlExternCheck IS { NONLOCAL d_webRawe d_webSite ; % ; EACH write '' 'clusterOps_Rsync : ' ; write 'stepin : webURLs_extract' ; webURLs_extract ; % write 'stepin : urls_check "extern"' ; urls_check 'extern' ; % operates on d_webSite only ; write 'stepin : webSite_link_counts' ; webSite_link_counts ; } #] 5. Overall doAll once everything is working well : webRaweSite_doAll IS { NONLOCAL d_webRawe d_webSite ; % ; % 04Jun2021 ??circular updates [webRawe, webURLs_extract] ; clusterOps_Rsync ; clusterOps_WebRawe ; clusterOps_WebSite ; clusterOps_URLcheck ; EACH write '' '' ; } #] 6. before uploads online - make sure that permissions are public!! #] 9. Upload to website - use "$d_PROJECTS"'Website secure/lftp update www-BillHowell-ca.sh' also [wget, curl, FileZilla]? 08********08 #] +-----+ #] Replacements involving '!!linkError!!' that won't cause massive damage. Just like what I have been doing, bypass hand stuff as it causes brain damage. 0n2021 I need a lot more useful [detail, examples] here Example - select paths from 'urls errors list.txt' : /media/bill/WebSite/!!linkError!!/national/nationalpost/ /media/bill/WebSite/!!linkError!!/national/nationalpost/search/ /media/bill/WebSite/!!linkError!!Software programming & code/bin/SSH/ /media/bill/WebSite/!!linkError!!Software programming & code/System_maintenance/ /media/bill/WebSite/!!linkError!!SP500 1928-2020 yahoo finance.dat p_fList_extractPathsWith_str.sh $ linkErrorSubDir_targetPathsReplace IS OP linkErrorSubDir { LOCAL ; NONLOCAL d_webRawe ; qnial> target_paths := strList_readFrom_path (link d_temp 'html target paths.txt') use text editor (geany) to convert form below, being careful to choose replacement subDirs : qnial> EACH str_replaceIn_pathList (o d_webRawe '!!linkError!!/national/nationalpost/' 'Climate - Kyoto Premise fraud/' target_paths) (o d_webRawe '!!linkError!!/national/nationalpost/search/' 'Climate - Kyoto Premise fraud/' target_paths) (o d_webRawe '!!linkError!!Software programming & code/bin/SSH/' 'bin/SSH/' target_paths) (o d_webRawe '!!linkError!!Software programming & code/System_maintenance/' 'System_maintenance/' target_paths) When working, convert to generalized Qial optr?>> nyet - simple enough as is 08********08 #] +-----+ #] Detailed instructions - [search, replace] '!!linkError!!s' due to an apostrophe in the [subDir, fname] # requires some adaptation #] 0. Setup - define variables to make processing easier, reduce errors that be catastrophic NOTICE : include ['!!linkError!!', subDir, fname] as needed to make specific to LINKS ONLY!! don't want regular text to be changed, don't bother with [#=; backtrack ;=#] - not necessary DON'T include "#d_webSite",as this is replace with "$d_webRawe"!! DO include [start,trail]ing slashes for subDir replacement term in the search term include trailing slash for subDir replacements ONLY For [apo,quote,&]s cannot use directly with unix : "!!linkError!!Paul L Vaughan/Vaughan 120324 The Solar Cycle's Footprint on Terrestrial Climate.PDF" >>[yes,no] is a problem here $ unix_noRootLinkErrorApoQuote='!!linkError!!Paul L Vaughan/Vaughan 120324 The Solar Cycle' $ unix_fixed='Paul L Vaughan/Vaughan 120324 The Solar Cycles Footprint on Terrestrial Climate.PDF' $ echo "$unix_noRootLinkErrorApoQuote" ; echo "$unix_fixed" Pay attention to [apo, quote, &]s in [subDir, fname]s qnial> qnial_error := (link '!!linkError!!Paul L Vaughan/Vaughan 120324 The Solar Cycle' chr_apo 's Footprint on Terrestrial Climate.PDF') qnial> qnial_replace := 'Paul L Vaughan/Vaughan 120324 The Solar Cycles Footprint on Terrestrial Climate.PDF' qnial> EACH write qnial_error qnial_replace ; #] 1. change the affected [subDir, fname]s in "$d_webRawe" (not $d_webSite") This is done manually via the fileManager, before creating pList #] 2. find pList - files with fname $ find "$d_webSite" -type f -name "*.html" | tr \\n \\0 | xargs -0 -IFILE grep --with-filename --line-number "$unix_noRootLinkErrorApoQuote" "FILE" | grep --invert-match "z_Old" | grep --invert-match "z_Archive" >"$d_temp""linkError fix temp1.txt" #] 3. prepare QNial "pList" variable assignment expression #] use htmlList of last step, regular expression search-replace with txt-editor geany geany regexpr multiline search-replace search: (/media/bill/WebSite/)(.*):(.*):(.*) replace : /media/bill/Dell2/Website - raw/\2 geany add apos to above list pre-pended by 'qnial> pList := ' : geany escSeq search: \n replace : ' ' qnial> pList := ' #] 4. replace erroneous [subDir, fname] in pList with QNial #] QNial is slow, but much safer with [apo, quote]s than [find, sed, grep], although the latter do work qnial> str_replaceIn_pathList o d_webRawe qnial_error qnial_replace pList #] 5. Re-list to temp2 file, but this time for d_webRawe where changes were made : $ find "$d_webRawe" -type f -name "*.html" | tr \\n \\0 | xargs -0 -IFILE grep --with-filename --line-number '$unix_fixed' "FILE" | grep --invert-match "z_Old" | grep --invert-match "z_Archive" >"$d_temp""linkError fix temp2.txt" 08********08 #] +-----+ #] Instructions for individual web-[page, site] updates (check at each step!) # for Quick [searches, fixes] for bad webSite links - example [QNial-bash] fixes # see "$d_Qndfs"'1_webSite instructions.txt' # 24Oct2020 This is the THIRD stage, after [creating, editing] cart [webPages, Menus, headers, footers] # 1. make structural changes to the website - adapt [directories, files, menus] a) menus! these are key changes affecting many webPages 25May2021 this is not working properly, probably a "hidden step" is required? maybe manual? b) do NOT include d_webRawe = "/media/bill/Dell2/Website - raw/" in links (or directories will not be linked properly) qnial> pathList := host_result (link 'find "$d_webRawe" -type f -name "*.html" | grep --invert-match "z_Old\|z_Archive" ') qnial> str_replaceIn_pathList l d_webRawe '/media/bill/Dell2/Website - raw/' '' pathList c) for analysis of problems, see section above : #] Quick [searches, fixes] for bad webSite links - example [QNial-bash] fixes # 2. check for errors and required updates in test mode a) 'webSite header.ndf' - normally won't require alteration b) 'webSite maintain [menu, header, footer, body] links, TableOfContents.ndf' (this file) # 3. mount "WebSite" USB drive!!! # 4. manually test "$d_bin""rsync website.sh" settings - see notes in "$d_SysMaint""Linux/rsync notes.txt" ONLY activate webRawe_to_webSite set options="test" and run manually, NOT from QNial!! set options="change" when ready to use this QNial program & make changes (eg run webRaweSite_doAll etc) true changes can take portions of an hour for a large-size transfer of many files 25May2021 this took ~15 minutes (1 year since last rsyn, many files) # 5. manually test bash "$d_bin""webSite check for [z_Archive, z_Old].sh" if there are [z_Archive, z_Old], then this is a problem! # Run a test for individual webPages : # For manual running of webPageRawe_update : # FIRST -run first lines of code below to make sure d_htmlBackup is present!!! # d_htmlBackup := link d_webRawe (link 'z_Archive/' timestamp_YYMMDD ' backups/') ; # IF (NOT path_exists "d d_htmlBackup) THEN host link 'mkdir "' d_htmlBackup '" ' ; ENDIF ; # replace with actual path) & run : # a) webPageRawe_update - for single webPage. Example : webPage := link d_webRawe 'economics, markets/currency-crypto/Cryptos versus [currencies, 10 year [rates, bonds]].html' OR more simply : webPageRawe_update l (link d_webRawe 'index.html') b) webPageSite_update l (link d_webRawe 'index.html') # Update whole d_[webRawe, webSite] ONLY when everything looks good : 25May2021 current problem - error messages for every z_[Archive, Old] perhaps caused by major directory revamps, 2 in the last year+ webAllRawOrSite_update l "webPageRawe_update webAllRawOrSite_update l "webPageSite_update CHECK results # Update whole d_webSite ONLY when everything is working OK : # webRaweSite_doAll # test several webPages on d_webSite - links, images etc # for uploads online - make sure that permissions are public!! # loaddefs link d_Qndfs 'webSite/webSite maintain [menu, header, footer, body] links, TableOfContents.ndf' 08********08 #] +-----+ #] other instructions 08********08 #] update a webPageRawe after editing - webPageRawe_update & webPageSite_update # webPageRawe_update l (link d_webRawe 'Neil Howell/_Neil Howell.html') # webPageSite_update l (link d_webRawe 'Neil Howell/_Neil Howell.html') 08********08 #] [find, grep] specific str # $ find "' d_webRawe '" -type f -name "*.html" | tr \\n \\0 | xargs -0 -IFILE grep --with-filename --line-number "!!linkError!!" "FILE" | grep --invert-match "z_Old" | grep --invert-match "z_Archive" 08********08 #] search allFnamesSortedByFname for an fname # (= (link 'Howell 120729 Why was France' chr_apo 's King Louis XIV called the Sun King.pdf') allFnamesSortedByFname ) sublist allPathsSortedByFname Howell 120729 Why was France's King Louis XIV called the Sun King.pdf 08********08 #] search allFnamesSortedByFname for part of an fname # ((link 'Howell 120729 Why was France' chr_apo 's King Louis XIV called the Sun King.pdf') subStr_in_str allFnamesSortedByFname ) sublist allPathsSortedByFname Howell 120729 Why was France's King Louis XIV called the Sun King.pdf 08********08 #] [find, grep, sed] to replace a specific str, example : !!linkError!! #] be careful - can screw up many files if str is not unique!!! # $ find "/media/bill/SWAPPER/Website - raw/" -type f -name "*.html" | tr \\n \\0 | xargs -0 -IFILE grep --with-filename --line-number "!!linkError!!" "FILE" | grep --invert-match "z_Old" | grep --invert-match "z_Archive" | sed "s/!!linkError!!/[#=; backtrack ;=#]/g" 08********08 #] change strOld to strNew in pathList, for strPattern, automatic path backups to d_backup # be careful - can screw up many files if str is not unique!!! # str_replaceIn_pathList l d_webRawe '!!linkError!!' '' htmlPathsSortedByPath # all webPages : str_replaceIn_pathList l d_webRawe (link '[#!: path_insertIn_fHand' chr_tab chr_tab chr_tab chr_tab chr_tab '(link d_webWork ' chr_apo 'confHead.html' chr_apo ')' chr_tab 'fout ;') (link '[#!: path_insertIn_fHand (link d_webWork ' chr_apo 'fin Head_one.html' chr_apo ') fout ; ' chr_newline ' Howell : [#=; fname ;=#] ' chr_newline '[#!: path_insertIn_fHand' chr_tab chr_tab chr_tab chr_tab chr_tab '(link d_webWork ' chr_apo 'fin Head_two.html' chr_apo ') fout ; ') htmlPathsSortedByPath # just test with one file str_replaceIn_pathList o d_webRawe (link '[#!: path_insertIn_fHand' chr_tab chr_tab chr_tab chr_tab chr_tab '(link d_webWork ' chr_apo 'confHead.html' chr_apo ')' chr_tab 'fout ;') (link '[#!: path_insertIn_fHand '(link d_webWork ' chr_apo 'fin Head_one.html' chr_apo ') fout ; ' chr_newline ' Howell : [#=; fname ;=#] ' chr_newline '[#!: path_insertIn_fHand' chr_tab chr_tab chr_tab chr_tab chr_tab '(link d_webWork ' chr_apo 'fin Head_two.html' chr_apo ') fout ; ') (solitary (link d_webRawe 'Neural nets/Conference guides/Author guide website/Author guide.html')) # confGuide webPages : str_replaceIn_pathList l d_webRawe (link '[#!: path_executeEmbedsInsertIn_fHand (link d_webWork ' chr_apo 'confMenu_publications.html' chr_apo ') phraseValueList ;') (link '[#!: path_executeEmbedsInsertIn_fHand (link d_webWork ' chr_apo 'confMenu_authors.html' chr_apo ') phraseValueList ;') htmlConfGuidePages qnial> str_replaceIn_pathList o d_webRawe (link '/120214 Venus et Mars, au dela d' chr_apo 'une histoire d amour/') '/120214 Venus et Mars, au dela d une histoire d amour/' pList Re-check to temp2 file, but this time for d_webRawe where changes were made : $ find "$d_webRawe" -type f -name "*.html" | tr \\n \\0 | xargs -0 -IFILE grep --with-filename --line-number '120214 Venus et Mars, au dela d' "FILE" | grep --invert-match "z_Old" | grep --invert-match "z_Archive" >"$d_temp""Venus et Mars line-links2.txt" >> still have 16 instead of 19 before!!??? >> but it's OK! the subDir has been corrected 08********08 #] test for path existance, example failure # qnial> (find_Howell 'Howell - Are we ready for global cooling.pdf' allFnamesSortedByFname) pick allPathsSortedByFname ?address #] +-----+ #] for an example, see "$d_Qndfs"'0_website notes.txt' : 0Jun2021 70 failed links I can handle manually >> quite possibly menu links? Check one-at-a-time and seek ways of fixing computer coding # enddoc