#!/bin/sh #] #] ********************* #] "$d_fileops"'fileops.sh' - library of handy bash functions for file needs #] # do NOT run this directly!, use, for example, "$d_bin"'fileops run.sh' # www.BillHowell.ca 01Jun2021 initial, 11Jan2025 declare most variables "local" # view in text editor, using constant-width font (eg courier), tabWidth = 3 # backup wrkSpc see : # "$d_bin"'starter/pgm_bash fileops.sh' # "$d_bin"'starter/start_webLocal hacked.sh' # # # # 48************************************************48 # 48************************************************48 # List of operators [view, generate] [in, with] : # $ grep "^#]" "$d_fileops"'fileops.sh' | sed 's/^#\]/ /' >"$d_fileops"'fileops.sh'' TblOfCont.txt' # # 48************************************************48 #] +-----+ #] Setup # bash sources [standardheader, strings [, header]].sh also loaded #] [func, process]*[Start, Ender] see : "$d_bin""standard header.sh" #] this is NOT used formain libraries [fileops, strings, etc].sh ([usefulness, efficiency] issues) source "$d_strings"'strings.sh' source "$d_fileops"'fileops header.sh' #] +-----+ #] ToDos #] 19Nov2025 pinnL_pSedExpL_get_pStrL: use tac (reverse of cat) to reverse whole file! # Python modules #p from pathlib import Path #p import os # #p import sys # #p import re # regular expressions #p #p import "$d_myPy"'stdHdr.py' # ermsg='' # ermsg="$ermsg\n"$( ) # echo "$ermsg" # 48************************************************48 #] 02Jun2021 I have assigned not-so-random numbers to FILE with read loops #] to avoid "crossover effects of nesting" of this bash library. # For example : # while read -u 9 FILE149; do # generate a suitable random number with : # qnial> floor (1000 * random 1) # 48************************************************48 #] +-----+ #] fprintf #() LMDE doesn't have this!! not worth doing... fprintf() { echo '' } #] +-----+ #] test function [tool, template]s - must see "$d_bin"'0_test/fileops/fileops test.sh' : #] funcNm_pOvrL_argL_execute #(funcNm pOvrL argL) flexArgs #] +-----+ #] [symbol, function] symbol codes - to prevent "cross-over" between library calls # 24Feb2024 initial #] code_get #(no args) [symbol, function] codes: list all being used #] to check against new random codes code_get() { local pCode="$d_bin"'codeL [symbol. function].txt' local pTemp="$d_temp"'codeL [symbol. function] temp.txt' local d_sec="$d_PROJECTS"'bin - secure/' # 12Jan2025 now encrypted - not easy to list d_Loc="$d_bin"'webLocal/' d_Onl="$d_bin"'webOnline/' grep '[0-9]\{3\}' "$d_fileops"'fileops.sh' | sed 's|.*\([0-9]\{3\}\).*|\1|g' >"$pTemp" grep '[0-9]\{3\}' "$d_fileops"'fileops header.sh' | sed 's|.*\([0-9]\{3\}\).*|\1|g' >>"$pTemp" grep '[0-9]\{3\}' "$d_strings"'strings.sh' | sed 's|.*\([0-9]\{3\}\).*|\1|g' >>"$pTemp" grep '[0-9]\{3\}' "$d_Loc"'webLocal library.sh' | sed 's|.*\([0-9]\{3\}\).*|\1|g' >>"$pTemp" grep '[0-9]\{3\}' "$d_Loc"'webLocal header.sh' | sed 's|.*\([0-9]\{3\}\).*|\1|g' >>"$pTemp" grep '[0-9]\{3\}' "$d_Onl"'webOnline library.sh' | sed 's|.*\([0-9]\{3\}\).*|\1|g' >>"$pTemp" grep '[0-9]\{3\}' "$d_Onl"'webOnline library header.sh' | sed 's|.*\([0-9]\{3\}\).*|\1|g' >>"$pTemp" cat "$pTemp" | sort -u >"$pCode" } #] code_make #(fractionSeed) new [symbol, function]s: generate [new, unique] code code_make() { local pCode="$d_bin"'codeL [symbol. function].txt' code_get #read -p 'input random fraction (eg. 0.142) : '"$seed" local bolOut=1 # while [ "$bolOut" ] for i in {0..10..1} do local codeNow=$((1 + RANDOM % 1000)) local bolSeed=$( grep "$codeNow" "$pCode" ) if [[ -z "$bolSeed" ]]; then bolOut=0; fi done echo "$codeNow" } #] +-----+ #] [dir, pth] basic functions #] I do NOT yet have [p,d][new, ovr, add, etc] distinctions that I created in QNial! #] chmod_Howell #() change standard access rights #] permissions [d_web, online] : root - don't touch!!, dir permissions = 755, file = 644 # shell files generally need 644, so they open in a txtEditor when clicked # in browser directory listing , and don't ask to be launched # executable files without extensions, permission 755 so they can run # 03Sep2023 initial, 03Oct2023 cleanup chmod_Howell() { local date_ymdhms=$(date +"%0y%0m%0d %0kh%0Mm%0Ss") echo >>"$p_log" "$date_ymdhms chmod_Howell" local d_webHtml="$d_PROJECTS"'webHtmlOnly/' local pLog="$d_bin""bin logs/chown_Howell log.txt" echo >>"$pLog" "" echo >>"$pLog" "************************" echo >>"$pLog" "chmod_Howell, date_ymdhms" echo >>"$pLog" "" # normal [dir, fil]s - txtEdit sudo find "$d_web" -type d -print0 | xargs -0 chmod 755 sudo find "$d_web" -type f -name "*.html" -print0 | xargs -0 chmod 644 sudo find "$d_webHtml" -type d -print0 | xargs -0 chmod 755 sudo find "$d_webHtml" -type f -name "*.html" -print0 | xargs -0 chmod 644 # executable files without extensions - run sudo chmod 755 "$d_bin"'linux' sudo chmod 755 "$d_bin"'qnial' sudo chmod 755 "$d_bin"'start' } #] chown_Howell #(d_target) change standard access rights #] permissions [d_web, online] : root - don't touch!!, dir permissions = 755, file = 644 # online [fil, dir] permissions handled by lftp (based on d_web permissions) # d_target="/media/bill/Midas" # d_target="/media/bill/HOWELL_BASE" # d_target="/media/bill/RaspPi_ext4_32Gb" # d_target="/home/bill/PROJECTS" # d_target="/home/bill/web" # 04Aug2018 initial - always having problem with permissions - nyet - my screw-ups # 03Oct2023 moved to fileops.sh chown_Howell() { local date_ymdhms=$(date +"%0y%0m%0d %0kh%0Mm%0Ss") echo >>"$p_log" "$date_ymdhms chown_Howell" local d_target="$1" local pLog="$d_bin""bin logs/chown_Howell log.txt" echo >>"$pLog" "" echo >>"$pLog" "************************" echo >>"$pLog" "chown_Howell, date_ymdhms" sudo chown >>"$pLog" 2>&1 -R -c bill:bill "$d_target" echo >>"$pLog" "" } #] pth_get_fnam #(pth) get fname with extension, pth must have at least one '/' # 13Jul2022 initial pth_get_fnam() { local fnam=$( echo "$1" | sed 's|.*\/\(.*\)|\1|' ) echo "$fnam" } #] pth_getExists_fnam #(pth) get fname with extension, pth must have at least one '/' # 13Jul2022 initial pth_getExists_fnam() { local pth="$1" local fnam='' local msg='' if [ -f "$pth" ]; then fnam=$( echo "$pth" | sed 's|.*\/\(.*\)|\1|' ) fnam=$( echo "$pth" | sed 's|.*[/]\(.*\)|\1|' ) msg="$fnam" else msg="pth_get_fnam error, OR[no-exist, isDir] $pth" fi echo "$msg" } # tests : # pth_get_fnam "$d_Qndfs"'arrays (copy).ndf' # OK - 'arrays (copy).ndf' # pth_get_fnam '/media/bill/Dell2/Website - raw/zzz/arrays (copy).ndf' # OK error : 'pth_get_fnam error, OR[no-exist, isDir] /media/bill/Dell2/Website - raw/zzz/arrays (copy).ndf' # pth_get_fnam "$d_Qndfs"'QNial [symbol, arg] changes/arrays/' # OK interesting : returns 'pth_get_fnam error, OR[no-exist, isDir] 565pth536' # so dirs ending with '/' give error - nice! (otherwise returns last subDir?) #] pth_get_fNamNoExt #() get fname without extension # 04Mar2023 initial pth_get_fNamNoExt() { local pth="$1" if [ -f "$pth" ]; then local fNamNoExt=$( echo "$pth" | sed 's|.*/\(.*\)\..*|\1|' ) local msg="$fNamNoExt" else local msg="pth_get_fnam error, OR[no-exist, isDir] $pth" fi echo "$msg" } # 04Mar2023 tests # echo "$d_temp"'port bash to python- funcArgNames.txt' | sed 's|.*/\(.*\)\.\(.*\)|\1|' #] pth_get_ext #() pth must exist or gives error # 26Feb2024 initial # 30Nov2024 search "Linux bash and how do U get the file extension?" # https://stackoverflow.com/questions/965053/extract-filename-and-extension-in-bash # Extract filename and extension in Bash # Asked 15 years, 5 months ago # Modified 9 months ago # Viewed 2.3m times # >> not better than what I have? # pth_get_ext() { local pth="$1" local msg='' local ext='' if [ -f "$pth" ]; then ext=$( echo "$pth" | sed 's|.*/.*\.\(.*\)|\1|' ) msg="$ext" else msg='pth_get_ext error, OR[no-exist, isDir]' fi echo "$msg" } #] fNam_get_fNamNoExt #(fNam) get fNam without extension (NOT pth!) # 23Sep2023 initial fNam_get_fNamNoExt() { local fNam="$1" local msg='' local bolPth=$( echo "$fNam" | grep "\/" ) if [[ -z "$bolPth" ]]; then msg=$( echo "$fNam" | sed 's|\(.*\)\..*|\1|' ) else msg="fNam_get_fNamNoExt error, isPth : $fNam" fi echo "$msg" } #] fNam_get_ext #(fNam) retrieve fName extension (NOT pth!) # 23Sep2023 initial fNam_get_ext() { local fNam="$1" local msg='' local bolPth=$( echo "$fNam" | grep "\/" ) if [[ -z "$bolPth" ]]; then msg=$( echo "$fNam" | sed 's|.*\.\(.*\)|\1|' ) else msg="fNam_get_ext error, isPth : $fNam" fi echo "$msg" } #] pth_get_dir #(pth) retrieve dir holding pth, dir must exist or error # 13Jul2022 initial, 28Nov2024 unix "dirname" does this!! pth_get_dir() { local pInn="$1" local dPth='' local result='' dPth=$( echo "$pInn" | sed 's|\(.*\)\/.*|\1\/|' ) if [ -d "$dPth" ]; then result="$dPth" else result='pth_get_dir error: dir not found' fi echo "$result" } #] dInn_ctime_get_pCtimeL_cp_dCtime #() find option "ctime" is in days, cp to dCtime #] critical for finding files potentially corrupted by hacking, so don't have to # look at entir # 27Jan2026 initial, 30Mar2026 [adapt, mv] to fileops.sh for general use # man find : # -atime n # File was last accessed less than, more than or exactly n*24 hours ago. # When find figures out how many 24-hour periods ago the file was last accessed, # any fractional part is ignored, so to match -atime +1, a file has to # have been accessed at least two days ago. # -ctime n # File's status was last changed less than, more than or exactly n*24 hours ago. # See the comments for -atime to understand how rounding affects the # interpretation of file status change times. # 07Mar2026 [cp, mv] quick tests [colon, hyphen] in fNam # no workee: see "$d_SysMaint"'Linux/cp notes.txt' dInn_ctime_get_pCtimeL_cp_dCtime() { dInn="$1" ctime="$2" pPthL="$3" dOvr="$4" pOvrL="$d_temp"'dInn_findCtime_pPthL_cp_dOvr pOvrL.txt' find "$dInn" -ctime -"$ctime" -type f -name "*" | grep --invert-match "z_Archive\|z_Old\|0_test" | sort -u >"$pPthL" sed "s|$dInn|$dOvr|" "$pPthL" >"$pOvrL" while IFS='' read -u 9 pPth && read -u 8 -r pOvr; do cp -p "$pPth" "$pOvr" done 9<"$pPthL" 8<"$pOvrL" } #] pth_get_dArchive #(pInn) returns dArc for pInn #] ensures that z_Archive is created if it doesn't exist, # 13Jul2022 initial pth_get_dArchive() { local pInn="$1" local dPth=$( pth_get_dir "$pInn" ) local result='' if [ -d "$dPth" ]; then local dArXiv="$dPth"'z_Archive/' if ! [ -d "$dArXiv" ]; then mkdir "$dArXiv" fi result="$dArXiv" else result='pth_get_dArchive error: dir not found' fi echo "$result" } #] pth_get_pArcDate #(pth) of form "$dArc$fNoX archive.$fExt", creates dArc if required # 20Jan2024 initial, 04May2025 upgraded and renamed to avoid confusion pth_get_pArcDate() { local pInn="$1" local dPth='' local result='' if [ -f "$pInn" ]; then dPth=$( pth_get_dir "$pInn" ) if [ -d "$dPth" ]; then dArc=$( pth_get_dArchive "$pInn" ) # ensures that z_Archive is created fNam=$( pth_get_fnam "$pInn" ) fNoX=$( fNam_get_fNamNoExt "$fNam" ) fExt=$( fNam_get_ext "$fNam" ) t_mod=$( pinn_timeModTo_yymmddhms "$pInn" ) # printf \n "t_mod= $t_mod\n" result="$dArc$fNoX $t_mod.$fExt" else result='pth_get_pArcDate error: dir not found' fi else result='pth_get_pArcDate error: pInn OR[dir, noExist]' fi echo "$result" } # 05May2025 tests , see 'fileops run.sh' # $ pInn="$d_bin"'z_Archive/startup Linux scripts 221208.sh' # $ t_mod=$( pinn_timeModTo_yymmddhms "$pInn" ) #] pth_get_pArcArch #(pinn) of form "$dArc$fNoX archive.$fExt", creates dArc if required #] end with ' archive.' # 20Jan2024 initial pth_get_pArcArch() { local pInn="$1" local dPth='' # dPth = directory of path local result='' if [ -f "$pInn" ]; then dPth=$( pth_get_dir "$pInn" ) if [ "$pInn" == "$dPth" ]; then result="$pInn"'z_Archive/' else if [ -d "$dPth" ]; then dArc=$( pth_get_dArchive "$pInn" ) # ensures that z_Archive is created fNam=$( pth_get_fnam "$pInn" ) fNoX=$( fNam_get_fNamNoExt "$fNam" ) fExt=$( fNam_get_ext "$fNam" ) result="$dArc$fNoX archive.$fExt" else result='pth_get_pArcArch error: dir not found' fi fi else result='pth_get_pArcArch error: pInn OR[dir, noExist]' fi echo "$result" } #] pth_get_byteSize #(pth)- res ipsa loquitor pth_get_byteSize() { ls -l "$1" | sed 's|^\([rwx-]*\) \([0-9]*\) \([a-zA-Z]*\) \([a-zA-Z]*\) \([0-9]*\) \([a-zA-Z]*\) \(.*\) \(.*\) \/\(.*\)|\5|' } #] pth1_pth2_isEqual #(pth1, pth2) file [tMod, size, content, etc] are same # 09Mar2026 initial # tests see "$d_bin"'0_test/fileops/pth1_pth2_isEqual/' # https://stackoverflow.com/questions/12900538/fastest-way-to-tell-if-two-files-have-the-same-contents-in-unix-linux # Fastest way to tell if two files have the same contents in Unix/Linux? # VasiliNovikov pth1_pth2_isEqual() { if cmp --silent -- "$1" "$2"; then echo 1 else echo 0 fi } # 09Mar2026 what was this for? Is it still used? # pth1_pth2_isEqual() # { # ls -l "$1" | sed 's|^\([rwx-]*\) \([0-9]*\) \([a-zA-Z]*\) \([a-zA-Z]*\) \([0-9]*\) \([a-zA-Z]*\) \(.*\) \(.*\) \/\(.*\)|\5|' # } #] pthL_fsizCompare_optr_process #(pth fsizeCompare optr)optr pthL based on fSizes (bytes) #] example : rm files <200 bytes (probably no corrections to make,rule of thumb only) pthL_fsizCompare_optr_process() { local date_ymdhms=$(date +"%0y%0m%0d %0kh%0Mm%0Ss") echo >>"$p_log" "$date_ymdhms pthL_fsizCompare_optr_process" local pthL="$1" local fsizeCompare="$2" local optr="$3" local fsiz='' local expression='' while read -u 9 pth; do fsiz=$( pth_get_byteSize "$pth" ) # echo "f_size= $fsiz" expression='if [ "$fsiz" '"$fsizeCompare"' ]; then '"$optr"' "$pth"; fi ' # echo "$expression" eval "$expression" done 9< "$pthL" } #] pSortedL_rename_as_intL #() required by ffmpeg to produce video from image files of same format #] 04Aug2024 must make generic with input args, create zero-pad func in strings.sh #] still setup for jpg files... pSortedL_rename_as_intL() { local d_imageL="$1" local d_imageTmp="$2" local f_imageL="$3" local p_imageL="$d_temp$f_imageL" local fnew='' local j=0 local n_chrs=0 local pout='' local strLeadZeros='' find "$d_imageL" -type f -name "*.jpg" | sort -u >"$p_imageL" local i="-1" while IFS='' read -r pth; do local fname=$( pth_get_fnam "$pth" ) let "i= $i + 1 " # add leading zeros to have consistent number list, max 100,000 files as-is!! n_chrs=${#i} let "n_zeros= 5 - $n_chrs" # echo "[n_chrs, n_zeros]= $n_chrs $n_zeros" j=0 strLeadZeros='' while [ $j -lt $n_zeros ] do let "j= $j + 1" strLeadZeros="$strLeadZeros"'0' done fnew="$strLeadZeros$i.jpg" # echo "fnew= $fnew" pout="$d_imageTmp$fnew" # echo "$pout" cp -p "$pth" "$pout" done < "$p_imageL" } # tests # i=0 # n_chrs=${#i} # let "n_zeros= 5 - $n_chrs" # $ echo "[n_chrs, n_zeros]= $n_chrs $n_zeros" #08********08 #] +-----+ #] SAFE [backup, restore] cart [pth, dir]s #] dir_archive_dirDated #(dir) archives ONLY files in dir # 24Apr2025 initial, 25Apr2025 mv to fileops.sh, generalized dir_archive_dirDated() { local dLocal="$1" local d_arXv='' date_ymdhm=$(date +"%0y%0m%0d %0kh%0Mm") if [ -d "$dLocal" ]; then if ! [ -d "$dLocal"'z_Archive/' ]; then mkdir "$dLocal"'z_Archive/'; fi d_arXv="$dLocal"'z_Archive/'"$date_ymdhm" mkdir "$d_arXv" cp -p "$dLocal"*.* "$d_arXv" else echo "dir_archive_dirDated error: noExist $dLocal" fi } # 26Apr2025 test - see 'fileops run.sh' #] pinn_arcArch #(pInn) NORMAL backup of form "$dArc$fNam archive.$fext", #] saves dated archive in 2nd (nested) z_Archive, creates z_Archive if doesn't exist # BENEFIT : overwrites same mod!! no more multiple copies of same file!!!! # WOW! cp doesn't change modified date if the file is the same [size, date]? beautiful # www.BillHowell.ca 20Jan2024 initial, based on mis-named pinn_arcArch # 27Sep2023 improve pinn_arcArch, two-step process to have [fixed, longTerm] fnames # 1. backup 'archive' in z_Archive (z_Archive will have a z_Archive!) # 2. save current file to fixed named 'archive' in z_Archive pinn_arcArch() { local pInn="$1" if [ -f "$pInn" ]; then local dArcArch=$( pth_get_dir "$pInn" ) # form "$dArcArch" local pArcArch=$( pth_get_pArcArch "$pInn" ) # form "$dArcArch$fNam archive.$fext" if ! [ -d "$dArcArch" ]; then mkdir "$dArcArch"; fi local fnam=$( pth_get_fNamNoExt "$pInn" ) local fext=$( pth_get_ext "$pInn" ) local pArcDate=$( pth_get_pArcDate "$pArcArch" ) # form "$dArcArc2$fnam archive.$fext" # echo "pInn= $pInn" # echo "fnam= $fnam" # echo "fext= $fext" # echo "dArcArch= $dArc" # echo "pArcDate= $pArcDate" # careful! no recursion!! don't forget pinn : if [ -f "$pArcArch" ]; then cp -p "$pArcArch" "$pArcDate"; fi cp -p "$pInn" "$pArcArch" else echo "pinn_arcArch error, no-exist: $pInn" fi } # tests # pinn_timeModTo_yymmddhms "$d_SysMaint"'timestamps notes.txt' #] pinn_arcArchRm #(pInn) archive then rm (clear) # 25Feb2024 finally did this to save annoying typing pinn_arcArchRm() { local pInn="$1" if [ -f "$pInn" ]; then pinn_arcArch "$pInn" rm "$pInn" else echo "pinn_arcArchRm error, no-exist: $pInn" fi } #] pinnL_arcArch #(pInn) normal archive of a list of pinn # 16Nov2025 initial pinnL_arcArch() { local pinnL="$1" local pinn='' while IFS='' read -u 9 pinn; do pinn_arcArch "$pinn" done 9<"$pinnL" } #] pinnL_arcArchRm #(pInn) archive then rm each of a list of pinn # 16Nov2025 initial pinnL_arcArchRm() { local pinnL="$1" local pinn='' while IFS='' read -u 9 pinn; do pinn_arcArchRm "$pinn" done 9<"$pinnL" } #] pBadL_archive_pArchiveL_restore #(pBadL) [archiveDated, restore archives as] current # Wow! this is easy with the constant " archive" labelling!! # tests - see "$d_bin"'0_test/fileops/fileops test.sh' # 03Oct2023 initial pBadL_archive_pArchiveL_restore() { local date_ymdhms=$(date +"%0y%0m%0d %0kh%0Mm%0Ss") echo >>"$p_log" "$date_ymdhms pBadL_archive_pArchiveL_restore" local pBadeL="$1" local pTempr="$d_temp"'pBadL_archive_pArchiveL_restore temp.txt' local pLoger="$d_bin"'bin logs/pBadL_archive_pArchiveL_restore log.txt' if [ -f "$pLoger" ];then pinn_arcArch "$pLoger" rm "$pLoger" fi echo >>"$pLoger" '' echo >>"$pLoger" '**************' echo >>"$pLoger" "pBadL_archive_pArchiveL_restore, $date_ymdhms" echo >>"$pLoger" '' local start="$date_ymdhms" if [ -f "$pBadeL" ];then while IFS='' read -u 9 pBader; do echo "$pBader" >>"$pLoger" local dircty=$( pth_get_dir "$pBader" ) local fNamer=$( pth_get_fnam "$pBader" ) local fNoExt=$( fNam_get_fNamNoExt "$fNamer" ) local fExten=$( fNam_get_ext "$fNamer" ) local pArXiv="$dircty"z_Archive/"$fNoExt archive.$fExten" cp -p "$pArXiv" "$pTempr" pinn_arcArch "$pBader" mv "$pTempr" "$pBader" done 9<"$pBadeL" else echo >>"$pLoger" 'pBadL_archive_pArchiveL_restore error: invalid pBadL' fi date_ymdhms=$(date +"%0y%0m%0d %0kh%0Mm%0Ss") echo >>"$pLoger" '' echo >>"$pLoger" "start = $start" echo >>"$pLoger" "end = $date_ymdhms" } #] dInn_pInnL_arcReplace_dOvr #(dInn pSubL dOvr) use pInnL to [archive, replace] same files in dOvr # 16Mar2026 initial dInn_pInnL_arcReplace_dOvr() { d_inn="$1" pInnL="$2" d_ovr="$3" while IFS='' read -r pinn; do povr=$( echo "$pinn" | sed "s|$dInn|$d_ovr|" ) if [ -s "$povr" ]; then pinn_arcArch "$povr" # cp -p "$pinn" "$povr" else echo "dInn_pInnL_arcReplace_dOvr noExist: $povr" fi done < "$pInnL" } #] wrkSpcL_arcArch #() easy backup of many workSpaces, see "$d_bin"'wrkSpcL_arcArch.sh' # 24Nov2025 initial # 25Nov2025 for some unknown reason, it's not archiving 0_test files (2 different dirs) # 26Jan2026 oops - now it works? #08********08 #] +-----+ #] SAFE [rm, rename] povrL, d_web transfer laptop[HP <-> TP]: # HP = Hewlett Packard (main) # TP = ThinkPad (d_web only) # (usually, a drive, eg d_web etc) #] dir_renameFileL #() rename an extracted list of files in dir using [sed, mv] # 05Jul2023 initial # examples : "$d_bin"'fileops run.sh' dir_renameFileL() { local date_ymdhms=$(date +"%0y%0m%0d %0kh%0Mm%0Ss") echo >>"$p_log" "$date_ymdhms dir_renameFileL() " local dinn="$1" local maxDepth="$2" local grepStr="$3" local sedStr="$4" local ptmp="$d_temp"'dir_renameFileL fNamL.txt' find "$dinn" -maxdepth "$maxDepth" | grep "$grepStr" | sort -u >"$ptmp" while IFS='' read -r line; do local pNew=$( echo "$line" | sed "$sedStr" ) echo "$pNew" # dangerous on first run!! comment-out first, then check [ptmp, # mv "$line" "$pNew" done < "$ptmp" } #] povrL_rm #(povrL) DANGEROUS!! rm all files in povrL #] must check list to make sure it's safe to delete all! (even with flatArch) #] dFlatArchive a safety measure, but povrL_rm will over-write files with same fNam # 04Mar2026 initial # will have problem with [colon, hyphen] in fNams? # I added '/home/bill/rm_bak/' to "$d_bin"'backupper/excludes home.txt' # 06Mar2026 ToDo: append # to fNam so same-fNam flatArch files not over-written povrL_rm() { dFlatArchive="$1" povrL="$2" if [ -d "$dFlatArchive" ]; then while IFS='' read -r povr; do if [ -f "$povr" ]; then fNam=$( pth_get_fname "$povr" ) cp "$povr" "$dFlatArchive$fNam" rm "$povr"; fi done < "$povrL" else echo 'povrL_rm() error: dFlatArchive doesnt exist' fi } #] dWebHPtrans_rsync_dWebTP #() many [bash, rsync] tricks, especially fNam with [colon, hyphen] #] see "$d_bin"'rsync/rsync directories.sh' #] fNamPatn_findMakeRun_pCmdL #(dWork...) find fNamPatn files, check, make mv bash file, run #] comment-out section "read -p 'CHECK pCmdL!" to make sure pCmdL OK before over-writes! # 01Mar2026 initial # povrL expressed WITHOUT dRoot of either drive d{Ovr, Out], # so it's easy to construct [source, destination] pths # eg ['/home/bill/web/', '/media/bill/WD8Tb_241224/251210 web/'] # dOut= single (flat) dir, can use pinnL to reverse, # EXCEPT for files sharing same fNam!! # the "find" must be customized: eg [dSrch, fPatn, etc] # 02Mar2026 WARNING! need to test, but I'm finished with .HoWeb.html, # so leave for next time. Problem was it saved to /home/bill/ not d_Out # draft fix is below 'sed "s|\$d_Out|$d_Out|" ', but does it work? # do this on "$d_computer" == 'laptopTP' ONLY!!!, run from 'fileops run.sh' # a kind of archive of files to backup drive, rm from original path # safer to pre-generate povrL, check that carefully, then use povrL_rm # 06Mar2026 needs [upgrade, check] next time I use!! fNamPatn_findMakeRun_pCmdL() { # create povrL of files to move into d_Out dWork="$1" fovrL="$2" dSrch="$3" fPatn="$4" d_Out="$5" fCmdL="$6" povrL="$dWork$fovrL" find "$dSrch" -type f -name "$fPatn" | grep --invert-match "z_Archive\|z_Old\|z_history\|z_historic" | sort -u >"$povrL" if [ -s "$povrL" ]; then # convert povrL to executable bash file # need three intermediate files, as I couldn't figure out one sed statement only pTmpCmd="$d_temp"'find_povrL_mv_dOut pTmpCmd.txt' pTmpCm2="$d_temp"'find_povrL_mv_dOut pTmpCm2.txt' pTmpFnm="$d_temp"'find_povrL_mv_dOut pTmpFnm.txt' sed 's|\/home\/bill\/web\/|mv "$d_web""|;s|$|" "$d_Out"|' "$povrL" >"$pTmpCmd" # 06Mar2026 not working? : sed "s|\$d_Out|$d_Out|" "$pTmpCmd" >"$pTmpCm2" sed 's|.*\/\(.*\)|\1|' "$povrL" >"$pTmpFnm" pCmdL="$dWork$fCmdL" if [ -f "$pCmdL" ]; then rm "$pCmdL"; fi while read -u 3 -r cmd && read -u 4 -r fnm; do echo "$cmd"'"'"$fnm"'"' >>"$pCmdL" done 3<"$pTmpCm2" 4<"$pTmpFnm" # read -p 'CHECK pCmdL! If K, enter Y to proceed, other chr to stop' proceed # if [ "$proceed" == 'Y' ]; then # cp "$povrL" "$d_Out" # bash "$pCmdL" else echo 'fNamPatn_findMakeRun_pCmdL: no povrL' fi } # 02Mar2026 quick tests # 11:04 2Mar2026$ dOut='/media/bill/WD8Tb_241224/260301 HoWebL removed/'; mv "$d_web""Bill Howells videos/160901 Big Data, Deep Learning, and Safety/0_Big Data, Deep Learning, and Safety.HoWeb.html" "$dOut" # >> worked!! #12:13 2Mar2026$ cmd='mv "$d_web""Bill Howells videos/220331 Hydrogen future Alberta/Howell - hydrogen future Alberta.HoWeb.html" "$d_Out"'; fnm='Howell - hydrogen future Alberta.HoWeb.html'; echo "$cmd"'"'"$fnm"'"' # mv "$d_web""Bill Howells videos/220331 Hydrogen future Alberta/Howell - hydrogen future Alberta.HoWeb.html" "$d_Out""Howell - hydrogen future Alberta.HoWeb.html" # >> OK, brilliant #] dOvr_pOvrL_arcReplaceWith_dInn #() pOvrL [archive, replaceFrom dInn] (eg dInn= a backup dir) #] pOvrL is with respect to dOvr, check pOvrL before proceeding! # 14Feb2026 initial, 30Mar2026 revamped after splitting function dOvr_pOvrL_arcReplaceWith_dInn() { dOvr="$1" pOvrL="$2" dInn="$3" if [ -s "$pOvrL" ]; then while IFS='' read -u 9 pOvr; do if [ -f "$pOvr" ]; then pinn_arcArch "$pOvr" pInn=$( echo "$pOvr" | sed "s|$dOvr|$dInn|" ) if [ -f "$pInn" ]; then becho "pInn= $pInn" pArc="$dInn$pInn" # date = now for uploads etc! cp --preserve=mode,ownership "$pInn" "$pOvr" fi fi done 9<"$pOvrL" else becho 'dInn_pOvrL_arcReplace_dOvr: pOvrL has nothing or doesnt exist' fi } #] +-----+ #] timestamp changes - all time, dates are Unix timevalue unless otherwise specified #] date %s = seconds since the Epoch (1970-01-01 00:00 UTC), good for lftp uploads #] see "$d_bin"'strings.sh' to convert from one time format to another, eg (see list) # statDate_to_timeUnix #() convert to linux stat format to timeUnix # yymmddhms_to_timeUnix #() convert to linux stat format, then $ date -d "$timeStat" +%s # note only works from 01Jan2000-present # timeUnix_tests #() I need to established fixed test file # tPthP_get_tDeltaDays #() returns days between file modified dates # yymmddhhmmP_to_tDeltaUnix #() returns the number of minutes between dates in YYMMDD_HM format # mmddYYY_to_ddmmmYYY #() convert mm/dd/YYYY to dd-MMM-YYYY (just use QNial!?!) #] pinn_get_timeAcs() #() for lftp uploads, access includes reads? # was pinn_timeAcsTo_timeUnix # from "$d_web"'CompLangs/PineScript/PuetzUWS IntlStkIdxs multi-fractals.txt' # 02Sep2023 initial, ignore UTC cirrection for now (not used for my p_log fNames) pinn_get_timeAcs() { local timeStat="$1" timeStat=$( echo "$timeStat" | grep 'Access: [0-9]' | sed 's|Access: ||' ) date -d "$timeStat" +%s } # 1 11 11 11 22 # chr (byte) positions 1 34 67 90 23 56 89 23 # $ date -d '2023-09-01 13:45:15.388594407 -0600' +%s # 1693597515 # seems to work OK? # $ stat "$d_SysMaint"'Linux/timestamps notes.txt' #] pinn_get_timeChg #() timeChg from stat "$1" #] %s =seconds since the Epoch (1970-01-01 00:00 UTC), good for lftp uploads # was timeChg_get_timeUnix # from "$d_web"'CompLangs/PineScript/PuetzUWS IntlStkIdxs multi-fractals.txt' # 02Sep2023 initial, ignore UTC cirrection for now (not used for my p_log fNames) pinn_get_timeChg() { local timeChg=$( stat "$1" | grep 'Change' | sed 's|Change: ||' ) date -d "$timeChg" +%s } #] pinn_get_timeMod #() timeChg from stat "$1" # was pinn_timeModTo_timeUnix # from "$d_web"'CompLangs/PineScript/PuetzUWS IntlStkIdxs multi-fractals.txt' # 02Sep2023 initial, 23Jan2025 redone pinn_get_timeMod() { local timeMod=$( stat "$1" | grep 'Modify' | sed 's|Modify: ||' ) date -d "$timeMod" +%s } #] pinn_getTimeModPut_pout #() set pout timeMod to that of pinn # was pinn_get_timeMod_pout #] just use touch --reference="$pinn" "$pout" # was pinn_timeModTo_pout # 20Oct2023 initial pinn_getTimeModPut_pout() { local pinn="$1" local pout="$2" touch --reference="$pinn" "$pout" #local timeModUnix=$( pinn_timeModTo_timeUnix "$pinn" ) #touch -m --date="$timeModUnix" "$pout" } #] pinn_timeModTo_yymmddhms #() archive in current dir #] trailing date so fNam sorts alphabetically with function # 21Sep2023 initial pinn_timeModTo_yymmddhms() { local timeMod=$( stat "$1" | grep 'Modify' | cut --bytes=11,12,14,15,17,18,19,20,21,22,23,24,25,26,27,28 | sed 's|:|h|g;s|\.|s|;s|h\([0-9]*\)h|h\1m|' ) echo "$timeMod" } #] pinn_timeChgTo_yymmddhms #() archive in current dir #] trailing date so fNam sorts alphabetically with function # 21Sep2023 initial pinn_timeChgTo_yymmddhms() { local pinn="$1" local timeChg=$( stat "$1" | grep 'Change' | cut --bytes=11,12,14,15,17,18,19,20,21,22,23,24,25,26,27,28 | sed 's|:|h|g;s|\.|s|;s|h\([0-9]*\)h|h\1m|' ) echo "$timeChg" } #08********08 #] +-----+ #] rename a selection of files in a dir # 05Jul2023 initial #] dir_renameFileL #() rename an extracted list of files in dir using [sed, mv] # 05Jul2023 initial # examples : "$d_bin"'fileops run.sh' dir_renameFileL() { local date_ymdhms=$(date +"%0y%0m%0d %0kh%0Mm%0Ss") echo >>"$p_log" "$date_ymdhms dir_renameFileL() " local dinn="$1" local maxDepth="$2" local grepStr="$3" local sedStr="$4" local ptmp="$d_temp"'dir_renameFileL fNamL.txt' find "$dinn" -maxdepth "$maxDepth" | grep "$grepStr" | sort -u >"$ptmp" while IFS='' read -r line; do local pNew=$( echo "$line" | sed "$sedStr" ) echo "$pNew" # dangerous on first run!! comment-out first, then check [ptmp, # mv "$line" "$pNew" done < "$ptmp" } #08********08 #] +-----+ #] str in [fil, dir][,L] - find str, results to pout #] pout is often a temporary file for further processing # forwardSlashes will likely be in str[Inn,Out], note that the `| c is used # that could also cause problems, but hopefully not in paths! #] pinn_pStrL_grep_dir #() find each str in pinn, results to dir, pNam "str.txt" # 10May2026 initial # must manually rm all pout in dir before starting (should do auto) # see 'fileops run.sh' for examples pinn_pStrL_grep_dout() { local pinn="$1" local pStrL="$2" local dout="$3" local str='' local pout='' while IFS='' read -u 9 str; do pout="$dout$str.txt" grep -w -i "$str" "$pinn" >"$pout" done 9<"$pStrL" } #] dir_fextn_pthLExclud_findPthL_pout #() find str in files with fextn in dir, results to pout #] skips [file, dir]s, eg "z_Old\|z_Archive\|webSite maintain \|file_ops.ndf" #] shows only subDirFname of $dir385 dir_fextn_pthLExclud_findPthL_pout() { local date_ymdhms=$(date +"%0y%0m%0d %0kh%0Mm%0Ss") echo >>"$p_log" "$date_ymdhms dir_fextn_pthLExclud_findPthL_pout" local dir="$1" local fextn="$2" local str="$3" local pthLExclud="$4" local pout="$5" find "$dir385" -type f -name "$fextn385" | grep --invert-match "$pthLExclud385" | tr \\n \\0 | xargs -0 -IFILE grep -i --with-filename --line-number "$str385" "FILE" | sed "s|$dir385||" >"$pout385" } #] str_getHowell_pout #() find str in Howell's [txt, html, email]s #] outputs [timeModUnix, tab, pth] -> sorted to pout # 07Nov2025 initial # grep [-i case insensitive, -l pth only output (doesn't just print pth!)] # 07Nov2025 Nuts! I often use `: in fNames!! this screws up output & isn't sure to fix str_getHowell_pout() { HowellPhrase="$1" local pout="$2" # d_temp=/media/bill/ramdisk/ defined by .bashrc p_temp="$d_temp"'str_getHowell_pout results.txt' p_tmp2="$d_temp"'str_getHowell_pout results sorted.txt' if [ -f "$p_temp" ]; then rm "$p_temp"; fi if [ -f "$p_tmp2" ]; then rm "$p_tmp2"; fi dRoot="$d_web" find "$dRoot" -type f \( -name "*.txt" -o -name "*.html" \) | grep --invert-match "z_Old\|z_Archive\|0_test" | tr \\n \\0 | xargs -0 -IFILE grep -i --with-filename "$HowellPhrase" "FILE" | rev | sed "s|.*\:||" | rev >>"$p_temp" echo >>"$p_temp" '' dRoot='/home/bill/.local/share/evolution/mail/local/cur/' find "$dRoot" -type f -name "*.*" | tr \\n \\0 | xargs -0 -IFILE grep -i --with-filename "$HowellPhrase" "FILE" | rev | sed "s|.*\:||" | rev >>"$p_temp" cat "$p_temp" | sort -u >"$p_tmp2" rm "$p_temp" while IFS='' read -u 9 pinn; do tUnixMod=$( pinn_get_timeMod "$pinn" ) echo "$tUnixMod $pinn" >>"$p_temp" done 9<"$p_tmp2" cat "$p_temp" | sort -u >"$p_tmp2" rm "$p_temp" while IFS='' read -u 9 strDateTabPth; do tUnixMod=$( echo "$strDateTabPth" | sed "s|:.*||" ) pth=$( echo "$strDateTabPth" | sed "s|.*:||" ) yyyymmmdd=$( timeUnix_to_yyyymmmdd "$tUnixMod" ) echo "$yyyymmmdd $pth" >>"$p_temp" done 9<"$p_tmp2" local date_ymdhms=$(date +"%0y%0m%0d %0kh%0Mm%0Ss") echo >"$pout" "$date_ymdhms str_getHowell_pout(): $HowellPhrase" echo >>"$pout" '' cat >>"$pout" "$p_temp" echo >>"$pout" '' } # 07Nov2025 quick test (note extra `:) : # $ echo '/home/bill/web/webOther/Howell, Steve/251021_0926 Steve Howell: Genesis creation - in a beginning.HoWeb.html: That"s the advantage of uncertainty, and "Multiple Conflicting Hypothesis". It"s not brittle. And especially "Multiple Conflicting Hypothesis". You don"t really care, "cause you would like to get towards truth, but you don"t really expect you"ll ever get there. But in terms of a scripture and a Bible, I think that probably : critically important because you"re dealing with rulers that will kill everybody if they don"t [do] anything that"s against the Bible, so you"re forming a whole new layer of religious organisations and beliefs : that counter the weight of a despot or king. Some of the time, but a lot of the time you can"t because they"re just too powerful.' | rev | sed "s|.*\:||" | rev # 07Nov2025 another quick test, new format to pick out timeUnix: # 11:27 7Nov2025$ echo '1416803774 /home/bill/web/ProjMini/JC-NPS/Webcasts/130608 NPA David de Hilster - NPA consensus or whatever.txt' | rev | sed "s|:Q:.*||" | rev # /home/bill/web/ProjMini/JC-NPS/Webcasts/130608 NPA David de Hilster - NPA consensus or whatever.txt # >> oops, backwards sed # 11:29 7Nov2025$ echo '1416803774 /home/bill/web/ProjMini/JC-NPS/Webcasts/130608 NPA David de Hilster - NPA consensus or whatever.txt' | rev | sed "s|.*:Q:||" | rev # 1416803774 /home/bill/web/ProjMini/JC-NPS/Webcasts/130608 NPA David de Hilster - NPA consensus or >> why doesn't work? use a tab in the sed expression? # 11:29 7Nov2025$ echo '1416803774 /home/bill/web/ProjMini/JC-NPS/Webcasts/130608 NPA David de Hilster - NPA consensus or whatever.txt' | rev | sed "s|.* ||" | rev # 1416803774 /home/bill/web/ProjMini/JC-NPS/Webcasts/130608 NPA David de Hilster - NPA consensus or whatever.txt # >> nyet # # 11:32 7Nov2025$ strTst='1416803774 /home/bill/web/ProjMini/JC-NPS/Webcasts/130608 NPA David de Hilster - NPA consensus or whatever.txt' # ~ # 11:38 7Nov2025$ echo "$strTst" | rev | sed "s|.*:Q:||" | rev # 1416803774 # >> OK!!! #] pinnL_str_extractPthLWithStr_pout #() grep 'st' --with-file-name, lineNum #] must escape problematic chrs in str, # www.BillHowell.ca 01Jun2021 initial # forwardSlashes will likely be in str[Inn,Out], note that the `| c is used # that could also cause problems, but hopefully not in paths! pinnL_str_extractPthLWithStr_pout() { local date_ymdhms=$(date +"%0y%0m%0d %0kh%0Mm%0Ss") echo >>"$p_log" "$date_ymdhms pinnL_str_extractPthLWithStr_pout" local pinnL="$1" local str="$2" local pout="$3" local pinn='' if [ -f "$pout" ]; then rm "$pout" fi while read -u 880 pinn; do grep -i --with-filename --line-number "$str" "$pinn" >>"$pout" done 880< "$pinnL" } # olde code # --count --with-filename # | sed "s|\($FILE880\)\(.*\)|\1|g" # | sort -u #] pinnL_str_extractCountPathsWithStrTo_pout #() [txtLines with str of pinnL] #] write [str count, path]L to pout # www.BillHowell.ca 03Jun2021 initial # forwardSlashes will likely be in str[Inn,Out], note that the `| c is used # that could also cause problems, but hopefully not in paths! pinnL_str_extractCountPathsWithStrTo_pout() { local date_ymdhms=$(date +"%0y%0m%0d %0kh%0Mm%0Ss") echo >>"$p_log" "$date_ymdhms pinnL_str_extractCountPathsWithStrTo_pout" local pinnL="$1" local str="$2" local pout="$3" if [ -f "$pout" ]; then rm "$pout" fi local p_extracted="$d_temp""pinnL_str_extractCountPathsWithStrTo_pout temp.txt" pinnL_str_extractPthLWithStr_pout "$pinnL" "$str" "$p_extracted" while read -u 1009 FILE; do grep -i --count --with-filename "$str" "$FILE" | sort -u >>"$pout" done 1009< "$p_extracted009" } #] pinnL_pStrL_getPthsWithOUTstrL_pout #() paths of pinnL withOUT str #] write [str count, path]L to pout # www.BillHowell.ca 14Nov2025 initial pinnL_pStrL_getPthsWithOUTstrL_pout() { local date_ymdhms=$(date +"%0y%0m%0d %0kh%0Mm%0Ss") echo >>"$p_log" "$date_ymdhms pinnL_pStrL_getPthsWithOUTstrL_pout" local pinnL="$1" local pStrL="$2" local pout_="$3" local p_Tmp="$d_temp"'pinnL_pStrL_getPthsWithOUTstrL_pout temp.txt' local p_Tm2="$d_temp"'pinnL_pStrL_getPthsWithOUTstrL_pout temp2.txt' local p_Tm3="$d_temp"'pinnL_pStrL_getPthsWithOUTstrL_pout temp3.txt' if [ -f "$pout_" ]; then pinn_arcArchRm "$pout_"; fi if [ -f "$p_Tm2" ]; then rm "$p_Tm2"; fi if [ -f "$p_Tm3" ]; then rm "$p_Tm3"; fi while read -u 349 str; do echo "$str" pinnL_str_extractPthLWithStr_pout "$pinnL" "$str" "$p_Tmp" cat "$p_Tmp" >>"$p_Tm2" done 349< "$pStrL" # extract paths only: sed 's|:[0-9]\+:.*||' "$p_Tm2" | sort -u >"$p_Tm3" diff "$pinnL" "$p_Tm3" --suppress-common-lines | sed 's|< ||' | grep --invert-match "^[0-9,]\+d[0-9]\+" >"$pout_" } #] pInnL_pChrP_get_pOut #(pInnL pChrP pOut) extract strL with chr[start, end] #] reverse line each item # 24Feb2024 initial adapted from pWebL_get_pLnkL_run(no args) in "$d_bin"'website.sh' # regExprs are greedy, therefore process with str_reverse [pLnkAll, sedStrt] # very trickey [yes, no] use of `\ !!! # 08Oct2023 unsuccessful sed caseInsensitive - remove grep -i option in pWebL_get_pLnkL_run # serious problem for the future pInnL_pChrP_get_pOut() { local pLogger="$1" local pLnkRaw="$2" local sedStrt="$3" local pLinker="$4" local pRevTmp="$d_temp"'webSite_get_links_run pRevTmp.txt' for sedEndd in '"' "'"; do local sedRvrs=$( echo "$sedStrt" | rev | sed "s|^|$sedEndd|" ) grep "$sedStrt" "$pLnkRaw" | rev >"$pRevTmp" local nsedRvrs=$( grep -o "$sedRvrs" "$pRevTmp" | wc -l ) echo "$sedRvrs $nsedRvrs" echo >>"$pLogger" "$sedRvrs $nsedRvrs " local nItrMax=0 while IFS='' read -u 577 linMix; do local nsedRvrs=$( echo "$linMix" | grep -o "$sedRvrs" | wc -l ) linMix=$( echo "$linMix" | sed "s|\(.*\)$sedRvrs.*|\1|" ) for (( nItr=0; nItr < $nsedRvrs; nItr++ )); do linLnk=$( echo "$linMix" | sed "s|.*$sedEndd\(.*\)|\1|" | rev ) echo "$linLnk" >>"$pLinker" linMix=$( echo "$linMix" | sed "s|\(.*\)$sedRvrs.*|\1|" ) done if [[ "$nItr" -gt "$nItrMax" ]]; then nItrMax="$nItr"; fi done 577<"$pRevTmp" done } #] pMain_strPthStrt_put_pOut #(strPthStrt, pMain, pOut) #] replace line strPthStrt with pOut # Howell default strPthStrt="#!pth " # strPth must not use ANY "sed delimiters", eg strPth "#|pth " will ruin 's|||' # 07Feb2025 initial based on pInnL_pChrP_get_pOut() pMain_strPthStrt_put_pOut() { local strPt="$1" local pMain="$2" local pOutt="$3" local date_ymdhms=$(date +"%0y%0m%0d %0kh%0Mm%0Ss") becho '+-----+' becho "pMain_strPthStrt_put_pOut, $date_ymdhms" local line='' local bol='' if [ -f "$pOutt" ]; then # printf 'pMain_strPthStrt_put_pOut() : pOutt already exists\n' rm "$pOutt" fi while IFS='' read -u 8 line; do bol=$( echo "$line" | grep "$strPt" ) if [ -n "$bol" ]; then pInst=$( echo "$bol" | sed "s|$strPt *||" ) if [ -f "$pInst" ]; then cat "$pInst" >>"$pOutt" else printf "pInsert no-exist: $pInst\n" fi else printf "$line\n" >>"$pOutt" fi done 8<"$pMain" becho '' } # tests # $ echo '#|pth /home/bill/PROJECTS/Personal/241115 Hacking damage/registeredLetters/' | sed 's/^#|pth *//' #] 08********08 #] replace in povr[L]: [chrP, strOld_strNew, [p]StrP, pStrP] #] rpl [strfixed strP , regXpr no!, sedBadChrs OK , # of [strP huge, pth huge]] #] sed [hand-escaped strP , regXpr , sedBadChrs no!, # of [strP tiny, pth huge]] #] chrCde [auto-chrCde [povr, strP], regXpr yes, sedBadChrs OK , # of [strP huge, pth huge]] slow #] ..... #] do NOT use povr functions without first checking that 0_tests povr still work!!! #] best to do USBbackup before calling povrL functions for >3 povrs!!!! #] WARNING! pStrP etc must have ONE :Q: per line to separate str[Old, New], but NO others!!! #] +-----+ #] rpl replace in povr[L]: [strfixed strP, regXpr no!, sedBadChrs OK , # of [strP huge, pth huge]] #] povr_strOld_strNew_replaceRpl #(povr strP) basic povr function # www.BillHowell.ca 01Jan2026 initial adapted from sed version povr_strOld_strNew_replaceRpl() { local povr="$1" local strOld="$2" local strNew="$3" rpl --fixed-strings --quiet "$strOld" "$strNew" "$povr" >"$povrStrPReplaceTmp337" mv "$povrStrPReplaceTmp337" "$povr" } #] povr_strP_replaceRpl #(povr strP) # www.BillHowell.ca 01Jan2026 initial adapted from sed version # 11May2026 careful: strP separated by ':Q:', not \t!!!! povr_strP_replaceRpl() { local povr="$1" local strP="$2" local strOld=$( echo "$strP" | sed 's#:Q:.*##' ) local strNew=$( echo "$strP" | sed 's#.*:Q:##' ) # becho "strOld= $strOld" # becho "strNew= $strNew" rpl --fixed-strings --quiet "$strOld" "$strNew" "$povr" } #] povrL_pStrPL_replaceRpl #(bolBecho d_log povrL pStrP) # www.BillHowell.ca 01Jan2026 initial adapted from sed version # 11May2026 careful: strP separated by ':Q:', not \t!!!! povrL_pStrPL_replaceRpl() { local povrL="$1" local pStrPL="$2" if [[ -s "$povrL" && -s "$pStrPL" ]]; then while IFS='' read -r -u 384 povr; do becho "povr= $povr" if [ -s "$povr" ]; then while IFS='' read -r -u 383 strP; do becho "strP= $strP" povr_strP_replaceRpl "$povr" "$strP" done 383< "$pStrPL" else # date_ymdhms=$(date +"%0y%0m%0d %0kh%0Mm%0Ss") becho '' # becho "$date_ymdhms povrL_pStrPL_replaceRpl error: povr doesnt exist :" becho "povrL_pStrPL_replaceRpl error: povr doesnt exist :" becho " $povr" becho '' fi done 384< "$povrL" else date_ymdhms=$(date +"%0y%0m%0d %0kh%0Mm%0Ss") becho '' becho "$date_ymdhms povrL_pStrPL_replaceRpl error: OR[povrL, pStrPL] doesnt exist" becho "povrL : $povrL" becho "pStrPL: $pStrPL" becho '' fi } #] dOvrL_pStrP_replaceRpl #(bolBecho d_log dOvrL pStrP) applies to all files in dOvrL #] not yet created, for now just see dOvrL_pStrP_replace_run() in 'fileops run.sh' #] +-----+ #] chrCode-based replace in povr[L]: #] povrL_pStrPL_replaceChrCde # see section below #] +-----+ #] sed replace in povr[L]: sed [hand-escaped strP, regXpr, sedBadChrs no!, # of [strP tiny, pth huge]] # single [str, strP] inputs (not lists): all args must be of same type: OR[normal, chrCde] # replaceChrCde: converts all args to "character coding" to avoid "problematic sed characters" # 11Oct2023 simplified extraction from povr_strP_replaceSed # 02Dec2025 [povrL, pStrP] versions retained, not [[povr, pStrP], [povrL, strP]] versions # leave these to calling function : # funcStart "$d_webWork" 'processWebSite_fix' 'povrL_pStrPL_replaceRpl' # funcEnder "$d_webWork" 'processWebSite_fix' 'povrL_pStrPL_replaceRpl' # 16Dec2025 careful with links, eg - as there are often [color, woidth, etc] specs that follow the actual link. # also, the start can vary too. # 01Jan2026 rename replace -> replaceSed #] povr_strOld_strNew_replaceSed #(povr strP) basic povr function # assumes pre-[arXiv, chrCde, povr, strP] checks are done BEFORE calling this function # www.BillHowell.ca 11Oct2023 # 30Nov2025 simplest version used by higher-level funcs # put in fileops header.sh : povrStrPReplaceTmp337="$d_temp"'povr_strP_replaceSed pOvrTmp.txt' povr_strOld_strNew_replaceSed() { local povr="$1" local strOld="$2" local strNew="$3" sed "s|$strOld|$strNew|" "$povr" >"$povrStrPReplaceTmp337" mv "$povrStrPReplaceTmp337" "$povr" } #] povr_strP_replaceSed #(povr strP) # www.BillHowell.ca 11Oct2023, convert to rpl for more generality with sedBadChrs # assumes [becho, arXiv, chrCde, povr, strP] [bol, check]s are done BEFORE calling this function # 11Oct2023 simplified extraction from povr_strP_replaceSed # 30Nov2025 simplest version here rename to be the basic func for higher-level # put in fileops header.sh : povrStrPReplaceTmp337="$d_temp"'povr_strP_replaceSed pOvrTmp.txt' # 09Dec2025 d_log set by calling function povr_strP_replaceSed() { local povr="$1" local strP="$2" local strOld=$( echo "$strP" | sed 's#:Q:.*##' ) local strNew=$( echo "$strP" | sed 's#.*:Q:##' ) sed "s|$strOld|$strNew|" "$povr" >"$povrStrPReplaceTmp337" mv "$povrStrPReplaceTmp337" "$povr" } #] povrL_pStrPL_replaceSed #(bolBecho d_log povrL pStrP) # escaping of pStrP done here, povrL escaped by povr_strP_replaceSed, if p[ov, st]Esc == 1 # example see : "$d_bin"'0_test/fileops test.sh' # 16Feb2023 initial # 16Feb2023 problem of many archives of files given length of pStrP? # 17Feb2023 addressed by use of ptemp384 # 23Feb2023 auto flagBackup - later will implement to opt out (now all of my wrk needs it!) # 28Jul2023 changed to use povr_pStrP_replace, no archive because in povr_pStrP_replace # 14Sep2023 inefficient but simple: povrL_pStr[P]_replace keep encoding pStrP with each pOvr # 10Oct2023 check if a pStrPtmp is in povr_tmp, if so do replace # avoid updates so track of changes obvious from pModDates # 30Nov2025 if statements not working, hard to debug. # make separate [becho, arXiv, chrCde] version, keep this version "very lean" # 02Dec2025 separate povrL_pStrPL_replaceChrCde (below) uses this func # #change the "process to OR[func, process] as needed when run # local processName='processWebSite_find' # local functionCmt='povrL_pStrPL_replace, used often to extract webSite links' # funcStart "$d_log" "$processName" "$functionCmt" povrL_pStrPL_replaceSed() { local povrL="$1" local pStrPL="$2" if [[ -s "$povrL" && -s "$pStrPL" ]]; then while IFS='' read -r -u 384 povr; do # becho "$povr" if [ -s "$povr" ]; then while IFS='' read -r -u 383 strP; do # becho "$povr" povr_strP_replaceSed "$povr" "$strP" done 383< "$pStrPL" else date_ymdhms=$(date +"%0y%0m%0d %0kh%0Mm%0Ss") becho '' becho "$date_ymdhms povrL_pStrPL_replaceSed error: povr doesnt exist :" becho " $povr" becho '' fi done 384< "$povrL" else date_ymdhms=$(date +"%0y%0m%0d %0kh%0Mm%0Ss") becho '' becho "$date_ymdhms povrL_pStrPL_replaceSed error: OR[povrL, pStrPL] doesnt exist" becho "povrL : $povrL" becho "pStrPL: $pStrPL" becho '' fi } #] dOvrL_pStrP_replaceSed #(bolBecho d_log dOvrL pStrP) applies to all files in dOvrL #] not yet created # for now just see dOvrL_pStrP_replace_run() in 'fileops run.sh' #] +-----+ #] chrCde replace in povr[L]: #] [auto-chrCde [povr, strP], regXpr yes, sedBadChrs OK, # of [strP huge, pth huge]] slow #] +--+ #] __This subsection is in 'strings.sh'. put here for easy reference : #] __#NOT chrSedBad?: `/ #] __chrSedBadEsy='&,.*\$^[]-' # escape-manageable (easy) #] __chrSedBadMid='&,.*\$^[]-{}|' # {}| are escape-problematic #] __chrSedBadAll='&,.*\$^[]-{}|' # use chrCde to avoid sed problems!! #] __ # very [slow, safe], missing {}|"' 03Jan2026 #] __chrGrepReplacOK='&*\$^[]-{}' #] __ #] __sed - convert between [human, code] forms of text #] __ Hum =human : human-readable, non-encoded format #] __ Cde =code : safe for sed expressions like "s|$strOld|$strNew|g" #] __ Apo =apostrophe ' : special "quote-enclosed" case, as apo can't be apo-enclosed #] __ Quo =quotation " : special "apostophe-enclosed" case, #] __ not used much here as almost all sed expressions are apo-enclosed #] __sedChrL_legend - chrL of special chrs that have to be encoded for sed #] __sed[x]To[y] - sedExpressions to [, de]-code chrSedL between [human, encoded] formats #] __strHum_sed_strCde #() convert str format [human -> code] #] __strCde_sed_strHum #() convert str format [code -> human] #] __str_strP_replace #() escape str[Old, New] then replace strOld with strNew in apo str #] __ str[, P] are escaped if bolChrCd == 1, returns same format as input #] +--+ #] p[Hum, Cde, ovr]_[chrP]_sed_p[Cde, Hum, ovr] #] search, replace] in pth[L], where strP of form "$strOld"$':Q:$strNew" # see "$d_bin"'0_test/fileops/fileops test.sh' # auto-create 'z_Archive' subDir if necessary # -i option for [capital, small] insensitive - might not want this? #] pHum_sed_pCde #() convert pth format [human -> code], eg for pStrP #] pHum must NOT already be escaped! #] pCde is overwritten # example # example with error handling see : "$d_bin"'0_test/fileops test.sh' # www.BillHowell.ca 04Aug2023 initial, from 30Jul2023 pStrP_sedEsc() pHum_sed_pCde() { local pHum="$1" local pCde="$2" sed "$sedHumToCde" "$pHum" | sed "$sedApoToCde" >"$pCde" } #] pCde_sed_pHum #() convert pth format [code -> human] #] pCde must NOT already be un-escaped! #] pCde is overwritten pCde_sed_pHum() { local pCde="$1" local pHum="$2" sed "$sedCdeToHum" "$pCde" | sed "$sedCdeToApo" >"$pHum" } #] +-----+ #] povrL_pStrPL_replaceChrCde #(povrL pStrP) slow processing ({}| problematic) # problematic chrs eg |{} : see 'strings.sh' and fileops.sh -> pHum_sed_pCde # archives are not a good way of securing a large number of files: too clumsy to reinstate! # 16Feb2023 initial as povrL_pStrPL_replaceSed(), or at least beore 14Sep2023 # 30Nov2025 split from povrL_pStrPL_replaceSed(): problems with if statements, don't need chrCde normally # 02Dec2025 further streamline to call povrL_pStrPL_replaceChrCde() { local povrL="$1" local pStrP="$2" if [[ -s "$povrL" && -s "$pStrP" ]]; then local povr_tmp="$d_temp"'povrL_pStrPL_replaceChrCde povr_tmp.txt' local pStrPtmp="$d_temp"'povrL_pStrPL_replaceChrCde pStrPtmp.txt' # convert pStrP to using character codes (chrCde), for use on all povL pHum_sed_pCde "$pStrP" "$pStrPtmp" # change each povr while IFS='' read -r -u 384 povr; do # becho "$povr" if [ -s "$povr" ]; then # convert povr to character codes (chrCde) pHum_sed_pCde "$povr" "$povr_tmp" while IFS='' read -r -u 385 strPtmp; do povr_strP_replaceSed "$povr_tmp" "$strPtmp" done 385< "$pStrPtmp" # revert to Human code in preparation for diff pCde_sed_pHum "$povr_tmp" "$povr" else date_ymdhms=$(date +"%0y%0m%0d %0kh%0Mm%0Ss") fi done 384< "$povrL" else date_ymdhms=$(date +"%0y%0m%0d %0kh%0Mm%0Ss") # for debugging # if [[ "$bolEcho" = 1 ]]; then # becho '' # becho "$date_ymdhms povrL_pStrPL_replaceChrCde error: OR[povrL, pStrP] doesnt exist" # becho "povrL : $povrL" # becho "pStrP : $pStrP" # becho '' # fi fi } # 29Nov2-25 quick test # genr=0 # blake=0 # if [[ "$genr" = 1 ]]; then echo "genr= $genr"; else echo 'genr ~= 1'; fi # if [[ "$genr" = 0 ]]; then echo 'genr= "$genr"'; else echo 'genr ~= 1'; fi #] 08********08 #] pinn[,L]_strP_replace_[d, p]out (pinn strP pout) , encoded [strP, pStrP] parallel series to above #] this targets extraction of LINKS from webPages, for which the file locations not needed # just cp whole povr section, create parallel functions # greatly reduces even more complicated conditionals, which are confusing # guarantees no write-overs!! # archives not such an issue # safest to cp all pinnL to temp files (in one dir to track) # - then only one set of code to maintain? # 12Jan2024 initial #] pinn_strP_replaceRpl_pout_sub #(pinn strP pout) # www.BillHowell.ca 12Jan2024 copy-over adapted from pinn[,L]_strP_replace() version pinn_strP_replaceRpl_pout_sub() { local pinn="$1" local strP="$2" local pout="$3" } #] pinn_strP_replaceSed_pout_sub #(pinn strP pout) # www.BillHowell.ca 12Jan2024 copy-over adapted from pinn[,L]_strP_replace() version pinn_strP_replaceSed_pout_sub() { local pinn="$1" local strP="$2" local pout="$3" } #] pinn_strP_replaceRpl_pout #(pinn strP pout) sed replace, eg LINKS! # www.BillHowell.ca 12Jan2024 copy-over adapted from pinn[,L]_strP_replace() version # final result is in : pin1Tmp892="$d_temp"'pinn_strP_replaceRpl temp1.txt' # pLogStrP337="$d_bin"'bin logs/pLog povrL_pStrPL_replace funcFamily.txt', defined header file # probably won't be used much, given cp pinnL in pinnL_pStrP_replaceRpl_poutL() pinn_strP_replaceRpl_pout() { local pinn="$1" local strP="$2" local pout="$3" if [ -s "$pinn" ]; then local strOld=$( echo "$strP" | sed 's|:Q:.*||' ) local strNew=$( echo "$strP" | sed 's|.*:Q:||' ) sed "s|$strOld|$strNew|g" "$pinn" >"$pout" else becho "$date_ymdhms pinn_strP_replaceRpl_pout error, could not get :" becho " $pinn" fi } #] pinn_strP_replaceSed_pout #(pinn strP pout) sed replace, eg LINKS! # www.BillHowell.ca 12Jan2024 copy-over adapted from pinn[,L]_strP_replace() version # final result is in : pin1Tmp892="$d_temp"'pinn_strP_replaceRpl temp1.txt' # pLogStrP337="$d_bin"'bin logs/pLog povrL_pStrPL_replace funcFamily.txt', defined header file # probably won't be used much, given cp pinnL in pinnL_pStrP_replaceRpl_poutL() pinn_strP_replaceSed_pout() { local pinn="$1" local strP="$2" local pout="$3" if [ -s "$pinn" ]; then local strOld=$( echo "$strP" | sed 's|:Q:.*||' ) local strNew=$( echo "$strP" | sed 's|.*:Q:||' ) sed "s|$strOld|$strNew|g" "$pinn" >"$pout" else becho "$date_ymdhms pinn_strP_replaceRpl_pout error, could not get :" becho " $pinn" fi } #] pinn_strP_replaceChrCde_pout #(pinn strP pout) replaceSed chrEncoded (strP, pinn) # www.BillHowell.ca 12Jan2024 copy-over adapted from pinn[,L]_strP_replace() version # final result is in : pin1Tmp892="$d_temp"'pinn_strP_replace temp1.txt' # pLogStrP337="$d_bin"'bin logs/pLog povrL_pStrPL_replace funcFamily.txt', defined header file # probably won't be used much, given cp pinnL in pinnL_pStrP_replace_poutL() # 20Dec2025 not needed for links! # 01Jan2026 because inputs are chrCoded, sed can be used (safe for sedChrBads) pinn_strP_replaceChrCde_pout() { local pinn="$1" local strP="$2" local pout="$3" if [ -s "$pinn" ]; then local pin1Tmp="$d_temp"'pinnstrP_replace temp1.txt' local pov2Tmp="$d_temp"'pinnstrP_replace temp2.txt' pHum_sed_pCde "$pinn" "$pin1Tmp" strP=$( strHum_sed_strCde "$strP" ) pinn_strP_replaceSed_pout "$pin1Tmp" "$strP" "$pov2Tmp" pCde_sed_pHum "$pov2Tmp" "$pout" else becho "$date_ymdhms pinnstrP_replace_pout error, could not get :" becho " $pinn" fi } # pinnL_strP_replace_pout(bolChrCd pinnL strP poutL) sed replace encoded (pStr, pinnL) UNTESTED! # USELESS? just use pinnL_pStrP_replace_poutL(bolChrCd pinnL pStrP poutL) #] pinnL_pStrP_replaceRpl_dOut #(bolChrCd pinnL pStrP dOut)sed replace encoded (pStrP, pinnL) LINKS! # www.BillHowell.ca 12Jan2024 copy-over adapted from pinn[,L]_strP_replace() version # PROBLEM - same fnames will be over-written in dOut # example use - extract links from webPages pinnL_pStrP_replaceRpl_dOut() { p_log="$pLogStrP337" local date_ymdhms=$(date +"%0y%0m%0d %0kh%0Mm%0Ss") becho "$date_ymdhms pinnL_pStrP_replaceRpl" local bolChrCd="$1" local pinnL="$2" local pStrP="$3" local dOut="$4" # must NOT overlap with dirs in pinnL!!!??? # dOut MUST exist - that is not done here!! as a safety if [[ -s "$pinnL" && -s "$pStrP384" && -d "$dOut" ]]; then local pOutL___="$dOut"'pinnL_pStrP_replaceRpl_dOut pOutL.txt' if [ -f "$pOutL___" ]; then rm "$pOutL___"; fi # cp all pinn in pinnL to dOut, so they are NOT changed while IFS='' read -r -u pinn; do if [ -f pinn ]; then local finn=$( ) local pout="$dOut$finn" echo >>"$pOutL___" "$pout" cp "$pinn" "$dOut$finn" else becho "noExist pinn: $pinn" fi done < "$pinnL" # now use povrL_pStrPL_replace(bolArXiv bolChrCd povrL pStrP) povrL_pStrPL_replace 0 "$bolChrCd" "$pOutL___" "$pStrP" else date_ymdhms=$(date +"%0y%0m%0d %0kh%0Mm%0Ss") becho "$date_ymdhms pinnL_pStrP_replaceRpl_dOut error: >=1 doesnt exist" becho "povrL : $povrL___" becho "pStrP : $pStrP" becho "dOut : $dOut" becho '' fi } #] +--+ #] __This subsection is in 'strings.sh' : #] __ pStrPL_rev "$pStrPL" # creates reversed pStrPL on startup #] __ strRev_sedStrtRev_sedEnddRev_get_pStrRevL #([str sed[Strt, Endd]]Rev) appends results to pStrRevL #] __ get multiple sedStr per line in strInn -> pLinkL #] __ one must REVERSE [str, sed[Strt, Endd]] because: #] __ the sequential cutting of strInn is based on "greedy sed", which finds the last occurence #] __ sedStart [] is NOT unique, and gives mixed up results (eg ] is NOT unique, and gives mixed up results (eg "$pinnRevCrop" grep -i "$StrtRev" "$pinnRevCrop" >"$pinnRev" while [ -s "$pinnRev" ]; do sed "s|\(.*\)$StrtRev.*|\1|i" "$pinnRev" >"$pinnRevBase" sed "s|.*$EnddRev\(.*\)|\1|i" "$pinnRevBase" >>"$pStrEnclRev" sed "s|\(.*\)$EnddRev.*|\1|i" "$pinnRevBase" >"$pinnRevCrop" grep -i "$StrtRev" "$pinnRevCrop" >"$pinnRev" done if [ -s "$pStrEnclRev" ]; then cat "$pStrEnclRev" | rev >"$pStrEncl"; fi } # 19Jan2026 test see 'fileops test.sh' -> pinn_Strt_Endd_get_pStrEncl_test #] pinnL_pStrPLStrtEndd_get_pPinQStr_pStrEncl_pStrEnclSortU #(args below) StrEncl = string within [Strt, Endd] # args: "$pinnL" "$pStrPL" "$poutStrPL" "$pStrWithinL" "$pStrSortUL" # DOES [pinn lines, pStrPL, pPthStrL, pStrL] intermediate reversal, [rev, fwd] double-sweep # does NOT chrCde problematic sedChrs ^$\&'",*.[]{}-\|/ -> will fail with these # maybe implement "unix replace cmd" for strs that need it? # 06Dec2025 initial # doesn't over-write pinnStrPL # 24Nov2025 would be nice to avoid reversal, but most often it is sedStrt that is unique, # whereas sedEndd isn't : # sedStr='"=FERH A<\|"=CRS GMI<\|'"'=FERH A<\|'=CRS GMI<" # sed 's|$sedStr||g' # 17Jan2026 create based on povrL_poutStrPL_get_povrStrPLtest (complete revamp to [efficient, fast]) pinnL_pStrPLStrtEndd_get_pPinQStr_pStrEncl_pStrEnclSortU() { local pinnL="$1" # don't rm, it's in dOriginal! local pStrPLStrtEndd="$2" # don't rm, it's in dOriginal! local pPinQStrL="$3"; if [ -s "$pPinQStrL" ]; then pinn_arcArchRm "$pPinQStrL" ; fi local pStrEncl="$4"; if [ -s "$pStrEncl" ]; then rm "$pStrEncl" ; fi local pStrEnclSortU="$5"; if [ -s "$pStrEnclSortU" ]; then pinn_arcArchRm "$pStrEnclSortU" ; fi local pStrEncl_temp="$d_temp"'pStrEncl_temp.txt' # not need to rm, over-written below local pinn='' local StrP='' local Strt='' local Endd='' while IFS='' read -u 074 pinn; do while IFS='' read -u 073 StrP; do Strt=$( echo "$StrP" | sed 's|\(.*\):Q:.*|\1|' ) Endd=$( echo "$StrP" | sed 's|.*:Q:\(.*\)|\1|' ) # over-writes pStrEncl_temp; multiple StrEncl per line to pStrEncl pinn_Strt_Endd_get_pStrEncl "$pinn" "$Strt" "$Endd" "$pStrEncl_temp" cat "$pStrEncl_temp" >>"$pStrEncl" sed "s|\(.*\)|$pinn:Q:\1|" "$pStrEncl_temp" >>"$pPinQStrL" done 073<"$pStrPLStrtEndd" done 074<"$pinnL" date_ymdhms=$(date +"%0y%0m%0d %0kh%0Mm%0Ss") becho "starting sort $date_ymdhms" cat "$pStrEncl" | sort -u >"$pStrEnclSortU" } # 17Jan2026 quick test # $ grep --with-filename --line-number ':Q:' "$pinn" #] povrStrPL_pStrPL_replaceRpl #(povrStrPL, pStrPL) replace strPL[Old -> New] in povrs # 28Nov2025 initial, # 29Nov2025 note that pinnL_pStrPL_getWithin_pPthStrPL_pStrL is "cluster bomb" ,whereas # povrStrPL_pStrPL_replaceRpl is "aim and fire" # 05Dec2025 [revamp, generaliz, mv]ed 'webLocal library.sh' -> fileops.sh # bolArXiv= (missing!) to force pinArchArc (lower-level funcs shouldm't always generate reports) # from: # pinnL_pStrPL_getWithin_pPthStrPL_pStrL "$pHtmAllTypL" "$pSedHrfImgL" "$pPovrLnkBadL" "$pLnkIntAllL" # 06Dec2026 I need a new function #pinnStrPL_pStrPL_get_pStrWithinL "$PinnStrPL" "$pStrPL" "$pStrWithinL" # sed 's|^.*\/\(.*\):\([0-9]\+\):\(.*\)||' "$povrOldPL" | sort -u >"$povrL" # sed 's|^.*\/\(.*\):\([0-9]\+\):\(.*\)||' "$povrOldPL" | sort -u >"$pOldNewPL" # 22Dec2025 povrStrPL order of appearance problems!?? # 25Jan2025 updated to avoid StrSrch in povrL povrStrPL_pStrPL_replaceRpl() { local povrStrPL="$1" local pStrPL="$2" local povr='' local povrL="$d_temp"'povrStrPL_pStrPL_replaceRpl povrL.txt' local povrStrPLtemp="$d_temp"'povrStrPL_pStrPL_replaceRpl povrStrPLtemp.txt' local pStrBadtemp="$d_temp"'povrStrPL_pStrPL_replaceRpl pStrBadtemp.txt' local pStrPLtemp="$d_temp"'povrStrPL_pStrPL_replaceRpl pStrPLtemp.txt' local pBadStrL="$d_temp"'povrStrPL_pStrPL_replaceRpl ' if [[ -s "$povrStrPL" && -s "$pStrPL" ]]; then sed 's|\(.*\):Q:.*|\1|' "$povrStrPL" | sort -u >"$povrL" while IFS='' read -u 386 povr; do echo "$povr" >"$povrL" grep "$povr:Q:" "$povrStrPL" | sort -u >"$povrStrPLtemp" sed 's|.*:Q:\(.*\)|\1|' "$povrStrPLtemp" | sort -u >"$pStrBadtemp" grep --regexp --FILE="$pStrBadtemp" "$pStrPL" | sort -u >"$pStrPLtemp" # ??? povrL_pStrPL_replaceRpl "$povrL" "$pStrPLtemp" done 386<"$povrL" else becho 'error: OR[povrStrPL, pStrPL] doesnt exist' becho "povrStrPL= $povrStrPL" becho "pStrPL = $pStrPL" becho '' fi } #] 08********08 #] [rgxP, pRgxP] sed [search, replace] in pth[L], rgxP of form "$rgxFind"$':Q:$rgxReplace" #] unlike strP, rgxP uses "pure sed" expressions that must NOT be encoded! #] uses triplicate form # 17Sep2023 coding has not yet been modified from rgxP format # www.BillHowell.ca 17Sep2023 initial placeholder, based on attempts ?Jun2023? #] povr_rgxP_replace #(bolArXiv bolChrCd povr rgxP)sed replace encoded (rgxP, povr) #] put `\ in front of [()] - bash not geany # www.BillHowell.ca 17Sep2023 initial copy of strP form, some adjustments # 15Mar2024 [upgrade, fix]ed povr_rgxP_replace() { local bolArXiv="$1" local povr="$2" local rgxP="$3" #pLogRgxP337 = 'povrL_pRgxP_replace funcFamily log.txt', defined in 'fileops header.sh' # bolTabInStr simple test - are any tabs in line? pRgxP_is: more complete testing of file # much of the time, tabs jave alredy been coded as ztbz, # but this test won't hurt, it's to handle cases that are NOT pre-encoded local bolTabInStr=$( echo "$rgxP" | sed 's|:Q:|ztbz|' | grep 'ztbz' ) # echo "bolTabInStr = $bolTabInStr" if [[ -n "$bolTabInStr" ]]; then if [ -s "$povr" ]; then if [ 1 -eq "$bolArXiv" ]; then pinn_arcArch "$povr" fi local povrTmp="$d_temp"'povr_rgxP_replace temp.txt' local rgxOld=$( echo "$rgxP" | sed 's#:Q:.*##' ) local rgxNew=$( echo "$rgxP" | sed 's#.*:Q:##' ) # comment out except for [test, debug] # echo "$rgxOld" # echo "$rgxNew" sed "s|$rgxOld|$rgxNew|g" "$povr" >"$povrTmp" mv "$povrTmp" "$povr" else echo >>"$pLogRgxP337" "$date_ymdhms povr_rgxP_replace error, could not get :" echo >>"$pLogRgxP337" " $povr" fi else # comment out except for [test, debug] echo >>"$pLogRgxP337" "$date_ymdhms povr_rgxP_replace error, bolTabInStr no-tab :" echo >>"$pLogRgxP337" " $rgxP" fi } #] povrL_pRgxP_replace #(bolArXiv bolChrCd povrL pRgxP)sed replace encoded (pRgxP, povrL) #] sed replaces each [strOld, strNew] in each povr, same pRgxP #] WARNING! all pRgxP cannot have :Q: other than to separate! #] put `\ in front of [()] - bash not geany # www.BillHowell.ca 17Sep2023 initial copy of strP form, some adjustments povrL_pRgxP_replace() { local date_ymdhms=$(date +"%0y%0m%0d %0kh%0Mm%0Ss") echo >>"$p_log" "$date_ymdhms povrL_pRgxP_replace" local bolArXiv="$1" local povrL="$2" local pRgxP="$3" #pLogRgxP337 = 'povrL_pRgxP_replace funcFamily log.txt', defined in 'fileops header.sh' local pRgxPTmp="$d_temp"'povrL_pRgxP_replace pRgxPTmp.txt' if [[ -s "$povrL" && -s "$pRgxP" ]]; then echo >>"$pLogRgxP337" '+-----+' echo >>"$pLogRgxP337" "povr[, L]_[, p]RgxP_replace start" if [ -f "$pRgxPTmp" ]; then rm "$pRgxPTmp" fi while IFS='' read -r -u 983 rgxP; do # ignores [comment, empty, nTab != 1] lines local commentChrs=$( echo "$rgxP" | cut --bytes=1,2,3 ) if ! [ '#] ' == "$commentChrs" ]; then nTab=$( echo "$rgxP" | tr \:Q: \\n | wc -l ) nTab=$(( $nTab - 1 )) if [[ 1 = "$nTab" ]]; then # both of the following are in Cde format echo "$rgxP" >>"$pRgxPTmp" fi fi done 983< "$pRgxP" while IFS='' read -r -u 982 povr; do echo "$povr" if [ -s "$povr" ]; then if [ 1 -eq "$bolArXiv" ]; then pinn_arcArch "$povr" fi while IFS='' read -r -u 983 rgxP; do povr_rgxP_replace 0 "$povr" "$rgxP" done 983< "$pRgxPTmp" else date_ymdhms=$(date +"%0y%0m%0d %0kh%0Mm%0Ss") echo >>"$pLogRgxP337" "$date_ymdhms povrL_pRgxP_replace error: povr doesnt exist :" echo >>"$pLogRgxP337" " $povr" echo >>"$pLogRgxP337" '' fi done 982< "$povrL" else date_ymdhms=$(date +"%0y%0m%0d %0kh%0Mm%0Ss") echo >>"$pLogRgxP337" "$date_ymdhms povrL_pRgxP_replace error: OR[povrL, pRgxP] doesnt exist" echo >>"$pLogRgxP337" "povrL : $povrL" echo >>"$pLogRgxP337" "pRgxP : $pRgxP" echo >>"$pLogRgxP337" '' fi echo >>"$pLogRgxP337" ' ' } #] povrL_pTypRgxT_replace #(type povrL pTypXprP) uses [(encoded) str, rgx] for sed #] type, either 'strP ' or 'rgxP ' #] trailing T in function name denotes "triple" (rather than pair) #] pTypXprT: #] strP of form "strOld\tstrNew" - expressions may [, not] be encoded #] rgxP of form "rgxFind\trgxReplace" - expressions must NOT be encoded! # obviates the issue of [comment, whiteSpace] lines # www.BillHowell.ca 17Sep2023 initial draft, based on attempts ?Jun2023? # 17Sep2023 still must fully translate! # 12Jan2025 problem with pStrRgxT ????? povrL_pTypRgxT_replace() { local date_ymdhms=$(date +"%0y%0m%0d %0kh%0Mm%0Ss") echo >>"$p_log" "$date_ymdhms povrL_pTypRgxT_replace" local bolArXiv="$1" local bolChrCd="$2" local povrL="$3" local pStrRgxT="$4" #pLogRgxP337 = 'povrL_pRgxP_replace funcFamily log.txt', defined in Setup section above local pStrPTmp="$d_temp"'povrL_pTypRgxT_replace pStrPTmp.txt' local pRgxPTmp="$d_temp"'povrL_pTypRgxT_replace pRgxPTmp.txt' if [[ -s "$povrL" && -s "$pStrRgxT" ]]; then # create "pure" pStrP and pRgxP files sed 's|^str||' "$pStrRgxT" >"$pStrPTmp" sed 's|^rgx||' "$pStrRgxT" >"$pStrPTmp" povrL_pStrPL_replace "$bolArXiv" "$bolChrCd" "$povrL" "$pStrPTmp" povrL_pRgxP_replace "$bolArXiv" "$povrL" "$pRgxPTmp" else date_ymdhms=$(date +"%0y%0m%0d %0kh%0Mm%0Ss") echo >>"$pLogRgxP337" "$date_ymdhms povrL_pTypRgxT_replace error: OR[povrL, pStrRgxT] doesnt exist" echo >>"$pLogRgxP337" "povrL : $povrL" echo >>"$pLogRgxP337" "pStrRgxT : $pStrRgxT" echo >>"$pLogRgxP337" '' fi } #] 08********08 #] lines - [extract, replace] * [line sequence, [func, optr] name]s] #] auto-create 'z_Archive' subDir if necessary #] -i option for [capital, small] insensitive - might not want this? #] forwardSlashes will likely be in str[Inn,Out], #] note that the `| c is used #] that could also cause problems, but hopefully not in paths! #] pinn_rmBlankLines_povr #(povr) removes [empty, with spaces] lines # 16May2026 initial pinn_rmBlankLines_povr() { local pinn="$1" local povr="$2" grep "\S" "$pinn" >"$povr" } #] pinn_getSortU_pWordL #() get sorted list of words in txt file # 16May2026 initial # see strings.sh for [sedChrProblem_to_return, sedApo_to_return, sedPunc_to_return] pinn_getSortU_pWordL() { pinn="$1" pWordL="$2" pNoProbApoPunc="$d_temp"'pinn_get_pWordL noProbApoPunc.txt' pNoBlankLines="$d_temp"'pNoBlankLines noProbApoPunc.txt' # sedChrProblem_to_return sed "$sedPunc_to_return" "$pinn" | sed "$sedApo_to_return" | sed "$sedChrProblem_to_return" >"$pNoProbApoPunc" pinn_rmBlankLines_povr "$pNoProbApoPunc" "$pNoBlankLines" sort -u "$pNoBlankLines" >"$pWordL" } #] pinnL_lineIdx_strTst_split_pYes_pNo #(pinnL lineIdx strTst pYes pNo pDifYes pDifNo)- #] split pinnL to [pYes, pNo] according to strTst in line# lineIdx #] cut takes characters #1 through strTst_len333 (could be generalized ...) # eg for non-webPages when line(idx) != "' # following : # # for now, all pin are backed up povrL_pPLstLines_replace() { local date_ymdhms=$(date +"%0y%0m%0d %0kh%0Mm%0Ss") echo >>"$p_log" "$date_ymdhms povrL_pPLstLines_replace" local flagBackup="$1" local povrL="$2" local pOldNewPL="$3" local ermsg='' # echo 'povrL_pOldNewPL_replace: ' echo "povrL: $povrL" echo "pOldNewPL: $pOldNewPL" # echo "$pStrP" # IFS='' if [[ -f "$povrL" && -f "$pOldNewPL" ]]; then IFS='' # check that pth_exists for ALL pOldNewPL while read -r -u 199 pPOldNew; do local pOld=$( printf '%s\n' "$pPOldNew" | sed 's|:Q:.*||' ) local pNew=$( printf '%s\n' "$pPOldNew" | sed 's|.*:Q:||' ) # echo "pOld: $pOld" # echo "pNew: $pNew" if [[ -f "$pOld" && -f "$pNew" ]]; then flag_pPOldNew=1 else echo 'povrL_pOldNewPL_replace error: OR(p[Old, New]) doesnt exist' flag_pPOldNew=0 fi done 199<"$pOldNewPL" # if [ $flag_pPOldNew == 1 ];then # go through povrL, skip povr that don't exist p2Tmp="$d_temp"'povrL_pOldNewPL_replace p2TmpOne.txt' IFS='' while read -r -u 198 povr; do if [ -f "$povr" ]; then if [ "$flagBackup" == "1" ]; then pinn_arcArch "$povr" fi cp "$povr" "$p2Tmp" echo "$povr" # initial repeat of flag_pOldNewPL section above, checks already done ( while read -r -u 197 pOldNew; do pOld=$( printf '%s\n' "$pOldNew" | sed 's|:Q:.*||' ) pNew=$( printf '%s\n' "$pOldNew" | sed 's|.*:Q:||' ) povr_pOld_pNew_replace 1 "$p2Tmp" "$pOld" "$pNew" done 197< "$pOldNewPL" mv "$p2Tmp" "$povr" else echo "povrL_pOldNewPL_replace error: noExist $povr" fi done 198< "$povrL" fi # else echo 'povrL_pOldNewPL_replace error: OR[povrL, pOldNewPL] doesnt exist' fi echo "$ermsg" } # 01Mar2023 prepare a list of pths that have NOT yet been processed : # cat "$d_webWork"'pinnL heads test done.txt' | sort -u >"$d_temp"'pinnL heads test done.txt' # mv "$d_temp"'pinnL heads test done.txt' "$d_webWork"'pinnL heads test done.txt' # diff "$d_webWork"'pinnL heads test done.txt' "$d_webWork"'pinnL heads test all.txt' --suppress-common-lines | grep "^>" | sed "s/^>\ //" >"$d_webWork"'pinnL heads test rest.txt' #] povr_pRgxL_cat_saveLines #() saves lineSeq to pRgxL #] used by pinn_sedFndGetPut_povr: find matching lineSeq of pinn, outputs new linSeq # relys on "all varables are global" in bash povr_pRgxL_cat_saveLines() { local date_ymdhms=$(date +"%0y%0m%0d %0kh%0Mm%0Ss") echo >>"$p_log" "$date_ymdhms povr_pRgxL_cat_saveLines" echo -e "$catLine$strEnd" >>"$pRep" # echo '+-----+' # echo "linPovr: $linPovr" # echo "rgxStr : $rgxStr" # echo "bolInrgxL: $bolInrgxL" echo -e "$catLine$strEnd: $catLine$strEnd" # echo "linNum: $linNum, idx: $idx" local catLine='' } #] povr_pRgxL_cat #() catenates lines of povr that regex match pRgxL (lineSeq) #] used by pinn_sedFndGetPut_povr: find matching lineSeq of pinn, outputs new linSeq #] pRgxL must [escape, []] sed special chrs # 04Mar2023 initial, adapted from povr_pPLines_replace() # 05Mar2023 empty lines aren't added to output # this can be commented out, depending on application # could add yet another argFlag to do it povr_pRgxL_cat() { local date_ymdhms=$(date +"%0y%0m%0d %0kh%0Mm%0Ss") echo >>"$p_log" "$date_ymdhms povr_pRgxL_cat" local flagBackup="$1" local povr="$2" local pRgx="$3" local strEnd="$4" # local pRep="$d_temp"'povr_pRgxL_replace temp1.txt' local pTmp="$d_temp"'povr_pRgxL_replace temp2.txt' local ermsg='' # echo "povr: $povr" echo "prgx: $pRgx" # if [[ -f "$povr" && -f "$pRgx" ]]; then # first do a backup! if [ "$flagBackup" == "1" ]; then pinn_arcArch "$povr" fi cp "$povr" "$pTmp" if [ -f "$pRep" ]; then rm "$pRep" fi # readarray -t rgxL <"$pRgx" local num=${#rgxL[@]} for (( i=0; i<${num}; i++ )); do echo "rgxL["$i"]: ${rgxL[i]}"; done echo "numTotal: $num" echo '' # very important to know empty lines - they will give bolInrgxL=='' # declare -a emptyLines=() local emptyLines=($(seq 0 1 "$num")) for (( i=0; i<${num}; i++ )); do # echo '+-----+' # echo "i: $i" local check="${rgxL[$i]}" # echo "check: $check" check=$( printf '%c' "$check" ) echo "check: $check" if [ "$check" == '' ]; then emptyLines[$i]=1; else emptyLines[$i]=0; fi echo "emptyLines["$i"]: ${emptyLines[i]}"; done echo '' # local linNum=0 local bolInrgxL=0 local idx=0 local catLine='' local rgxStr='' local strrgx='' IFS='' while read -r -u 252 linPovr; do linNum=$(( $linNum + 1 )) rgxStr="${rgxL[$idx]}" if [ "$linPovr" == '' ]; then if [ "${emptyLines[idx]}" == 1 ]; then bolInrgxL=1; else bolInrgxL=0; fi else strrgx=$( echo "$linPovr" | sed "s|$rgxStr||" ) if [ "$strrgx" == '' ]; then bolInrgxL=1; else bolInrgxL=0; fi fi # decide what to do with linPovr : if [ "$bolInrgxL" == 0 ]; then # check if catLine is empty if [ "$catLine" == '' ]; then echo "$linPovr" >>"$pRep" else povr_pRgxL_cat_saveLines echo "$linPovr" >>"$pRep" fi idx=0 else # 05Mar2023 empty lines aren't added to output - automatic effect of null catLine+="$linPovr" idx=$(( $idx + 1 )) fi done 252< "$povr" # mv "$pRep" "$povr" # else ermsg='povr_pRgxL_replace error: no-exist(OR p[ovr, prgx])' fi echo "$ermsg" } # tests - see "$d_bin"'fileops notes.txt' # 04Mar2023 rgxStr_tests - rgxStr_tests() { local date_ymdhms=$(date +"%0y%0m%0d %0kh%0Mm%0Ss") echo >>"$p_log" "$date_ymdhms rgxStr_tests" local str="$1" local idx="$2" local prgx="$d_bin"'port python funcDecln via rgx.txt' readarray -t rgxL <"$prgx" local rgxStr="${rgxL["$idx"]}" local bolrgx=$( echo "$str" | sed "s|$rgxStr||" ) # echo "str: $str" echo "idx: $idx" echo "prgx: $prgx" echo '+-----+' echo 'for (( c=0; c<=2; c++ )); do echo "${rgxL[c]}"; done : ' for (( c=0; c<=2; c++ )); do echo "${rgxL[c]}"; done echo '+-----+' echo "rgxStr: $rgxStr" echo "bolrgx: $bolrgx" } # rgxStr_tests 'pth536: str, ' 2 # rgxStr_tests 'elephant' 2 # rgxStr_tests 'def pth_get_dir(' 0 #] pinn_sedFndGetPut_povr #() finds matching sequences of lines of pinn, outputs new linSeq #] (Fnd=Find, rgx= regular expression, linSeq = line sequence) #] pFnd - line-level rgxL [search, replace] to test if each line in a pinn linSeq matches #] pGet - line-level rgxL that extracts components of each line into a strLst #] pPut - multi-line rgxL stating how components are to be combined into output lines # files are used, rather than strings, because often the sedExprs are very LONG!!! # sedExprs can also be difficult to assemble [eyes, complexity, etc], easier to catenate # typically, copy-paste a section of a text file to d_temp to process (eg blog sequence) # rgxs must [escape, []] sed special chrs (rgx = regular expressions) # [pinn, povr] are NOT backed up!! # p[inn, Fnd, Get, Put] & [org, sed]Lin # 14Mar2023 initial adapted from povr_pRgxL_cat() pinn_sedFndGetPut_povr() { local date_ymdhms=$(date +"%0y%0m%0d %0kh%0Mm%0Ss") echo >>"$p_log" "$date_ymdhms pinn_sedFndGetPut_povr" local flag="$1" local pinn="$2" local pFnd="$3" local pGet="$4" local pPut="$5" local povr="$6" # local p1Tmp="$d_temp"'pinn_sedFndGetPut_povr() temp1.txt' local p2Tmp="$d_temp"'pinn_sedFndGetPut_povr() temp2.txt' local ermsg='' # echo "pinn : $pinn" # if [[ -f "$pinn" && -f "$pFnd" && -f "$pGet" && -f "$pPut" ]]; then cp "$pinn" "$p1Tmp" # # pFnd to find the next linSeq that matches desires readarray -t fndL <"$pFnd" local nFnd=${#fndL[@]} # echo '+-----+' # echo 'fndL : ' # printf "%s\n" "${fndL[@]}" # echo "nFnd: $nFnd" # echo '' # flag "empty lines" # very important to know empty fndL - they will give false signal bolInrgxL==''? local bolEmpty=($(seq 0 1 "$nFnd")) for (( i=0; i<${nFnd}; i++ )); do if [ "${fndL[i]}" == '' ]; then bolEmpty[$i]=1; else bolEmpty[$i]=0; fi # echo "bolEmpty["$i"]: ${bolEmpty[i]}"; done echo '' # # pGetL to [extract, label] components of a linSeq into a list of lists readarray -t getL <"$pGet" local nGet=${#getL[@]} # echo '+-----+' # echo 'getL : ' # printf "%s\n" "${getL[@]}" # echo "nGet: $nGet" # echo '' # # pPutL assembles components into a new linSeq for output readarray -t putL <"$pPut" local nPut=${#putL[@]} # echo '+-----+' # echo 'putL : ' # printf "%s\n" "${putL[@]}" # echo "nPut: $nPut" # echo '' # # [org, sed]Lin setup with dummy values - each of nFnd length local nSeq=$(( $nFnd - 1 )) local orgLin=($(seq 0 1 $nSeq)) # echo '+-----+' # echo 'orgLin : ' # printf "%s\n" "${orgLin[@]}" # echo "nOrgLin=nFnd: $nFnd" # echo '' local sedLin=("${orgLin[@]}") # echo '+-----+' # echo 'sedLin : ' # printf "%s\n" "${sedLin[@]}" # echo "nSedLin=nFnd: $nFnd" # echo '' local linNum=0 local nFndLast=0 IFS='' while read -r -u 611 linPinn; do linNum=$(( $linNum + 1 )) nFndLast=$(( $nFnd - 1 )) local bol=1 echo '' echo '+-----+' # echo "nFnd: $nFnd" echo "linNum : $linNum" # echo "linPinn : $linPinn" # echo "orgLin[$nFndLast] : ${orgLin[$nFndLast]}" # update the stack orgLin, when i>${nFnd ?+ 1?}, for is done, and i is at max # 0 is "top of stack", remember nSeq is one less than nFnd for (( i=0; i<$nSeq; i++ )); do iPlus=$(( $i + 1 )) orgLin[$i]="${orgLin[$iPlus]}" # echo "i: $i, iPlus: $iPlus" echo "orgLin[$i]: ${orgLin[$i]}" done orgLin[$nFndLast]="$linPinn" echo "orgLin[$nFndLast]: ${orgLin[$nFndLast]}" for (( i=0; i<$nSeq; i++ )); do if [ "${bolEmpty[$i]}" == 1 ]; then echo 'bolEmpty[$i] == 1' sedLin[$i]='' if [ "${orgLin[$i]}" != '' ]; then echo "break at : bolEmpty[$i] == 1 && orgLin[$i] != null" bol=0 break ; fi else # check if orgLin[] fit fndL pattern # stop as soon as one fails, because then entire orgLin[] fails pattern # results aren't kept - just the boolean value fndy="${fndL[$i]}" echo "fndy : $fndy" if [ "$fndy" == '' ]; then if [ "${orgLin[$i]}" != '' ]; then echo "break at :fndL[$i] == null && orgLin[$i] != null" bol=0 break ; else ??? fi else sedLin[$i]=$( eval "$fndy" ) echo "sedLin[$i] : ${sedLin[$i]}" fi if [ "$sedLin[$i]" != '' ]; then echo "break at sedLin[$i] != null" bol=0 break ; fi fi done # output to povr, clear lines if there is a match, else just top-of-stack to povr if [ "$bol" == 1 ]; then for (( i=0; i<$nSeq; i++ )); do echo "${orgLin[$i]}" >"$povr" done orgLin=($(seq 0 1 $nSeq)) else echo "${orgLin[0]}" >"$povr" fi # # # # paste new code here from comment-out part # # # # done 611< "$pinn" fi # +-------------------------------+ # comment-out parts being revamped if [ 4 == 5 ]; then if [ "$linPinn" == '' ]; then if [ "${emptyLines[idx]}" == 1 ]; then bolInpFnd=1; else bolInpFnd=0; fi fi # pGetL - extract components of linSeqCurrent into strGetL #strOrgStk1[0] #strOrgStk1[1] #strOrgStk1[n] # #strOrgStk2[0] #strOrgStk2[1] #strOrgStk2[n] # #... # #linOut1="sedLin1" #linOut2 #linOut3 # pPutL - assemble strGetL (components) plus other text into strOutPutL if [ "$bolInpFnd" == 0 ]; then # check if catLine is empty if [ "$catLine" == '' ]; then echo "$linPinn" >>"$p1Tmp" else povr_pRgxL_cat_saveLines echo "$linPinn" >>"$p1Tmp" fi idx=0 else # 05Mar2023 empty lines aren't added to output - automatic effect of null catLine+="$linPinn" idx=$(( $idx + 1 )) fi # #mv "$p1Tmp" "$povr" # else ermsg='pinn_sedFndGetPut_povr() error: no-exist(OR p[ovr, RgxPL])' fi echo "$ermsg" } #] pinn_reverseLines_pout #() reverse the sequence of lines # 08Jul2024 initial # see "$d_bin"'tail notes.txt' - therre are several ways to do it pinn_reverseLines_pout() { local date_ymdhms=$(date +"%0y%0m%0d %0kh%0Mm%0Ss") echo >>"$p_log" "$date_ymdhms pinn_reverseLines_pout" local pinn="$1" local pout="$2" echo "pinn = $pinn" echo "pout = $pout" tac "$pinn" >"$pout" } #] rm_lineSeq_start_to_end #() cp file, then remove all occurrences of a sequence of lines # 08Mar2022 initial # danger - of end being missing and much of file being deleted!!! # could put in seq_line_count as a check # pinn1="$d_webRawe"'History/Ukraine-Russia/kyivindependent.com news log.html' # pinn2="$d_temp"'kyivindependent.com news log.html' # pout="$d_temp"'diff.txt' # $ diff "$pinn1" "$pinn2" --suppress-common-lines | grep "<" | sed 's/< //' >"$pout" # rm_lineSeq_start_to_end - much safer to use cryptic special lines to code # start : <[!---- end : ---!]> i.e search line_strt replace <[!----\nline_strt rm_lineSeq_start_to_end() { local date_ymdhms=$(date +"%0y%0m%0d %0kh%0Mm%0Ss") echo >>"$p_log" "$date_ymdhms rm_lineSeq_start_to_end" local dinn="$1" local dout="$2" local fnam="$3" local pinn="$dinn$fnam" local pout="$dout$fnam" local ermsg='' echo "pinn = $pinn" echo "pout = $pout" local line_strt="$4" local line_endd="$5" echo "" >"$pout" if [ -f "$pinn" ]; then local in_seq="" while read -u 295 line; do at_strt=$( echo "$line" | grep -i "$line_strt" ) at_endd=$( echo "$line" | grep -i "$line_endd" ) if [ -n "$at_strt" ]; then in_seq="1" elif [ -n "$at_endd" ]; then in_seq="" else if [ -z "$in_seq" ]; then echo "$line" >>"$pout" fi fi done 295< "$pinn" else local ermsg="rm_lineSeq_start_to_end error: $pinn doesn't exist" fi echo "$ermsg" } #08********08 #] +-----+ #] bash-> python ports #] pBash_pSymL_dPy_stripQuotes_pPython #() strip "$..." from syms in pBash # www.BillHowell.ca 03Mar2023 initial adapted from povr_strPL_replace # 03Mar2023 - should make more general, feed sedExpr # 04Mar2023 used only to help me track syms without codes for "$d_bin"'port bash to python.sh' pBash_pSymL_dPyth_stripQuotes_pPython() { local date_ymdhms=$(date +"%0y%0m%0d %0kh%0Mm%0Ss") echo >>"$p_log" "$date_ymdhms pBash_pSymL_dPyth_stripQuotes_pPython" local bolArXiv="$1" local pBash="$2" local pSymL="$3" local dPyth="$4" echo "pBash: $pBash" echo "pSymL: $pSymL" echo "dPyth: $dPyth" if [[ -f "$pBash" && -f "$pSymL" ]]; then if [ "$bolArXiv" == "1" ]; then pinn_arcArch "$pBash" fi local p1tmp="$d_temp"'pBash_symL_stripQuotes_pPython temp1.txt' local p2tmp="$d_temp"'pBash_symL_stripQuotes_pPython temp2.txt' local ermsg='' local fnm=$( pth_get_fNamNoExt "$pBash" ) local pPy="$dPyth$fnm"'.py' echo "pPy: $pPy" cp "$pBash" "$p1tmp" IFS='' while read -r -u 908 sym; do sed "s|\"\$\($sym\)\"|\1|" "$p1tmp" >"$p2tmp" mv "$p2tmp" "$p1tmp" done 908< "$pSymL" mv "$p1tmp" "$pPy" else local ermsg="$ermsg\n"$( 'pBash_pSymL_stripQuotes_pPython error: OR[pBash, pSymL] doesnt exist' ) ermsg="$ermsg\n"$( "pBash: $pBash") ermsg="$ermsg\n"$( "pSymL: $pSymL") fi if ! [ -z "$ermsg" ]; then printf "$ermsg" fi } # 03Mar2023 tests # see "$d_bin"'port bash to python.sh' # sym="pBash" # echo 'cp "$pBash" "$p1tmp" ' | sed "s|["]\$\($sym\)["]|\1|" # >> returns whole str # $ echo 'cp "$pBash" "$p1tmp" ' | sed "s|.*\($sym\).*|\1|" # pBash # >> OK # echo 'cp "$pBash" "$p1tmp" ' | sed "s|\"\$\($sym\)\"|\1|" #08********08 #] +-----+ #] wrd[Old, New] replace in path[,L] #] MUCH safer than str versions!!, creates 'z_Archive' subDir if necessary #] -i option for [capital, small] insensitive - might not want this? #] forwardSlashes will likely be in str[Inn,Out], note that the `| c is used #] that could also cause problems, but hopefully not in paths! # 07Jul2022 failure of file read for tabs?!? # use an easy short string to separate strPL!! : ' ~:~ ' ( enclosed by apos) # see : "$d_SysMaint"'Linux/read file & loops.txt' # also : https://unix.stackexchange.com/questions/614708/linux-bash-shell-read-with-tab-support-t # >> none of the appraoches worked for me #] 02Mar2023 tests of flagWrd: specialChrs [ \^\$'",*,\\[]{}-\n] # flagWrd_test #() # flagSpecChrs_test #() must escape ^$[]{}\, enclose all with `' except `' which uses `" # see bash dir "$d_bin"'code develop/' #] flagWrd_test #() flagWrd_test() { local date_ymdhms=$(date +"%0y%0m%0d %0kh%0Mm%0Ss") echo >>"$p_log" "$date_ymdhms flagWrd_test" local symChrs="[^a-zA-Z0-9_]" echo '1wrdOld123' | grep "$symChrs" >"$d_temp"'flagWrd_test results.txt' echo '2wrd Old123' | grep "$symChrs" >>"$d_temp"'flagWrd_test results.txt' echo '3wrd\^Old123' | grep "$symChrs" >>"$d_temp"'flagWrd_test results.txt' echo '4wrd\$Old123' | grep "$symChrs" >>"$d_temp"'flagWrd_test results.txt' # note the use of `" instead of `' around the wrd echo "5wrd'Old123" | grep "$symChrs" >>"$d_temp"'flagWrd_test results.txt' echo '6wrd"Old123' | grep "$symChrs" >>"$d_temp"'flagWrd_test results.txt' echo '7wrd,Old123' | grep "$symChrs" >>"$d_temp"'flagWrd_test results.txt' echo '8wrd*Old123' | grep "$symChrs" >>"$d_temp"'flagWrd_test results.txt' echo '9wrd.Old123' | grep "$symChrs" >>"$d_temp"'flagWrd_test results.txt' echo '10wrd\Old123' | grep "$symChrs" >>"$d_temp"'flagWrd_test results.txt' echo '11wrd[Old123' | grep "$symChrs" >>"$d_temp"'flagWrd_test results.txt' echo '12wrd]Old123' | grep "$symChrs" >>"$d_temp"'flagWrd_test results.txt' echo '13wrd{Old123' | grep "$symChrs" >>"$d_temp"'flagWrd_test results.txt' echo '14wrd}Old123' | grep "$symChrs" >>"$d_temp"'flagWrd_test results.txt' echo '15wrd-Old123' | grep "$symChrs" >>"$d_temp"'flagWrd_test results.txt' echo '16wrd\\Old123' | grep "$symChrs" >>"$d_temp"'flagWrd_test results.txt' echo '17wrd\nOld123' | grep "$symChrs" >>"$d_temp"'flagWrd_test results.txt' } # 02Mar2023 tests # bash "$d_fileops"'fileops.sh' # flagWrd_test # $ flagWrd=$( echo "$wrd\^Old123" | grep "[^a-zA-Z0-9_]" ) # ~ # $ echo "$flagWrd" # \^Old123 # $ echo "$wrd\^Old123" # \^Old123 # >> oops - use `' # $ echo 'wrd\^Old123' # wrd\^Old123 #] flagSpecChrs_test #() must escape ^$[]{}\, enclose all with `' except `' which uses `" #] still doesn't work with `' flagSpecChrs_test() { local date_ymdhms=$(date +"%0y%0m%0d %0kh%0Mm%0Ss") echo >>"$p_log" "$date_ymdhms flagSpecChrs_test() " local specChrs=' \|\^\|\$\|\"\|\,\|\*\|\.\|\\[\|\\]|\\{\|\\}\|\-\|\\\|\n' echo '1wrdOld123' | grep "$specChrs" >"$d_temp"'flagSpecChrs_test results.txt' echo '2wrd Old123' | grep "$specChrs" >>"$d_temp"'flagSpecChrs_test results.txt' echo '3wrd\^Old123' | grep "$specChrs" >>"$d_temp"'flagSpecChrs_test results.txt' echo '4wrd\$Old123' | grep "$specChrs" >>"$d_temp"'flagSpecChrs_test results.txt' echo "5wrd'Old123" | grep "$specChrs" >>"$d_temp"'flagSpecChrs_test results.txt' echo '6wrd"Old123' | grep "$specChrs" >>"$d_temp"'flagSpecChrs_test results.txt' echo '7wrd,Old123' | grep "$specChrs" >>"$d_temp"'flagSpecChrs_test results.txt' echo '8wrd*Old123' | grep "$specChrs" >>"$d_temp"'flagSpecChrs_test results.txt' echo '9wrd.Old123' | grep "$specChrs" >>"$d_temp"'flagSpecChrs_test results.txt' echo '10wrd\Old123' | grep "$specChrs" >>"$d_temp"'flagSpecChrs_test results.txt' echo '11wrd\[Old123' | grep "$specChrs" >>"$d_temp"'flagSpecChrs_test results.txt' echo '12wrd\]Old123' | grep "$specChrs" >>"$d_temp"'flagSpecChrs_test results.txt' echo '13wrd\{Old123' | grep "$specChrs" >>"$d_temp"'flagSpecChrs_test results.txt' echo '14wrd\}Old123' | grep "$specChrs" >>"$d_temp"'flagSpecChrs_test results.txt' echo '15wrd-Old123' | grep "$specChrs" >>"$d_temp"'flagSpecChrs_test results.txt' echo '16wrd\\Old123' | grep "$specChrs" >>"$d_temp"'flagSpecChrs_test results.txt' echo '17wrd\nOld123' | grep "$specChrs" >>"$d_temp"'flagSpecChrs_test results.txt' } # grep " \^\$\'\"\,\*\.\[\][{][}]\-\\|\n" # 02Mar2023 tests # bash "$d_fileops"'fileops.sh' # flagSpecChrs_test # specChrs=' \|\^\|\$\|\"\|\,\|\*\|\.\|\\[\|\\]|\\{\|\\}\|\-\|\\\|\n' # echo "$specChrs" # ' \|\^\|\$\|\"\|\,\|\*\|\.\|\\[\|\\]|\\{\|\\}\|\-\|\\\|\n' #] povr_wrdP_replace #() z_Archive povr, wrdP=(wrdOld, wrdNew) replace in povr #] error if wrds are not actually words! #] 01Mar2023 what about case sensitivity? # www.BillHowell.ca 13Jul2022 initial # ptmp123 needed to allow sed to progress (can't write to same file!) # mv - comment out to debug or view povr_wrdP_replace() { local flagBackup="$1" local povr="$2" local wrdOld="$3" local wrdNew="$4" local ptmp="$d_temp""povr_strP_replaceSed temp.txt" local ermsg='' # if [ -f "$povr" ]; then if [ "$flagBackup" == "1" ]; then pinn_arcArch "$povr" fi cp "$povr" "$pTmp" # ensures that wrds are wrds! local flagWrd=$( echo "$wrdOld" | grep "[^a-zA-Z0-9_]" ) if [ "$flagWrd" == null ]; then sed "s|$wrdOld|$wrdNew|g" "$povr" >>"$ptmp" mv "$ptmp" "$povr" else ermsg="povr_wrdP_replace error: not a word $wrdOld" fi else ermsg="povr_wrdP_replace error: noexist $povr" fi echo "$ermsg" } #] povr_wrdPL_replace #() as listed in the lst of files wrdPL # 13Jul2022 haven't created replacement for this yet (there iss quivalent for strPL #] povrL_wrdP_replace #() each povrL [z_Archive, (wrdOld, wrdNew), over-write] # wrdP=(wrdOld, wrdNew) replace in each povrL # www.BillHowell.ca 13Jul2022 initial povrL_wrdP_replace() { local date_ymdhms=$(date +"%0y%0m%0d %0kh%0Mm%0Ss") echo >>"$p_log" "$date_ymdhms povrL_wrdP_replace" local flagBackup="$1" local povrL="$2" local wrdOld="$3" local wrdNew="$4" local ermsg='' # while read -u 882 FILE; do echo "$FILE" ermsg="$ermsg\n"$( povr_wrdP_replace "$flagBackup" "$FILE" "$wrdOld" "$wrdNew" ) done 882< "$povrL" echo "$ermsg" } #08********08 #] +-----+ #] Paragraph - contiguous lines, with line breaks but no whitespace lines #] pinn_uniteParagraphs_povr #(pinn povr) consolidate paragraphs # must convert file to Unix LF, not CRLF # 10May2026 initial # line endings: geany # check via Menu -> View -> show line endings # convert Menu -> Document -> Set line endings -> convert & set to LF (Unix) pinn_uniteParagraphs_povr() { local pinn="$1" local povr="$2" if [ -f "$povr" ]; then rm "$povr"; fi paragraph='' bolInn=0 while IFS='' read -u 9 line; do if [ '' == "$line" ]; then if [ "$bolInn" == 1 ]; then echo "$paragraph" >>"$povr" echo '' >>"$povr" fi bolInn=0 paragraph='' else if [ "$bolInn" == 1 ]; then paragraph="$paragraph $line" else paragraph="$line" bolInn=1 fi fi done 9<"$pinn" } #] transcript_unite_nLines #() keep timestamp each n lines of transcript # 08Jun initial transcript_unite_nLines() { local nMax="$1" local pinn="$2" local pout="$3" if [ -f "$pout" ]; then rm "$pout"; fi # proceed to 1st timeStamp, writing lines along way exec 681<"$pinn" while IFS='' read -u 681 line; do printf "$line\n" >>"$pout" isTimeStamp=$( echo "$line" | grep '\([0-9]*:[0-9]*\)' ) if [ -z "$isTimeStamp" ]; then printf "$line\n" >>"$pout" break fi done # notice that reads continue from pinn line above # each line is either [timestamp, text] : # output timestamp (TS), collect nLines ignoring mid-TS, output paragraph nLine=0 parag='' exec 681<"$pinn" while IFS='' read -u 681 line; do isTimeStamp=$( echo "$line" | grep '\([0-9]*:[0-9]*\)' ) if [ -z "$isTimeStamp" ]; then let "nLine = $nLine + 1" parag="$parag $line" if [ "$nLine" -ge "$nMax" ]; then printf "\t$parag\n" >>"$pout" parag='' nLine=0 fi else if [ "$nLine" == 0 ]; then printf "$line\n" >>"$pout" fi fi done printf "\t$parag\n" >>"$pout" } # echo "00:06" | grep '\([0-9]*:[0-9]*\)' #] transcript_unite_each_paragraph - res ipsa loquitor # date? initial # see "$d_web"'My sports & clubs/natural- Suspicious/230915 Bosnian Pyramids: 30,000-Year-Old Enigma and Expansive Subterrenean Tunnels15 Bosnian Pyramids: 30,000-Year-Old Enigma and Expansive Subterrenean Tunnels.txt' transcript_unite_each_paragraph() { local date_ymdhms=$(date +"%0y%0m%0d %0kh%0Mm%0Ss") echo >>"$p_log" "$date_ymdhms transcript_unite_each_paragraph" local pinn="$1" local pout="$2" echo >"$pout" 'transcript_unite_each_paragraph()' echo >>"$pout" "$date_ymdhms: $pinn" echo >>"$pout" '' echo >>"$pout" 'manual steps : ' echo >>"$pout" ' press twice in text editor to separate paragraphs (sometimes within the line!)' echo >>"$pout" ' eg different [speaker, subject]' echo >>"$pout" ' this inserts empty line between paragraphs' echo >>"$pout" ' if the first line of a paratgraph doesnt have a time-stamp (eg 05:39) then ' echo >>"$pout" ' type in a timestamp at the beginning of the first line' echo >>"$pout" ' interpolate approximastely between [earlier, l;ater] timestamps' echo >>"$pout" 'bash script to unify paragraphs : ' echo >>"$pout" ' "$d_bin""fileops run.sh" -> use transcript_unite_each_paragraph' echo >>"$pout" ' starts after line beggining with "Transcript"' echo >>"$pout" '' echo >>"$pout" '+--------+' echo >>"$pout" 'Transcript: times are mostly within +-00:05 (5 seconds)' local ptmp="$d_temp"'transcript_unite_each_paragraph temp.txt' if [ -f "$ptmp" ]; then rm "$ptmp"; fi # proceed to start of transcript, writing lines along way exec 681<"$pinn" while IFS='' read -u 681 line; do if [ 'Transcript' == "$line" ]; then break; fi done # consolidate each paragraph, remove [timestamp, newLine]s lines along way # notice that reads continue from line above bolInn=0 exec 681<"$pinn" while IFS='' read -u 681 line; do if [ '' == "$line" ]; then if [ "$bolInn" == 1 ]; then echo ' ' >>"$pout" # save paragraph cat "$ptmp" | tr \\n ' ' | sed 's| \([0-9]*:[0-9]*\) | |g' >>"$pout" rm "$ptmp" bolInn=0 echo ' ' >>"$pout" fi else echo "$line" >>"$ptmp" bolInn=1 fi done } #*************************** #] +-----+ #] [read, write] special [table, Unix cmdResult] formats 06Jan2026 QNial port not done yet!! # IF flag_debug THEN write '+-----+' ; ENDIF ; # IF flag_debug THEN write '[read, write] special [table, Unix cmdResult] formats ' ; ENDIF ; # IF flag_debug THEN write 'loading pinn_read_csvTable' ; ENDIF ; #] pinn_read_csvTable IS OP pinn - read a csv table from file, cells to [apo-str, numeric] # (tab-cols, apostrophe-txt) # www.BillHowell.ca 24Jul2018 initial # 09Jan2021 doesn't work, needs an upgrade # 06Nov2021 not revamped - maybe later, much later (focus on core for 'QNial usrWkSpc symbol changes.ndf') # # automatic conversion of number types?, all others to string # # later develop : # # paired "" string delimiters converted to '' # # handle commas within strings # # handle phrases # # 09Jan2021 - fhen extra commas are in line ... fix later as for now I can at lst go ahead with my project # pinn_read_csvTable IS OP pinn # { # lineL := pinn_read_strL pinn ; # strsInLinesL := `, EACHRIGHT ary_itm_cut_subWithNullsL lineL ; # n_strs := EACH (gage shape) strsInLinesL ; # add_m_strs := (max n_strs) - (EACH (gage shape) strsInLinesL) ; # strsInLinesL := strsInLinesL EACHBOTH append (add_m_strs EACHLEFT reshape ['n/a']) ; # table := mix strsInLinesL ; # EACH str_convertTo_numOrStr table # } # # # p_data := link d_Puetz 'SP500 1870-2030 y-log detrend.dat' # # /media/bill/SWAPPER/Website - raw/Projects - mini/Puetz & Borchardt/Puetz - Universal Waves Series 2011 tabTable.dat # # /media/bill/SWAPPER/Website - raw/Projects - mini/Puetz & Borchardt/Puetz - Universal Waves Series 2011 csvTable.dat # # c := pinn_read_csvTable '/media/bill/SWAPPER/Website - raw/Projects - mini/Puetz & Borchardt/Puetz - Universal Waves Series 2011 csvTable.dat' #] pinnTblCsv_put_povrTblHtm #(pinnTblCsv_put_povrTblHtm) convert table (csv -> html) # from file, cells to [apo-str, numeric], (tab-cols, apostrophe-txt) # www.BillHowell.ca 24Jul2018 initial adapted from QNial # automatic conversion of number types?, all others to string # later develop : # paired "" string delimiters converted to '' # handle commas within strings # handle phrases # 09Jan2021 - fhen extra commas are in line ... fix later as for now I can at lst go ahead with my project # pinnTblCsv_put_povrTblHtm() #{ # lineL= pinn_read_strL pinn # strsInLinesL= `, EACHRIGHT ary_itm_cut_subWithNullsL lineL # n_strs= EACH (gage shape) strsInLinesL # add_m_strs= (max n_strs) - (EACH (gage shape) strsInLinesL) # strsInLinesL= strsInLinesL EACHBOTH append (add_m_strs EACHLEFT reshape ['n/a']) # table= mix strsInLinesL # EACH str_convertTo_numOrStr table #} # p_data := link d_Puetz 'SP500 1870-2030 y-log detrend.dat' # /media/bill/SWAPPER/Website - raw/Projects - mini/Puetz & Borchardt/Puetz - Universal Waves Series 2011 tabTable.dat # /media/bill/SWAPPER/Website - raw/Projects - mini/Puetz & Borchardt/Puetz - Universal Waves Series 2011 csvTable.dat # c := pinn_read_csvTable '/media/bill/SWAPPER/Website - raw/Projects - mini/Puetz & Borchardt/Puetz - Universal Waves Series 2011 csvTable.dat' #] tbl_writeCSVformatTo_pnew IS OP table pnew - writefile table to path, [tab-separated cols, apo-text] #] important - assumes 2D table of ONLY [string, atomic data] types (eg not lists of numbers...) # www.BillHowell.ca 24Jul2018 initial, incomplete - must handle integers etc, better to use "picture"? # 09Oct2020 legacy version, keep it # 06Nov2021 not revamped - maybe later, much later (focus on core for 'QNial usrWkSpc symbol changes.ndf') # # tbl_writeCSVformatTo_pnew IS OP table pnew # { LOCAL fout tabRows ; # % verify that paths are allowable ; # IF (NOT pth_typ_exists "pnew pnew) # THEN EACH write # '?tbl_writeCSVformatTo_pnew error with path : ' # pnew # '' ; # ELSE # fout := open pnew "w ; # tabRows := EACH link (rows ((EACH string table) EACHLEFT link ', ')) ; # fout EACHRIGHT writefile tabRows ; # close fout ; # ENDIF ; # } # # writefile_table IS tbl_writeCSVformatTo_pnew # # csvTable_writeTo_pth IS tbl_writeCSVformatTo_pnew # # # # # # table_writeCSVformatTo_pth (random 4 5) (link d_temp 'table_writeCSVformatTo_pth test.txt') # # # a := 'Hello' 'Dolly" 'today' # # (chr_tab chr_tab chr_tab) EACHBOTH link a # # # csvTable_writeTo_pth (trans_old_Lucas trans_new_Lucas trans_msg_Lucas) (link d_QNial_temp 'tranList HFLN temp.txt') # # csvTable_readFrom_pth (link d_Lucas 'tranList HFLN.txt') # IF flag_debug THEN write 'loading pHeaderTitleTbl_quoteTitles' ; ENDIF ; #] pHeaderTitleTabTbl_quoteTitles IS OP povr - quote titles, as per gnuplot #] header is at top of file, each line starts with '# ' # 04Aug2022 initial for fit_SLRgress # 04Aug2022 missing pth exists checks!!!, this is actually useless for Puetz project # but may be useful later? # need to test subStrL := ary_subP_put_aryOut subStrL ((`" `#) (`" null)) # not working currently...fix optr some day # pHeaderTitleTabTbl_quoteTitles IS OP povr # { LOCAL finn fout line pTmp subStrL ; # % ; # pTmp := link d_temp 'pHeaderTitleTabTbl_quoteTitles.txt' ; # finn := open povr "r ; # fout := open pTmp "w ; # % ; # WHILE (~= ??eof (line := readfile finn)) DO # IF (= `# (first line)) # THEN writefile fout line ; # ELSE exit 'done' ; # ENDIF ; # ENDWHILE ; # % ; # subStrL := ary_itm_cut_subL line chr_tab ; # subStrL := (`" EACHRIGHT link subStrL) EACHLEFT link `" ; # % if subStrL itms already enclosed in quotes, replace double quote with quote ; # subStrL := ary_subP_put_aryOut subStrL ((`" `#) (`" null)) ; # % 04Aug2022 might need a 'first' before link? ; # line := link (subStrL EACHLEFT link chr_tab) ; # writefile fout line ; # % ; # WHILE (~= ??eof (line := readfile finn)) DO # writefile fout line ; # ENDWHILE ; # % ; # EACH close finn fout ; # host link 'cp "' pTmp '" "' povr '" ' ; # } # # # # subStrL := link '""Hello my ""' chr_tab '"darling""' chr_tab 'jump together"' # # b := # IF flag_debug THEN write 'loading str_removeRepeat_chr' ; ENDIF ; #] str_removeRepeat_chr IS OP str chrr - temporary optr until #] ary_subP_put_aryOut is working # str_removeRepeat_chr IS OP str chrr # { LOCAL flag_quote str2 ; # flag_quote := o ; # str2 := null ; # FOR chrr WITH str DO # IF chrr = chr_quote THEN # IF flag_quote # THEN null ; # ELSE str2 := link str2 chrr ; # ENDIF ; # flag_quote := l ; # ELSE str2 := link str2 chrr ; # flag_quote := o ; # ENDIF ; # ENDFOR ; # str2 # } # # # test # qnial> str := link '""Hello my ""' chr_tab '"darling""' chr_tab 'jump together"' # ""Hello my "" "darling"" jump together" # qnial> diagram str # +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ # |"|"|H|e|l|l|o| |m|y| |"|"| |"|d|a|r|l|i|n|g|"|"| |j|u|m|p| |t|o|g|e|t|h|e|r|"| # +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ # qnial> chr_tab EACHRIGHT = str # ooooooooooooolooooooooooloooooooooooooo # qnial> a := str_removeRepeat_chr str `" # "Hello my " "darling" jump together" # qnial> diagram a # +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ # |"|H|e|l|l|o| |m|y| |"| |"|d|a|r|l|i|n|g|"| |j|u|m|p| |t|o|g|e|t|h|e|r|"| # +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ # qnial> chr_tab EACHRIGHT = a # oooooooooooloooooooooloooooooooooooo # >> OK, works # IF flag_debug THEN write 'loading pTitleOneLineTabTbl_quoteTitles' ; ENDIF ; #] pTitleOneLineTabTbl_quoteTitles IS OP povr - quote titles, as per gnuplot #] header is at top of ile, each line starts with '# ' # 04Aug2022 initial for fit_SLRgress # 04Aug2022 missing pth exists checks!!!, this is actually useless for Puetz project # but may be useful later? # need to test subStrL := ary_subP_put_aryOut subStrL ((`" `#) (`" null)) # not working currently...fix optr some day # pTitleOneLineTabTbl_quoteTitles IS OP povr # { LOCAL finn fout line pTmp subStrL ; # % ; # % 04Aug2022 need to test for [nonexistant, empty] file ; # pTmp := link d_temp 'pTitleOneLineTabTbl_quoteTitles.txt' ; # finn := open povr "r ; # fout := open pTmp "w ; # % ; # % specifically for TradingView data, may be of use elsewhere ; # % if subStrL itms already enclosed in quotes, replace double quote with quote ; # line := readfile finn ; # subStrL := ary_itm_cut_subL line chr_tab ; # subStrL := (`" EACHRIGHT link subStrL) EACHLEFT link `" ; # % 04Aug2022 ary_subP_put_aryOut not working properly ; # % subStrL := ary_subP_put_aryOut subStrL ((`" `") (`" null)) ; # line := link (subStrL EACHLEFT link chr_tab) ; # % ; # % sed version, \d34=`" ; # % line := first host_result 'cat "' line '" | sed "s|\d34\d34|\d34|" ' ; # % ; # % try QNial from another angle ; # line := str_removeRepeat_chr line `" ; # % ; # writefile fout line ; # % ; # WHILE (~= ??eof (line := readfile finn)) DO # writefile fout line ; # ENDWHILE ; # % ; # EACH close finn fout ; # host link 'cp "' pTmp '" "' povr '" ' ; # } # # # # test # qnial> subStrL := link '""Hello my ""' chr_tab '"darling""' chr_tab 'jump together"' # ""Hello my "" "darling"" jump together" # qnial> diagram subStrL # +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ # |"|"|H|e|l|l|o| |m|y| |"|"| |"|d|a|r|l|i|n|g|"|"| |j|u|m|p| |t|o|g|e|t|h|e|r|"| # +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ # qnial> chr_tab EACHRIGHT = subStrL # ooooooooooooolooooooooooloooooooooooooo # # b := first host_result link 'cat "' subStrL '" | tr \d34 : | sed "s|\d34\d34|\d34|" ' # qnial> 'cat "' subStrL '" | sed "s|\d34\d34|\d34|" ' # +------+---------------------------------------+----------------------------+ # |cat "|""Hello my "" "darling"" jump together"|" | sed "s|\d34\d34|\d34|" | # +------+---------------------------------------+----------------------------+ # qnial> link 'cat "' subStrL '" | sed "s|\d34\d34|\d34|" ' # cat """Hello my "" "darling"" jump together"" | sed "s|\d34\d34|\d34|" # qnial> link 'cat "' subStrL '" | tr \d34 : # ?SCAN - missing ' at end of string: '" | # >> what a mess!! bash not good at this # # # try a file # pTitleOneLineTabTbl_quoteTitles (link d_work '') # IF flag_debug THEN write 'loading pinn_readExecute_tabTbl' ; ENDIF ; #] pinn_readExecute_tabTbl IS OP pinn - read tbl of tab delineated cols #] convert "cells" to proper type, eg spreadsheet of : #] - [str, phr, int, real, bool]s, NO chr_tabs within strings, just to separate cols #] ignores 'comment lines' that start with `#, must have NO empty lines! # see str_to_typ in 'strings.ndf' for type-conversion rules (will have to update with experience!) # 06Nov2021 not revamped - maybe later, much later (focus on core for 'QNial usrWkSpc symbol changes.ndf') # Note : vulnerable to tabs within strings! # BEST - for copy-paste of an area of a from spreadsheet, with tab-separated columns # there cannot be tabs within the cols! # www.BillHowell.ca 09Oct2020 initial, 09Aug2022 re[vamp, name]ed # alternate approaches to reading a table from a text file (eg txt from spreadsheet) # 1. loaddefs with [rows, cols] calc'd from expressions (most flexible, can copy from spreadsheet) # - copy-paste loaddef format from spreadsheet is most [general, powerful]!, but not for bashCmd out # 2. csv save format - quotation-enclosed text (strings can't include [quotes, apos]) # IF flag_break THEN BREAK ; ENDIF ; # pinn_readExecute_tabTbl IS OP pinn # { LOCAL line n_rows n_cols resFlt rowStrL ; # rowStrL := null ; # IF (NOT (pth_typ_exists pinn "pinn)) # THEN resFlt := fault '?pinn_readExecute_tabTbl file unknown error : ' pinn ; # ELSE # finn := open pinn "r ; # WHILE (NOT isfault (line := readfile finn)) DO # IF (`# ~= line@0) THEN # rowStrL := link rowStrL [ary_itm_cut_subWithNullsL line chr_tab] ; # ENDIF ; # ENDWHILE ; # close finn ; # % ; # n_rows := gage shape rowStrL ; # n_cols := gage shape (first rowStrL) ; # rawTbl := n_rows n_cols reshape (link rowStrL) ; # resFlt := EACH str_to_typ rawTbl ; # ENDIF ; # resFlt # } # # # symbols 0 # # ?date? test # # pinn := link d_web 'economics, markets/SP500/PuetzUWS intnl stocks/default/semi-log DAX40 Jan1970-Jul2022 TradingView regression fit.txt' # # tbl := pinn_readExecute_tabTbl pinn # # # 08Oct2022 tests # qnial> pinn := link d_Qmkts 'market tabWhole.dat' # /home/bill/SG6/web/Qnial/MY_NDFS/economics, markets/market tabWhole.dat # qnial> tblWhole := pinn_readExecute_tabTbl pinn # >> tblWhole looks good # qnial> povr := link d_temp 'tblWhole_writeTabSep_povr market tabWhole temp.txt' # /mnt/ramdisk/tblWhole_writeTabSep_povr market tabWhole temp.txt # qnial> tblWhole_writeTabSep_povr tblWhole povr # $ pinn='/home/bill/SG6/web/Qnial/MY_NDFS/economics, markets/market tabWhole.dat' # $ povr='/mnt/ramdisk/tblWhole_writeTabSep_povr market tabWhole temp.txt' # $ diff "$pinn" "$povr" --suppress-common-lines # >> all lines are shown because of real number formats # # # Do parallel test & diff # qnial> povr2 := link d_temp 'tblWhole_writeTabSep_povr market tabWhole temp2.txt' # qnial> tblWhole_writeTabSep_povr tblWhole povr2 # $ povr2='/mnt/ramdisk/tblWhole_writeTabSep_povr market tabWhole temp2.txt' # $ diff "$povr" "$povr2" --suppress-common-lines # >> null, so perfect match. Also - quick visual check # # # # 18Jul2022 olde code # % if the last col is empty, I must add a tab chr during read ; # eachShape_rowTxtList := EACH (gage shape) rowTxtList ; # n_cols := max eachShape_rowTxtList ; # % corrects for null content in last column only ; # augment_rows := (eachShape_rowTxtList < n_cols) sublist (tell n_cols) ; # rowTxtList#augment_rows := (augment_rows EACHLEFT pick rowTxtList) EACHLEFT append chr_tab ; # % 18Jul2022 olde code : rowTxtList := append rowTxtList (ary_itm_cut_subWithNullsL line chr_tab) ; # rowTxtList := append rowTxtList (ary_itm_cut_subWithNullsL line chr_tab) ; # IF flag_debug THEN write 'loading pinn_readStr_tabTbl' ; ENDIF ; #] pinn_readStr_tabTbl IS OP pinn - read file as a tbl with tab delineated cols #] eg spreadsheet, ignores lines that start with `#, must have NO empty lines! #] all itms are strings, NO chr_tabs within strings, just to separate cols # 09Aug2022 initial # IF flag_break THEN BREAK ; ENDIF ; # pinn_readStr_tabTbl IS OP pinn # { LOCAL line n_rows n_cols resFlt rowStr rowStrL ; # rowStrL := null ; # IF (NOT (pth_typ_exists pinn "pinn)) # THEN resFlt := fault '?pinn_readStr_tabTbl file unknown error : ' pinn ; # ELSE # finn := open pinn "r ; # WHILE (NOT isfault (line := readfile finn)) DO # IF (`# ~= line@0) THEN # rowStr := ary_itm_cut_subWithNullsL line chr_tab ; # rowStrL := append rowStrL rowStr ; # % write (gage shape rowStr) rowStr ; # ENDIF ; # ENDWHILE ; # close finn ; # % ; # n_rows := gage shape rowStrL ; # n_cols := gage shape (first rowStrL) ; # resFlt := n_rows n_cols reshape (link rowStrL) ; # ENDIF ; # resFlt # } # # # # 09Oct2020 test # # pinn := link d_Qtest 'test table - tab delineated, [numbers, strings].txt' # qnial> pinn := link d_Qtest 'test table - tab delineated, [numbers, strings].txt' # /home/bill/SG6/web/Qnial/code develop_test/test table - tab delineated, [numbers, strings].txt # qnial> tbl1 := pinn_readStr_tabTbl pinn # >> 08Oct2022 now only 2 lines wrong gage shape : # +--+-----------------------------------------------------------------+ # |12|+---------+---+-----+-+---+-++------+----------+++--------------+| # | ||09-Oct-20|Fri|07:05|E|1.2|d||Social|Sundowners|||morning coffee|| # | |+---------+---+-----+-+---+-++------+----------+++--------------+| # +--+-----------------------------------------------------------------+ # ~:~ # +--+--------------------------------------------------+ # |12|+---------+---+-----++---+-++--------+-----+---+++| # | ||09-Oct-20|Fri|14:46||1.7|d||Personal|break|nap|||| # | |+---------+---+-----++---+-++--------+-----+---+++| # +--+--------------------------------------------------+ # >> trailing abs don't give [null] - probably leading tabs as well # IF flag_debug THEN write 'loading tblWhole_writeTabSep_padd' ; ENDIF ; #] tblWhole_writeTabSep_padd IS OP tblWhole padd - append tab-separated "whole table" to pth # Note : vulnerable to tabs within strings! # 28Dec2020 used for link d_Qndfs 'Puetz - Universal Waves Series.ndf' # 06Nov2021 not revamped - maybe later, much later (focus on core for 'QNial usrWkSpc symbol changes.ndf') # tblWhole_writeTabSep_padd IS OP tblWhole padd # { LOCAL fout tblStrRows ; # IF (NOT pth_typ_exists padd "padd) THEN # write fault '?tblWhole_writeTabSep_padd file error : padd' ; # ELSE # fout := open padd "a ; # tblStrRows := EACH link (rows ((EACH string tblWhole) EACHLEFT link chr_tab)) ; # fout EACHRIGHT writefile tblStrRows ; # close fout ; # ENDIF ; # } # IF flag_debug THEN write 'loading tblWhole_writeTabSep_povr' ; ENDIF ; #] tblWhole_writeTabSep_povr IS OP tblWhole povr - overwrite mode #] handles [chr, phr, bol, str] so far # Note : vulnerable to tabs within strings! # 28Dec2020 used for link d_Qndfs 'Puetz - Universal Waves Series.ndf' # 06Nov2021 not revamped - maybe later, much later (focus on core for 'QNial usrWkSpc symbol changes.ndf') # IF flag_break THEN BREAK ; ENDIF ; # tblWhole_writeTabSep_povr IS OP tblWhole povr # { LOCAL fout tblStrRows ; # IF (NOT pth_typ_exists povr "povr) THEN # write fault '?tblWhole_writeTabSep_povr file error : povr' ; # ELSE # tblStr := EACH typ_to_str tblWhole ; # fout := open povr "w ; # tblStrRows := EACH (front link) (rows (tblStr EACHLEFT link chr_tab)) ; # fout EACHRIGHT writefile tblStrRows ; # close fout ; # ENDIF ; # } # # # tblWhole_writeTabSep_povr mktTblWhole (link d_temp 'tblWhole_writeTabSep_povr test.txt') # # # IF flag_debug THEN write 'loading pinn_read_wcUnix' ; ENDIF ; #] pinn_read_wcUnix IS OP pinn - read file of Unix "wc" command outputs ONLY! #] read file as a table with tab delineated cols (eg txt from spreadsheet) # 06Nov2021 not revamped - maybe later, much later (focus on core for 'QNial usrWkSpc symbol changes.ndf') # www.BillHowell.ca 09Oct2020 initial, 19Oct2020 - now I screwed it up - fix later! # alternate approaches to reading a table from a text file (eg txt from spreadsheet) # 1. loaddefs with [rows, cols] calc'd from expressions (most flexible, can copy from spreadsheet) # - copy-paste loaddef format from spreadsheet is most [general, powerful]!, but not for bashCmd out # 2. csv save format - quotation-enclosed text (strings can't include [quotes, apos]) # IF flag_break THEN BREAK ; ENDIF ; # pinn_read_wcUnix IS OP pinn # { LOCAL augment_rows fout line n_rows n_cols rowTxtList ; # rowTxtList := null ; # IF (NOT pth_typ_exists "pinn pinn) # THEN EACH write # (fault '?pinn_read_wcUnix file unknown error : pinn') # pinn ; # ELSE # finn := open pinn "r ; # WHILE (NOT isfault (line := readfile finn)) DO # % if the last col is empty, I must add a tab chr during read ; # line := # rowTxtList := append rowTxtList line ; # ENDWHILE ; # close finn ; # n_rows := gage shape rowTxtList ; # eachShape_rowTxtList := EACH (gage shape) rowTxtList ; # n_cols := max eachShape_rowTxtList ; # % corrects for null contant in last column only ; # augment_rows := (eachShape_rowTxtList < n_cols) sublist (tell n_cols) ; # rowTxtList#augment_rows := (augment_rows EACHLEFT pick rowTxtList) EACHLEFT append chr_tab ; # result := n_rows n_cols reshape (link rowTxtList) ; # ENDIF ; # result # } #] line count of file : tonumber first (host_result link 'wc -l "' '" | sed '' # tonumber first (host_result link 'wc -l "' (link d_temp 'extract_pFragsAndSubs raweFrags.txt') '" | sed "s/^\([0-9]\+\)\(.*\)/\1/" ') # # test # # $ wc -l "' (link d_webRaw fname) # # pth := link d_Qroot 'code develop_test/test table - website linkType fname counts 201004 15h43m.txt' # # tab1 := wcUnix_writeTo_pth pth # # # old code # # not needed for wcUnix_writeTo_pth # % corrects for null contant in last column only ; # augment_rows := (eachShape_rowTxtList < n_cols) sublist (tell n_cols) ; # rowTxtList#augment_rows := (augment_rows EACHLEFT pick rowTxtList) EACHLEFT append chr_tab ; # eachShape_rowTxtList := EACH (gage shape) rowTxtList ; #] winID_copyAllTextTo_pnew IS OP winID pname - copy text of webPage and pre-pend to file #] see 'windows [open, close, ID, title], [get,set,move] cart [posn,size].ndf' #] 20Oct2021 I have lost this!!??!! #08********08 #] +-----+ #] path[,L] - check HowellLibrarys for (optrName arg1 arg2) based on " IS OP " definitions # instructions, see "$d_Qtest"'7_HowellCoreLib arg[Order, Number] instructions.txt' # 09Jul2022 initial, 12Jul2022 moved dirs from "$d_Qtest" to "$d_libWork" # needs [cleanup, verification] #] dirSrc_fbase_fxtn_str_appendTo_pStrUse #() append list of uses of optrNam #] in "$dirSrc" to pStrUse # www.BillHowell.ca 09Jul2022 intitial # NOTE!!! : must use different varNames in povr_strP_replaceSed() versus povr_strPL_replace() dirSrc_fbase_fxtn_str_appendTo_pStrUse() { local date_ymdhms=$(date +"%0y%0m%0d %0kh%0Mm%0Ss") echo >>"$p_log" "$date_ymdhms dirSrc_fbase_fxtn_str_appendTo_pStrUse" local dirSrc="$1" local libFbase="$2" local libFxtn="$3" local str="$4" local pthSrc="$dirSrc$libFbase$libFxtn" local pStrUse="$d_libFbase"'raw/'"$str"' usage.txt' # echo "dirSrc = $dirSrc" # echo "str = $str" # echo "pStrUse = $pStrUse" local blurb=$( grep "$str" "$pthSrc" | grep --invert-match '^#\|flag_debug' ) if [ -n "$blurb" ]; then echo "$blurb" >>"$pStrUse" fi } # Table of contents included without \|^ \| - but this is OK, won't miss omothers, maybe # blurb=$( grep "$optrNam" "$pthSrc" | grep --invert-match '^#\| IS OP \|^ \|flag_debug' ) #] fNdfL_subDirRef_argOrderNum_dout #() extract list of uses of each of pthOptrNamL #] in each of pthL, results for each OptrNam are saved in a optrNam-specific file # www.BillHowell.ca 09Jul2022 initial # NOTE!!! : must use different varNames in povr_strP_replaceSed() versus povr_strPL_replace() # how do I "force execution" of path strings like : "$d_Qndfs"'QNial setup.ndf' ??? # 09Jul2022 idiot! : forget about the complexities of librarys etc, just do : dirSrc_fnamSrcL_fbase_appendTo_dirStrUse() { local date_ymdhms=$(date +"%0y%0m%0d %0kh%0Mm%0Ss") echo >>"$p_log" "$date_ymdhms dirSrc_fnamSrcL_fbase_appendTo_dirStrUse" local dirSrc="$1" local fnamSrcL="$2" local pthSrcL="$dirSrc$fnamSrcL" local dirStrUse="$4$fbase"'raw/' local pthLStrUse="$d_libWork337$libFbase337"'pStrPL optrNamL/optrNamL.txt' while read -u 9 optrNam; do if [ -f "$d_libWork337$fbase"'optr usage/'"$optrNam"' usage.txt' ]; then rm "$d_libWork337$fbase"'optr usage/'"$optrNam"' usage.txt' fi while read -u 8 pthSrc; do # echo "pthSrc= $pthSrc" evalStr="echo ""$pthSrc" pthSrcFull=$( eval "$evalStr" ) # echo "pthSrcFull= $pthSrcFull" dirSrc_fbase_fxtn_str_appendTo_pStrUse "$pthSrcFull" "$libFbase337" "$optrNam" done 8< "$fnamSrcL" done 9< "$pthLStrUse" } #08********08 #] +-----+ #] variables [read, assignment] from file # www.BillHowell.ca 22Nov2025 no functions yet... #08********08 #] +-----+ #] '* run.sh' files - which are the "active lines" that will be run? # www.BillHowell.ca 22Nov2025 #] pRun_get_linActive #(pRun) which are the "active lines" that will be run? pRun_get_linActive() { grep "^[ ]*[A-Za-z0-9_]\+[ ]\+.*#" "$1" } # 22Nov2025 quick tests, see "$d_bin"'fileops run.sh' -> pRun_get_linActive # this simply uses command line setup, so don't have ro run "$d_bin"'fileops run.sh' #08********08 #] +-----+ #] path[,L] - redistributes StrUse from-str-to-Src basis output files #] using results from [pthSrc_subDirRef_optrNam_getArgOrderNum_pout, #] fNdfL_subDirRef_argOrderNum_dout] # fbase assumes a pre-established framework of [pth, subDir]s # following a standard nomenclature # www.BillHowell.ca 11Jul2022 initial # 09Jul2022 initial, 12Jul2022 moved dirs from "$d_Qtest" to "$d_libWork" # needs [cleanup, verification] # pthSrc is typically one of Howell's "library" ndf files (like arrays.ndf, etc) # instructions, see "$d_Qtest"'7_HowellCoreLib arg[Order, Number] instructions.txt' #] fSrc_pStrUse_appendTo_pSrcChk #() redistributes strUse from-str-to-Src basis #] append lines in pthLStrUse from fnamSrc to pthSrcStrUse # www.BillHowell.ca 11Jul2022 intitial # pthGrp identifies lines that show the path for which lines containing any str of pthStrL are shown # 13Jul2022 removed as redundant : bol_pth210=$( echo "$line210" | grep '/media/bill/' ) # but it might not be... # fSrc_pStrUse_appendTo_pSrcChk "$pStrUse476" "$fnamSrc476" "$pout476" # using results from [pthSrc_subDirRef_optrNam_getArgOrderNum_pout, fNdfL_subDirRef_argOrderNum_dout] fSrc_pStrUse_appendTo_pSrcChk() { local pStrUse="$1" local pSrcChk="$2" local bol_fbase='' # echo "pStrUse = $pStrUse" # echo "fSrc = $libFbase337" # echo "pSrcChk = $pSrcChk" while read -u 9 line; do bol_fbase=$( echo "$line" | grep "$libFbase337" ) if [ -n "$bol_fbase" ]; then # echo "line = $line" # echo "bol_fbase = $bol_fbase" echo "$line" >>"$pSrcChk" fi done 9< "$pStrUse" } #] pLibSrcPthVarL_pLibStrUseL_appendTo_pLSrcChk #() #] redistributes StrUse from-str-to-Src basis output files #] using results from [pthSrc_subDirRef_optrNam_getArgOrderNum_pout, #] fNdfL_subDirRef_argOrderNum_dout] #] append lines in pthLStrUse from fnamSrc to pthSrcStrUse #] nor arguments - [global, internal] variables only # www.BillHowell.ca 09Jul2022 initial # 14Jul2022 : # global variables reduce chances of mixups and cross-over between different [project, fbase] # They are set differently : # for each project : [d_libWork, pStrUsePthL, pLibSrcPthVarL] # for each fbase : [libFbase, d_libFbase, d_libData, p_libStrPL, d_libSrcChk] # NOTE! : SrcPthVarL are given as vars, eg : "$d_Qndfs"'arrays.ndf' # 'eval' to get fullPths : /media/bill/Dell2/Website - raw/Qnial/MY_NDFS/arrays.ndf # "$pLibSrcPthVarL337" is manually taken from 'QNial setup.ndf' for HowellQNialCoreLibs # need also at some time to regenerate for all .ndfs # I probably meant to regenerate pLibStrUseL.txt ?? # is pLibStrUseL supposed to be strUseL.txt?? (used to be optrNamL.txt) # ls -1 "$d_libData337"*.* >"$pLibSrcPthVarL337" # test only one : # strUse - must re-assign pLibStrUseL="$d_libData337"'strUse one only.txt' # pLibStrUseL="$d_libData337"'StrUse.txt' # these rm earlier results, so be careful! pLibSrcPthVarL_pLibStrUseL_appendTo_pLSrcChk() { local date_ymdhms=$(date +"%0y%0m%0d %0kh%0Mm%0Ss") local fSrc='' local pSrcChk='' local fSrc_pStrUse_appendTo_pSrcCh='' echo >>"$p_log" "$date_ymdhms pLibSrcPthVarL_pLibStrUseL_appendTo_pLSrcChk" while read -u 9 pthSrcVar; do fSrc=$( echo "$pthSrcVar" | sed 's|\(.*\)\x27\(.*\)\x27|\2|' | sed 's|\(.*\)\/\(.*\)|\2|') echo "$fSrc" pSrcChk="$d_libSrcChk$fSrc"' argOrderNum toCheck.txt' if [ -f "$pSrcChk" ]; then rm "$pSrcChk" fi while read -u 8 pStrUse; do fSrc_pStrUse_appendTo_pSrcChk "$fSrc" "$pStrUse" "$pSrcChk" done 8< "$pLibStrUseL" done 9< "$pLibSrcPthVarL337" } #] pOvrL_pStrP_pIns_replaceLines #(pOvrL pStrP pIns)replace ONE strPStrEnd linSeq in each pOvrL #] pStrP MUST be properly escaped (problem for arbitrary pStrP -> need pHum_sed_pCde) #] n[Strt, Endd] are set manually: to fix "addition" in prodn version instead of "replace" # 11Nov2023 still has mistake!!! don't use until fixed! # example : replace Google Analytics tag # 08Nov2023 adapted from 22Apr2020 old code in # "$d_SysMaint"'webSite/0_website bash notes.txt' # "$d_bin"'z_History/webSite update.sh' # start str must be unique to replacement need # may be only part of a line - but entire line is copied # [ignore, insert] continues to first occurrence of end str # 09Nov2023 auto-archive! may want to make that a boolean later # 10Nov2023 following failure of pOvrL_pStrP_pIns_replaceLines # close 734 "command not found" how do I close a file? maybe bash does that... pOvrL_pStrP_pIns_replaceLines() { # 11Nov2023 still has mistake!!! don't use until fixed! local pOvL="$1" local pStP="$2" local pIns="$3" local pTmp="$d_temp"'pOvrL_pStrP_pInsert_replace temp.txt' local pLog="$d_bin"'bin logs/pOvrL_pStrP_pIns_replaceLines log.txt' if [ -s "$pLog" ]; then pinn_arcArch "$pLog"; rm "$pLog"; fi if [[ -s "$pOvL" && -s "$pStP" ]]; then pinn_arcArch "$pOvrL" # 11Nov2023 currently only one pStrP works in file exec <"$pStP" read -u strP local strt=$( echo "$strP" | sed 's|:Q:.*||' ) local endd=$( echo "$strP" | sed 's|.*:Q:||' ) # echo "strt = $strt" # echo "endd = $endd" echo >>"$pLog" "strt = $strt" echo >>"$pLog" "endd = $endd" while IFS='' read -u 734 pOvr; do echo >>"$pLog" "$pOvr" if [ -s "$pTmp" ]; then rm "$pTmp"; fi bolStrt=$( grep "$strt" "$pOvr" ) bolEndd=$( grep "$endd" "$pOvr" ) # echo "bolStrt = $bolStrt" # echo "bolEndd = $bolEndd" if [[ -n "$bolStrt" && -n "$bolEndd" ]]; then bolIn=0 # initial read up to strt while IFS='' read -u 584 line; do bolStrt=$( echo "$line" | grep "$strt" ) if [[ -n "$bolStrt" ]]; then bolIn=1 cat "$pIns" >>"$pTmp" while [[ 1 == "$bolIn" ]]; do IFS='' read -u 584 line bolEndd=$( echo "$line" | grep "$endd" ) # echo "line = $line" # echo "bolEndd = $bolEndd" if [[ -n "$bolEndd" ]]; then bolIn=0; # 11Nov2023 fixTmp: remove for normal cases when this function works!! read -u 584 line # get rid of extra fi done elif [[ 0 == "$bolIn" ]]; then echo "$line" >>"$pTmp" fi done 584<"$pOvr" # echo "pTmp = $pTmp" # echo "pOvr = $pOvr" cp "$pTmp" "$pOvr" else echo >>"$pLog" ' pOvrL_pStrP_pInsert_replace, file missing >= one of : ' echo >>"$pLog" " bolStrt = $bolStrt, bolEndd = $bolEndd" fi done 734<"$pOvL" else echo >>"$pLog" 'pOvrL_pStrP_pInsert_replace, one of noexist : ' echo >>"$pLog" " $pOvL" echo >>"$pLog" " $pStP" fi } #] +-----+ #] notes files: #] pNotes_str_get_dateBefore_StrLine_dateAfter #() extract [dateBefore, str] lines # 18Dec2025 initial pNotes_str_get_dateBefore_StrLine_dateAfter() { dItems="$1" fNotes="$2" str="$3" date_ymd=$(date +%y%m%d) povr="$dItems$date_ymd $pNotes $str extracts.txt" pNotes="$dItems$fNotes" pDat="$d_temp"'pNotes_str_get_dateAfter_StrLine pDat.txt' pStr="$d_temp"'pNotes_str_get_dateAfter_StrLine pStr.txt' if [ -f "$povr" ]; then rm "$povr"; fi linNumStr=0 linNumDat=0 date='' grep --line-number "[0-9]\+[A-Za-z]\{3\}[0-9]\+" "$pNotes" | sed 's|^\([0-9]\+\):.*\([0-9]\+\)\([A-Za-z]\{3\}\)\([0-9]\+\).*|\1:\2\3\4|' >"$pDat" grep -i --line-number "$str" "$pNotes" | grep --invert-match "\] *: *$\|^[0-9]*:_" >"$pStr" echo >>"$povr" '24************************24' echo >>"$povr" "pNotes_str_get_dateBefore_StrLine_dateAfter() $date_ymd" echo >>"$povr" "pNotes= $pNotes" echo >>"$povr" "strSearch= $str" echo >>"$povr" '' exec 7<"$pDat" read -u 7 linDat linNumDat=$( echo "$linDat" | sed 's|^\([0-9]\+\):.*|\1|' ) while IFS='' read -u 9 linStr; do linNumStr=$( echo "$linStr" | sed 's|^\([0-9]\+\):.*|\1|' ) # echo "linNumStr= $linNumStr $sgn linNumDat= $linNumDat" while [ "$linNumDat" -le "$linNumStr" ]; do read -u 7 linDat linNumDat=$( echo "$linDat" | sed 's|^\([0-9]\+\):.*|\1|' ) done date=$( echo "$linDat" | sed 's|.*\([0-9]\+\)\([A-Za-z]\+\)\([0-9]\+\).*|\1\2\3|' ) # echo "$date $linNumDat $linNumStr" echo >>"$povr" "$date $linStr" done 9<"$pStr" } # 18Dec2025 quick test - see 'fileops run.sh' # pNotes_str_get_dateBefore_StrLine_dateAfter "$d_projMaj"'History/Ukraine-Russia/Howell - Ukraine and Russia notes.txt' 'Myrnohrad' #] pNotes_divider_get_pKeyWrdLL #() extract sections of notes, save each to one of pSectionLL # 24Dec2025 initial # pDivSecPL = file of [Div=section divider line, Sec=path for a Div] pNotes_divider_get_pKeyWrdLL() { # small test # pNotes="$d_bin"'0_test/fileops/dOriginal/inn Howell - Ukraine and Russia notes 652 lines.txt' # 24Dec2025 actual notes - split into [milSum, other] pNotes="$d_projMaj"'History/Ukraine-Russia/Howell - Ukraine and Russia notes.txt' divider='+-----+' divEsc='\+\-\-\-\-\-\+' pMilSum="$d_temp"'pNotes_pDivSecPL_get_pSectionLL milSum.txt' pOthers="$d_temp"'pNotes_pDivSecPL_get_pSectionLL others.txt' keyWrdMilSum='Military Summary\|@milSum\|@RuslanBelov\|Ruslan Belov' #sedOther='???' if [ -f "$pMilSum" ]; then rm "$pMilSum"; fi if [ -f "$pOthers" ]; then rm "$pOthers"; fi pTemp="$pOthers" # using Divs, collect pTemp for each section, classify & file # It would probably be better to get line#, then [get, save] section? #24Dec2025 rather than an AI-like matching, for now just look for keywords in 1st line of Section inSection=0 while IFS='' read -u 9 line; do bolSec=$( echo "$line" | grep "$divEsc" ) if [ "$bolSec" == '' ]; then echo "$line" >>"$pTemp" else read -u 9 line pointsMilSum=$( echo "$line" | grep "$keyWrdMilSum" | wc -l ) if [ 1 -le "$pointsMilSum" ]; then pTemp="$pMilSum"; else # echo "pointsMilSum= $pointsMilSum" pTemp="$pOthers"; fi echo "$divider" >>"$pTemp" echo "$line" >>"$pTemp" fi done 9<"$pNotes" } # 24Dec2025 # 18:33 24Dec2025$ keyWrdMilSum='Military Summary\|@milSum\|@RuslanBelov\|Ruslan Belov' # 18:33 24Dec2025$ line='#] @milSum am: Tue 23Dec2025 Massive Strike on Energy Infrastructure💥Intensity of Assault Operations' # 18:33 24Dec2025$ echo "$keyWrdMilSum" # Military Summary\|@milSum\|@RuslanBelov\|Ruslan Belov # 18:34 24Dec2025$ echo "$line" # #] @milSum am: Tue 23Dec2025 Massive Strike on Energy Infrastructure💥Intensity of Assault Operations # 18:34 24Dec2025$ echo "$line" | grep "$keyWrdMilSum" # #] @milSum am: Tue 23Dec2025 Massive Strike on Energy Infrastructure💥Intensity of Assault Operations # 18:34 24Dec2025$ echo "$line" | grep "$keyWrdMilSum" | wc -l # 1 # ~ #************ # do NOT run this directly!, use, for example, "$d_bin"'fileops run.sh' # enddoc