#] 
#] *********************
#] "$d_SysMaint"'Linux/rsync notes.txt'  - mirror local [fil, dir]s, for backups etc
# www.BillHowell.ca  10Sep2018 initial
# view in text editor, using constant-width font (eg courier), tabWidth = 3


#	see also : 
#		"$d_SysMaint"'Linux/chown & chmod notes.txt'  - [fil, dir] access [local, remote]
#		"$d_SysMaint"'Linux/permissions notes.link.txt'
#	REMOTE [doc, app]s : 
#		"$d_SysMaint"'Linux/lftp notes.txt'  - [upload, maintain] webSite, my main app for this!!
#		"$d_webWork"'fileops run commentary, webSite.html' - documentation, instructions
#		"$d_webWork"'fileops run webSite general.sh' - process to update [, sub]webSites
#		"$d_PROJECTS"'bin - secure/lftp update specified dir.sh'
#	LOCAL [doc, app]s : 
#		"$d_SysMaint"'Linux/rsync notes.txt'  - mirror local [fil, dir]s, for backups etc
#		"$d_bin"'backup.sh'  - backup drives, see instructions to run near end of this file
#		"$d_bin"'fileops.sh' -> chmod_Howell() for standard permissions on my system
#		"$d_bin"'webSite update notes.txt'
#		"$d_SysMaint"'Linux/rm notes.txt'
#48************************************************48




#24************************24
# Table of Contents, generate with :
# $ grep  "^#]"  "$d_SysMaint"'Linux/rsync notes.txt' |  sed "s/^#\]/  /" 
#
   *********************
   "$d_SysMaint"'Linux/rsync notes.txt'  - ???
   30Jun2023 rsync contents of d_src/<some part> -> d_out : no ending `/ for d_out
   ddJan2023 search 'Linux rsync and how to backup hidden files'
   28Sep2021 man rsync  - delete
   28Sep2021  search "Linux rsync backup option"
   26Mar2021  rsync d_webSite to http://www.BillHowell.ca
   10Sep2018  rsync QNial to website
   https://stackoverflow.com/questions/21060705/how-to-parse-rsync-stats-data


#24************************24
# Setup, ToDos,   

+-----+
https://stackoverflow.com/questions/4493525/what-does-f-mean-in-rsync-logs
Coding of transfers :  
The general format is like the string YXcstpoguax, where Y is replaced by the type of update being done, X is replaced by the file-type, and the other letters represent attributes that may be output if they are being modified.
YXcstpoguax
	. 	- the item is not being updated (though it might have attributes that are being modified)
	+++++++++ - this is a newly created item replaces each letter with a "+"
	identical item replaces the dots with spaces
	an unknown attribute replaces each letter with a "?" (this can happen when talking to an older rsync)
	Y	the type of update being done : 
		<	means that a file is being transferred to the remote host (sent)
		>	means that a file is being transferred to the local host (received).
	X 	the file-type
		f	it is a regular file, 
		d	directory
		L	for a symlink
		D	for a device
		S	for a special file (e.g. named sockets and fifos)
	a	means that the ACL information changed
	c 	means that a local change/creation is occurring for the item 
		(such as the creation of a directory or the changing of a symlink, etc.).
	g	means the group is different and is being updated to the sender’s value 
		(requires --group and the authority to set the group)
	h 	means that the item is a hard link to another item (requires --hard-links)
	o	means the owner is different and is being updated to the sender’s value 
		(requires --owner and super-user privileges)
	p	means the permissions are different and are being updated to the sender’s value
	s	the file size is different
	t	the time stamp is different
	u	slot is reserved for future use
	x	means that the extended attribute information changed
	*	means that the rest of the itemized-output area contains a message (e.g. "deleting")
examples : 
>f+++++++++
>f..t......
24************************24




#08********08
#] ??Oct2023 


#08********08
#] ??Oct2023 


#08********08
#] ??Oct2023 


#08********08
#] ??Oct2023 


#08********08
#] ??Oct2023 


#08********08
#] ??Oct2023 


#08********08
#] 20Oct2023 [rm, rsync] dir, touch files to old timeMod 
see pWebPageL_upload_online() in : 
	"$d_PROJECTS"'bin - secure/lftp update entire webSite.sh'  

	# ensure d_webOnly up-to-date, this resets all d_webOnly html to today's date? 
	rm  --recursive  "$d_webOnly"				# clear all [dir, webPage]s in d_webOnly
	sed  "s|$d_web||"   "$pWebPageL"  >"$pWebPageL_no_dWeb"
	rsync --files-from="$pWebPageL_no_dWeb"  "$d_web"  "$d_webOnly" 

	# overwrite webPages : must change dRoot
	while IFS='' read -u 9 pWebPage; do
		pWebOnly=$( echo  "$pWebPage"  |  sed  's|\/home\/bill\/web\/|/home/bill/PROJECTS/webHtmlOnly/|' )
		sed  's|\/home\/bill\/web\/|http://www.BillHowell.ca/|g'  "$pWebPage"  >"$pWebOnly" 
		# 20Oct2023 change pWebOnly timeMod to pWebPage's
		touch  --reference="$pWebPage"  "$pWebOnly"
	done 9<"$pWebPageL"  


#08********08
#] 17Sep2023 rsyn --include-from="$d_webWork"'pHtmlPathAll_L.txt'

	backer_webPageLOnly()
{
dater=$(date +"%y%m%d %kh%Mm")
d_src="$d_web"
d_out="$d_PROJECTS"'webHtmlOnly/' 

becho "+---------------------------------------------------------+"
becho "backer_rsync() - $dater  rsync of webPageLOnly $d_src  to  $d_out, "
becho ""

# 17Sep2023 see if a pLst can work??  Will I have to start include-from at d_src?
# I'm afraid of major damage to d_PROJECTS
beval 'rsync '"$options"'  --include-from="'"$d_webWork"'pHtmlPathAll_L.txt"     
  "'"$d_src"'"  "'"$d_out"'"  >>"'"$p_log"'"  '
becho ""

# --delete  option

becho ""
dater=$(date +"%y%m%d %kh%Mm")
becho "backer_rsync() - $dater  end of operation $dater"
becho ""
}

15:57$ bash "$d_bin""rsync directories.sh" 
...
rsync error: syntax or usage error (code 1) at main.c(1804) [Receiver=3.2.3]
/home/bill/web/bin/standard header.sh: line 142: /home/bill/web/: Is a directory
...
>> 

+-----+
search "Linux rsyn  from list of files"

+-----+
https://stackoverflow.com/questions/16647476/how-to-rsync-only-a-specific-list-of-files
How to rsync only a specific list of files?
Asked 10 years, 4 months ago
Modified 1 year, 2 months ago
Viewed 178k times

There is a flag --files-from that does exactly what you want. From man rsync:

try : 
options='-rup'
options='-up'
beval 'rsync '"$options"'  --include-from="'"$d_webWork"'pHtmlPathAll_L.txt"     
  "'"$d_src"'"  "'"$d_out"'"  >>"'"$p_log"'"  '
16:26$ bash "$d_bin""rsync directories.sh" 
rsync error: syntax or usage error (code 1) at main.c(1804) [Receiver=3.2.3]
/home/bill/web/bin/standard header.sh: line 142: /home/bill/web/: Is a directory
>> blah

rsync isn't a good tool?


#08********08
#] 02Jun2023 search "Linux rsync and how do I exclude multiple dirs?"

# see "$d_SysMaint"'Linux/rm notes.txt' :
# backer_htmlOnly() :
# rm "z_Archive$\|z_archive$\|z_Old$\|z_old$", also high-volume dirs 
	# check result : 
		# $ find  "$d_PROJECTS""webHtmlOnly/"  -type d  |  grep  "z_Archive$\|z_archive$\|z_Old$\|z_old$\|System_maintenance\/tex$\|ProjMajor\/OPM\/OPM\/matlab_mrst$\|ProjMajor\/OPM\/OPM/Octave$\|References\/Climategate $\|garrybog\/pine$"  >"$d_temp"'rm webHtmlOnly z_[Archive, Old, high-volume, etc].txt'
	# rm dirs : 
		# $ find  "$d_PROJECTS""webHtmlOnly/" -type d  |  grep  "z_Archive$\|z_archive$\|z_Old$\|z_old$\|System_maintenance\/tex$\|ProjMajor\/OPM\/OPM\/matlab_mrst$\|ProjMajor\/OPM\/OPM/Octave$\|References\/Climategate $\|garrybog\/pine$"  |  tr  \\n  \\0  |  xargs -0  -IFILE  rm  -r  "FILE"

# rm files without extension 
	# examples - see "$d_temp"'rm webHtmlOnly files without extension list.txt'
		# eg '/home/bill/PROJECTS/webHtmlOnly/economics, markets/Econometric models/Indexes'
	# check result : 
		# $ find  "$d_PROJECTS""webHtmlOnly/"  -type f  |  grep  "\/[A-Za-z0-9]*$"   >"$d_temp"'rm webHtmlOnly files without extension list.txt'
	# rm files without extension 
		# $ find  "$d_PROJECTS""webHtmlOnly/"  -type f  |  grep  "\/[A-Za-z0-9]*$"  |  tr  \\n  \\0  |  xargs -0  -IFILE  rm  -r  "FILE"

man rsync
	--exclude=PATTERN		exclude files matching PATTERN
>> use find-grep-xargs-rm - much better control

+-----+
https://linuxize.com/post/how-to-exclude-files-and-directories-with-rsync/
How to Exclude Files and Directories with Rsync
Posted  Feb 20, 2019

Exclude Multiple Files or Directories
To exclude multiple files or directories simply specify multiple --exclude options:
	rsync -a --exclude 'file1.txt' --exclude 'dir1/*' --exclude 'dir2' src_directory/ dst_directory/
If you prefer to use a single --exclude option you can list the files and directories you want to exclude in curly braces {} separated by a comma as shown below:
	rsync -a --exclude={'file1.txt','dir1/*','dir2'} src_directory/ dst_directory/



#08********08
#] 30Jun2023 rsync contents of d_src/<some part> -> d_out : no ending `/ for d_out
can use find with cp

see "$d_bin""rsync directories.sh"  
>> doesn't work?  -  maybe, just unbelievably SLOOOWWW!!!
>> end of day, all was copied via nemo file manager, kept original dates even

08********08
#] ddJan2023 search 'Linux rsync and how to backup hidden files'
https://www.linuxquestions.org/questions/linux-software-2/rsync-how-to-include-hidden-something-files-and-directories-501153/


08********08 
#] 28Sep2021 man rsync - delete
				 --delete            delete extraneous files from dest dirs
            --delete-excluded   also delete excluded files from dest dirs
>> both we used

08********08
#] 28Sep2021  search "Linux rsync backup option"
https://unix.stackexchange.com/questions/658918/using-or-not-using-rsync-with-backup-option
>> confused user

search "Linux rsync and hidden file?"
https://www.linuxquestions.org/questions/linux-software-2/rsync-how-to-include-hidden-something-files-and-directories-501153/
>> hmm, obscure...

...unsolved, I ust copied the hidden file...

08********08
#] 26Mar2021  rsync d_webSite to http://www.BillHowell.ca
Check lexi instructions - nyet... email
I will email lexicom.ca

+-----+
https://medium.com/@catquarks/how-to-upload-update-your-site-with-sftp-rsync-5043b1f897fa
How to Upload & Update Your Site With SFTP & Rsync
T.J.
Sep 21, 2016·12 min read
Rsync is not the same as cp. Rysnc was built to add CHANGES to files whenever possible, rather than overwrite entire files. That’s part of what makes it so fast.
	rsync -vnae ‘ssh -p 21098’ /home/bernardo/website/* bernardo@shared-moose-server360.com:/home/bernardo/www
>> So I will try that 

First, let’s make sure we have ssh access by running this:
	ssh bernardo@shared-moose-server360.com -p 21098
If you don’t explicitly specify a port like we did above, ssh will attempt to log into port 22 by default. Anyway, we know we’re in when we see a prompt like this:
	[bernardo@shared-moose-server360.com ~]$
>> for my ssh server, see "$bin""ssh_logon"  : 
	ssh -X root@bh.cust.lexi.net 
>> so try without specifying port :
	ssh billhowellweb@billhowell.ca 
>> no response, so try : 
	ssh -X billhowellweb@billhowell.ca 
>> no response again.  I'm stuck

+-----+
https://www.howtoforge.com/mirroring_with_rsync
Mirror Your Web Site With rsync
Falko Timme
>> excellent instruction, a bit over my head.

+-----+
https://www.sitepoint.com/community/t/rsync-beats-filezilla-hands-down/323150
Rsync beats FileZilla hands down
John_Betong, Mentor, Mar '19


08********08
#] 10Sep2018  rsync QNial to website

https://unix.stackexchange.com/questions/326680/how-to-properly-use-rsync

no go rsync with ftp site
lftp

08********08
#] https://stackoverflow.com/questions/21060705/how-to-parse-rsync-stats-data
# grep -e "Number of files transferred" -e "Total transferred file size"

# rsync - vastly preferred over cp
# rsync options 
#    By default, rsync does INCREMENTAL backups!!!
# man rsync 
#        -A, --acls          		preserve ACLs (implies -p)
#        -H, --hard-links    		preserve hard links
#        -a, --archive       		archive mode; equals -rlptgoD (no -H,-A,-X)
#            --no-OPTION     		turn off an implied OPTION (e.g. --no-D)
#				 --delete            delete extraneous files from dest dirs
#            --delete-excluded   also delete excluded files from dest dirs
#        -D                  		same as --devices --specials
#        -g, --group         		preserve group
#            --devices       		preserve device files (super-user only)
#            --specials      		preserve special files
#        -l, --links         		copy symlinks as symlinks
#        -n, --dry-run				perform a trial run that doesn’t make any changes             #			#				 --itemize-changes	see what an rsync command is going to do before one actually runs it
#        -o, --owner         		preserve owner (super-user only)
#        -p, --perms         		preserve permissions
#        -r, --recursive     		recurse into directories
#        -t, --times         		preserve modification times
#        -u, --update        		skip files that are newer on the receiver
#            --inplace       		update destination files in-place
#            --append        		append data onto shorter files
#            --append-verify 		--append w/old data in file checksum
#			-v, --verbose				increase verbosity
#        -X, --xattrs        		preserve extended attributes
# 04Aug2018 - always having problem with SWAPPER permissions.  
# Maybe get rid of rsync -a (archive=-rlptgoD (no -H,-A,-X))
# use -rltgu
#	--timeout=SECONDS	- set I/O timeout in seconds
#	--ignore-errors	delete even if there are I/O errors
#            --stats                 give some file-transfer stats
#       This tells rsync to print a verbose set of statistics on the file transfer, allowing  you  to  tell
#       how  effective  rsync’s  delta-transfer  algorithm  is for your data.  This option is equivalent to
#       --info=stats2 if combined with 0 or 1 -v options, or --info=stats3 if combined with 2  or  more  -v
#       options.

# major problem - many files :
# rsync error: received SIGINT, SIGTERM, or SIGHUP (code 20) at rsync.c(632) [sender=3.1.1]

# old code 
# |  grep -e 'Number of files transferred'  |  grep -e 'Total transferred file size' 
# Note : the links option -l may be causing freezes?  (something is!) 
#		 rsync --stats  -rltgu  --exclude-from="$p_excl"  "$d_src"  "$d_out"   >>"$p_log"  |  grep -e 'size'  |  grep  -e  'Total'  >>"$p_log"  


# tests
#echo "'"	"$d_PROJECTS/Videos of note"	"'"	 

# Problematic if included (BIG!?)
# "/home/bill/.mozilla/firefox/mwad0hks.default"
# 




# enddoc