Log to destination (#280)

* mod fn_parse_date

combine identical cases, rm oft-failing darwin, subproc syntax

* repair my default spaces to not expand tabs

.github/FUNDING.yml

@@ -0,0 +1 @@
+github: laurent22

README.md

@@ -15,17 +15,25 @@ On macOS, it has a few disadvantages compared to Time Machine - in particular it
 	Usage: rsync_tmbackup.sh [OPTION]... <[USER@HOST:]SOURCE> <[USER@HOST:]DESTINATION> [exclude-pattern-file]
 
 	Options
-	 -p, --port           SSH port.
-	 -h, --help           Display this help message.
-	 --rsync-get-flags    Display the default rsync flags that are used for backup.
-	 --rsync-set-flags    Set the rsync flags that are going to be used for backup.
-	 --log-dir            Set the log file directory. If this flag is set, generated files will
-	                      not be managed by the script - in particular they will not be
-	                      automatically deleted.
-	 --strategy           Set the expiration strategy. Default: "1:1 30:7 365:30" means after one
-	                      day, keep one backup per day. After 30 days, keep one backup every 7 days.
-	                      After 365 days keep one backup every 30 days.
-	 --no-auto-expire     Set option to disable automatically purging old backups when out of space.
+	 -p, --port             SSH port.
+	 -h, --help             Display this help message.
+	 -i, --id_rsa           Specify the private ssh key to use.
+	 --rsync-get-flags      Display the default rsync flags that are used for backup. If using remote
+	                        drive over SSH, --compress will be added.
+	 --rsync-set-flags      Set the rsync flags that are going to be used for backup.
+	 --rsync-append-flags   Append the rsync flags that are going to be used for backup.
+	 --log-dir              Set the log file directory. If this flag is set, generated files will
+	                        not be managed by the script - in particular they will not be
+	                        automatically deleted.
+	                        Default: /home/backuper/.rsync_tmbackup
+	 --log-to-destination   Set the log file directory to the destination directory. If this flag
+	                        is set, generated files will not be managed by the script - in particular
+				they will not be automatically deleted.
+	 --strategy             Set the expiration strategy. Default: "1:1 30:7 365:30" means after one
+	                        day, keep one backup per day. After 30 days, keep one backup every 7 days.
+	                        After 365 days keep one backup every 30 days.
+	 --no-auto-expire       Disable automatically deleting backups when out of space. Instead an error
+	                        is logged, and the backup is aborted.
 
 ## Features
 
@@ -65,8 +73,8 @@ On macOS, it has a few disadvantages compared to Time Machine - in particular it
 		rsync_tmbackup.sh user@example.com:/home /mnt/backup_drive
 
 * To mimic Time Machine's behaviour, a cron script can be setup to backup at regular interval. For example, the following cron job checks if the drive "/mnt/backup" is currently connected and, if it is, starts the backup. It does this check every 1 hour.
-
-		0 */1 * * * if [[ -d /mnt/backup ]]; then rsync_tmbackup.sh /home /mnt/backup; fi
+		
+		0 */1 * * * if grep -qs /mnt/backup /proc/mounts; then rsync_tmbackup.sh /home /mnt/backup; fi
 
 ## Backup expiration logic
 
@@ -80,7 +88,7 @@ Before the first interval (i.e. by default within the first 24h) it is implied t
 
 ## Exclusion file
 
-An optional exclude file can be provided as a third parameter. It should be compatible with the `--exclude-from` parameter of rsync. See [this tutorial](https://sites.google.com/site/rsync2u/home/rsync-tutorial/the-exclude-from-option) for more information.
+An optional exclude file can be provided as a third parameter. It should be compatible with the `--exclude-from` parameter of rsync. See [this tutorial](https://web.archive.org/web/20230126121643/https://sites.google.com/site/rsync2u/home/rsync-tutorial/the-exclude-from-option) for more information.
 
 ## Built-in lock
 
@@ -88,14 +96,13 @@ The script is designed so that only one backup operation can be active for a giv
 
 ## Rsync options
 
-To display the rsync options that are used for backup, run `./rsync_tmbackup.sh --rsync-get-flags`. It is also possible to add or remove options using the `--rsync-set-flags` option. For example, to exclude backing up permissions and groups:
+To display the rsync options that are used for backup, run `./rsync_tmbackup.sh --rsync-get-flags`. It is also possible to add or remove options using the `--rsync-append-flags` or `--rsync-set-flags` option. For example, to exclude backing up permissions and groups:
 
-	rsync_tmbackup --rsync-set-flags "--numeric-ids --links --hard-links \
-	--one-file-system --archive --no-perms --no-groups --itemize-changes" /src /dest
+	rsync_tmbackup --rsync-append-flags "--no-perms --no-group" /src /dest
 
 ## No automatic backup expiration
 
-An option to diable the default behaviour to purge old backups when out of space. This option is set with the `--no-auto-expire` flag.
+An option to disable the default behaviour to purge old backups when out of space. This option is set with the `--no-auto-expire` flag.
 	
 	
 ## How to restore
@@ -117,7 +124,7 @@ The script creates a backup in a regular directory so you can simply copy the fi
 
 The MIT License (MIT)
 
-Copyright (c) 2013-2018 Laurent Cozic
+Copyright (c) 2013-2024 Laurent Cozic
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal

rsync_tmbackup.sh

@@ -1,6 +1,6 @@
 #!/usr/bin/env bash
 
-APPNAME=$(basename $0 | sed "s/\.sh$//")
+APPNAME=$(basename "$0" | sed "s/\.sh$//")
 
 # -----------------------------------------------------------------------------
 # Log functions
@@ -32,22 +32,28 @@ trap 'fn_terminate_script' SIGINT
 # Small utility functions for reducing code duplication
 # -----------------------------------------------------------------------------
 fn_display_usage() {
-	echo "Usage: $(basename $0) [OPTION]... <[USER@HOST:]SOURCE> <[USER@HOST:]DESTINATION> [exclude-pattern-file]"
+	echo "Usage: $(basename "$0") [OPTION]... <[USER@HOST:]SOURCE> <[USER@HOST:]DESTINATION> [exclude-pattern-file]"
 	echo ""
 	echo "Options"
-	echo " -p, --port           SSH port."
-	echo " -h, --help           Display this help message."
-	echo " --rsync-get-flags    Display the default rsync flags that are used for backup."
-	echo " --rsync-set-flags    Set the rsync flags that are going to be used for backup."
-	echo " --log-dir            Set the log file directory. If this flag is set, generated files will"
-	echo "                      not be managed by the script - in particular they will not be"
-	echo "                      automatically deleted."
-	echo "                      Default: $LOG_DIR"
-	echo " --strategy           Set the expiration strategy. Default: \"1:1 30:7 365:30\" means after one"
-	echo "                      day, keep one backup per day. After 30 days, keep one backup every 7 days."
-	echo "                      After 365 days keep one backup every 30 days."
-	echo " --no-auto-expire     Disable automatically deleting backups when out of space. Instead an error"
-	echo "                      is logged, and the backup is aborted."
+	echo " -p, --port             SSH port."
+	echo " -h, --help             Display this help message."
+	echo " -i, --id_rsa           Specify the private ssh key to use."
+	echo " --rsync-get-flags      Display the default rsync flags that are used for backup. If using remote"
+	echo "                        drive over SSH, --compress will be added."
+	echo " --rsync-set-flags      Set the rsync flags that are going to be used for backup."
+	echo " --rsync-append-flags   Append the rsync flags that are going to be used for backup."
+	echo " --log-dir              Set the log file directory. If this flag is set, generated files will"
+	echo "                        not be managed by the script - in particular they will not be"
+	echo "                        automatically deleted."
+	echo "                        Default: $LOG_DIR"
+	echo " --log-to-destination   Set the log file directory to the destination directory. If this flag"
+	echo "                        is set, generated files will not be managed by the script - in particular"
+	echo "                        they will not be automatically deleted."
+	echo " --strategy             Set the expiration strategy. Default: \"1:1 30:7 365:30\" means after one"
+	echo "                        day, keep one backup per day. After 30 days, keep one backup every 7 days."
+	echo "                        After 365 days keep one backup every 30 days."
+	echo " --no-auto-expire       Disable automatically deleting backups when out of space. Instead an error"
+	echo "                        is logged, and the backup is aborted."
 	echo ""
 	echo "For more detailed help, please see the README file:"
 	echo ""
@@ -57,18 +63,20 @@ fn_display_usage() {
 fn_parse_date() {
 	# Converts YYYY-MM-DD-HHMMSS to YYYY-MM-DD HH:MM:SS and then to Unix Epoch.
 	case "$OSTYPE" in
-		linux*) date -d "${1:0:10} ${1:11:2}:${1:13:2}:${1:15:2}" +%s ;;
-		cygwin*) date -d "${1:0:10} ${1:11:2}:${1:13:2}:${1:15:2}" +%s ;;
-		darwin8*) yy=`expr ${1:0:4}`
-			mm=`expr ${1:5:2} - 1`
-			dd=`expr ${1:8:2}`
-			hh=`expr ${1:11:2}`
-			mi=`expr ${1:13:2}`
-			ss=`expr ${1:15:2}`
-			# Because under MacOS X Tiger 'date -j' doesn't work, we do this:
-			perl -e 'use Time::Local; print timelocal('$ss','$mi','$hh','$dd','$mm','$yy'),"\n";' ;;
-		darwin*) date -j -f "%Y-%m-%d-%H%M%S" "$1" "+%s" ;;
+		linux*|cygwin*|netbsd*)
+			date -d "${1:0:10} ${1:11:2}:${1:13:2}:${1:15:2}" +%s ;;
 		FreeBSD*) date -j -f "%Y-%m-%d-%H%M%S" "$1" "+%s" ;;
+		darwin*)
+			# Under MacOS X Tiger
+			# Or with GNU 'coreutils' installed (by homebrew)
+			#   'date -j' doesn't work, so we do this:
+			yy=$(expr ${1:0:4})
+			mm=$(expr ${1:5:2} - 1)
+			dd=$(expr ${1:8:2})
+			hh=$(expr ${1:11:2})
+			mi=$(expr ${1:13:2})
+			ss=$(expr ${1:15:2})
+			perl -e 'use Time::Local; print timelocal('$ss','$mi','$hh','$dd','$mm','$yy'),"\n";' ;;
 	esac
 }
 
@@ -92,8 +100,14 @@ fn_expire_backups() {
 	local current_timestamp=$EPOCH
 	local last_kept_timestamp=9999999999
 
-	# Process each backup dir from most recent to oldest
-	for backup_dir in $(fn_find_backups | sort -r); do
+	# we will keep requested backup
+	backup_to_keep="$1"
+	# we will also keep the oldest backup
+	oldest_backup_to_keep="$(fn_find_backups | sort | sed -n '1p')"
+
+	# Process each backup dir from the oldest to the most recent
+	for backup_dir in $(fn_find_backups | sort); do
+
 		local backup_date=$(basename "$backup_dir")
 		local backup_timestamp=$(fn_parse_date "$backup_date")
 
@@ -103,36 +117,60 @@ fn_expire_backups() {
 			continue
 		fi
 
+		if [ "$backup_dir" == "$backup_to_keep" ]; then
+			# this is the latest backup requsted to be kept. We can finish pruning
+			break
+		fi
+
+		if [ "$backup_dir" == "$oldest_backup_to_keep" ]; then
+			# We dont't want to delete the oldest backup. It becomes first "last kept" backup
+			last_kept_timestamp=$backup_timestamp
+			# As we keep it we can skip processing it and go to the next oldest one in the loop
+			continue
+		fi
+
 		# Find which strategy token applies to this particular backup
 		for strategy_token in $(echo $EXPIRATION_STRATEGY | tr " " "\n" | sort -r -n); do
 			IFS=':' read -r -a t <<< "$strategy_token"
 
-			# After which date (relative to today) this token applies (X)
+			# After which date (relative to today) this token applies (X) - we use seconds to get exact cut off time
 			local cut_off_timestamp=$((current_timestamp - ${t[0]} * 86400))
 
-			# Every how many days should a backup be kept past the cut off date (Y)
-			local cut_off_interval=$((${t[1]} * 86400))
+			# Every how many days should a backup be kept past the cut off date (Y) - we use days (not seconds)
+			local cut_off_interval_days=$((${t[1]}))
 
 			# If we've found the strategy token that applies to this backup
 			if [ "$backup_timestamp" -le "$cut_off_timestamp" ]; then
 
 				# Special case: if Y is "0" we delete every time
-				if [ $cut_off_interval -eq "0" ]; then
+				if [ $cut_off_interval_days -eq "0" ]; then
 					fn_expire_backup "$backup_dir"
 					break
 				fi
 
+				# we calculate days number since last kept backup
+				local last_kept_timestamp_days=$((last_kept_timestamp / 86400))
+				local backup_timestamp_days=$((backup_timestamp / 86400))
+				local interval_since_last_kept_days=$((backup_timestamp_days - last_kept_timestamp_days))
+
 				# Check if the current backup is in the interval between
 				# the last backup that was kept and Y
-				local interval_since_last_kept=$((last_kept_timestamp - backup_timestamp))
-				if [ "$interval_since_last_kept" -lt "$cut_off_interval" ]; then
+				# to determine what to keep/delete we use days difference
+				if [ "$interval_since_last_kept_days" -lt "$cut_off_interval_days" ]; then
+
 					# Yes: Delete that one
 					fn_expire_backup "$backup_dir"
+					# backup deleted no point to check shorter timespan strategies - go to the next backup
+					break
+
 				else
-					# No: Keep it
+
+					# No: Keep it.
+					# this is now the last kept backup
 					last_kept_timestamp=$backup_timestamp
+					# and go to the next backup
+					break
 				fi
-				break
 			fi
 		done
 	done
@@ -145,14 +183,22 @@ fn_parse_ssh() {
 		SSH_USER=$(echo "$DEST_FOLDER" | sed -E  's/^([A-Za-z0-9\._%\+\-]+)@([A-Za-z0-9.\-]+)\:(.+)$/\1/')
 		SSH_HOST=$(echo "$DEST_FOLDER" | sed -E  's/^([A-Za-z0-9\._%\+\-]+)@([A-Za-z0-9.\-]+)\:(.+)$/\2/')
 		SSH_DEST_FOLDER=$(echo "$DEST_FOLDER" | sed -E  's/^([A-Za-z0-9\._%\+\-]+)@([A-Za-z0-9.\-]+)\:(.+)$/\3/')
-		SSH_CMD="ssh -p $SSH_PORT ${SSH_USER}@${SSH_HOST}"
+		if [ -n "$ID_RSA" ] ; then
+			SSH_CMD="ssh -p $SSH_PORT -i $ID_RSA ${SSH_USER}@${SSH_HOST}"
+		else
+			SSH_CMD="ssh -p $SSH_PORT ${SSH_USER}@${SSH_HOST}"
+		fi
 		SSH_DEST_FOLDER_PREFIX="${SSH_USER}@${SSH_HOST}:"
 	elif echo "$SRC_FOLDER"|grep -Eq '^[A-Za-z0-9\._%\+\-]+@[A-Za-z0-9.\-]+\:.+$'
 	then
 		SSH_USER=$(echo "$SRC_FOLDER" | sed -E  's/^([A-Za-z0-9\._%\+\-]+)@([A-Za-z0-9.\-]+)\:(.+)$/\1/')
 		SSH_HOST=$(echo "$SRC_FOLDER" | sed -E  's/^([A-Za-z0-9\._%\+\-]+)@([A-Za-z0-9.\-]+)\:(.+)$/\2/')
 		SSH_SRC_FOLDER=$(echo "$SRC_FOLDER" | sed -E  's/^([A-Za-z0-9\._%\+\-]+)@([A-Za-z0-9.\-]+)\:(.+)$/\3/')
-		SSH_CMD="ssh -p $SSH_PORT ${SSH_USER}@${SSH_HOST}"
+		if [ -n "$ID_RSA" ] ; then
+			SSH_CMD="ssh -p $SSH_PORT -i $ID_RSA ${SSH_USER}@${SSH_HOST}"
+		else
+			SSH_CMD="ssh -p $SSH_PORT ${SSH_USER}@${SSH_HOST}"
+		fi
 		SSH_SRC_FOLDER_PREFIX="${SSH_USER}@${SSH_HOST}:"
 	fi
 }
@@ -166,6 +212,15 @@ fn_run_cmd() {
 	fi
 }
 
+fn_run_cmd_src() {
+	if [ -n "$SSH_SRC_FOLDER_PREFIX" ]
+	then
+		eval "$SSH_CMD '$1'"
+	else
+		eval $1
+	fi
+}
+
 fn_find() {
 	fn_run_cmd "find '$1'"  2>/dev/null
 }
@@ -195,6 +250,18 @@ fn_ln() {
 	fn_run_cmd "ln -s -- '$1' '$2'"
 }
 
+fn_test_file_exists_src() {
+	fn_run_cmd_src "test -e '$1'"
+}
+
+fn_df_t_src() {
+	fn_run_cmd_src "df -T '${1}'"
+}
+
+fn_df_t() {
+	fn_run_cmd "df -T '${1}'"
+}
+
 # -----------------------------------------------------------------------------
 # Source and destination information
 # -----------------------------------------------------------------------------
@@ -206,16 +273,18 @@ SSH_CMD=""
 SSH_DEST_FOLDER_PREFIX=""
 SSH_SRC_FOLDER_PREFIX=""
 SSH_PORT="22"
+ID_RSA=""
 
 SRC_FOLDER=""
 DEST_FOLDER=""
 EXCLUSION_FILE=""
 LOG_DIR="$HOME/.$APPNAME"
 AUTO_DELETE_LOG="1"
+LOG_TO_DEST="0"
 EXPIRATION_STRATEGY="1:1 30:7 365:30"
 AUTO_EXPIRE="1"
 
-RSYNC_FLAGS="-D --compress --numeric-ids --links --hard-links --one-file-system --itemize-changes --times --recursive --perms --owner --group --stats --human-readable"
+RSYNC_FLAGS="-D --numeric-ids --links --hard-links --one-file-system --itemize-changes --times --recursive --perms --owner --group --stats --human-readable"
 
 while :; do
 	case $1 in
@@ -227,15 +296,23 @@ while :; do
 			shift
 			SSH_PORT=$1
 			;;
+		-i|--id_rsa)
+			shift
+			ID_RSA="$1"
+			;;
 		--rsync-get-flags)
 			shift
-			echo $RSYNC_FLAGS
+			echo "$RSYNC_FLAGS"
 			exit
 			;;
 		--rsync-set-flags)
 			shift
 			RSYNC_FLAGS="$1"
 			;;
+		--rsync-append-flags)
+			shift
+			RSYNC_FLAGS="$RSYNC_FLAGS $1"
+			;;
 		--strategy)
 			shift
 			EXPIRATION_STRATEGY="$1"
@@ -245,6 +322,10 @@ while :; do
 			LOG_DIR="$1"
 			AUTO_DELETE_LOG="0"
 			;;
+		--log-to-destination)
+			LOG_TO_DEST="1"
+			AUTO_DELETE_LOG="0"
+			;;
 		--no-auto-expire)
 			AUTO_EXPIRE="0"
 			;;
@@ -296,6 +377,12 @@ if [ -n "$SSH_SRC_FOLDER" ]; then
 	SRC_FOLDER="$SSH_SRC_FOLDER"
 fi
 
+# Exit if source folder does not exist.
+if ! fn_test_file_exists_src "${SRC_FOLDER}"; then
+	fn_log_error "Source folder \"${SRC_FOLDER}\" does not exist - aborting."
+	exit 1
+fi
+
 # Now strip off last slash from source folder.
 SRC_FOLDER="${SRC_FOLDER%/}"
 
@@ -324,6 +411,22 @@ if [ -z "$(fn_find_backup_marker "$DEST_FOLDER")" ]; then
 	exit 1
 fi
 
+# Check source and destination file-system (df -T /dest).
+# If one of them is FAT, use the --modify-window rsync parameter
+# (see man rsync) with a value of 1 or 2.
+#
+# The check is performed by taking the second row
+# of the output of the first command.
+if [[ "$(fn_df_t_src "${SRC_FOLDER}" | awk '{print $2}' | grep -c -i -e "fat")" -gt 0 ]]; then
+	fn_log_info "Source file-system is a version of FAT."
+	fn_log_info "Using the --modify-window rsync parameter with value 2."
+	RSYNC_FLAGS="${RSYNC_FLAGS} --modify-window=2"
+elif [[ "$(fn_df_t "${DEST_FOLDER}" | awk '{print $2}' | grep -c -i -e "fat")" -gt 0 ]]; then
+	fn_log_info "Destination file-system is a version of FAT."
+	fn_log_info "Using the --modify-window rsync parameter with value 2."
+	RSYNC_FLAGS="${RSYNC_FLAGS} --modify-window=2"
+fi
+
 # -----------------------------------------------------------------------------
 # Setup additional variables
 # -----------------------------------------------------------------------------
@@ -344,6 +447,10 @@ MYPID="$$"
 # Create log folder if it doesn't exist
 # -----------------------------------------------------------------------------
 
+if [[ $LOG_TO_DEST == "1" ]]; then
+	LOG_DIR="$DEST_FOLDER/.$APPNAME"
+fi
+
 if [ ! -d "$LOG_DIR" ]; then
 	fn_log_info "Creating log folder in '$LOG_DIR'..."
 	mkdir -- "$LOG_DIR"
@@ -369,9 +476,16 @@ if [ -n "$(fn_find "$INPROGRESS_FILE")" ]; then
 			fn_log_error "Previous backup task is still active - aborting (command: $RUNNINGCMD)."
 			exit 1
 		fi
+	elif [[ "$OSTYPE" == "netbsd"* ]]; then
+		RUNNINGPID="$(fn_run_cmd "cat $INPROGRESS_FILE")"
+		if ps -axp "$RUNNINGPID" -o "command" | grep "$APPNAME" > /dev/null; then
+			fn_log_error "Previous backup task is still active - aborting."
+			exit 1
+		fi
 	else
 		RUNNINGPID="$(fn_run_cmd "cat $INPROGRESS_FILE")"
-		if [ "$RUNNINGPID" = "$(pgrep -o -f "$APPNAME")" ]; then
+		if ps -p "$RUNNINGPID" -o command | grep "$APPNAME"
+		then
 			fn_log_error "Previous backup task is still active - aborting."
 			exit 1
 		fi
@@ -423,7 +537,13 @@ while : ; do
 	# Purge certain old backups before beginning new backup.
 	# -----------------------------------------------------------------------------
 
-	fn_expire_backups
+	if [ -n "$PREVIOUS_DEST" ]; then
+		# regardless of expiry strategy keep backup used for --link-dest
+		fn_expire_backups "$PREVIOUS_DEST"
+	else
+		# keep latest backup
+		fn_expire_backups "$DEST"
+	fi
 
 	# -----------------------------------------------------------------------------
 	# Start backup
@@ -437,7 +557,12 @@ while : ; do
 
 	CMD="rsync"
 	if [ -n "$SSH_CMD" ]; then
-		CMD="$CMD  -e 'ssh -p $SSH_PORT -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null'"
+		RSYNC_FLAGS="$RSYNC_FLAGS --compress"
+		if [ -n "$ID_RSA" ] ; then
+			CMD="$CMD  -e 'ssh -p $SSH_PORT -i $ID_RSA -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null'"
+		else
+			CMD="$CMD  -e 'ssh -p $SSH_PORT -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null'"
+		fi
 	fi
 	CMD="$CMD $RSYNC_FLAGS"
 	CMD="$CMD --log-file '$LOG_FILE'"
@@ -500,11 +625,14 @@ while : ; do
 	# -----------------------------------------------------------------------------
 	# Add symlink to last backup
 	# -----------------------------------------------------------------------------
+	if [ "$EXIT_CODE" = 0 ]; then
+		# Create the latest symlink only when rsync succeeded
+		fn_rm_file "$DEST_FOLDER/latest"
+		fn_ln "$(basename -- "$DEST")" "$DEST_FOLDER/latest"
 
-	fn_rm_file "$DEST_FOLDER/latest"
-	fn_ln "$(basename -- "$DEST")" "$DEST_FOLDER/latest"
-
-	fn_rm_file "$INPROGRESS_FILE"
+		# Remove .inprogress file only when rsync succeeded
+		fn_rm_file "$INPROGRESS_FILE"
+	fi
 
 	exit $EXIT_CODE
 done