Merge pull request #23 from robru/simplify-logfiles

Simplify logfile parsing and drop unreachable codepath.

README.md

@@ -1,6 +1,6 @@
 # Rsync time backup
 
-Time Machine style backup with rsync. Should work on Linux, Mac OS X and Windows with Cygwin.
+Time Machine style backup with rsync. Should work on Linux (tested on some distros), Mac OS X (fully tested) and Windows with Cygwin (not tested yet but feeback would be welcome).
 
 # Installation
 
@@ -36,18 +36,20 @@ An optional exclude file can be provided as a third parameter. It should be comp
 
 * Exclude file - support for pattern-based exclusion via the `--exclude-from` rsync parameter.
 
-* The application is one bash script that can be easily edited.
+* Automatically purge old backups - within 24 hours, all backups are kept. Within one month, the most recent backup for each day is kept. For all previous backups, the most recent of each month is kept.
+
+* "latest" symlink that points to the latest successful backup.
+
+* The application is just one bash script that can be easily edited.
 
 # TODO
 
-* Check if there's enough space in the destination before doing the backup. Also automatically delete old backups.
+* Minor changes (see TODO comments in the source).
 
-* Manage the backups in a way similar to Time Machine - hourly backups for the past 24 hours; daily backups for the past month; weekly backups for the previous months.
+* Backup to remote drive?
 
 # LICENSE
 
 [MIT](http://opensource.org/licenses/MIT)
 
-
 [![Bitdeli Badge](https://d2weczhvl823v0.cloudfront.net/laurent22/rsync-time-backup/trend.png)](https://bitdeli.com/free "Bitdeli Badge")
-

rsync_tmbackup.sh

@@ -1,39 +1,68 @@
 #!/usr/bin/env bash
 
+APPNAME=$(basename $0 | sed "s/\.sh$//")
+
 # -----------------------------------------------------------------------------
 # Log functions
 # -----------------------------------------------------------------------------
 
-fn_log_info() {
+fn_log_info()  { echo "$APPNAME: $1"; }
-	echo "[RB INFO] $1"
+fn_log_warn()  { echo "$APPNAME: [WARNING] $1"; }
-}
+fn_log_error() { echo "$APPNAME: [ERROR] $1"; }
-
-fn_log_warn() {
-	echo "[RB WARN] $1"
-}
-
-fn_log_error() {
-	echo "[RB ERROR] $1"
-}
 
 # -----------------------------------------------------------------------------
 # Make sure everything really stops when CTRL+C is pressed
 # -----------------------------------------------------------------------------
 
 fn_terminate_script() {
-	echo "SIGINT caught"
+	fn_log_info "SIGINT caught."
 	exit 1
 }
 
 trap 'fn_terminate_script' SIGINT
 
+# -----------------------------------------------------------------------------
+# Small utility functions for reducing code duplication
+# -----------------------------------------------------------------------------
+
+fn_parse_date() {
+	# Converts YYYY-MM-DD-HHMMSS to YYYY-MM-DD HH:MM:SS and then to Unix Epoch.
+	case "$OSTYPE" in
+		linux*) date -d "${1:0:10} ${1:11:2}:${1:13:2}:${1:15:2}" +%s ;;
+		darwin*) date -j -f "%Y-%m-%d-%H%M%S" "$1" "+%s" ;;
+	esac
+}
+
+fn_find_backups() {
+	find "$DEST_FOLDER" -type d -name "????-??-??-??????" -prune | sort -r
+}
+
+fn_expire_backup() {
+	# Double-check that we're on a backup destination to be completely
+	# sure we're deleting the right folder
+	if [ -z "$(fn_find_backup_marker "$(dirname -- "$1")")" ]; then
+		fn_log_error "$1 is not on a backup destination - aborting."
+		exit 1
+	fi
+
+	fn_log_info "Expiring $1"
+	rm -rf -- "$1"
+}
+
 # -----------------------------------------------------------------------------
 # Source and destination information
 # -----------------------------------------------------------------------------
 
-SRC_FOLDER=${1%/}
+SRC_FOLDER="${1%/}"
-DEST_FOLDER=${2%/}
+DEST_FOLDER="${2%/}"
-EXCLUSION_FILE=$3
+EXCLUSION_FILE="$3"
+
+for ARG in "$SRC_FOLDER" "$DEST_FOLDER" "$EXCLUSION_FILE"; do
+if [[ "$ARG" == *"'"* ]]; then
+		fn_log_error 'Arguments may not have any single quote characters.'
+		exit 1
+	fi
+done
 
 # -----------------------------------------------------------------------------
 # Check that the destination drive is a backup drive
@@ -41,24 +70,14 @@ EXCLUSION_FILE=$3
 
 # TODO: check that the destination supports hard links
 
-fn_backup_marker_path() {
+fn_backup_marker_path() { echo "$1/backup.marker"; }
-	echo "$1/backup.marker"
+fn_find_backup_marker() { find "$(fn_backup_marker_path "$1")" 2>/dev/null; }
-}
 
-fn_is_backup_destination() {
+if [ -z "$(fn_find_backup_marker "$DEST_FOLDER")" ]; then
-	DEST_MARKER_FILE="$(fn_backup_marker_path $1)"
-	if [ -f "$DEST_MARKER_FILE" ]; then
-		echo "1"
-	else
-		echo "0"
-	fi
-}
-
-if [ "$(fn_is_backup_destination $DEST_FOLDER)" != "1" ]; then
 	fn_log_info "Safety check failed - the destination does not appear to be a backup folder or drive (marker file not found)."
 	fn_log_info "If it is indeed a backup folder, you may add the marker file by running the following command:"
 	fn_log_info ""
-	fn_log_info "touch \"$(fn_backup_marker_path $DEST_FOLDER)\""
+	fn_log_info "touch \"$(fn_backup_marker_path "$DEST_FOLDER")\""
 	fn_log_info ""
 	exit 1
 fi
@@ -67,14 +86,17 @@ fi
 # Setup additional variables
 # -----------------------------------------------------------------------------
 
-BACKUP_FOLDER_PATTERN="\d\d\d\d-\d\d-\d\d-\d\d\d\d\d\d"
+# Date logic
 NOW=$(date +"%Y-%m-%d-%H%M%S")
-PROFILE_FOLDER="$HOME/.rsync_tmbackup"
+EPOCH=$(date "+%s")
-LOG_FILE="$PROFILE_FOLDER/$NOW.log"
+KEEP_ALL_DATE=$(($EPOCH - 86400))       # 1 day ago
-DEST=$DEST_FOLDER/$NOW
+KEEP_DAILIES_DATE=$(($EPOCH - 2678400)) # 31 days ago
-LAST_TIME=$(ls -1 $DEST_FOLDER | grep "$BACKUP_FOLDER_PATTERN" | tail -n 1)
+
-PREVIOUS_DEST=$DEST_FOLDER/$LAST_TIME
+export IFS=$'\n' # Better for handling spaces in filenames.
-INPROGRESS_FILE=$DEST_FOLDER/backup.inprogress
+PROFILE_FOLDER="$HOME/.$APPNAME"
+DEST="$DEST_FOLDER/$NOW"
+PREVIOUS_DEST="$(fn_find_backups | head -n 1)"
+INPROGRESS_FILE="$DEST_FOLDER/backup.inprogress"
 
 # -----------------------------------------------------------------------------
 # Create profile folder if it doesn't exist
@@ -90,52 +112,79 @@ fi
 # -----------------------------------------------------------------------------
 
 if [ -f "$INPROGRESS_FILE" ]; then
-	if [ "$LAST_TIME" != "" ]; then
+	if [ -n "$PREVIOUS_DEST" ]; then
 		# - Last backup is moved to current backup folder so that it can be resumed.
 		# - 2nd to last backup becomes last backup.
 		fn_log_info "$INPROGRESS_FILE already exists - the previous backup failed or was interrupted. Backup will resume from there."
-		LINE_COUNT=$(ls -1 $DEST_FOLDER | grep "$BACKUP_FOLDER_PATTERN" | tail -n 2 | wc -l)
+		mv -- "$PREVIOUS_DEST" "$DEST"
-		mv $PREVIOUS_DEST $DEST
+		if [ "$(fn_find_backups | wc -l)" -gt 1 ]; then
-		if [ "$LINE_COUNT" -gt 1 ]; then
+			PREVIOUS_DEST="$(fn_find_backups | sed -n '2p')"
-			SECOND_LAST_TIME=$(ls -1 $DEST_FOLDER | grep "$BACKUP_FOLDER_PATTERN" | tail -n 2 | head -n 1)
-			LAST_TIME=$SECOND_LAST_TIME
 		else
-			LAST_TIME=""
+			PREVIOUS_DEST=""
 		fi
-		PREVIOUS_DEST=$DEST_FOLDER/$LAST_TIME
 	fi
 fi
 
-# -----------------------------------------------------------------------------
+# Run in a loop to handle the "No space left on device" logic.
-# Check if we are doing an incremental backup (if previous backup exists) or not
+while : ; do
-# -----------------------------------------------------------------------------
 
-LINK_DEST_OPTION=""
+	# -----------------------------------------------------------------------------
-if [ "$LAST_TIME" == "" ]; then
+	# Check if we are doing an incremental backup (if previous backup exists).
-	fn_log_info "No previous backup - creating new one."
+	# -----------------------------------------------------------------------------
-else
-	# If the path is relative, it needs to be relative to the destination. To keep
-	# it simple, just use an absolute path. See http://serverfault.com/a/210058/118679
-	PREVIOUS_DEST=`cd \`dirname "$PREVIOUS_DEST"\`; pwd`"/"`basename "$PREVIOUS_DEST"`
-	fn_log_info "Previous backup found - doing incremental backup from $PREVIOUS_DEST"
-	LINK_DEST_OPTION="--link-dest=$PREVIOUS_DEST"
-fi
 
-# -----------------------------------------------------------------------------
+	LINK_DEST_OPTION=""
-# Create destination folder if it doesn't already exists
+	if [ -z "$PREVIOUS_DEST" ]; then
-# -----------------------------------------------------------------------------
+		fn_log_info "No previous backup - creating new one."
+	else
+		# If the path is relative, it needs to be relative to the destination. To keep
+		# it simple, just use an absolute path. See http://serverfault.com/a/210058/118679
+		PREVIOUS_DEST="$(cd "$PREVIOUS_DEST"; pwd)"
+		fn_log_info "Previous backup found - doing incremental backup from $PREVIOUS_DEST"
+		LINK_DEST_OPTION="--link-dest='$PREVIOUS_DEST'"
+	fi
 
-if [ ! -d "$DEST" ]; then
+	# -----------------------------------------------------------------------------
-	fn_log_info "Creating destination $DEST"
+	# Create destination folder if it doesn't already exists
-	mkdir -p $DEST
+	# -----------------------------------------------------------------------------
-fi
 
-# -----------------------------------------------------------------------------
+	if [ ! -d "$DEST" ]; then
-# Start backup
+		fn_log_info "Creating destination $DEST"
-# -----------------------------------------------------------------------------
+		mkdir -p -- "$DEST"
+	fi
+
+	# -----------------------------------------------------------------------------
+	# Purge certain old backups before beginning new backup.
+	# -----------------------------------------------------------------------------
+
+	# Default value for $PREV ensures that the most recent backup is never deleted.
+	PREV="0000-00-00-000000"
+	for FILENAME in $(fn_find_backups | sort -r); do
+		BACKUP_DATE=$(basename "$FILENAME")
+		TIMESTAMP=$(fn_parse_date $BACKUP_DATE)
+
+		# Skip if failed to parse date...
+		if [ -z "$TIMESTAMP" ]; then
+			fn_log_warn "Could not parse date: $FILENAME"
+			continue
+		fi
+
+		if   [ $TIMESTAMP -ge $KEEP_ALL_DATE ]; then
+			true
+		elif [ $TIMESTAMP -ge $KEEP_DAILIES_DATE ]; then
+			# Delete all but the most recent of each day.
+			[ "${BACKUP_DATE:0:10}" == "${PREV:0:10}" ] && fn_expire_backup "$FILENAME"
+		else
+			# Delete all but the most recent of each month.
+			[ "${BACKUP_DATE:0:7}" == "${PREV:0:7}" ] && fn_expire_backup "$FILENAME"
+		fi
+
+		PREV=$BACKUP_DATE
+	done
+
+	# -----------------------------------------------------------------------------
+	# Start backup
+	# -----------------------------------------------------------------------------
 
-# Run in a loop to handle the "No space left on device" logic
-while [ "1" ]; do
 	LOG_FILE="$PROFILE_FOLDER/$(date +"%Y-%m-%d-%H%M%S").log"
 
 	fn_log_info "Starting backup..."
@@ -147,25 +196,23 @@ while [ "1" ]; do
 	CMD="$CMD --numeric-ids"
 	CMD="$CMD --links"
 	CMD="$CMD --hard-links"
-	CMD="$CMD --delete"
-	CMD="$CMD --delete-excluded"
 	CMD="$CMD --one-file-system"
 	CMD="$CMD --archive"
-	CMD="$CMD --progress"
 	CMD="$CMD --itemize-changes"
 	CMD="$CMD --verbose"
 	CMD="$CMD --log-file '$LOG_FILE'"
-	if [ "$EXCLUSION_FILE" != "" ]; then
+	if [ -n "$EXCLUSION_FILE" ]; then
-		CMD="$CMD --exclude-from \"$EXCLUSION_FILE\""
+		# We've already checked that $EXCLUSION_FILE doesn't contain a single quote
+		CMD="$CMD --exclude-from '$EXCLUSION_FILE'"
 	fi
 	CMD="$CMD $LINK_DEST_OPTION"
-	CMD="$CMD $SRC_FOLDER/ $DEST/"
+	CMD="$CMD -- '$SRC_FOLDER/' '$DEST/'"
 	CMD="$CMD | grep -E '^deleting|[^/]$'"
 
 	fn_log_info "Running command:"
 	fn_log_info "$CMD"
 
-	touch $INPROGRESS_FILE
+	touch -- "$INPROGRESS_FILE"
 	eval $CMD
 	RSYNC_EXIT_CODE=$?
 
@@ -174,50 +221,20 @@ while [ "1" ]; do
 	# -----------------------------------------------------------------------------
 
 	# TODO: find better way to check for out of space condition without parsing log.
-	grep --quiet "No space left on device (28)" "$LOG_FILE"
+	NO_SPACE_LEFT="$(grep "No space left on device (28)\|Result too large (34)" "$LOG_FILE")"
-	NO_SPACE_LEFT="$?"
-	if [ "$NO_SPACE_LEFT" != "0" ]; then
-		# This error might also happen if there is no space left
-		grep --quiet "Result too large (34)" "$LOG_FILE"
-		NO_SPACE_LEFT="$?"
-	fi
-		
-	rm -- "$LOG_FILE"
-	
-	if [ "$NO_SPACE_LEFT" == "0" ]; then
-		read -p "It looks like there is no space left on the destination. Delete old backup? (Y/n) " yn
-		case $yn in
-			[Nn]* ) exit 0;;
-		esac
 
+	rm -- "$LOG_FILE"
+
+	if [ -n "$NO_SPACE_LEFT" ]; then
 		fn_log_warn "No space left on device - removing oldest backup and resuming."
-		
+
-		BACKUP_FOLDER_COUNT=$(ls -1 $DEST_FOLDER | grep "$BACKUP_FOLDER_PATTERN" | wc -l)
+		if [[ "$(fn_find_backups | wc -l)" -lt "2" ]]; then
-		if [ "$BACKUP_FOLDER_COUNT" -lt "2" ]; then
 			fn_log_error "No space left on device, and no old backup to delete."
 			exit 1
 		fi
-		
+
-		# TODO: handle case where only two backup folders are left. In which case, need to remove --link-dest option
+		fn_expire_backup "$(fn_find_backups | tail -n 1)"
-		
+
-		OLDEST_BACKUP=$(ls -1 $DEST_FOLDER | grep "$BACKUP_FOLDER_PATTERN" | head -n 1)
-		if [ "$OLDEST_BACKUP" == "" ]; then
-			fn_log_error "No space left on device, and cannot get path to oldest backup to delete."
-			exit 1
-		fi
-		
-		OLD_BACKUP_PATH="$DEST_FOLDER/$OLDEST_BACKUP"
-		
-		# Double-check that we're on a backup destination to be completely sure we're deleting the right folder
-		OLD_BACKUP_PARENT_PATH=$(dirname -- "$OLD_BACKUP_PATH")
-		if [ "$(fn_is_backup_destination $OLD_BACKUP_PARENT_PATH)" != "1" ]; then
-			fn_log_error "'$OLD_BACKUP_PATH' is not on a backup destination - aborting."
-			exit 1
-		fi
-		
-		fn_log_info "Deleting '$OLD_BACKUP_PATH'..."
-		rm -rf -- "$OLD_BACKUP_PATH"
-		
 		# Resume backup
 		continue
 	fi
@@ -226,8 +243,16 @@ while [ "1" ]; do
 		fn_log_error "Exited with error code $RSYNC_EXIT_CODE"
 		exit $RSYNC_EXIT_CODE
 	fi
-	
+
-	rm $INPROGRESS_FILE
+	# -----------------------------------------------------------------------------
+	# Add symlink to last successful backup
+	# -----------------------------------------------------------------------------
+
+	rm -rf -- "$DEST_FOLDER/latest"
+	ln -vs -- "$(basename -- "$DEST")" "$DEST_FOLDER/latest"
+
+	rm -f -- "$INPROGRESS_FILE"
+	# TODO: grep for "^rsync error:.*$" in log
 	fn_log_info "Backup completed without errors."
 	exit 0
 done