Delete old backup when no more space on destination

rsync_tmbackup.sh

@@ -1,15 +1,31 @@
 #!/usr/bin/env bash
 
+# -----------------------------------------------------------------------------
+# Log functions
+# -----------------------------------------------------------------------------
+
+fn_log_info() {
+	echo "[RB INFO] $1"
+}
+
+fn_log_warn() {
+	echo "[RB WARN] $1"
+}
+
+fn_log_error() {
+	echo "[RB ERROR] $1"
+}
+
 # -----------------------------------------------------------------------------
 # Make sure everything really stops when CTRL+C is pressed
 # -----------------------------------------------------------------------------
 
-terminate_script() {
+fn_terminate_script() {
 	echo "SIGINT caught"
 	exit 1
 }
 
-trap 'terminate_script' SIGINT
+trap 'fn_terminate_script' SIGINT
 
 # -----------------------------------------------------------------------------
 # Source and destination information
@@ -23,13 +39,27 @@ EXCLUSION_FILE=$3
 # Check that the destination drive is a backup drive
 # -----------------------------------------------------------------------------
 
-DEST_MARKER_FILE=$DEST_FOLDER/backup.marker
+# TODO: check that the destination supports hard links
-if [ ! -f "$DEST_MARKER_FILE" ]; then
+
-	echo "Safety check failed - the destination does not appear to be a backup folder or drive (marker file not found)."
+fn_backup_marker_path() {
-	echo "If it is indeed a backup folder, you may add the marker file by running the following command:"
+	echo "$1/backup.marker"
-	echo ""
+}
-	echo "touch \"$DEST_MARKER_FILE\""
+
-	echo ""
+fn_is_backup_destination() {
+	DEST_MARKER_FILE="$(fn_backup_marker_path $1)"
+	if [ -f "$DEST_MARKER_FILE" ]; then
+		echo "1"
+	else
+		echo "0"
+	fi
+}
+
+if [ "$(fn_is_backup_destination $DEST_FOLDER)" != "1" ]; then
+	fn_log_info "Safety check failed - the destination does not appear to be a backup folder or drive (marker file not found)."
+	fn_log_info "If it is indeed a backup folder, you may add the marker file by running the following command:"
+	fn_log_info ""
+	fn_log_info "touch \"$(fn_backup_marker_path $DEST_FOLDER)\""
+	fn_log_info ""
 	exit 1
 fi
 
@@ -37,12 +67,24 @@ fi
 # Setup additional variables
 # -----------------------------------------------------------------------------
 
+BACKUP_FOLDER_PATTERN="\d\d\d\d-\d\d-\d\d-\d\d\d\d\d\d"
 NOW=$(date +"%Y-%m-%d-%H%M%S")
+PROFILE_FOLDER="$HOME/.rsync_tmbackup"
+LOG_FILE="$PROFILE_FOLDER/$NOW.log"
 DEST=$DEST_FOLDER/$NOW
-LAST_TIME=$(ls -1 $DEST_FOLDER | grep "\d\d\d\d-\d\d-\d\d-\d\d\d\d\d\d" | tail -n 1)
+LAST_TIME=$(ls -1 $DEST_FOLDER | grep "$BACKUP_FOLDER_PATTERN" | tail -n 1)
 PREVIOUS_DEST=$DEST_FOLDER/$LAST_TIME
 INPROGRESS_FILE=$DEST_FOLDER/backup.inprogress
 
+# -----------------------------------------------------------------------------
+# Create profile folder if it doesn't exist
+# -----------------------------------------------------------------------------
+
+if [ ! -d "$PROFILE_FOLDER" ]; then
+	fn_log_info "Creating profile folder in '$PROFILE_FOLDER'..."
+	mkdir -- "$PROFILE_FOLDER"
+fi
+
 # -----------------------------------------------------------------------------
 # Handle case where a previous backup failed or was interrupted.
 # -----------------------------------------------------------------------------
@@ -51,11 +93,11 @@ if [ -f "$INPROGRESS_FILE" ]; then
 	if [ "$LAST_TIME" != "" ]; then
 		# - Last backup is moved to current backup folder so that it can be resumed.
 		# - 2nd to last backup becomes last backup.
-		echo "$INPROGRESS_FILE already exists - the previous backup failed or was interrupted. Backup will resume from there."
+		fn_log_info "$INPROGRESS_FILE already exists - the previous backup failed or was interrupted. Backup will resume from there."
-		LINE_COUNT=$(ls -1 $DEST_FOLDER | grep "\d\d\d\d-\d\d-\d\d-\d\d\d\d\d\d" | tail -n 2 | wc -l)
+		LINE_COUNT=$(ls -1 $DEST_FOLDER | grep "$BACKUP_FOLDER_PATTERN" | tail -n 2 | wc -l)
 		mv $PREVIOUS_DEST $DEST
 		if [ "$LINE_COUNT" -gt 1 ]; then
-			SECOND_LAST_TIME=$(ls -1 $DEST_FOLDER | grep "\d\d\d\d-\d\d-\d\d-\d\d\d\d\d\d" | tail -n 2 | head -n 1)
+			SECOND_LAST_TIME=$(ls -1 $DEST_FOLDER | grep "$BACKUP_FOLDER_PATTERN" | tail -n 2 | head -n 1)
 			LAST_TIME=$SECOND_LAST_TIME
 		else
 			LAST_TIME=""
@@ -70,12 +112,12 @@ fi
 
 LINK_DEST_OPTION=""
 if [ "$LAST_TIME" == "" ]; then
-	echo "No previous backup - creating new one."
+	fn_log_info "No previous backup - creating new one."
 else
 	# If the path is relative, it needs to be relative to the destination. To keep
 	# it simple, just use an absolute path. See http://serverfault.com/a/210058/118679
 	PREVIOUS_DEST=`cd \`dirname "$PREVIOUS_DEST"\`; pwd`"/"`basename "$PREVIOUS_DEST"`
-	echo "Previous backup found - doing incremental backup from $PREVIOUS_DEST"
+	fn_log_info "Previous backup found - doing incremental backup from $PREVIOUS_DEST"
 	LINK_DEST_OPTION="--link-dest=$PREVIOUS_DEST"
 fi
 
@@ -84,7 +126,7 @@ fi
 # -----------------------------------------------------------------------------
 
 if [ ! -d "$DEST" ]; then
-	echo "Creating destination $DEST"
+	fn_log_info "Creating destination $DEST"
 	mkdir -p $DEST
 fi
 
@@ -92,9 +134,13 @@ fi
 # Start backup
 # -----------------------------------------------------------------------------
 
-echo "Starting backup..."
+# Run in a loop to handle the "No space left on device" logic
-echo "From: $SRC_FOLDER"
+while [ "1" ]; do
-echo "To:   $DEST"
+	LOG_FILE="$PROFILE_FOLDER/$(date +"%Y-%m-%d-%H%M%S").log"
+
+	fn_log_info "Starting backup..."
+	fn_log_info "From: $SRC_FOLDER"
+	fn_log_info "To:   $DEST"
 
 	CMD="rsync"
 	CMD="$CMD --compress"
@@ -106,20 +152,82 @@ CMD="$CMD --delete-excluded"
 	CMD="$CMD --one-file-system"
 	CMD="$CMD --archive"
 	CMD="$CMD --progress"
+	CMD="$CMD --itemize-changes"
+	CMD="$CMD --verbose"
+	CMD="$CMD --log-file '$LOG_FILE'"
 	if [ "$EXCLUSION_FILE" != "" ]; then
 		CMD="$CMD --exclude-from \"$EXCLUSION_FILE\""
 	fi
-CMD="$CMD $LINK_DEST_OPTION $SRC_FOLDER/ $DEST/"
+	CMD="$CMD $LINK_DEST_OPTION"
+	CMD="$CMD $SRC_FOLDER/ $DEST/"
 	CMD="$CMD | grep -E '^deleting|[^/]$'"
 
-echo "Running command:"
+	fn_log_info "Running command:"
-echo $CMD
+	fn_log_info "$CMD"
 
 	touch $INPROGRESS_FILE
 	eval $CMD
-EXIT_CODE=$?
+	RSYNC_EXIT_CODE=$?
-if [ "$EXIT_CODE" == "0" ]; then
+
-	rm $INPROGRESS_FILE
+	# -----------------------------------------------------------------------------
-else
+	# Check if we ran out of space
-	echo "Error: Exited with error code $EXIT_CODE"
+	# -----------------------------------------------------------------------------
+
+	# TODO: find better way to check for out of space condition without parsing log.
+	grep --quiet "No space left on device (28)" "$LOG_FILE"
+	NO_SPACE_LEFT="$?"
+	if [ "$NO_SPACE_LEFT" != "0" ]; then
+		# This error might also happen if there is no space left
+		grep --quiet "Result too large (34)" "$LOG_FILE"
+		NO_SPACE_LEFT="$?"
 	fi
+		
+	rm -- "$LOG_FILE"
+	
+	if [ "$NO_SPACE_LEFT" == "0" ]; then
+		read -p "It looks like there is no space left on the destination. Delete old backup? (Y/n) " yn
+		case $yn in
+			[Nn]* ) exit 0;;
+		esac
+
+		fn_log_warn "No space left on device - removing oldest backup and resuming."
+		
+		BACKUP_FOLDER_COUNT=$(ls -1 $DEST_FOLDER | grep "$BACKUP_FOLDER_PATTERN" | wc -l)
+		if [ "$BACKUP_FOLDER_COUNT" -lt "2" ]; then
+			fn_log_error "No space left on device, and no old backup to delete."
+			exit 1
+		fi
+		
+		# TODO: handle case where only two backup folders are left. In which case, need to remove --link-dest option
+		
+		OLDEST_BACKUP=$(ls -1 $DEST_FOLDER | grep "$BACKUP_FOLDER_PATTERN" | head -n 1)
+		if [ "$OLDEST_BACKUP" == "" ]; then
+			fn_log_error "No space left on device, and cannot get path to oldest backup to delete."
+			exit 1
+		fi
+		
+		OLD_BACKUP_PATH="$DEST_FOLDER/$OLDEST_BACKUP"
+		
+		# Double-check that we're on a backup destination to be completely sure we're deleting the right folder
+		OLD_BACKUP_PARENT_PATH=$(dirname -- "$OLD_BACKUP_PATH")
+		if [ "$(fn_is_backup_destination $OLD_BACKUP_PARENT_PATH)" != "1" ]; then
+			fn_log_error "'$OLD_BACKUP_PATH' is not on a backup destination - aborting."
+			exit 1
+		fi
+		
+		fn_log_info "Deleting '$OLD_BACKUP_PATH'..."
+		rm -rf -- "$OLD_BACKUP_PATH"
+		
+		# Resume backup
+		continue
+	fi
+
+	if [ "$RSYNC_EXIT_CODE" != "0" ]; then
+		fn_log_error "Exited with error code $RSYNC_EXIT_CODE"
+		exit $RSYNC_EXIT_CODE
+	fi
+	
+	rm $INPROGRESS_FILE
+	fn_log_info "Backup completed without errors."
+	exit 0
+done

rsync_tmbackup.sh.bak

@@ -0,0 +1,215 @@
+#!/bin/bash
+
+# -----------------------------------------------------------------------------
+# Log functions
+# -----------------------------------------------------------------------------
+
+fn_log_info() {
+	echo "[RB INFO] $1"
+}
+
+fn_log_warn() {
+	echo "[RB WARN] $1"
+}
+
+fn_log_error() {
+	echo "[RB ERROR] $1"
+}
+
+# -----------------------------------------------------------------------------
+# Make sure everything really stops when CTRL+C is pressed
+# -----------------------------------------------------------------------------
+
+fn_terminate_script() {
+	echo "SIGINT caught"
+	exit 1
+}
+
+trap 'fn_terminate_script' SIGINT
+
+# -----------------------------------------------------------------------------
+# Source and destination information
+# -----------------------------------------------------------------------------
+
+SRC_FOLDER=${1%/}
+DEST_FOLDER=${2%/}
+EXCLUSION_FILE=$3
+
+# -----------------------------------------------------------------------------
+# Check that the destination drive is a backup drive
+# -----------------------------------------------------------------------------
+
+# TODO: check that the destination supports hard links
+
+fn_backup_marker_path() {
+	echo "$1/backup.marker"
+}
+
+fn_is_backup_destination() {
+	DEST_MARKER_FILE="$(fn_backup_marker_path $1)"
+	if [ -f "$DEST_MARKER_FILE" ]; then
+		echo "1"
+	else
+		echo "0"
+	fi
+}
+
+if [ "$(fn_is_backup_destination $DEST_FOLDER)" != "1" ]; then
+	fn_log_info "Safety check failed - the destination does not appear to be a backup folder or drive (marker file not found)."
+	fn_log_info "If it is indeed a backup folder, you may add the marker file by running the following command:"
+	fn_log_info ""
+	fn_log_info "touch \"$(fn_backup_marker_path $DEST_FOLDER)\""
+	fn_log_info ""
+	exit 1
+fi
+
+# -----------------------------------------------------------------------------
+# Setup additional variables
+# -----------------------------------------------------------------------------
+
+BACKUP_FOLDER_PATTERN="\d\d\d\d-\d\d-\d\d-\d\d\d\d\d\d"
+NOW=$(date +"%Y-%m-%d-%H%M%S")
+PROFILE_FOLDER="$HOME/.rsync_tmbackup"
+LOG_FILE="$PROFILE_FOLDER/$NOW.log"
+DEST=$DEST_FOLDER/$NOW
+LAST_TIME=$(ls -1 $DEST_FOLDER | grep "$BACKUP_FOLDER_PATTERN" | tail -n 1)
+PREVIOUS_DEST=$DEST_FOLDER/$LAST_TIME
+INPROGRESS_FILE=$DEST_FOLDER/backup.inprogress
+
+# -----------------------------------------------------------------------------
+# Create profile folder if it doesn't exist
+# -----------------------------------------------------------------------------
+
+if [ ! -d "$PROFILE_FOLDER" ]; then
+	fn_log_info "Creating profile folder in '$PROFILE_FOLDER'..."
+	mkdir -- "$PROFILE_FOLDER"
+fi
+
+# -----------------------------------------------------------------------------
+# Handle case where a previous backup failed or was interrupted.
+# -----------------------------------------------------------------------------
+
+if [ -f "$INPROGRESS_FILE" ]; then
+	if [ "$LAST_TIME" != "" ]; then
+		# - Last backup is moved to current backup folder so that it can be resumed.
+		# - 2nd to last backup becomes last backup.
+		fn_log_info "$INPROGRESS_FILE already exists - the previous backup failed or was interrupted. Backup will resume from there."
+		LINE_COUNT=$(ls -1 $DEST_FOLDER | grep "$BACKUP_FOLDER_PATTERN" | tail -n 2 | wc -l)
+		mv $PREVIOUS_DEST $DEST
+		if [ "$LINE_COUNT" -gt 1 ]; then
+			SECOND_LAST_TIME=$(ls -1 $DEST_FOLDER | grep "$BACKUP_FOLDER_PATTERN" | tail -n 2 | head -n 1)
+			LAST_TIME=$SECOND_LAST_TIME
+		else
+			LAST_TIME=""
+		fi
+		PREVIOUS_DEST=$DEST_FOLDER/$LAST_TIME
+	fi
+fi
+
+# -----------------------------------------------------------------------------
+# Check if we are doing an incremental backup (if previous backup exists) or not
+# -----------------------------------------------------------------------------
+
+LINK_DEST_OPTION=""
+if [ "$LAST_TIME" == "" ]; then
+	fn_log_info "No previous backup - creating new one."
+else
+	# If the path is relative, it needs to be relative to the destination. To keep
+	# it simple, just use an absolute path. See http://serverfault.com/a/210058/118679
+	PREVIOUS_DEST=`cd \`dirname "$PREVIOUS_DEST"\`; pwd`"/"`basename "$PREVIOUS_DEST"`
+	fn_log_info "Previous backup found - doing incremental backup from $PREVIOUS_DEST"
+	LINK_DEST_OPTION="--link-dest=$PREVIOUS_DEST"
+fi
+
+# -----------------------------------------------------------------------------
+# Create destination folder if it doesn't already exists
+# -----------------------------------------------------------------------------
+
+if [ ! -d "$DEST" ]; then
+	fn_log_info "Creating destination $DEST"
+	mkdir -p $DEST
+fi
+
+# -----------------------------------------------------------------------------
+# Start backup
+# -----------------------------------------------------------------------------
+
+# Run in a loop to handle the "No space left on device" logic
+while [ "1" ]; do
+	LOG_FILE="$PROFILE_FOLDER/$(date +"%Y-%m-%d-%H%M%S").log"
+
+	fn_log_info "Starting backup..."
+	fn_log_info "From: $SRC_FOLDER"
+	fn_log_info "To:   $DEST"
+
+	CMD="rsync"
+	CMD="$CMD --compress"
+	CMD="$CMD --numeric-ids"
+	CMD="$CMD --links"
+	CMD="$CMD --hard-links"
+	CMD="$CMD --delete"
+	CMD="$CMD --delete-excluded"
+	CMD="$CMD --one-file-system"
+	CMD="$CMD --archive"
+	CMD="$CMD --progress"
+	CMD="$CMD --log-file '$LOG_FILE'"
+	if [ "$EXCLUSION_FILE" != "" ]; then
+		CMD="$CMD --exclude-from \"$EXCLUSION_FILE\""
+	fi
+	CMD="$CMD $LINK_DEST_OPTION $SRC_FOLDER/ $DEST/"
+	CMD="$CMD | grep -E '^deleting|[^/]$'"
+
+	fn_log_info "Running command:"
+	fn_log_info "$CMD"
+
+	touch $INPROGRESS_FILE
+	eval $CMD
+	EXIT_CODE=$?
+
+	# -----------------------------------------------------------------------------
+	# Check if we ran out of space
+	# -----------------------------------------------------------------------------
+
+	grep --quiet "No space left on device (28)" "$LOG_FILE"
+	# TODO: delete log file
+	
+	if [ "$?" == "0" ]; then
+		fn_log_warn "No space left on device - removing oldest backup and resuming."
+		
+		BACKUP_FOLDER_COUNT=$(ls -1 $DEST_FOLDER | grep "$BACKUP_FOLDER_PATTERN" | wc -l)
+		if [ "$BACKUP_FOLDER_COUNT" -lt "2" ]; then
+			fn_log_error "No space left on device, and no old backup to delete."
+			exit 1
+		fi
+		
+		OLDEST_BACKUP=$(ls -1 $DEST_FOLDER | grep "$BACKUP_FOLDER_PATTERN" | head -n 1)
+		if [ "$OLDEST_BACKUP" == "" ]; then
+			fn_log_error "No space left on device, and cannot get path to oldest backup to delete."
+			exit 1
+		fi
+		
+		OLD_BACKUP_PATH="$DEST_FOLDER/$OLDEST_BACKUP"
+		
+		# Double-check that we're on a backup destination to be completely sure we're deleting the right folder
+		OLD_BACKUP_PARENT_PATH=$(dirname -- "$OLD_BACKUP_PATH")
+		if [ "$(fn_is_backup_destination $OLD_BACKUP_PARENT_PATH)" != "1" ]; then
+			fn_log_error "'$OLD_BACKUP_PATH' is not on a backup destination - aborting."
+			exit 1
+		fi
+		
+		fn_log_info "Deleting '$OLD_BACKUP_PATH'..."
+		rm -rf -- "$OLD_BACKUP_PATH"
+		
+		# Resume backup
+		continue
+	fi
+
+	if [ "$EXIT_CODE" != "0" ]; then
+		fn_log_error "Exited with error code $EXIT_CODE"
+		exit $EXIT_CODE
+	fi
+	
+	rm $INPROGRESS_FILE
+	fn_log_info "Backup completed without errors."
+	exit 0
+done