Merge pull request #114 from neddy70/no-auto-expire

Add option to disable automatic purging of backups

.gitignore

@@ -1,3 +1,5 @@
 .idea
 test.sh
 *~
+tests/TestDest/
+tests/TestSource/

README.md

@@ -22,9 +22,10 @@ On macOS, it has a few disadvantages compared to Time Machine - in particular it
 	 --log-dir            Set the log file directory. If this flag is set, generated files will
 	                      not be managed by the script - in particular they will not be
 	                      automatically deleted.
-	 --strategy           Set the expiration strategy. Default: "365:30 30:7 1:1" means after one
+	 --strategy           Set the expiration strategy. Default: "1:1 30:7 365:30" means after one
 	                      day, keep one backup per day. After 30 days, keep one backup every 7 days.
 	                      After 365 days keep one backup every 30 days.
+	 --no-auto-expire     Set option to disable automatically purging old backups when out of space.
 
 ## Features
 
@@ -69,13 +70,15 @@ On macOS, it has a few disadvantages compared to Time Machine - in particular it
 
 ## Backup expiration logic
 
-The script automatically deletes old backups using the following logic:
-- Within the last 24 hours, all the backups are kept.
-- Within the last 31 days, the most recent backup of each day is kept.
-- After 31 days, only the most recent backup of each month is kept.
-- Additionally, if the backup destination directory is full, the oldest backups are deleted until enough space is available.
+Backup sets are automatically deleted following a simple expiration strategy defined with the `--strategy` flag. This strategy is a series of time intervals with each item being defined as `x:y`, which means "after x days, keep one backup every y days". The default strategy is `1:1 30:7 365:30`, which means:
 
-## Exclude file
+- After **1** day, keep one backup every **1** day (**1:1**).
+- After **30** days, keep one backup every **7** days (**30:7**).
+- After **365** days, keep one backup every **30** days (**365:30**).
+
+Before the first interval (i.e. by default within the first 24h) it is implied that all backup sets are kept. Additionally, if the backup destination directory is full, the oldest backups are deleted until enough space is available.
+
+## Exclusion file
 
 An optional exclude file can be provided as a third parameter. It should be compatible with the `--exclude-from` parameter of rsync. See [this tutorial](https://sites.google.com/site/rsync2u/home/rsync-tutorial/the-exclude-from-option) for more information.
 
@@ -89,6 +92,10 @@ To display the rsync options that are used for backup, run `./rsync_tmbackup.sh
 
 	rsync_tmbackup --rsync-set-flags "--numeric-ids --links --hard-links \
 	--one-file-system --archive --no-perms --no-groups --itemize-changes" /src /dest
+
+## No automatic backup expiration
+
+An option to diable the default behaviour to purge old backups when out of space. This option is set with the `--no-auto-expire` flag.
 	
 	
 ## How to restore
@@ -110,7 +117,7 @@ The script creates a backup in a regular directory so you can simply copy the fi
 
 The MIT License (MIT)
 
-Copyright (c) 2013-2017 Laurent Cozic
+Copyright (c) 2013-2018 Laurent Cozic
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal

rsync_tmbackup.sh

@@ -43,9 +43,11 @@ fn_display_usage() {
 	echo "                      not be managed by the script - in particular they will not be"
 	echo "                      automatically deleted."
 	echo "                      Default: $LOG_DIR"
-	echo " --strategy           Set the expiration strategy. Default: \"365:30 30:7 1:1\" means after one"
+	echo " --strategy           Set the expiration strategy. Default: \"1:1 30:7 365:30\" means after one"
 	echo "                      day, keep one backup per day. After 30 days, keep one backup every 7 days."
 	echo "                      After 365 days keep one backup every 30 days."
+	echo " --no-auto-expire     Disable automatically deleting backups when out of space. Instead an error"
+	echo "                      is logged, and the backup is aborted."
 	echo ""
 	echo "For more detailed help, please see the README file:"
 	echo ""
@@ -53,39 +55,25 @@ fn_display_usage() {
 }
 
 fn_parse_date() {
-	local date_string="$1"
-	local date_format="$2"
-
 	# Converts YYYY-MM-DD-HHMMSS to YYYY-MM-DD HH:MM:SS and then to Unix Epoch.
-
-	if [[ -z "$date_format" || "$date_format" == "Y-m-d H:i:s" ]]; then
-		case "$OSTYPE" in
-			linux*) date -d "${date_string:0:10} ${date_string:1date_string:2}:${date_string:13:2}:${date_string:15:2}" +%s ;;
-			cygwin*) date -d "${date_string:0:10} ${date_string:1date_string:2}:${date_string:13:2}:${date_string:15:2}" +%s ;;
-			darwin*) date -j -f "%Y-%m-%d-%H%M%S" "$date_string" "+%s" ;;
-			FreeBSD*) date -j -f "%Y-%m-%d-%H%M%S" "$date_string" "+%s" ;;
-		esac
-	else
-		case "$OSTYPE" in
-			linux*) date -d "${date_string:0:10} 00:00:00" +%s ;;
-			cygwin*) date -d "${date_string:0:10} 00:00:00" +%s ;;
-			darwin*) date -j -f "%Y-%m-%d" "$date_string" "+%s" ;;
-			FreeBSD*) date -j -f "%Y-%m-%d" "$date_string" "+%s" ;;
-		esac
-	fi
-}
-
-fn_run_cmd() {
-	if [ -n "$SSH_DEST_FOLDER_PREFIX" ] 
-	then
-		eval "$SSH_CMD '$1'"
-	else
-		eval $1
-	fi
+	case "$OSTYPE" in
+		linux*) date -d "${1:0:10} ${1:11:2}:${1:13:2}:${1:15:2}" +%s ;;
+		cygwin*) date -d "${1:0:10} ${1:11:2}:${1:13:2}:${1:15:2}" +%s ;;
+		darwin8*) yy=`expr ${1:0:4}`
+			mm=`expr ${1:5:2} - 1`
+			dd=`expr ${1:8:2}`
+			hh=`expr ${1:11:2}`
+			mi=`expr ${1:13:2}`
+			ss=`expr ${1:15:2}`
+			# Because under MacOS X Tiger 'date -j' doesn't work, we do this:
+			perl -e 'use Time::Local; print timelocal('$ss','$mi','$hh','$dd','$mm','$yy'),"\n";' ;;
+		darwin*) date -j -f "%Y-%m-%d-%H%M%S" "$1" "+%s" ;;
+		FreeBSD*) date -j -f "%Y-%m-%d-%H%M%S" "$1" "+%s" ;;
+	esac
 }
 
 fn_find_backups() {
-	fn_run_cmd "find "$DEST_FOLDER" -maxdepth 1 -type d -name \"????-??-??-??????\" -prune | sort -r"
+	fn_run_cmd "find "$DEST_FOLDER/" -maxdepth 1 -type d -name \"????-??-??-??????\" -prune | sort -r"
 }
 
 fn_expire_backup() {
@@ -107,8 +95,7 @@ fn_expire_backups() {
 	# Process each backup dir from most recent to oldest
 	for backup_dir in $(fn_find_backups | sort -r); do
 		local backup_date=$(basename "$backup_dir")
-		local backup_day=${backup_date:0:10}
-		local backup_timestamp=$(fn_parse_date $backup_day "Y-m-d")
+		local backup_timestamp=$(fn_parse_date "$backup_date")
 
 		# Skip if failed to parse date...
 		if [ -z "$backup_timestamp" ]; then
@@ -152,14 +139,15 @@ fn_expire_backups() {
 }
 
 fn_parse_ssh() {
-	if [[ "$DEST_FOLDER" =~ ^[A-Za-z0-9\._%\+\-]+@[A-Za-z0-9.\-]+\:.+$ ]]
+	# To keep compatibility with bash version < 3, we use grep
+	if echo "$DEST_FOLDER"|grep -Eq '^[A-Za-z0-9\._%\+\-]+@[A-Za-z0-9.\-]+\:.+$'
 	then
 		SSH_USER=$(echo "$DEST_FOLDER" | sed -E  's/^([A-Za-z0-9\._%\+\-]+)@([A-Za-z0-9.\-]+)\:(.+)$/\1/')
 		SSH_HOST=$(echo "$DEST_FOLDER" | sed -E  's/^([A-Za-z0-9\._%\+\-]+)@([A-Za-z0-9.\-]+)\:(.+)$/\2/')
 		SSH_DEST_FOLDER=$(echo "$DEST_FOLDER" | sed -E  's/^([A-Za-z0-9\._%\+\-]+)@([A-Za-z0-9.\-]+)\:(.+)$/\3/')
 		SSH_CMD="ssh -p $SSH_PORT ${SSH_USER}@${SSH_HOST}"
 		SSH_DEST_FOLDER_PREFIX="${SSH_USER}@${SSH_HOST}:"
-	elif [[ "$SRC_FOLDER" =~ ^[A-Za-z0-9\._%\+\-]+@[A-Za-z0-9.\-]+\:.+$ ]]
+	elif echo "$SRC_FOLDER"|grep -Eq '^[A-Za-z0-9\._%\+\-]+@[A-Za-z0-9.\-]+\:.+$'
 	then
 		SSH_USER=$(echo "$SRC_FOLDER" | sed -E  's/^([A-Za-z0-9\._%\+\-]+)@([A-Za-z0-9.\-]+)\:(.+)$/\1/')
 		SSH_HOST=$(echo "$SRC_FOLDER" | sed -E  's/^([A-Za-z0-9\._%\+\-]+)@([A-Za-z0-9.\-]+)\:(.+)$/\2/')
@@ -169,6 +157,15 @@ fn_parse_ssh() {
 	fi
 }
 
+fn_run_cmd() {
+	if [ -n "$SSH_DEST_FOLDER_PREFIX" ]
+	then
+		eval "$SSH_CMD '$1'"
+	else
+		eval $1
+	fi
+}
+
 fn_find() {
 	fn_run_cmd "find '$1'"  2>/dev/null
 }
@@ -215,7 +212,8 @@ DEST_FOLDER=""
 EXCLUSION_FILE=""
 LOG_DIR="$HOME/.$APPNAME"
 AUTO_DELETE_LOG="1"
-EXPIRATION_STRATEGY="365:30 1:1 30:7"
+EXPIRATION_STRATEGY="1:1 30:7 365:30"
+AUTO_EXPIRE="1"
 
 RSYNC_FLAGS="-D --compress --numeric-ids --links --hard-links --one-file-system --itemize-changes --times --recursive --perms --owner --group --stats --human-readable"
 
@@ -240,13 +238,16 @@ while :; do
 			;;
 		--strategy)
 			shift
-			STRATEGY="$1"
+			EXPIRATION_STRATEGY="$1"
 			;;
 		--log-dir)
 			shift
 			LOG_DIR="$1"
 			AUTO_DELETE_LOG="0"
 			;;
+		--no-auto-expire)
+			AUTO_EXPIRE="0"
+			;;
 		--)
 			shift
 			SRC_FOLDER="$1"
@@ -368,9 +369,9 @@ if [ -n "$(fn_find "$INPROGRESS_FILE")" ]; then
 			fn_log_error "Previous backup task is still active - aborting (command: $RUNNINGCMD)."
 			exit 1
 		fi
-	else 
+	else
 		RUNNINGPID="$(fn_run_cmd "cat $INPROGRESS_FILE")"
-		if [ "$RUNNINGPID" = "$(pgrep "$APPNAME")" ]; then
+		if [ "$RUNNINGPID" = "$(pgrep -o -f "$APPNAME")" ]; then
 			fn_log_error "Previous backup task is still active - aborting."
 			exit 1
 		fi
@@ -460,6 +461,12 @@ while : ; do
 	NO_SPACE_LEFT="$(grep "No space left on device (28)\|Result too large (34)" "$LOG_FILE")"
 
 	if [ -n "$NO_SPACE_LEFT" ]; then
+
+		if [[ $AUTO_EXPIRE == "0" ]]; then
+			fn_log_error "No space left on device, and automatic purging of old backups is disabled."
+			exit 1
+		fi
+
 		fn_log_warn "No space left on device - removing oldest backup and resuming."
 
 		if [[ "$(fn_find_backups | wc -l)" -lt "2" ]]; then

tests/populate_dest.php

@@ -0,0 +1,30 @@
+<?php
+
+// This PHP script can be used to test the expiration strategy.
+// It is going to populate a directory with fake backup sets (directories named Y-m-d-His) over several months.
+// Then the backup script can be run on it to check what directories are going to be deleted.
+
+// rm -rf ./tests/TestDest/201* && php ./tests/populate_dest.php && ./rsync_tmbackup.sh ./tests/TestSource/ ./tests/TestDest/
+
+$baseDir = dirname(__FILE__);
+$destDir = $baseDir . '/TestDest';
+
+$backupsPerDay = 2;
+$totalDays = 500;
+
+$intervalBetweenBackups = null;
+if ($backupsPerDay === 1) {
+	$intervalBetweenBackups = 'PT1D';
+} else if ($backupsPerDay === 2) {
+	$intervalBetweenBackups = 'PT12H';
+} else {
+	throw new Exception('Not implemented');
+}
+
+$d = new DateTime();
+$d->sub(new DateInterval('P' . $totalDays . 'D'));
+
+for ($i = 0; $i < $backupsPerDay * $totalDays; $i++) {
+	$d->add(new DateInterval($intervalBetweenBackups));
+	mkdir($destDir . '/' . $d->format('Y-m-d-His'), 0777, true);
+}