schiwagoa/rsync-time-backup
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Formatting - not sure when that was changed to spaces

Browse Source
This commit is contained in:
Laurent Cozic
2016-08-12 17:26:44 +01:00
parent b905b61a86
commit 4fcbb0eab5
1 changed files with 182 additions and 182 deletions
Download Patch File Download Diff File Expand all files Collapse all files
rsync_tmbackup.sh
+182 -182
View File
@@ -10,11 +10,11 @@ fn_log_info() { echo "$APPNAME: $1"; }
fn_log_warn() { echo "$APPNAME: [WARNING] $1" 1>&2; } fn_log_warn() { echo "$APPNAME: [WARNING] $1" 1>&2; }
fn_log_error() { echo "$APPNAME: [ERROR] $1" 1>&2; } fn_log_error() { echo "$APPNAME: [ERROR] $1" 1>&2; }
fn_log_info_cmd() { fn_log_info_cmd() {
if [ -n "$SSH_CMD" ]; then if [ -n "$SSH_CMD" ]; then
echo "$APPNAME: $SSH_CMD '$1'"; echo "$APPNAME: $SSH_CMD '$1'";
else else
echo "$APPNAME: $1"; echo "$APPNAME: $1";
fi fi
} }
# ----------------------------------------------------------------------------- # -----------------------------------------------------------------------------
@@ -22,8 +22,8 @@ fn_log_info_cmd() {
# ----------------------------------------------------------------------------- # -----------------------------------------------------------------------------
fn_terminate_script() { fn_terminate_script() {
fn_log_info "SIGINT caught." fn_log_info "SIGINT caught."
exit 1 exit 1
} }
trap 'fn_terminate_script' SIGINT trap 'fn_terminate_script' SIGINT
@@ -36,72 +36,72 @@ fn_display_usage() {
} }
fn_parse_date() { fn_parse_date() {
# Converts YYYY-MM-DD-HHMMSS to YYYY-MM-DD HH:MM:SS and then to Unix Epoch. # Converts YYYY-MM-DD-HHMMSS to YYYY-MM-DD HH:MM:SS and then to Unix Epoch.
case "$OSTYPE" in case "$OSTYPE" in
linux*) date -d "${1:0:10} ${1:11:2}:${1:13:2}:${1:15:2}" +%s ;; linux*) date -d "${1:0:10} ${1:11:2}:${1:13:2}:${1:15:2}" +%s ;;
cygwin*) date -d "${1:0:10} ${1:11:2}:${1:13:2}:${1:15:2}" +%s ;; cygwin*) date -d "${1:0:10} ${1:11:2}:${1:13:2}:${1:15:2}" +%s ;;
darwin*) date -j -f "%Y-%m-%d-%H%M%S" "$1" "+%s" ;; darwin*) date -j -f "%Y-%m-%d-%H%M%S" "$1" "+%s" ;;
esac esac
} }
fn_find_backups() { fn_find_backups() {
fn_run_cmd "find "$DEST_FOLDER" -type d -name "????-??-??-??????" -prune | sort -r" fn_run_cmd "find "$DEST_FOLDER" -type d -name "????-??-??-??????" -prune | sort -r"
} }
fn_expire_backup() { fn_expire_backup() {
# Double-check that we're on a backup destination to be completely # Double-check that we're on a backup destination to be completely
# sure we're deleting the right folder # sure we're deleting the right folder
if [ -z "$(fn_find_backup_marker "$(dirname -- "$1")")" ]; then if [ -z "$(fn_find_backup_marker "$(dirname -- "$1")")" ]; then
fn_log_error "$1 is not on a backup destination - aborting." fn_log_error "$1 is not on a backup destination - aborting."
exit 1 exit 1
fi fi
fn_log_info "Expiring $1" fn_log_info "Expiring $1"
fn_rm "$1" fn_rm "$1"
} }
fn_parse_ssh() { fn_parse_ssh() {
if [[ "$DEST_FOLDER" =~ ^[A-Za-z0-9\._%\+\-]+@[A-Za-z0-9.\-]+\:.+$ ]] if [[ "$DEST_FOLDER" =~ ^[A-Za-z0-9\._%\+\-]+@[A-Za-z0-9.\-]+\:.+$ ]]
then then
SSH_USER=$(echo "$DEST_FOLDER" | sed -E 's/^([A-Za-z0-9\._%\+\-]+)@([A-Za-z0-9.\-]+)\:(.+)$/\1/') SSH_USER=$(echo "$DEST_FOLDER" | sed -E 's/^([A-Za-z0-9\._%\+\-]+)@([A-Za-z0-9.\-]+)\:(.+)$/\1/')
SSH_HOST=$(echo "$DEST_FOLDER" | sed -E 's/^([A-Za-z0-9\._%\+\-]+)@([A-Za-z0-9.\-]+)\:(.+)$/\2/') SSH_HOST=$(echo "$DEST_FOLDER" | sed -E 's/^([A-Za-z0-9\._%\+\-]+)@([A-Za-z0-9.\-]+)\:(.+)$/\2/')
SSH_DEST_FOLDER=$(echo "$DEST_FOLDER" | sed -E 's/^([A-Za-z0-9\._%\+\-]+)@([A-Za-z0-9.\-]+)\:(.+)$/\3/') SSH_DEST_FOLDER=$(echo "$DEST_FOLDER" | sed -E 's/^([A-Za-z0-9\._%\+\-]+)@([A-Za-z0-9.\-]+)\:(.+)$/\3/')
SSH_CMD="ssh ${SSH_USER}@${SSH_HOST}" SSH_CMD="ssh ${SSH_USER}@${SSH_HOST}"
SSH_FOLDER_PREFIX="${SSH_USER}@${SSH_HOST}:" SSH_FOLDER_PREFIX="${SSH_USER}@${SSH_HOST}:"
fi fi
} }
fn_run_cmd() { fn_run_cmd() {
if [ -n "$SSH_CMD" ] if [ -n "$SSH_CMD" ]
then then
eval "$SSH_CMD '$1'" eval "$SSH_CMD '$1'"
else else
eval $1 eval $1
fi fi
} }
fn_find() { fn_find() {
fn_run_cmd "find $1" 2>/dev/null fn_run_cmd "find $1" 2>/dev/null
} }
fn_get_absolute_path() { fn_get_absolute_path() {
fn_run_cmd "cd $1;pwd" fn_run_cmd "cd $1;pwd"
} }
fn_mkdir() { fn_mkdir() {
fn_run_cmd "mkdir -p -- $1" fn_run_cmd "mkdir -p -- $1"
} }
fn_rm() { fn_rm() {
fn_run_cmd "rm -rf -- $1" fn_run_cmd "rm -rf -- $1"
} }
fn_touch() { fn_touch() {
fn_run_cmd "touch -- $1" fn_run_cmd "touch -- $1"
} }
fn_ln() { fn_ln() {
fn_run_cmd "ln -s -- $1 $2" fn_run_cmd "ln -s -- $1 $2"
} }
# ----------------------------------------------------------------------------- # -----------------------------------------------------------------------------
@@ -113,7 +113,7 @@ SSH_DEST_FOLDER=""
SSH_CMD="" SSH_CMD=""
SSH_FOLDER_PREFIX="" SSH_FOLDER_PREFIX=""
#display usage information if required arguments are not passed # display usage information if required arguments are not passed
if [[ "${#@}" -lt 2 ]]; then if [[ "${#@}" -lt 2 ]]; then
fn_display_usage fn_display_usage
exit 1 exit 1
@@ -125,14 +125,14 @@ EXCLUSION_FILE="$3"
fn_parse_ssh fn_parse_ssh
if [ -n "$SSH_DEST_FOLDER" ]; then if [ -n "$SSH_DEST_FOLDER" ]; then
DEST_FOLDER="$SSH_DEST_FOLDER" DEST_FOLDER="$SSH_DEST_FOLDER"
fi fi
for ARG in "$SRC_FOLDER" "$DEST_FOLDER" "$EXCLUSION_FILE"; do for ARG in "$SRC_FOLDER" "$DEST_FOLDER" "$EXCLUSION_FILE"; do
if [[ "$ARG" == *"'"* ]]; then if [[ "$ARG" == *"'"* ]]; then
fn_log_error 'Arguments may not have any single quote characters.' fn_log_error 'Arguments may not have any single quote characters.'
exit 1 exit 1
fi fi
done done
# ----------------------------------------------------------------------------- # -----------------------------------------------------------------------------
@@ -145,12 +145,12 @@ fn_backup_marker_path() { echo "$1/backup.marker"; }
fn_find_backup_marker() { fn_find "$(fn_backup_marker_path "$1")" 2>/dev/null; } fn_find_backup_marker() { fn_find "$(fn_backup_marker_path "$1")" 2>/dev/null; }
if [ -z "$(fn_find_backup_marker "$DEST_FOLDER")" ]; then if [ -z "$(fn_find_backup_marker "$DEST_FOLDER")" ]; then
fn_log_info "Safety check failed - the destination does not appear to be a backup folder or drive (marker file not found)." fn_log_info "Safety check failed - the destination does not appear to be a backup folder or drive (marker file not found)."
fn_log_info "If it is indeed a backup folder, you may add the marker file by running the following command:" fn_log_info "If it is indeed a backup folder, you may add the marker file by running the following command:"
fn_log_info "" fn_log_info ""
fn_log_info_cmd "mkdir -p -- \"$DEST_FOLDER\" ; touch \"$(fn_backup_marker_path "$DEST_FOLDER")\"" fn_log_info_cmd "mkdir -p -- \"$DEST_FOLDER\" ; touch \"$(fn_backup_marker_path "$DEST_FOLDER")\""
fn_log_info "" fn_log_info ""
exit 1 exit 1
fi fi
# ----------------------------------------------------------------------------- # -----------------------------------------------------------------------------
@@ -175,8 +175,8 @@ MYPID="$$"
# ----------------------------------------------------------------------------- # -----------------------------------------------------------------------------
if [ ! -d "$PROFILE_FOLDER" ]; then if [ ! -d "$PROFILE_FOLDER" ]; then
fn_log_info "Creating profile folder in '$PROFILE_FOLDER'..." fn_log_info "Creating profile folder in '$PROFILE_FOLDER'..."
mkdir -- "$PROFILE_FOLDER" mkdir -- "$PROFILE_FOLDER"
fi fi
# ----------------------------------------------------------------------------- # -----------------------------------------------------------------------------
@@ -190,165 +190,165 @@ if [ -n "$(fn_find "$INPROGRESS_FILE")" ]; then
# TODO: the script name, and extract the process ID from it. # TODO: the script name, and extract the process ID from it.
fn_log_warn "Cygwin only: Previous backup task has either been interrupted or it might still be active, but there is currently no check for this. Assuming that the task was simply interrupted." fn_log_warn "Cygwin only: Previous backup task has either been interrupted or it might still be active, but there is currently no check for this. Assuming that the task was simply interrupted."
else else
RUNNINGPID="$(fn_run_cmd "cat $INPROGRESS_FILE")" RUNNINGPID="$(fn_run_cmd "cat $INPROGRESS_FILE")"
if [ "$RUNNINGPID" = "$(pgrep "$APPNAME")" ]; then if [ "$RUNNINGPID" = "$(pgrep "$APPNAME")" ]; then
fn_log_error "Previous backup task is still active - aborting." fn_log_error "Previous backup task is still active - aborting."
exit 1 exit 1
fi fi
fi fi
if [ -n "$PREVIOUS_DEST" ]; then if [ -n "$PREVIOUS_DEST" ]; then
# - Last backup is moved to current backup folder so that it can be resumed. # - Last backup is moved to current backup folder so that it can be resumed.
# - 2nd to last backup becomes last backup. # - 2nd to last backup becomes last backup.
fn_log_info "$SSH_FOLDER_PREFIX$INPROGRESS_FILE already exists - the previous backup failed or was interrupted. Backup will resume from there." fn_log_info "$SSH_FOLDER_PREFIX$INPROGRESS_FILE already exists - the previous backup failed or was interrupted. Backup will resume from there."
fn_run_cmd "mv -- $PREVIOUS_DEST $DEST" fn_run_cmd "mv -- $PREVIOUS_DEST $DEST"
if [ "$(fn_find_backups | wc -l)" -gt 1 ]; then if [ "$(fn_find_backups | wc -l)" -gt 1 ]; then
PREVIOUS_DEST="$(fn_find_backups | sed -n '2p')" PREVIOUS_DEST="$(fn_find_backups | sed -n '2p')"
else else
PREVIOUS_DEST="" PREVIOUS_DEST=""
fi fi
# update PID to current process to avoid multiple concurrent resumes # update PID to current process to avoid multiple concurrent resumes
fn_run_cmd "echo $MYPID > $INPROGRESS_FILE" fn_run_cmd "echo $MYPID > $INPROGRESS_FILE"
fi fi
fi fi
# Run in a loop to handle the "No space left on device" logic. # Run in a loop to handle the "No space left on device" logic.
while : ; do while : ; do
# ----------------------------------------------------------------------------- # -----------------------------------------------------------------------------
# Check if we are doing an incremental backup (if previous backup exists). # Check if we are doing an incremental backup (if previous backup exists).
# ----------------------------------------------------------------------------- # -----------------------------------------------------------------------------
LINK_DEST_OPTION="" LINK_DEST_OPTION=""
if [ -z "$PREVIOUS_DEST" ]; then if [ -z "$PREVIOUS_DEST" ]; then
fn_log_info "No previous backup - creating new one." fn_log_info "No previous backup - creating new one."
else else
# If the path is relative, it needs to be relative to the destination. To keep # If the path is relative, it needs to be relative to the destination. To keep
# it simple, just use an absolute path. See http://serverfault.com/a/210058/118679 # it simple, just use an absolute path. See http://serverfault.com/a/210058/118679
PREVIOUS_DEST="$(fn_get_absolute_path "$PREVIOUS_DEST")" PREVIOUS_DEST="$(fn_get_absolute_path "$PREVIOUS_DEST")"
fn_log_info "Previous backup found - doing incremental backup from $SSH_FOLDER_PREFIX$PREVIOUS_DEST" fn_log_info "Previous backup found - doing incremental backup from $SSH_FOLDER_PREFIX$PREVIOUS_DEST"
LINK_DEST_OPTION="--link-dest='$PREVIOUS_DEST'" LINK_DEST_OPTION="--link-dest='$PREVIOUS_DEST'"
fi fi
# ----------------------------------------------------------------------------- # -----------------------------------------------------------------------------
# Create destination folder if it doesn't already exists # Create destination folder if it doesn't already exists
# ----------------------------------------------------------------------------- # -----------------------------------------------------------------------------
if [ -z "$(fn_find "$DEST -type d" 2>/dev/null)" ]; then if [ -z "$(fn_find "$DEST -type d" 2>/dev/null)" ]; then
fn_log_info "Creating destination $SSH_FOLDER_PREFIX$DEST" fn_log_info "Creating destination $SSH_FOLDER_PREFIX$DEST"
fn_mkdir "$DEST" fn_mkdir "$DEST"
fi fi
# ----------------------------------------------------------------------------- # -----------------------------------------------------------------------------
# Purge certain old backups before beginning new backup. # Purge certain old backups before beginning new backup.
# ----------------------------------------------------------------------------- # -----------------------------------------------------------------------------
# Default value for $PREV ensures that the most recent backup is never deleted. # Default value for $PREV ensures that the most recent backup is never deleted.
PREV="0000-00-00-000000" PREV="0000-00-00-000000"
for FILENAME in $(fn_find_backups | sort -r); do for FILENAME in $(fn_find_backups | sort -r); do
BACKUP_DATE=$(basename "$FILENAME") BACKUP_DATE=$(basename "$FILENAME")
TIMESTAMP=$(fn_parse_date $BACKUP_DATE) TIMESTAMP=$(fn_parse_date $BACKUP_DATE)
# Skip if failed to parse date... # Skip if failed to parse date...
if [ -z "$TIMESTAMP" ]; then if [ -z "$TIMESTAMP" ]; then
fn_log_warn "Could not parse date: $FILENAME" fn_log_warn "Could not parse date: $FILENAME"
continue continue
fi fi
if [ $TIMESTAMP -ge $KEEP_ALL_DATE ]; then if [ $TIMESTAMP -ge $KEEP_ALL_DATE ]; then
true true
elif [ $TIMESTAMP -ge $KEEP_DAILIES_DATE ]; then elif [ $TIMESTAMP -ge $KEEP_DAILIES_DATE ]; then
# Delete all but the most recent of each day. # Delete all but the most recent of each day.
[ "${BACKUP_DATE:0:10}" == "${PREV:0:10}" ] && fn_expire_backup "$FILENAME" [ "${BACKUP_DATE:0:10}" == "${PREV:0:10}" ] && fn_expire_backup "$FILENAME"
else else
# Delete all but the most recent of each month. # Delete all but the most recent of each month.
[ "${BACKUP_DATE:0:7}" == "${PREV:0:7}" ] && fn_expire_backup "$FILENAME" [ "${BACKUP_DATE:0:7}" == "${PREV:0:7}" ] && fn_expire_backup "$FILENAME"
fi fi
PREV=$BACKUP_DATE PREV=$BACKUP_DATE
done done
# ----------------------------------------------------------------------------- # -----------------------------------------------------------------------------
# Start backup # Start backup
# ----------------------------------------------------------------------------- # -----------------------------------------------------------------------------
LOG_FILE="$PROFILE_FOLDER/$(date +"%Y-%m-%d-%H%M%S").log" LOG_FILE="$PROFILE_FOLDER/$(date +"%Y-%m-%d-%H%M%S").log"
fn_log_info "Starting backup..." fn_log_info "Starting backup..."
fn_log_info "From: $SRC_FOLDER" fn_log_info "From: $SRC_FOLDER"
fn_log_info "To: $SSH_FOLDER_PREFIX$DEST" fn_log_info "To: $SSH_FOLDER_PREFIX$DEST"
CMD="rsync" CMD="rsync"
if [ -n "$SSH_CMD" ]; then if [ -n "$SSH_CMD" ]; then
CMD="$CMD -e 'ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null'" CMD="$CMD -e 'ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null'"
fi fi
CMD="$CMD --compress" CMD="$CMD --compress"
CMD="$CMD --numeric-ids" CMD="$CMD --numeric-ids"
CMD="$CMD --links" CMD="$CMD --links"
CMD="$CMD --hard-links" CMD="$CMD --hard-links"
CMD="$CMD --one-file-system" CMD="$CMD --one-file-system"
CMD="$CMD --archive" CMD="$CMD --archive"
CMD="$CMD --itemize-changes" CMD="$CMD --itemize-changes"
CMD="$CMD --verbose" CMD="$CMD --verbose"
CMD="$CMD --log-file '$LOG_FILE'" CMD="$CMD --log-file '$LOG_FILE'"
if [ -n "$EXCLUSION_FILE" ]; then if [ -n "$EXCLUSION_FILE" ]; then
# We've already checked that $EXCLUSION_FILE doesn't contain a single quote # We've already checked that $EXCLUSION_FILE doesn't contain a single quote
CMD="$CMD --exclude-from '$EXCLUSION_FILE'" CMD="$CMD --exclude-from '$EXCLUSION_FILE'"
fi fi
CMD="$CMD $LINK_DEST_OPTION" CMD="$CMD $LINK_DEST_OPTION"
CMD="$CMD -- '$SRC_FOLDER/' '$SSH_FOLDER_PREFIX$DEST/'" CMD="$CMD -- '$SRC_FOLDER/' '$SSH_FOLDER_PREFIX$DEST/'"
CMD="$CMD | grep -E '^deleting|[^/]$'" CMD="$CMD | grep -E '^deleting|[^/]$'"
fn_log_info "Running command:" fn_log_info "Running command:"
fn_log_info "$CMD" fn_log_info "$CMD"
fn_run_cmd "echo $MYPID > $INPROGRESS_FILE" fn_run_cmd "echo $MYPID > $INPROGRESS_FILE"
eval $CMD eval $CMD
# ----------------------------------------------------------------------------- # -----------------------------------------------------------------------------
# Check if we ran out of space # Check if we ran out of space
# ----------------------------------------------------------------------------- # -----------------------------------------------------------------------------
# TODO: find better way to check for out of space condition without parsing log. # TODO: find better way to check for out of space condition without parsing log.
NO_SPACE_LEFT="$(grep "No space left on device (28)\|Result too large (34)" "$LOG_FILE")" NO_SPACE_LEFT="$(grep "No space left on device (28)\|Result too large (34)" "$LOG_FILE")"
if [ -n "$NO_SPACE_LEFT" ]; then if [ -n "$NO_SPACE_LEFT" ]; then
fn_log_warn "No space left on device - removing oldest backup and resuming." fn_log_warn "No space left on device - removing oldest backup and resuming."
if [[ "$(fn_find_backups | wc -l)" -lt "2" ]]; then if [[ "$(fn_find_backups | wc -l)" -lt "2" ]]; then
fn_log_error "No space left on device, and no old backup to delete." fn_log_error "No space left on device, and no old backup to delete."
exit 1 exit 1
fi fi
fn_expire_backup "$(fn_find_backups | tail -n 1)" fn_expire_backup "$(fn_find_backups | tail -n 1)"
# Resume backup # Resume backup
continue continue
fi fi
# ----------------------------------------------------------------------------- # -----------------------------------------------------------------------------
# Check whether rsync reported any errors # Check whether rsync reported any errors
# ----------------------------------------------------------------------------- # -----------------------------------------------------------------------------
if [ -n "$(grep "rsync:" "$LOG_FILE")" ]; then if [ -n "$(grep "rsync:" "$LOG_FILE")" ]; then
fn_log_warn "Rsync reported a warning, please check '$LOG_FILE' for more details." fn_log_warn "Rsync reported a warning, please check '$LOG_FILE' for more details."
fi fi
if [ -n "$(grep "rsync error:" "$LOG_FILE")" ]; then if [ -n "$(grep "rsync error:" "$LOG_FILE")" ]; then
fn_log_error "Rsync reported an error, please check '$LOG_FILE' for more details." fn_log_error "Rsync reported an error, please check '$LOG_FILE' for more details."
exit 1 exit 1
fi fi
# ----------------------------------------------------------------------------- # -----------------------------------------------------------------------------
# Add symlink to last successful backup # Add symlink to last successful backup
# ----------------------------------------------------------------------------- # -----------------------------------------------------------------------------
fn_rm "$DEST_FOLDER/latest" fn_rm "$DEST_FOLDER/latest"
fn_ln "$(basename -- "$DEST")" "$DEST_FOLDER/latest" fn_ln "$(basename -- "$DEST")" "$DEST_FOLDER/latest"
fn_rm "$INPROGRESS_FILE" fn_rm "$INPROGRESS_FILE"
rm -f -- "$LOG_FILE" rm -f -- "$LOG_FILE"
fn_log_info "Backup completed without errors." fn_log_info "Backup completed without errors."
exit 0 exit 0
done done