From 88dce0e3f9e1853f42f2610c499e6f6017934c04 Mon Sep 17 00:00:00 2001 From: markalston Date: Wed, 20 Sep 2017 09:22:18 -0600 Subject: [PATCH 1/5] Updated rsync_tmbackup.sh to allow remote source Changes to allow rsync_tmbackup.sh to pull files from remote server. --- rsync_tmbackup.sh | 30 ++++++++++++++++++++++-------- 1 file changed, 22 insertions(+), 8 deletions(-) diff --git a/rsync_tmbackup.sh b/rsync_tmbackup.sh index fc8ceb8..416c8e5 100755 --- a/rsync_tmbackup.sh +++ b/rsync_tmbackup.sh @@ -32,7 +32,7 @@ trap 'fn_terminate_script' SIGINT # Small utility functions for reducing code duplication # ----------------------------------------------------------------------------- fn_display_usage() { - echo "Usage: $(basename $0) [OPTION]... <[USER@HOST:]DESTINATION> [exclude-pattern-file]" + echo "Usage: $(basename $0) [OPTION]... <[USER@HOST:]SOURCE> <[USER@HOST:]DESTINATION> [exclude-pattern-file]" echo "" echo "Options" echo " -p, --port SSH port." @@ -82,7 +82,15 @@ fn_parse_ssh() { SSH_HOST=$(echo "$DEST_FOLDER" | sed -E 's/^([A-Za-z0-9\._%\+\-]+)@([A-Za-z0-9.\-]+)\:(.+)$/\2/') SSH_DEST_FOLDER=$(echo "$DEST_FOLDER" | sed -E 's/^([A-Za-z0-9\._%\+\-]+)@([A-Za-z0-9.\-]+)\:(.+)$/\3/') SSH_CMD="ssh -p $SSH_PORT ${SSH_USER}@${SSH_HOST}" - SSH_FOLDER_PREFIX="${SSH_USER}@${SSH_HOST}:" + SSH_DEST_FOLDER_PREFIX="${SSH_USER}@${SSH_HOST}:" + + elif [[ "$SRC_FOLDER" =~ ^[A-Za-z0-9\._%\+\-]+@[A-Za-z0-9.\-]+\:.+$ ]] + then + SSH_USER=$(echo "$SRC_FOLDER" | sed -E 's/^([A-Za-z0-9\._%\+\-]+)@([A-Za-z0-9.\-]+)\:(.+)$/\1/') + SSH_HOST=$(echo "$SRC_FOLDER" | sed -E 's/^([A-Za-z0-9\._%\+\-]+)@([A-Za-z0-9.\-]+)\:(.+)$/\2/') + SSH_SRC_FOLDER=$(echo "$SRC_FOLDER" | sed -E 's/^([A-Za-z0-9\._%\+\-]+)@([A-Za-z0-9.\-]+)\:(.+)$/\3/') + SSH_CMD="ssh -p $SSH_PORT ${SSH_USER}@${SSH_HOST}" + SSH_SRC_FOLDER_PREFIX="${SSH_USER}@${SSH_HOST}:" fi } @@ -130,8 +138,10 @@ fn_ln() { SSH_USER="" SSH_HOST="" SSH_DEST_FOLDER="" +SSH_SRC_FOLDER="" SSH_CMD="" -SSH_FOLDER_PREFIX="" +SSH_DEST_FOLDER_PREFIX="" +SSH_SRC_FOLDER_PREFIX="" SSH_PORT="22" SRC_FOLDER="" @@ -210,6 +220,10 @@ if [ -n "$SSH_DEST_FOLDER" ]; then DEST_FOLDER="$SSH_DEST_FOLDER" fi +if [ -n "$SSH_SRC_FOLDER" ]; then + SRC_FOLDER="$SSH_SRC_FOLDER" +fi + for ARG in "$SRC_FOLDER" "$DEST_FOLDER" "$EXCLUSION_FILE"; do if [[ "$ARG" == *"'"* ]]; then fn_log_error 'Source and destination directories may not contain single quote characters.' @@ -291,7 +305,7 @@ if [ -n "$(fn_find "$INPROGRESS_FILE")" ]; then if [ -n "$PREVIOUS_DEST" ]; then # - Last backup is moved to current backup folder so that it can be resumed. # - 2nd to last backup becomes last backup. - fn_log_info "$SSH_FOLDER_PREFIX$INPROGRESS_FILE already exists - the previous backup failed or was interrupted. Backup will resume from there." + fn_log_info "$SSH_DEST_FOLDER_PREFIX$INPROGRESS_FILE already exists - the previous backup failed or was interrupted. Backup will resume from there." fn_run_cmd "mv -- $PREVIOUS_DEST $DEST" if [ "$(fn_find_backups | wc -l)" -gt 1 ]; then PREVIOUS_DEST="$(fn_find_backups | sed -n '2p')" @@ -317,7 +331,7 @@ while : ; do # If the path is relative, it needs to be relative to the destination. To keep # it simple, just use an absolute path. See http://serverfault.com/a/210058/118679 PREVIOUS_DEST="$(fn_get_absolute_path "$PREVIOUS_DEST")" - fn_log_info "Previous backup found - doing incremental backup from $SSH_FOLDER_PREFIX$PREVIOUS_DEST" + fn_log_info "Previous backup found - doing incremental backup from $SSH_DEST_FOLDER_PREFIX$PREVIOUS_DEST" LINK_DEST_OPTION="--link-dest='$PREVIOUS_DEST'" fi @@ -326,7 +340,7 @@ while : ; do # ----------------------------------------------------------------------------- if [ -z "$(fn_find "$DEST -type d" 2>/dev/null)" ]; then - fn_log_info "Creating destination $SSH_FOLDER_PREFIX$DEST" + fn_log_info "Creating destination $SSH_DEST_FOLDER_PREFIX$DEST" fn_mkdir "$DEST" fi @@ -367,7 +381,7 @@ while : ; do fn_log_info "Starting backup..." fn_log_info "From: $SRC_FOLDER/" - fn_log_info "To: $SSH_FOLDER_PREFIX$DEST/" + fn_log_info "To: $SSH_DEST_FOLDER_PREFIX$DEST/" CMD="rsync" if [ -n "$SSH_CMD" ]; then @@ -380,7 +394,7 @@ while : ; do CMD="$CMD --exclude-from '$EXCLUSION_FILE'" fi CMD="$CMD $LINK_DEST_OPTION" - CMD="$CMD -- '$SRC_FOLDER/' '$SSH_FOLDER_PREFIX$DEST/'" + CMD="$CMD -- '$SRC_FOLDER/' '$SSH_DEST_FOLDER_PREFIX$DEST/'" fn_log_info "Running command:" fn_log_info "$CMD" From 6b05dca48afc9917568a308f79f56c79526d8160 Mon Sep 17 00:00:00 2001 From: markalston Date: Wed, 20 Sep 2017 09:36:36 -0600 Subject: [PATCH 2/5] fixed missing SSH_SRC_FOLDER_PREFIXs --- rsync_tmbackup.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/rsync_tmbackup.sh b/rsync_tmbackup.sh index 416c8e5..20fa05d 100755 --- a/rsync_tmbackup.sh +++ b/rsync_tmbackup.sh @@ -380,7 +380,7 @@ while : ; do LOG_FILE="$LOG_DIR/$(date +"%Y-%m-%d-%H%M%S").log" fn_log_info "Starting backup..." - fn_log_info "From: $SRC_FOLDER/" + fn_log_info "From: $SSH_SRC_FOLDER_PREFIX$SRC_FOLDER/" fn_log_info "To: $SSH_DEST_FOLDER_PREFIX$DEST/" CMD="rsync" @@ -394,7 +394,7 @@ while : ; do CMD="$CMD --exclude-from '$EXCLUSION_FILE'" fi CMD="$CMD $LINK_DEST_OPTION" - CMD="$CMD -- '$SRC_FOLDER/' '$SSH_DEST_FOLDER_PREFIX$DEST/'" + CMD="$CMD -- '$SSH_SRC_FOLDER_PREFIX$SRC_FOLDER/' '$SSH_DEST_FOLDER_PREFIX$DEST/'" fn_log_info "Running command:" fn_log_info "$CMD" From 13357c1cc97469f751ada41725163318bc56c026 Mon Sep 17 00:00:00 2001 From: markalston Date: Wed, 20 Sep 2017 10:40:34 -0600 Subject: [PATCH 3/5] fixes for ssh source folder / --- rsync_tmbackup.sh | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/rsync_tmbackup.sh b/rsync_tmbackup.sh index 20fa05d..9662476 100755 --- a/rsync_tmbackup.sh +++ b/rsync_tmbackup.sh @@ -10,7 +10,7 @@ fn_log_info() { echo "$APPNAME: $1"; } fn_log_warn() { echo "$APPNAME: [WARNING] $1" 1>&2; } fn_log_error() { echo "$APPNAME: [ERROR] $1" 1>&2; } fn_log_info_cmd() { - if [ -n "$SSH_CMD" ]; then + if [ -n "$SSH_DEST_FOLDER_PREFIX" ]; then echo "$APPNAME: $SSH_CMD '$1'"; else echo "$APPNAME: $1"; @@ -83,7 +83,6 @@ fn_parse_ssh() { SSH_DEST_FOLDER=$(echo "$DEST_FOLDER" | sed -E 's/^([A-Za-z0-9\._%\+\-]+)@([A-Za-z0-9.\-]+)\:(.+)$/\3/') SSH_CMD="ssh -p $SSH_PORT ${SSH_USER}@${SSH_HOST}" SSH_DEST_FOLDER_PREFIX="${SSH_USER}@${SSH_HOST}:" - elif [[ "$SRC_FOLDER" =~ ^[A-Za-z0-9\._%\+\-]+@[A-Za-z0-9.\-]+\:.+$ ]] then SSH_USER=$(echo "$SRC_FOLDER" | sed -E 's/^([A-Za-z0-9\._%\+\-]+)@([A-Za-z0-9.\-]+)\:(.+)$/\1/') @@ -95,7 +94,7 @@ fn_parse_ssh() { } fn_run_cmd() { - if [ -n "$SSH_CMD" ] + if [ -n "$SSH_DEST_FOLDER_PREFIX" ] then eval "$SSH_CMD '$1'" else @@ -205,13 +204,13 @@ if [[ -z "$SRC_FOLDER" || -z "$DEST_FOLDER" ]]; then exit 1 fi -# Strips off last slash. Note that it means the root folder "/" +# Strips off last slash from dest. Note that it means the root folder "/" # will be represented as an empty string "", which is fine # with the current script (since a "/" is added when needed) # but still something to keep in mind. -# Don't think it would with DEST_FOLDER set to "/" though, -# but there's probably not a use case for this anyway. -SRC_FOLDER="${SRC_FOLDER%/}" +# However, due to this behavior we delay stripping the last slash for +# the source folder until after parsing for ssh usage. + DEST_FOLDER="${DEST_FOLDER%/}" fn_parse_ssh @@ -224,6 +223,9 @@ if [ -n "$SSH_SRC_FOLDER" ]; then SRC_FOLDER="$SSH_SRC_FOLDER" fi +# Now strip off last slash from source folder. +SRC_FOLDER="${SRC_FOLDER%/}" + for ARG in "$SRC_FOLDER" "$DEST_FOLDER" "$EXCLUSION_FILE"; do if [[ "$ARG" == *"'"* ]]; then fn_log_error 'Source and destination directories may not contain single quote characters.' From 5a8ed7b008ef95c4cae031892e31c0d252a2a31a Mon Sep 17 00:00:00 2001 From: markalston Date: Wed, 20 Sep 2017 10:54:52 -0600 Subject: [PATCH 4/5] Update rsync_tmbackup.sh --- rsync_tmbackup.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/rsync_tmbackup.sh b/rsync_tmbackup.sh index 9662476..8f3392e 100755 --- a/rsync_tmbackup.sh +++ b/rsync_tmbackup.sh @@ -402,7 +402,6 @@ while : ; do fn_log_info "$CMD" fn_run_cmd "echo $MYPID > $INPROGRESS_FILE" - eval $CMD # ----------------------------------------------------------------------------- From 732ad83ed80b9c9b7cd0eebcb1695132673eda81 Mon Sep 17 00:00:00 2001 From: markalston Date: Tue, 26 Sep 2017 08:03:48 -0600 Subject: [PATCH 5/5] Update Readme.md to reflect remote pull method --- README.md | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 99cc635..54091d0 100644 --- a/README.md +++ b/README.md @@ -12,7 +12,7 @@ On macOS, it has a few disadvantages compared to Time Machine - in particular it ## Usage - Usage: rsync_tmbackup.sh [OPTION]... <[USER@HOST:]DESTINATION> [exclude-pattern-file] + Usage: rsync_tmbackup.sh [OPTION]... <[USER@HOST:]SOURCE> <[USER@HOST:]DESTINATION> [exclude-pattern-file] Options -p, --port SSH port. @@ -37,6 +37,11 @@ On macOS, it has a few disadvantages compared to Time Machine - in particular it rsync_tmbackup.sh -p 2222 /home user@example.com:/mnt/backup_drive + +* Backup from remote drive over SSH: + + rsync_tmbackup.shuser@example.com:/home /mnt/backup_drive + * To mimic Time Machine's behaviour, a cron script can be setup to backup at regular interval. For example, the following cron job checks if the drive "/mnt/backup" is currently connected and, if it is, starts the backup. It does this check every 1 hour. 0 */1 * * * if [[ -d /mnt/backup ]]; then rsync_tmbackup.sh /home /mnt/backup; fi @@ -70,6 +75,8 @@ To display the rsync options that are used for backup, run `./rsync_tmbackup.sh * Backup to remote destinations over SSH. +* Backup from remote destinations over SSH. + * Files that haven't changed from one backup to the next are hard-linked to the previous backup so take very little extra space. * Safety check - the backup will only happen if the destination has explicitly been marked as a backup destination.