schiwagoa/rsync-time-backup
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'upstream/master'

Browse Source 
Conflicts:
	rsync_tmbackup.sh
This commit is contained in:
Bhuvanesh Kumar
2015-12-08 00:25:50 +05:30
parent 9bc6be475a 9c36702657
commit 5e4250d5bc
3 changed files with 263 additions and 160 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.gitignore
+2
View File
@@ -0,0 +1,2 @@
.idea
test.sh
README.md
+27 -7
View File
@@ -1,6 +1,10 @@
# Rsync time backup
Time Machine style backup with rsync. Should work on Linux (tested), Mac OS X (tested) and Windows with Cygwin (not tested yet but feedback would be welcome).
This script offers Time Machine-style backup using rsync. It creates incremental backups of files and directories to the destination of your choice. The backups are structured in a way that makes it easy to recover any file at any point in time.
It should work on Linux, OS X and Windows with Cygwin. The main advantage over Time Machine is the flexibility as it can backup from/to any filesystem and works on any platform. You can also backup, for example, to a Truecrypt drive without any problem.
On OS X, it has a few disadvantages compared to Time Machine - in particular it doesn't auto-start when the backup drive is plugged (though it can be achieved using a launch agent), it requires some knowledge of the command line, and no specific GUI is provided to restore files. Instead files can be restored by using any file explorer, including Finder, or the command line.
# Installation
@@ -28,6 +32,8 @@ An optional exclude file can be provided as a third parameter. It should be comp
* Each backup is on its own folder named after the current timestamp. Files can be copied and restored directly, without any intermediate tool.
* Backup to remote destinations over SSH.
* Files that haven't changed from one backup to the next are hard-linked to the previous backup so take very little extra space.
* Safety check - the backup will only happen if the destination has explicitly been marked as a backup destination.
@@ -40,18 +46,32 @@ An optional exclude file can be provided as a third parameter. It should be comp
* "latest" symlink that points to the latest successful backup.
* The application is just one bash script that can be easily edited.
# TODO
* Check source and destination file-system. If one of them is FAT, use the --modify-window rsync parameter (see `man rsync`) with a value of 1 or 2.
* Minor changes (see TODO comments in the source).
* Backup to remote drive?
# LICENSE
[MIT](http://opensource.org/licenses/MIT)
The MIT License (MIT)
[![Bitdeli Badge](https://d2weczhvl823v0.cloudfront.net/laurent22/rsync-time-backup/trend.png)](https://bitdeli.com/free "Bitdeli Badge")
Copyright (c) 2013-2016 Laurent Cozic
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
THE SOFTWARE.
rsync_tmbackup.sh
+105 -24
View File
@@ -9,6 +9,13 @@ APPNAME=$(basename $0 | sed "s/\.sh$//")
fn_log_info() { echo "$APPNAME: $1"; }
fn_log_warn() { echo "$APPNAME: [WARNING] $1" 1>&2; }
fn_log_error() { echo "$APPNAME: [ERROR] $1" 1>&2; }
fn_log_info_cmd() {
if [ -n "$SSH_CMD" ]; then
echo "$APPNAME: $SSH_CMD '$1'";
else
echo "$APPNAME: $1";
fi
}
# -----------------------------------------------------------------------------
# Make sure everything really stops when CTRL+C is pressed
@@ -25,7 +32,7 @@ trap 'fn_terminate_script' SIGINT
# Small utility functions for reducing code duplication
# -----------------------------------------------------------------------------
fn_display_usage() {
fn_log_info "Usage : $(basename $0) <source> [destination] [exclude-pattern file]"
fn_log_info "Usage : $(basename $0) <source> [[[user@host:]destination] exclude-pattern-file]"
}
fn_parse_date() {
@@ -38,7 +45,7 @@ fn_parse_date() {
}
fn_find_backups() {
find "$DEST_FOLDER" -type d -name "????-??-??-??????" -prune | sort -r
fn_run_cmd "find "$DEST_FOLDER" -type d -name "????-??-??-??????" -prune | sort -r"
}
fn_expire_backup() {
@@ -50,28 +57,83 @@ fn_expire_backup() {
fi
fn_log_info "Expiring $1"
rm -rf -- "$1"
fn_rm "$1"
}
fn_parse_ssh() {
if [[ "$DEST_FOLDER" =~ ^[A-Za-z0-9\._%\+\-]+@[A-Za-z0-9.\-]+\:.+$ ]]
then
SSH_USER=$(echo "$DEST_FOLDER" | sed -E 's/^([A-Za-z0-9\._%\+\-]+)@([A-Za-z0-9.\-]+)\:(.+)$/\1/')
SSH_HOST=$(echo "$DEST_FOLDER" | sed -E 's/^([A-Za-z0-9\._%\+\-]+)@([A-Za-z0-9.\-]+)\:(.+)$/\2/')
SSH_DEST_FOLDER=$(echo "$DEST_FOLDER" | sed -E 's/^([A-Za-z0-9\._%\+\-]+)@([A-Za-z0-9.\-]+)\:(.+)$/\3/')
SSH_CMD="ssh ${SSH_USER}@${SSH_HOST}"
SSH_FOLDER_PREFIX="${SSH_USER}@${SSH_HOST}:"
fi
}
fn_run_cmd() {
if [ -n "$SSH_CMD" ]
then
eval "$SSH_CMD '$1'"
else
eval $1
fi
}
fn_find() {
fn_run_cmd "find $1" 2>/dev/null
}
fn_get_absolute_path() {
fn_run_cmd "cd $1;pwd"
}
fn_mkdir() {
fn_run_cmd "mkdir -p -- $1"
}
fn_rm() {
fn_run_cmd "rm -rf -- $1"
}
fn_touch() {
fn_run_cmd "touch -- $1"
}
fn_ln() {
fn_run_cmd "ln -vs -- $1 $2"
}
# -----------------------------------------------------------------------------
# Source and destination information
# -----------------------------------------------------------------------------
SSH_USER=""
SSH_HOST=""
SSH_DEST_FOLDER=""
SSH_CMD=""
SSH_FOLDER_PREFIX=""
#display usage information if required arguments are not passed
if [[ "${#@}" -lt 1 ]]; then
fn_display_usage
exit 1
fi
SRC_FOLDER="${1%/}"
#if destination folder is not specified assume current folder
if [ -z ${2} ]; then
DEST_FOLDER="."
echo ${2}
else
DEST_FOLDER="${2%/}"
fi
EXCLUSION_FILE="$3"
fn_parse_ssh
if [ -n "$SSH_DEST_FOLDER" ]; then
DEST_FOLDER="$SSH_DEST_FOLDER"
fi
for ARG in "$SRC_FOLDER" "$DEST_FOLDER" "$EXCLUSION_FILE"; do
if [[ "$ARG" == *"'"* ]]; then
fn_log_error 'Arguments may not have any single quote characters.'
@@ -86,13 +148,13 @@ done
# TODO: check that the destination supports hard links
fn_backup_marker_path() { echo "$1/backup.marker"; }
fn_find_backup_marker() { find "$(fn_backup_marker_path "$1")" 2>/dev/null; }
fn_find_backup_marker() { fn_find "$(fn_backup_marker_path "$1")" 2>/dev/null; }
if [ -z "$(fn_find_backup_marker "$DEST_FOLDER")" ]; then
fn_log_info "Safety check failed - the destination does not appear to be a backup folder or drive (marker file not found)."
fn_log_info "If it is indeed a backup folder, you may add the marker file by running the following command:"
fn_log_info ""
fn_log_info "touch \"$(fn_backup_marker_path "$DEST_FOLDER")\""
fn_log_info_cmd "mkdir -p -- \"$DEST_FOLDER\" ; touch \"$(fn_backup_marker_path "$DEST_FOLDER")\""
fn_log_info ""
exit 1
fi
@@ -104,14 +166,15 @@ fi
# Date logic
NOW=$(date +"%Y-%m-%d-%H%M%S")
EPOCH=$(date "+%s")
KEEP_ALL_DATE=$(($EPOCH - 86400)) # 1 day ago
KEEP_DAILIES_DATE=$(($EPOCH - 2678400)) # 31 days ago
KEEP_ALL_DATE=$((EPOCH - 86400)) # 1 day ago
KEEP_DAILIES_DATE=$((EPOCH - 2678400)) # 31 days ago
export IFS=$'\n' # Better for handling spaces in filenames.
PROFILE_FOLDER="$HOME/.$APPNAME"
DEST="$DEST_FOLDER/$NOW"
PREVIOUS_DEST="$(fn_find_backups | head -n 1)"
INPROGRESS_FILE="$DEST_FOLDER/backup.inprogress"
MYPID="$$"
# -----------------------------------------------------------------------------
# Create profile folder if it doesn't exist
@@ -126,17 +189,32 @@ fi
# Handle case where a previous backup failed or was interrupted.
# -----------------------------------------------------------------------------
if [ -f "$INPROGRESS_FILE" ]; then
if [ -n "$(fn_find "$INPROGRESS_FILE")" ]; then
if [ "$OSTYPE" == "cygwin" ]; then
# TODO: Cygwin reports the name of currently running Bash scripts as just "/usr/bin/bash"
# TODO: so the pgrep solution below won't work. Need to use "procps -wwFAH", grep
# TODO: the script name, and extract the process ID from it.
fn_log_warn "Cygwin only: Previous backup task has either been interrupted or it might still be active, but there is currently no check for this. Assuming that the task was simply interrupted."
else
RUNNINGPID="$(fn_run_cmd "cat $INPROGRESS_FILE")"
if [ "$RUNNINGPID"="$(pgrep "$APPNAME")" ]; then
fn_log_error "Previous backup task is still active - aborting."
exit 1
fi
fi
if [ -n "$PREVIOUS_DEST" ]; then
# - Last backup is moved to current backup folder so that it can be resumed.
# - 2nd to last backup becomes last backup.
fn_log_info "$INPROGRESS_FILE already exists - the previous backup failed or was interrupted. Backup will resume from there."
mv -- "$PREVIOUS_DEST" "$DEST"
fn_log_info "$SSH_FOLDER_PREFIX$INPROGRESS_FILE already exists - the previous backup failed or was interrupted. Backup will resume from there."
fn_run_cmd "mv -- $PREVIOUS_DEST $DEST"
if [ "$(fn_find_backups | wc -l)" -gt 1 ]; then
PREVIOUS_DEST="$(fn_find_backups | sed -n '2p')"
else
PREVIOUS_DEST=""
fi
# update PID to current process to avoid multiple concurrent resumes
fn_run_cmd "echo $MYPID > $INPROGRESS_FILE"
fi
fi
@@ -153,8 +231,8 @@ while : ; do
else
# If the path is relative, it needs to be relative to the destination. To keep
# it simple, just use an absolute path. See http://serverfault.com/a/210058/118679
PREVIOUS_DEST="$(cd "$PREVIOUS_DEST"; pwd)"
fn_log_info "Previous backup found - doing incremental backup from $PREVIOUS_DEST"
PREVIOUS_DEST="$(fn_get_absolute_path "$PREVIOUS_DEST")"
fn_log_info "Previous backup found - doing incremental backup from $SSH_FOLDER_PREFIX$PREVIOUS_DEST"
LINK_DEST_OPTION="--link-dest='$PREVIOUS_DEST'"
fi
@@ -162,9 +240,9 @@ while : ; do
# Create destination folder if it doesn't already exists
# -----------------------------------------------------------------------------
if [ ! -d "$DEST" ]; then
fn_log_info "Creating destination $DEST"
mkdir -p -- "$DEST"
if [ -z "$(fn_find "$DEST -type d" 2>/dev/null)" ]; then
fn_log_info "Creating destination $SSH_FOLDER_PREFIX$DEST"
fn_mkdir "$DEST"
fi
# -----------------------------------------------------------------------------
@@ -204,9 +282,12 @@ while : ; do
fn_log_info "Starting backup..."
fn_log_info "From: $SRC_FOLDER"
fn_log_info "To: $DEST"
fn_log_info "To: $SSH_FOLDER_PREFIX$DEST"
CMD="rsync"
if [ -n "$SSH_CMD" ]; then
CMD="$CMD -e 'ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null'"
fi
CMD="$CMD --compress"
CMD="$CMD --numeric-ids"
CMD="$CMD --links"
@@ -221,13 +302,14 @@ while : ; do
CMD="$CMD --exclude-from '$EXCLUSION_FILE'"
fi
CMD="$CMD $LINK_DEST_OPTION"
CMD="$CMD -- '$SRC_FOLDER/' '$DEST/'"
CMD="$CMD -- '$SRC_FOLDER/' '$SSH_FOLDER_PREFIX$DEST/'"
CMD="$CMD | grep -E '^deleting|[^/]$'"
fn_log_info "Running command:"
fn_log_info "$CMD"
touch -- "$INPROGRESS_FILE"
fn_run_cmd "echo $MYPID > $INPROGRESS_FILE"
eval $CMD
# -----------------------------------------------------------------------------
@@ -254,7 +336,6 @@ while : ; do
# -----------------------------------------------------------------------------
# Check whether rsync reported any errors
# -----------------------------------------------------------------------------
if [ -n "$(grep "rsync:" "$LOG_FILE")" ]; then
fn_log_warn "Rsync reported a warning, please check '$LOG_FILE' for more details."
fi
@@ -267,10 +348,10 @@ while : ; do
# Add symlink to last successful backup
# -----------------------------------------------------------------------------
rm -rf -- "$DEST_FOLDER/latest"
ln -vs -- "$(basename -- "$DEST")" "$DEST_FOLDER/latest"
fn_rm "$DEST_FOLDER/latest"
fn_ln "$(basename -- "$DEST")" "$DEST_FOLDER/latest"
rm -f -- "$INPROGRESS_FILE"
fn_rm "$INPROGRESS_FILE"
rm -f -- "$LOG_FILE"
fn_log_info "Backup completed without errors."