From 72042bc3d5f5de4ce02d8f4f66cd13c1886b19ef Mon Sep 17 00:00:00 2001 From: Bernard `Guyzmo` Pratz Date: Thu, 14 Jan 2016 19:38:04 +0000 Subject: [PATCH] Added support for backup to a remote server using rsync MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * updated web interface to get data from user * added way to list files from server It’s not using the “username” field of the yaml configuration file to minimise the amount of patches needed. So the username is actually sorted within the rsync URL. Signed-off-by: Bernard `Guyzmo` Pratz --- management/backup.py | 27 +++++++++ management/templates/system-backup.html | 76 ++++++++++++++++++++----- 2 files changed, 90 insertions(+), 13 deletions(-) diff --git a/management/backup.py b/management/backup.py index 1612fcbc..c0751e9e 100755 --- a/management/backup.py +++ b/management/backup.py @@ -351,6 +351,33 @@ def list_target_files(config): if p.scheme == "file": return [(fn, os.path.getsize(os.path.join(p.path, fn))) for fn in os.listdir(p.path)] + elif p.scheme == "rsync": + env = load_environment() + rsync_fn_size_re = re.compile(r'.* ([^ ]*) [^ ]* [^ ]* (.*)') + rsync_target = '{host}:{path}' + + _, target_host, target_path = config['target'].split('//') + target_path = '/' + target_path + + rsync_command = [ 'rsync', + '-e', + '/usr/bin/ssh -o StrictHostKeyChecking=no -oBatchMode=yes', + '--list-only', + '-r', + rsync_target.format( + host=target_host, + path=target_path) + ] + + code, listing = shell('check_output', rsync_command, get_env(env), trap=True) + if code == 0: + for l in listing.split('\n'): + match = rsync_fn_size_re.match(l) + if match: + yield (match.groups()[1], int(match.groups()[0].replace(',',''))) + else: + raise ValueError("Connection to rsync host failed") + elif p.scheme == "s3": # match to a Region fix_boto() # must call prior to importing boto diff --git a/management/templates/system-backup.html b/management/templates/system-backup.html index 1e538ecd..3be0e54b 100644 --- a/management/templates/system-backup.html +++ b/management/templates/system-backup.html @@ -16,16 +16,52 @@ +

Backups are stored on this machine’s own hard disk. You are responsible for periodically using SFTP (FTP over SSH) to copy the backup files from to a safe location. These files are encrypted, so they are safe to store anywhere.

Separately copy the encryption password from to a safe and secure location. You will need this file to decrypt backup files.

+ +
+
+ +

Backups synced to a remote machine using rsync over SSH, with local + copies in . These files are encrypted, so + they are safe to store anywhere.

Separately copy the encryption + password from to a safe and + secure location. You will need this file to decrypt backup files.

+

Separately copy the SSH public key from /root/.ssh/id_rsa.pub + and add it to your backup account on your remote server, in the + ~/.ssh/authorized_keys file.

+ +
+
+
+ +
+ +
+
+
+ +
+ +
+
+
+ +
+ +
+
+

Backups are stored in an Amazon Web Services S3 bucket. You must have an AWS account already.

@@ -92,7 +128,7 @@ function toggle_form() { var target_type = $("#backup-target-type").val(); - $(".backup-target-local, .backup-target-s3").hide(); + $(".backup-target-local, .backup-target-rsync, .backup-target-s3").hide(); $(".backup-target-" + target_type).show(); } @@ -160,28 +196,36 @@ function show_system_backup() { } function show_custom_backup() { - $(".backup-target-local, .backup-target-s3").hide(); + $(".backup-target-local, .backup-target-rsync, .backup-target-s3").hide(); api( "/system/backup/config", "GET", { }, function(r) { - if (r.target == "file://" + r.file_target_directory) { - $("#backup-target-type").val("local"); - } else if (r.target == "off") { - $("#backup-target-type").val("off"); - } else if (r.target.substring(0, 5) == "s3://") { - $("#backup-target-type").val("s3"); - var hostpath = r.target.substring(5).split('/'); - var host = hostpath.shift(); - $("#backup-target-s3-host").val(host); - $("#backup-target-s3-path").val(hostpath.join('/')); - } $("#backup-target-user").val(r.target_user); $("#backup-target-pass").val(r.target_pass); $("#min-age").val(r.min_age_in_days); $('#backup-location').text(r.file_target_directory); $('.backup-encpassword-file').text(r.enc_pw_file); + + if (r.target == "file://" + r.file_target_directory) { + $("#backup-target-type").val("local"); + } else if (r.target == "off") { + $("#backup-target-type").val("off"); + } else if (r.target.substring(0, 5) == "rsync://") { + $("#backup-target-type").val("rsync"); + var path = r.target.substring(8).split('//'); + var [ user, host ] = path.shift().split('@'); + $("#backup-target-rsync").val(user); + $("#backup-target-rsync-host").val(host); + $("#backup-target-rsync-path").val('/'+path); + } else if (r.target.substring(0, 5) == "s3://") { + $("#backup-target-type").val("s3"); + var hostpath = r.target.substring(5).split('/'); + var host = hostpath.shift(); + $("#backup-target-s3-host").val(host); + $("#backup-target-s3-path").val(hostpath.join('/')); + } toggle_form() }) } @@ -196,6 +240,12 @@ function set_custom_backup() { target = target_type; else if (target_type == "s3") target = "s3://" + $("#backup-target-s3-host").val() + "/" + $("#backup-target-s3-path").val(); + else if (target_type == "rsync") { + target = "rsync://" + $("#backup-target-user").val() + "@" + $("#backup-target-rsync-host").val() + + "/" + $("#backup-target-rsync-path").val(); + target_user = ''; + } + var min_age = $("#min-age").val(); api(