diff --git a/management/backup.py b/management/backup.py index be87a429..c1ba03c5 100755 --- a/management/backup.py +++ b/management/backup.py @@ -30,6 +30,11 @@ def backup_status(env): backups = { } backup_cache_dir = os.path.join(backup_root, 'cache') + rsync_ssh_options = [ + "--ssh-options='-i /root/.ssh/id_rsa_miab'", + "--rsync-options=-e \"/usr/bin/ssh -oStrictHostKeyChecking=no -oBatchMode=yes -p 22 -i /root/.ssh/id_rsa_miab\"", + ] + def reldate(date, ref, clip): if ref < date: return clip rd = dateutil.relativedelta.relativedelta(ref, date) @@ -52,6 +57,7 @@ def backup_status(env): "size": 0, # collection-status doesn't give us the size "volumes": keys[2], # number of archive volumes for this backup (not really helpful) } + code, collection_status = shell('check_output', [ "/usr/bin/duplicity", "collection-status", @@ -59,7 +65,7 @@ def backup_status(env): "--gpg-options", "--cipher-algo=AES256", "--log-fd", "1", config["target"], - ], + ] + rsync_ssh_options, get_env(env), trap=True) if code != 0: @@ -177,24 +183,24 @@ def get_passphrase(env): with open(os.path.join(backup_root, 'secret_key.txt')) as f: passphrase = f.readline().strip() if len(passphrase) < 43: raise Exception("secret_key.txt's first line is too short!") - + return passphrase def get_env(env): config = get_backup_config(env) - + env = { "PASSPHRASE" : get_passphrase(env) } - + if get_target_type(config) == 's3': env["AWS_ACCESS_KEY_ID"] = config["target_user"] env["AWS_SECRET_ACCESS_KEY"] = config["target_pass"] - + return env - + def get_target_type(config): protocol = config["target"].split(":")[0] return protocol - + def perform_backup(full_backup): env = load_environment() @@ -204,7 +210,7 @@ def perform_backup(full_backup): backup_cache_dir = os.path.join(backup_root, 'cache') backup_dir = os.path.join(backup_root, 'encrypted') - # Are backups dissbled? + # Are backups disabled? if config["target"] == "off": return @@ -283,7 +289,7 @@ def perform_backup(full_backup): env["STORAGE_ROOT"], config["target"], "--allow-source-mismatch" - ], + ] + rsync_ssh_options, get_env(env)) finally: # Start services again. @@ -305,7 +311,7 @@ def perform_backup(full_backup): "--archive-dir", backup_cache_dir, "--force", config["target"] - ], + ] + rsync_ssh_options, get_env(env)) # From duplicity's manual: @@ -320,7 +326,7 @@ def perform_backup(full_backup): "--archive-dir", backup_cache_dir, "--force", config["target"] - ], + ] + rsync_ssh_options, get_env(env)) # Change ownership of backups to the user-data user, so that the after-bcakup @@ -359,7 +365,7 @@ def run_duplicity_verification(): "--exclude", backup_root, config["target"], env["STORAGE_ROOT"], - ], get_env(env)) + ] + rsync_ssh_options, get_env(env)) def run_duplicity_restore(args): env = load_environment() @@ -370,7 +376,7 @@ def run_duplicity_restore(args): "restore", "--archive-dir", backup_cache_dir, config["target"], - ] + args, + ] + rsync_ssh_options + args, get_env(env)) def list_target_files(config): @@ -383,6 +389,34 @@ def list_target_files(config): if p.scheme == "file": return [(fn, os.path.getsize(os.path.join(p.path, fn))) for fn in os.listdir(p.path)] + elif p.scheme == "rsync": + rsync_fn_size_re = re.compile(r'.* ([^ ]*) [^ ]* [^ ]* (.*)') + rsync_target = '{host}:{path}' + + _, target_host, target_path = config['target'].split('//') + target_path = '/' + target_path + if not target_path.endswith('/'): + target_path += '/' + + rsync_command = [ 'rsync', + '-e', + '/usr/bin/ssh -i /root/.ssh/id_rsa_miab -oStrictHostKeyChecking=no -oBatchMode=yes', + '--list-only', + '-r', + rsync_target.format( + host=target_host, + path=target_path) + ] + + code, listing = shell('check_output', rsync_command, trap=True) + if code == 0: + for l in listing.split('\n'): + match = rsync_fn_size_re.match(l) + if match: + yield (match.groups()[1], int(match.groups()[0].replace(',',''))) + else: + raise ValueError("Connection to rsync host failed") + elif p.scheme == "s3": # match to a Region fix_boto() # must call prior to importing boto @@ -425,7 +459,7 @@ def list_target_files(config): def backup_set_custom(env, target, target_user, target_pass, min_age): config = get_backup_config(env, for_save=True) - + # min_age must be an int if isinstance(min_age, str): min_age = int(min_age) @@ -443,11 +477,11 @@ def backup_set_custom(env, target, target_user, target_pass, min_age): list_target_files(config) except ValueError as e: return str(e) - + write_backup_config(env, config) return "OK" - + def get_backup_config(env, for_save=False, for_ui=False): backup_root = os.path.join(env["STORAGE_ROOT"], 'backup') @@ -482,6 +516,9 @@ def get_backup_config(env, for_save=False, for_ui=False): if config["target"] == "local": # Expand to the full URL. config["target"] = "file://" + config["file_target_directory"] + ssh_pub_key = os.path.join('/root', '.ssh', 'id_rsa_miab.pub') + if os.path.exists(ssh_pub_key): + config["ssh_pub_key"] = open(ssh_pub_key, 'r').read() return config diff --git a/management/templates/system-backup.html b/management/templates/system-backup.html index 8fceafe6..63a220e8 100644 --- a/management/templates/system-backup.html +++ b/management/templates/system-backup.html @@ -16,16 +16,60 @@ +
-

Backups are stored on this machine’s own hard disk. You are responsible for periodically using SFTP (FTP over SSH) to copy the backup files from to a safe location. These files are encrypted, so they are safe to store anywhere.

+

Backups are stored on this machine’s own hard disk. You are responsible for periodically using SFTP (FTP over SSH) to copy the backup files from to a safe location. These files are encrypted, so they are safe to store anywhere.

Separately copy the encryption password from to a safe and secure location. You will need this file to decrypt backup files.

+ +
+
+ +

Backups synced to a remote machine using rsync over SSH, with local + copies in . These files are encrypted, so + they are safe to store anywhere.

Separately copy the encryption + password from to a safe and + secure location. You will need this file to decrypt backup files.

+ +
+
+
+ +
+ +
+
+
+ +
+ +
+
+
+ +
+ +
+
+
+ +
+ +
+ Copy the Public SSH Key above, and paste it within the ~/.ssh/authorized_keys + of target user on the backup server specified above. That way you'll enable secure and + passwordless authentication from your mail-in-a-box server and your backup server. +
+
+
+

Backups are stored in an Amazon Web Services S3 bucket. You must have an AWS account already.

@@ -60,7 +104,8 @@
-
+ +
@@ -92,7 +137,7 @@ function toggle_form() { var target_type = $("#backup-target-type").val(); - $(".backup-target-local, .backup-target-s3").hide(); + $(".backup-target-local, .backup-target-rsync, .backup-target-s3").hide(); $(".backup-target-" + target_type).show(); } @@ -114,7 +159,7 @@ function nice_size(bytes) { function show_system_backup() { show_custom_backup() - + $('#backup-status tbody').html("Loading...") api( "/system/backup/status", @@ -160,28 +205,37 @@ function show_system_backup() { } function show_custom_backup() { - $(".backup-target-local, .backup-target-s3").hide(); + $(".backup-target-local, .backup-target-rsync, .backup-target-s3").hide(); api( "/system/backup/config", "GET", { }, function(r) { + $("#backup-target-user").val(r.target_user); + $("#backup-target-pass").val(r.target_pass); + $("#min-age").val(r.min_age_in_days); + $(".backup-location").text(r.file_target_directory); + $(".backup-encpassword-file").text(r.enc_pw_file); + $("#ssh-pub-key").val(r.ssh_pub_key); + if (r.target == "file://" + r.file_target_directory) { $("#backup-target-type").val("local"); } else if (r.target == "off") { $("#backup-target-type").val("off"); + } else if (r.target.substring(0, 8) == "rsync://") { + $("#backup-target-type").val("rsync"); + var path = r.target.substring(8).split('//'); + var [ user, host ] = path.shift().split('@'); + $("#backup-target-rsync-user").val(user); + $("#backup-target-rsync-host").val(host); + $("#backup-target-rsync-path").val('/'+path); } else if (r.target.substring(0, 5) == "s3://") { $("#backup-target-type").val("s3"); - var hostpath = r.target.substring(5).split('/'); + var hostpath = r.target.substring(5).split('/'); var host = hostpath.shift(); $("#backup-target-s3-host").val(host); $("#backup-target-s3-path").val(hostpath.join('/')); } - $("#backup-target-user").val(r.target_user); - $("#backup-target-pass").val(r.target_pass); - $("#min-age").val(r.min_age_in_days); - $('#backup-location').text(r.file_target_directory); - $('.backup-encpassword-file').text(r.enc_pw_file); toggle_form() }) } @@ -190,12 +244,18 @@ function set_custom_backup() { var target_type = $("#backup-target-type").val(); var target_user = $("#backup-target-user").val(); var target_pass = $("#backup-target-pass").val(); - + var target; if (target_type == "local" || target_type == "off") target = target_type; else if (target_type == "s3") target = "s3://" + $("#backup-target-s3-host").val() + "/" + $("#backup-target-s3-path").val(); + else if (target_type == "rsync") { + target = "rsync://" + $("#backup-target-rsync-user").val() + "@" + $("#backup-target-rsync-host").val() + + "/" + $("#backup-target-rsync-path").val(); + target_user = ''; + } + var min_age = $("#min-age").val(); api( diff --git a/setup/system.sh b/setup/system.sh index bde013a3..53fae9d6 100755 --- a/setup/system.sh +++ b/setup/system.sh @@ -214,6 +214,12 @@ pollinate -q -r # Between these two, we really ought to be all set. +# We need an ssh key to store backups via rsync, if it doesn't exist create one +if [ ! -f /root/.ssh/id_rsa_miab ]; then + echo 'Creating SSH key for backup…' + ssh-keygen -t rsa -b 2048 -a 100 -f /root/.ssh/id_rsa_miab -N '' -q +fi + # ### Package maintenance # # Allow apt to install system updates automatically every day.