diff --git a/management/backup.py b/management/backup.py index be87a429..c1ba03c5 100755 --- a/management/backup.py +++ b/management/backup.py @@ -30,6 +30,11 @@ def backup_status(env): backups = { } backup_cache_dir = os.path.join(backup_root, 'cache') + rsync_ssh_options = [ + "--ssh-options='-i /root/.ssh/id_rsa_miab'", + "--rsync-options=-e \"/usr/bin/ssh -oStrictHostKeyChecking=no -oBatchMode=yes -p 22 -i /root/.ssh/id_rsa_miab\"", + ] + def reldate(date, ref, clip): if ref < date: return clip rd = dateutil.relativedelta.relativedelta(ref, date) @@ -52,6 +57,7 @@ def backup_status(env): "size": 0, # collection-status doesn't give us the size "volumes": keys[2], # number of archive volumes for this backup (not really helpful) } + code, collection_status = shell('check_output', [ "/usr/bin/duplicity", "collection-status", @@ -59,7 +65,7 @@ def backup_status(env): "--gpg-options", "--cipher-algo=AES256", "--log-fd", "1", config["target"], - ], + ] + rsync_ssh_options, get_env(env), trap=True) if code != 0: @@ -177,24 +183,24 @@ def get_passphrase(env): with open(os.path.join(backup_root, 'secret_key.txt')) as f: passphrase = f.readline().strip() if len(passphrase) < 43: raise Exception("secret_key.txt's first line is too short!") - + return passphrase def get_env(env): config = get_backup_config(env) - + env = { "PASSPHRASE" : get_passphrase(env) } - + if get_target_type(config) == 's3': env["AWS_ACCESS_KEY_ID"] = config["target_user"] env["AWS_SECRET_ACCESS_KEY"] = config["target_pass"] - + return env - + def get_target_type(config): protocol = config["target"].split(":")[0] return protocol - + def perform_backup(full_backup): env = load_environment() @@ -204,7 +210,7 @@ def perform_backup(full_backup): backup_cache_dir = os.path.join(backup_root, 'cache') backup_dir = os.path.join(backup_root, 'encrypted') - # Are backups dissbled? + # Are backups disabled? if config["target"] == "off": return @@ -283,7 +289,7 @@ def perform_backup(full_backup): env["STORAGE_ROOT"], config["target"], "--allow-source-mismatch" - ], + ] + rsync_ssh_options, get_env(env)) finally: # Start services again. @@ -305,7 +311,7 @@ def perform_backup(full_backup): "--archive-dir", backup_cache_dir, "--force", config["target"] - ], + ] + rsync_ssh_options, get_env(env)) # From duplicity's manual: @@ -320,7 +326,7 @@ def perform_backup(full_backup): "--archive-dir", backup_cache_dir, "--force", config["target"] - ], + ] + rsync_ssh_options, get_env(env)) # Change ownership of backups to the user-data user, so that the after-bcakup @@ -359,7 +365,7 @@ def run_duplicity_verification(): "--exclude", backup_root, config["target"], env["STORAGE_ROOT"], - ], get_env(env)) + ] + rsync_ssh_options, get_env(env)) def run_duplicity_restore(args): env = load_environment() @@ -370,7 +376,7 @@ def run_duplicity_restore(args): "restore", "--archive-dir", backup_cache_dir, config["target"], - ] + args, + ] + rsync_ssh_options + args, get_env(env)) def list_target_files(config): @@ -383,6 +389,34 @@ def list_target_files(config): if p.scheme == "file": return [(fn, os.path.getsize(os.path.join(p.path, fn))) for fn in os.listdir(p.path)] + elif p.scheme == "rsync": + rsync_fn_size_re = re.compile(r'.* ([^ ]*) [^ ]* [^ ]* (.*)') + rsync_target = '{host}:{path}' + + _, target_host, target_path = config['target'].split('//') + target_path = '/' + target_path + if not target_path.endswith('/'): + target_path += '/' + + rsync_command = [ 'rsync', + '-e', + '/usr/bin/ssh -i /root/.ssh/id_rsa_miab -oStrictHostKeyChecking=no -oBatchMode=yes', + '--list-only', + '-r', + rsync_target.format( + host=target_host, + path=target_path) + ] + + code, listing = shell('check_output', rsync_command, trap=True) + if code == 0: + for l in listing.split('\n'): + match = rsync_fn_size_re.match(l) + if match: + yield (match.groups()[1], int(match.groups()[0].replace(',',''))) + else: + raise ValueError("Connection to rsync host failed") + elif p.scheme == "s3": # match to a Region fix_boto() # must call prior to importing boto @@ -425,7 +459,7 @@ def list_target_files(config): def backup_set_custom(env, target, target_user, target_pass, min_age): config = get_backup_config(env, for_save=True) - + # min_age must be an int if isinstance(min_age, str): min_age = int(min_age) @@ -443,11 +477,11 @@ def backup_set_custom(env, target, target_user, target_pass, min_age): list_target_files(config) except ValueError as e: return str(e) - + write_backup_config(env, config) return "OK" - + def get_backup_config(env, for_save=False, for_ui=False): backup_root = os.path.join(env["STORAGE_ROOT"], 'backup') @@ -482,6 +516,9 @@ def get_backup_config(env, for_save=False, for_ui=False): if config["target"] == "local": # Expand to the full URL. config["target"] = "file://" + config["file_target_directory"] + ssh_pub_key = os.path.join('/root', '.ssh', 'id_rsa_miab.pub') + if os.path.exists(ssh_pub_key): + config["ssh_pub_key"] = open(ssh_pub_key, 'r').read() return config diff --git a/management/templates/system-backup.html b/management/templates/system-backup.html index 8fceafe6..63a220e8 100644 --- a/management/templates/system-backup.html +++ b/management/templates/system-backup.html @@ -16,16 +16,60 @@ +
Backups are stored on this machine’s own hard disk. You are responsible for periodically using SFTP (FTP over SSH) to copy the backup files from to a safe location. These files are encrypted, so they are safe to store anywhere.
+Backups are stored on this machine’s own hard disk. You are responsible for periodically using SFTP (FTP over SSH) to copy the backup files from to a safe location. These files are encrypted, so they are safe to store anywhere.
Separately copy the encryption password from to a safe and secure location. You will need this file to decrypt backup files.
Backups synced to a remote machine using rsync over SSH, with local + copies in . These files are encrypted, so + they are safe to store anywhere.
Separately copy the encryption + password from to a safe and + secure location. You will need this file to decrypt backup files.
+ +Backups are stored in an Amazon Web Services S3 bucket. You must have an AWS account already.
@@ -60,7 +104,8 @@