From 2e6c4103361e9924f90d6b22a11fdf3bb1c854b0 Mon Sep 17 00:00:00 2001 From: Leo Koppelkamm Date: Sun, 26 Jul 2015 18:25:52 +0200 Subject: [PATCH 01/18] Make backups more configurable Backup location and maximum age can now be configured in the admin panel. For now only S3 is supported, but adding other duplicity supported backends should be straightforward. --- management/backup.py | 182 +++++++++++++++++------- management/daemon.py | 19 ++- management/templates/system-backup.html | 101 ++++++++++++- setup/management.sh | 5 +- 4 files changed, 246 insertions(+), 61 deletions(-) diff --git a/management/backup.py b/management/backup.py index 40d70458..26426313 100755 --- a/management/backup.py +++ b/management/backup.py @@ -10,20 +10,29 @@ import os, os.path, shutil, glob, re, datetime import dateutil.parser, dateutil.relativedelta, dateutil.tz +import rtyaml from utils import exclusive_process, load_environment, shell, wait_for_service +# Root folder +backup_root = os.path.join(load_environment()["STORAGE_ROOT"], 'backup') + +# Default settings # Destroy backups when the most recent increment in the chain # that depends on it is this many days old. -keep_backups_for_days = 3 +default_config = { + "max_age_in_days": 3, + "target": "file://" + os.path.join(backup_root, 'encrypted'), + "target_type": "file" +} def backup_status(env): # What is the current status of backups? - # Loop through all of the files in STORAGE_ROOT/backup/encrypted to - # get a list of all of the backups taken and sum up file sizes to - # see how large the storage is. - + # Query duplicity to get a list of all backups. + # Use the number of volumes to estimate the size. + config = get_backup_config() now = datetime.datetime.now(dateutil.tz.tzlocal()) + def reldate(date, ref, clip): if ref < date: return clip rd = dateutil.relativedelta.relativedelta(ref, date) @@ -33,29 +42,42 @@ def backup_status(env): if rd.days > 1: return "%d days, %d hours" % (rd.days, rd.hours) if rd.days == 1: return "%d day, %d hours" % (rd.days, rd.hours) return "%d hours, %d minutes" % (rd.hours, rd.minutes) + + def parse_line(line): + keys = line.strip().split() + date = dateutil.parser.parse(keys[1]) + return { + "date": keys[1], + "date_str": date.strftime("%x %X"), + "date_delta": reldate(date, now, "the future?"), + "full": keys[0] == "full", + "size": int(keys[2]) * 250 * 1000000, + } + + # Write duplicity status to file + shell('check_call', [ + "/usr/bin/duplicity", + "collection-status", + "--log-file", os.path.join(backup_root, "duplicity_status"), + "--gpg-options", "--cipher-algo=AES256", + config["target"], + ], + get_env()) + backups = { } - backup_root = os.path.join(env["STORAGE_ROOT"], 'backup') backup_dir = os.path.join(backup_root, 'encrypted') - os.makedirs(backup_dir, exist_ok=True) # os.listdir fails if directory does not exist - for fn in os.listdir(backup_dir): - m = re.match(r"duplicity-(full|full-signatures|(inc|new-signatures)\.(?P\d+T\d+Z)\.to)\.(?P\d+T\d+Z)\.", fn) - if not m: raise ValueError(fn) - - key = m.group("date") - if key not in backups: - date = dateutil.parser.parse(m.group("date")) - backups[key] = { - "date": m.group("date"), - "date_str": date.strftime("%x %X"), - "date_delta": reldate(date, now, "the future?"), - "full": m.group("incbase") is None, - "previous": m.group("incbase"), - "size": 0, - } - - backups[key]["size"] += os.path.getsize(os.path.join(backup_dir, fn)) + + # Parse backup data from status file + with open(os.path.join(backup_root, "duplicity_status"),'r') as status_file: + for line in status_file: + if line.startswith(" full") or line.startswith(" inc"): + backup = parse_line(line) + backups[backup["date"]] = backup + # Remove status file + os.remove(os.path.join(backup_root, "duplicity_status")) + # Ensure the rows are sorted reverse chronologically. # This is relied on by should_force_full() and the next step. backups = sorted(backups.values(), key = lambda b : b["date"], reverse=True) @@ -79,11 +101,11 @@ def backup_status(env): # when the threshold is met. deleted_in = None if incremental_count > 0 and first_full_size is not None: - deleted_in = "approx. %d days" % round(keep_backups_for_days + (.5 * first_full_size - incremental_size) / (incremental_size/incremental_count) + .5) + deleted_in = "approx. %d days" % round(config["max_age_in_days"] + (.5 * first_full_size - incremental_size) / (incremental_size/incremental_count) + .5) # When will a backup be deleted? saw_full = False - days_ago = now - datetime.timedelta(days=keep_backups_for_days) + days_ago = now - datetime.timedelta(days=config["max_age_in_days"]) for bak in backups: if deleted_in: # Subsequent backups are deleted when the most recent increment @@ -124,12 +146,35 @@ def should_force_full(env): # (I love for/else blocks. Here it's just to show off.) return True +def get_passphrase(): + # Get the encryption passphrase. secret_key.txt is 2048 random + # bits base64-encoded and with line breaks every 65 characters. + # gpg will only take the first line of text, so sanity check that + # that line is long enough to be a reasonable passphrase. It + # only needs to be 43 base64-characters to match AES256's key + # length of 32 bytes. + with open(os.path.join(backup_root, 'secret_key.txt')) as f: + passphrase = f.readline().strip() + if len(passphrase) < 43: raise Exception("secret_key.txt's first line is too short!") + + return passphrase + +def get_env(): + config = get_backup_config() + + env = { "PASSPHRASE" : get_passphrase() } + + if config["target_type"] == 's3': + env["AWS_ACCESS_KEY_ID"] = config["target_user"] + env["AWS_SECRET_ACCESS_KEY"] = config["target_pass"] + + return env + def perform_backup(full_backup): env = load_environment() exclusive_process("backup") - - backup_root = os.path.join(env["STORAGE_ROOT"], 'backup') + config = get_backup_config() backup_cache_dir = os.path.join(backup_root, 'cache') backup_dir = os.path.join(backup_root, 'encrypted') @@ -169,17 +214,6 @@ def perform_backup(full_backup): shell('check_call', ["/usr/sbin/service", "dovecot", "stop"]) shell('check_call', ["/usr/sbin/service", "postfix", "stop"]) - # Get the encryption passphrase. secret_key.txt is 2048 random - # bits base64-encoded and with line breaks every 65 characters. - # gpg will only take the first line of text, so sanity check that - # that line is long enough to be a reasonable passphrase. It - # only needs to be 43 base64-characters to match AES256's key - # length of 32 bytes. - with open(os.path.join(backup_root, 'secret_key.txt')) as f: - passphrase = f.readline().strip() - if len(passphrase) < 43: raise Exception("secret_key.txt's first line is too short!") - env_with_passphrase = { "PASSPHRASE" : passphrase } - # Run a backup of STORAGE_ROOT (but excluding the backups themselves!). # --allow-source-mismatch is needed in case the box's hostname is changed # after the first backup. See #396. @@ -192,10 +226,10 @@ def perform_backup(full_backup): "--volsize", "250", "--gpg-options", "--cipher-algo=AES256", env["STORAGE_ROOT"], - "file://" + backup_dir, - "--allow-source-mismatch" + config["target"], + "--allow-source-mismatch" ], - env_with_passphrase) + get_env()) finally: # Start services again. shell('check_call', ["/usr/sbin/service", "dovecot", "start"]) @@ -210,12 +244,12 @@ def perform_backup(full_backup): shell('check_call', [ "/usr/bin/duplicity", "remove-older-than", - "%dD" % keep_backups_for_days, + "%dD" % config["max_age_in_days"], "--archive-dir", backup_cache_dir, "--force", - "file://" + backup_dir + config["target"] ], - env_with_passphrase) + get_env()) # From duplicity's manual: # "This should only be necessary after a duplicity session fails or is @@ -227,13 +261,14 @@ def perform_backup(full_backup): "cleanup", "--archive-dir", backup_cache_dir, "--force", - "file://" + backup_dir + config["target"] ], - env_with_passphrase) + get_env()) # Change ownership of backups to the user-data user, so that the after-bcakup # script can access them. - shell('check_call', ["/bin/chown", "-R", env["STORAGE_USER"], backup_dir]) + if config["target_type"] == 'file': + shell('check_call', ["/bin/chown", "-R", env["STORAGE_USER"], backup_dir]) # Execute a post-backup script that does the copying to a remote server. # Run as the STORAGE_USER user, not as root. Pass our settings in @@ -241,8 +276,8 @@ def perform_backup(full_backup): post_script = os.path.join(backup_root, 'after-backup') if os.path.exists(post_script): shell('check_call', - ['su', env['STORAGE_USER'], '-c', post_script], - env=env) + ['su', env['STORAGE_USER'], '-c', post_script, config["target"]], + env=get_env()) # Our nightly cron job executes system status checks immediately after this # backup. Since it checks that dovecot and postfix are running, block for a @@ -253,10 +288,10 @@ def perform_backup(full_backup): def run_duplicity_verification(): env = load_environment() - backup_root = os.path.join(env["STORAGE_ROOT"], 'backup') + config = get_backup_config() backup_cache_dir = os.path.join(backup_root, 'cache') backup_dir = os.path.join(backup_root, 'encrypted') - env_with_passphrase = { "PASSPHRASE" : open(os.path.join(backup_root, 'secret_key.txt')).read() } + shell('check_call', [ "/usr/bin/duplicity", "--verbosity", "info", @@ -264,9 +299,47 @@ def run_duplicity_verification(): "--compare-data", "--archive-dir", backup_cache_dir, "--exclude", backup_root, - "file://" + backup_dir, + config["target"], env["STORAGE_ROOT"], - ], env_with_passphrase) + ], get_env()) + + +def backup_set_custom(target, target_user, target_pass, target_type, max_age): + config = get_backup_config() + + # max_age must be an int + if isinstance(max_age, str): + max_age = int(max_age) + + config["target"] = target + config["target_user"] = target_user + config["target_pass"] = target_pass + config["target_type"] = target_type + config["max_age_in_days"] = max_age + + write_backup_config(config) + + return "Updated backup config" + +def get_backup_config(): + try: + config = rtyaml.load(open(os.path.join(backup_root, 'custom.yaml'))) + if not isinstance(config, dict): raise ValueError() # caught below + except: + return default_config + + merged_config = default_config.copy() + merged_config.update(config) + + # max_age must be an int + if isinstance(merged_config["max_age_in_days"], str): + merged_config["max_age_in_days"] = int(merged_config["max_age_in_days"]) + + return config + +def write_backup_config(newconfig): + with open(os.path.join(backup_root, 'custom.yaml'), "w") as f: + f.write(rtyaml.dump(newconfig)) if __name__ == "__main__": import sys @@ -274,6 +347,7 @@ if __name__ == "__main__": # Run duplicity's verification command to check a) the backup files # are readable, and b) report if they are up to date. run_duplicity_verification() + else: # Perform a backup. Add --full to force a full backup rather than # possibly performing an incremental backup. diff --git a/management/daemon.py b/management/daemon.py index af15b1c3..46ba9685 100755 --- a/management/daemon.py +++ b/management/daemon.py @@ -402,6 +402,24 @@ def backup_status(): from backup import backup_status return json_response(backup_status(env)) +@app.route('/system/backup/get-custom') +@authorized_personnel_only +def backup_get_custom(): + from backup import get_backup_config + return json_response(get_backup_config()) + +@app.route('/system/backup/set-custom', methods=["POST"]) +@authorized_personnel_only +def backup_set_custom(): + from backup import backup_set_custom + return json_response(backup_set_custom( + request.form.get('target', ''), + request.form.get('target_user', ''), + request.form.get('target_pass', ''), + request.form.get('target_type', ''), + request.form.get('max_age', '') + )) + # MUNIN @app.route('/munin/') @@ -432,4 +450,3 @@ if __name__ == '__main__': # Start the application server. Listens on 127.0.0.1 (IPv4 only). app.run(port=10222) - diff --git a/management/templates/system-backup.html b/management/templates/system-backup.html index 01682cf3..bf34759d 100644 --- a/management/templates/system-backup.html +++ b/management/templates/system-backup.html @@ -7,12 +7,53 @@

Copying Backup Files

-

The box makes an incremental backup each night. The backup is stored on the machine itself. You are responsible for copying the backup files off of the machine.

- -

Many cloud providers make this easy by allowing you to take snapshots of the machine's disk.

+

The box makes an incremental backup each night. By default the backup is stored on the machine itself, but you can also have it stored on Amazon S3

You can also use SFTP (FTP over SSH) to copy files from . These files are encrypted, so they are safe to store anywhere. Copy the encryption password from also but keep it in a safe location.

+

Backup Configuration

+ +
+
+ +
+ +
+
+
+ +
+ +
+
+
+ +
+ +
+
+
+ +
+ +
+
+
+ +
+ +
+
+
+
+ +
+
+
+

Current Backups

The backup directory currently contains the backups listed below. The total size on disk of the backups is currently .

@@ -27,8 +68,17 @@ - diff --git a/setup/management.sh b/setup/management.sh index 518a2ad6..817818fb 100755 --- a/setup/management.sh +++ b/setup/management.sh @@ -4,8 +4,9 @@ source setup/functions.sh # build-essential libssl-dev libffi-dev python3-dev: Required to pip install cryptography. apt_install python3-flask links duplicity libyaml-dev python3-dnspython python3-dateutil \ - build-essential libssl-dev libffi-dev python3-dev -hide_output pip3 install --upgrade rtyaml email_validator idna cryptography + build-essential libssl-dev libffi-dev python3-dev python-pip +hide_output pip3 install --upgrade rtyaml email_validator idna cryptography boto +hide_output pip install --upgrade boto # email_validator is repeated in setup/questions.sh # Create a backup directory and a random key for encrypting backups. From 96fb0f78f7b651a2f4e88db1e1c0217c0550de43 Mon Sep 17 00:00:00 2001 From: Leo Koppelkamm Date: Mon, 27 Jul 2015 21:56:08 +0200 Subject: [PATCH 02/18] Add comment regarding the use of pip instead of pip3 --- setup/management.sh | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/setup/management.sh b/setup/management.sh index 817818fb..3df2c72e 100755 --- a/setup/management.sh +++ b/setup/management.sh @@ -6,8 +6,11 @@ source setup/functions.sh apt_install python3-flask links duplicity libyaml-dev python3-dnspython python3-dateutil \ build-essential libssl-dev libffi-dev python3-dev python-pip hide_output pip3 install --upgrade rtyaml email_validator idna cryptography boto + +# duplicity uses python 2 so we need to use the python 2 package of boto hide_output pip install --upgrade boto - # email_validator is repeated in setup/questions.sh + +# email_validator is repeated in setup/questions.sh # Create a backup directory and a random key for encrypting backups. mkdir -p $STORAGE_ROOT/backup From 1e3e34f15f186b653053194ed9f2e89aa9e57c55 Mon Sep 17 00:00:00 2001 From: Leo Koppelkamm Date: Mon, 27 Jul 2015 22:00:36 +0200 Subject: [PATCH 03/18] Make backup API RESTful --- management/daemon.py | 4 ++-- management/templates/system-backup.html | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/management/daemon.py b/management/daemon.py index 46ba9685..c8d98eb1 100755 --- a/management/daemon.py +++ b/management/daemon.py @@ -402,13 +402,13 @@ def backup_status(): from backup import backup_status return json_response(backup_status(env)) -@app.route('/system/backup/get-custom') +@app.route('/system/backup/config', methods=["GET"]) @authorized_personnel_only def backup_get_custom(): from backup import get_backup_config return json_response(get_backup_config()) -@app.route('/system/backup/set-custom', methods=["POST"]) +@app.route('/system/backup/config', methods=["POST"]) @authorized_personnel_only def backup_set_custom(): from backup import backup_set_custom diff --git a/management/templates/system-backup.html b/management/templates/system-backup.html index bf34759d..fbad719f 100644 --- a/management/templates/system-backup.html +++ b/management/templates/system-backup.html @@ -138,7 +138,7 @@ function show_system_backup() { function show_custom_backup() { api( - "/system/backup/get-custom", + "/system/backup/custom", "GET", { }, function(r) { @@ -158,7 +158,7 @@ function set_custom_backup() { var target_pass = $("#target-pass").val(); var max_age = $("#max-age").val(); api( - "/system/backup/set-custom", + "/system/backup/custom", "POST", { target: target, From 91e4ea6e2fd4a112301adf64dc6f6fc7df1c64bd Mon Sep 17 00:00:00 2001 From: Leo Koppelkamm Date: Mon, 27 Jul 2015 22:09:58 +0200 Subject: [PATCH 04/18] Infer target_type from url --- management/backup.py | 16 +++++++++------- management/daemon.py | 1 - management/templates/system-backup.html | 4 ++-- 3 files changed, 11 insertions(+), 10 deletions(-) diff --git a/management/backup.py b/management/backup.py index 26426313..b974902b 100755 --- a/management/backup.py +++ b/management/backup.py @@ -22,8 +22,7 @@ backup_root = os.path.join(load_environment()["STORAGE_ROOT"], 'backup') # that depends on it is this many days old. default_config = { "max_age_in_days": 3, - "target": "file://" + os.path.join(backup_root, 'encrypted'), - "target_type": "file" + "target": "file://" + os.path.join(backup_root, 'encrypted') } def backup_status(env): @@ -164,12 +163,16 @@ def get_env(): env = { "PASSPHRASE" : get_passphrase() } - if config["target_type"] == 's3': + if get_target_type(config) == 's3': env["AWS_ACCESS_KEY_ID"] = config["target_user"] env["AWS_SECRET_ACCESS_KEY"] = config["target_pass"] return env - + +def get_target_type(config): + protocol = config["target"].split(":")[0] + return protocol + def perform_backup(full_backup): env = load_environment() @@ -267,7 +270,7 @@ def perform_backup(full_backup): # Change ownership of backups to the user-data user, so that the after-bcakup # script can access them. - if config["target_type"] == 'file': + if get_target_type(config) == 'file': shell('check_call', ["/bin/chown", "-R", env["STORAGE_USER"], backup_dir]) # Execute a post-backup script that does the copying to a remote server. @@ -304,7 +307,7 @@ def run_duplicity_verification(): ], get_env()) -def backup_set_custom(target, target_user, target_pass, target_type, max_age): +def backup_set_custom(target, target_user, target_pass, max_age): config = get_backup_config() # max_age must be an int @@ -314,7 +317,6 @@ def backup_set_custom(target, target_user, target_pass, target_type, max_age): config["target"] = target config["target_user"] = target_user config["target_pass"] = target_pass - config["target_type"] = target_type config["max_age_in_days"] = max_age write_backup_config(config) diff --git a/management/daemon.py b/management/daemon.py index c8d98eb1..3380e757 100755 --- a/management/daemon.py +++ b/management/daemon.py @@ -416,7 +416,6 @@ def backup_set_custom(): request.form.get('target', ''), request.form.get('target_user', ''), request.form.get('target_pass', ''), - request.form.get('target_type', ''), request.form.get('max_age', '') )) diff --git a/management/templates/system-backup.html b/management/templates/system-backup.html index fbad719f..7ff67d0d 100644 --- a/management/templates/system-backup.html +++ b/management/templates/system-backup.html @@ -142,8 +142,9 @@ function show_custom_backup() { "GET", { }, function(r) { + var target_type = r.target.split(':')[0] $("#target").val(r.target); - $("#target-type").val(r.target_type); + $("#target-type").val(target_type); $("#target-user").val(r.target_user); $("#target-pass").val(r.target_pass); $("#max-age").val(r.max_age_in_days); @@ -162,7 +163,6 @@ function set_custom_backup() { "POST", { target: target, - target_type: target_type, target_user: target_user, target_pass: target_pass, max_age: max_age From 43fb7fe635b1e7093bf3a927f663c1bd0de7df6c Mon Sep 17 00:00:00 2001 From: Leo Koppelkamm Date: Mon, 27 Jul 2015 22:11:43 +0200 Subject: [PATCH 05/18] Remove unused variable --- management/backup.py | 1 - 1 file changed, 1 deletion(-) diff --git a/management/backup.py b/management/backup.py index b974902b..b86b70f3 100755 --- a/management/backup.py +++ b/management/backup.py @@ -293,7 +293,6 @@ def run_duplicity_verification(): env = load_environment() config = get_backup_config() backup_cache_dir = os.path.join(backup_root, 'cache') - backup_dir = os.path.join(backup_root, 'encrypted') shell('check_call', [ "/usr/bin/duplicity", From fa0dd684da05a800da867782f44106292086f833 Mon Sep 17 00:00:00 2001 From: Leo Koppelkamm Date: Mon, 27 Jul 2015 22:13:28 +0200 Subject: [PATCH 06/18] Add archive-dir argument to collection-status --- management/backup.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/management/backup.py b/management/backup.py index b86b70f3..8448bbce 100755 --- a/management/backup.py +++ b/management/backup.py @@ -31,7 +31,11 @@ def backup_status(env): # Use the number of volumes to estimate the size. config = get_backup_config() now = datetime.datetime.now(dateutil.tz.tzlocal()) - + + backups = { } + backup_dir = os.path.join(backup_root, 'encrypted') + backup_cache_dir = os.path.join(backup_root, 'cache') + def reldate(date, ref, clip): if ref < date: return clip rd = dateutil.relativedelta.relativedelta(ref, date) @@ -57,16 +61,13 @@ def backup_status(env): shell('check_call', [ "/usr/bin/duplicity", "collection-status", + "--archive-dir", backup_cache_dir, "--log-file", os.path.join(backup_root, "duplicity_status"), "--gpg-options", "--cipher-algo=AES256", config["target"], ], get_env()) - - backups = { } - backup_dir = os.path.join(backup_root, 'encrypted') - # Parse backup data from status file with open(os.path.join(backup_root, "duplicity_status"),'r') as status_file: for line in status_file: From e693802091c04e88da0755d0d3d5ea6afc3996c2 Mon Sep 17 00:00:00 2001 From: Leo Koppelkamm Date: Mon, 27 Jul 2015 22:18:19 +0200 Subject: [PATCH 07/18] Rename max_age to min_age Also clarify a comment and remove an unneeded type check --- management/backup.py | 29 +++++++++++-------------- management/templates/system-backup.html | 8 +++---- 2 files changed, 17 insertions(+), 20 deletions(-) diff --git a/management/backup.py b/management/backup.py index 8448bbce..3cccf1ff 100755 --- a/management/backup.py +++ b/management/backup.py @@ -18,10 +18,11 @@ from utils import exclusive_process, load_environment, shell, wait_for_service backup_root = os.path.join(load_environment()["STORAGE_ROOT"], 'backup') # Default settings -# Destroy backups when the most recent increment in the chain -# that depends on it is this many days old. +# min_age_in_days is the minimum amount of days a backup will be kept before +# it is eligble to be removed. Backups might be kept much longer if there's no +# new full backup yet. default_config = { - "max_age_in_days": 3, + "min_age_in_days": 3, "target": "file://" + os.path.join(backup_root, 'encrypted') } @@ -101,11 +102,11 @@ def backup_status(env): # when the threshold is met. deleted_in = None if incremental_count > 0 and first_full_size is not None: - deleted_in = "approx. %d days" % round(config["max_age_in_days"] + (.5 * first_full_size - incremental_size) / (incremental_size/incremental_count) + .5) + deleted_in = "approx. %d days" % round(config["min_age_in_days"] + (.5 * first_full_size - incremental_size) / (incremental_size/incremental_count) + .5) # When will a backup be deleted? saw_full = False - days_ago = now - datetime.timedelta(days=config["max_age_in_days"]) + days_ago = now - datetime.timedelta(days=config["min_age_in_days"]) for bak in backups: if deleted_in: # Subsequent backups are deleted when the most recent increment @@ -248,7 +249,7 @@ def perform_backup(full_backup): shell('check_call', [ "/usr/bin/duplicity", "remove-older-than", - "%dD" % config["max_age_in_days"], + "%dD" % config["min_age_in_days"], "--archive-dir", backup_cache_dir, "--force", config["target"] @@ -307,17 +308,17 @@ def run_duplicity_verification(): ], get_env()) -def backup_set_custom(target, target_user, target_pass, max_age): +def backup_set_custom(target, target_user, target_pass, min_age): config = get_backup_config() - # max_age must be an int - if isinstance(max_age, str): - max_age = int(max_age) + # min_age must be an int + if isinstance(min_age, str): + min_age = int(min_age) config["target"] = target config["target_user"] = target_user config["target_pass"] = target_pass - config["max_age_in_days"] = max_age + config["min_age_in_days"] = min_age write_backup_config(config) @@ -332,11 +333,7 @@ def get_backup_config(): merged_config = default_config.copy() merged_config.update(config) - - # max_age must be an int - if isinstance(merged_config["max_age_in_days"], str): - merged_config["max_age_in_days"] = int(merged_config["max_age_in_days"]) - + return config def write_backup_config(newconfig): diff --git a/management/templates/system-backup.html b/management/templates/system-backup.html index 7ff67d0d..ed344281 100644 --- a/management/templates/system-backup.html +++ b/management/templates/system-backup.html @@ -26,7 +26,7 @@
- +
@@ -147,7 +147,7 @@ function show_custom_backup() { $("#target-type").val(target_type); $("#target-user").val(r.target_user); $("#target-pass").val(r.target_pass); - $("#max-age").val(r.max_age_in_days); + $("#min-age").val(r.min_age_in_days); toggle_form() }) } @@ -157,7 +157,7 @@ function set_custom_backup() { var target_type = $("#target-type").val(); var target_user = $("#target-user").val(); var target_pass = $("#target-pass").val(); - var max_age = $("#max-age").val(); + var min_age = $("#min-age").val(); api( "/system/backup/custom", "POST", @@ -165,7 +165,7 @@ function set_custom_backup() { target: target, target_user: target_user, target_pass: target_pass, - max_age: max_age + min_age: min_age }, function(r) { // Responses are multiple lines of pre-formatted text. From ba9065cada64d23963403400fd9f57ceebe99d2d Mon Sep 17 00:00:00 2001 From: Leo Koppelkamm Date: Mon, 27 Jul 2015 22:30:22 +0200 Subject: [PATCH 08/18] Don't write collection_status output to file but parse it directly --- management/backup.py | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/management/backup.py b/management/backup.py index 3cccf1ff..3a7a29cc 100755 --- a/management/backup.py +++ b/management/backup.py @@ -46,7 +46,7 @@ def backup_status(env): if rd.days > 1: return "%d days, %d hours" % (rd.days, rd.hours) if rd.days == 1: return "%d day, %d hours" % (rd.days, rd.hours) return "%d hours, %d minutes" % (rd.hours, rd.minutes) - + def parse_line(line): keys = line.strip().split() date = dateutil.parser.parse(keys[1]) @@ -57,28 +57,28 @@ def backup_status(env): "full": keys[0] == "full", "size": int(keys[2]) * 250 * 1000000, } - - # Write duplicity status to file - shell('check_call', [ + + # Get duplicity collection status + collection_status = shell('check_output', [ "/usr/bin/duplicity", "collection-status", "--archive-dir", backup_cache_dir, "--log-file", os.path.join(backup_root, "duplicity_status"), "--gpg-options", "--cipher-algo=AES256", + "--log-fd", "1", config["target"], ], get_env()) - # Parse backup data from status file - with open(os.path.join(backup_root, "duplicity_status"),'r') as status_file: - for line in status_file: - if line.startswith(" full") or line.startswith(" inc"): - backup = parse_line(line) - backups[backup["date"]] = backup + # Split multi line string into list + collection_status = collection_status.split('\n') + + # Parse backup data from status file + for line in collection_status: + if line.startswith(" full") or line.startswith(" inc"): + backup = parse_line(line) + backups[backup["date"]] = backup - # Remove status file - os.remove(os.path.join(backup_root, "duplicity_status")) - # Ensure the rows are sorted reverse chronologically. # This is relied on by should_force_full() and the next step. backups = sorted(backups.values(), key = lambda b : b["date"], reverse=True) From 606cf6a941eb0f50513f185a34e26482a5ec45d4 Mon Sep 17 00:00:00 2001 From: Leo Koppelkamm Date: Tue, 28 Jul 2015 00:34:11 +0200 Subject: [PATCH 09/18] Fix API typo --- management/templates/system-backup.html | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/management/templates/system-backup.html b/management/templates/system-backup.html index ed344281..59c7357b 100644 --- a/management/templates/system-backup.html +++ b/management/templates/system-backup.html @@ -138,7 +138,7 @@ function show_system_backup() { function show_custom_backup() { api( - "/system/backup/custom", + "/system/backup/config", "GET", { }, function(r) { @@ -159,7 +159,7 @@ function set_custom_backup() { var target_pass = $("#target-pass").val(); var min_age = $("#min-age").val(); api( - "/system/backup/custom", + "/system/backup/config", "POST", { target: target, From 0d8a4099c1e4ee88339661f73936dbdc67698be5 Mon Sep 17 00:00:00 2001 From: Leo Koppelkamm Date: Tue, 28 Jul 2015 00:37:43 +0200 Subject: [PATCH 10/18] Add placeholder attribute; use input instead of textarea --- management/templates/system-backup.html | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/management/templates/system-backup.html b/management/templates/system-backup.html index 59c7357b..8542f100 100644 --- a/management/templates/system-backup.html +++ b/management/templates/system-backup.html @@ -32,19 +32,19 @@
- +
- +
- +
From 77099b3bce80b39c98b11d69b07f7796d8e66b0e Mon Sep 17 00:00:00 2001 From: Leo Koppelkamm Date: Tue, 28 Jul 2015 00:42:00 +0200 Subject: [PATCH 11/18] Reword backup min_time label --- management/templates/system-backup.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/management/templates/system-backup.html b/management/templates/system-backup.html index 8542f100..615ba980 100644 --- a/management/templates/system-backup.html +++ b/management/templates/system-backup.html @@ -24,7 +24,7 @@
- +
From 1cdd205eb763dc829ac10e3bdd08d4be5d1e5aca Mon Sep 17 00:00:00 2001 From: Leo Koppelkamm Date: Tue, 28 Jul 2015 20:58:39 +0200 Subject: [PATCH 12/18] Missed one max_age --- management/daemon.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/management/daemon.py b/management/daemon.py index 3380e757..4b63f71b 100755 --- a/management/daemon.py +++ b/management/daemon.py @@ -416,7 +416,7 @@ def backup_set_custom(): request.form.get('target', ''), request.form.get('target_user', ''), request.form.get('target_pass', ''), - request.form.get('max_age', '') + request.form.get('min_age', '') )) # MUNIN From 3f15879578d4785020d54ecfd4aebd5240f55671 Mon Sep 17 00:00:00 2001 From: Joshua Tauberer Date: Sun, 9 Aug 2015 16:56:33 +0000 Subject: [PATCH 13/18] remove global variables in backup.py --- management/backup.py | 75 ++++++++++++++++++++++---------------------- management/daemon.py | 4 +-- 2 files changed, 39 insertions(+), 40 deletions(-) diff --git a/management/backup.py b/management/backup.py index 3a7a29cc..17cc54f6 100755 --- a/management/backup.py +++ b/management/backup.py @@ -14,23 +14,14 @@ import rtyaml from utils import exclusive_process, load_environment, shell, wait_for_service -# Root folder -backup_root = os.path.join(load_environment()["STORAGE_ROOT"], 'backup') - -# Default settings -# min_age_in_days is the minimum amount of days a backup will be kept before -# it is eligble to be removed. Backups might be kept much longer if there's no -# new full backup yet. -default_config = { - "min_age_in_days": 3, - "target": "file://" + os.path.join(backup_root, 'encrypted') -} - def backup_status(env): + # Root folder + backup_root = os.path.join(env["STORAGE_ROOT"], 'backup') + # What is the current status of backups? # Query duplicity to get a list of all backups. # Use the number of volumes to estimate the size. - config = get_backup_config() + config = get_backup_config(env) now = datetime.datetime.now(dateutil.tz.tzlocal()) backups = { } @@ -63,12 +54,11 @@ def backup_status(env): "/usr/bin/duplicity", "collection-status", "--archive-dir", backup_cache_dir, - "--log-file", os.path.join(backup_root, "duplicity_status"), "--gpg-options", "--cipher-algo=AES256", "--log-fd", "1", config["target"], ], - get_env()) + get_env(env)) # Split multi line string into list collection_status = collection_status.split('\n') @@ -147,23 +137,24 @@ def should_force_full(env): # (I love for/else blocks. Here it's just to show off.) return True -def get_passphrase(): +def get_passphrase(env): # Get the encryption passphrase. secret_key.txt is 2048 random # bits base64-encoded and with line breaks every 65 characters. # gpg will only take the first line of text, so sanity check that # that line is long enough to be a reasonable passphrase. It # only needs to be 43 base64-characters to match AES256's key # length of 32 bytes. + backup_root = os.path.join(env["STORAGE_ROOT"], 'backup') with open(os.path.join(backup_root, 'secret_key.txt')) as f: passphrase = f.readline().strip() if len(passphrase) < 43: raise Exception("secret_key.txt's first line is too short!") return passphrase -def get_env(): - config = get_backup_config() +def get_env(env): + config = get_backup_config(env) - env = { "PASSPHRASE" : get_passphrase() } + env = { "PASSPHRASE" : get_passphrase(env) } if get_target_type(config) == 's3': env["AWS_ACCESS_KEY_ID"] = config["target_user"] @@ -179,7 +170,8 @@ def perform_backup(full_backup): env = load_environment() exclusive_process("backup") - config = get_backup_config() + config = get_backup_config(env) + backup_root = os.path.join(env["STORAGE_ROOT"], 'backup') backup_cache_dir = os.path.join(backup_root, 'cache') backup_dir = os.path.join(backup_root, 'encrypted') @@ -234,7 +226,7 @@ def perform_backup(full_backup): config["target"], "--allow-source-mismatch" ], - get_env()) + get_env(env)) finally: # Start services again. shell('check_call', ["/usr/sbin/service", "dovecot", "start"]) @@ -254,7 +246,7 @@ def perform_backup(full_backup): "--force", config["target"] ], - get_env()) + get_env(env)) # From duplicity's manual: # "This should only be necessary after a duplicity session fails or is @@ -268,7 +260,7 @@ def perform_backup(full_backup): "--force", config["target"] ], - get_env()) + get_env(env)) # Change ownership of backups to the user-data user, so that the after-bcakup # script can access them. @@ -282,7 +274,7 @@ def perform_backup(full_backup): if os.path.exists(post_script): shell('check_call', ['su', env['STORAGE_USER'], '-c', post_script, config["target"]], - env=get_env()) + env=get_env(env)) # Our nightly cron job executes system status checks immediately after this # backup. Since it checks that dovecot and postfix are running, block for a @@ -293,7 +285,8 @@ def perform_backup(full_backup): def run_duplicity_verification(): env = load_environment() - config = get_backup_config() + backup_root = os.path.join(env["STORAGE_ROOT"], 'backup') + config = get_backup_config(env) backup_cache_dir = os.path.join(backup_root, 'cache') shell('check_call', [ @@ -305,11 +298,11 @@ def run_duplicity_verification(): "--exclude", backup_root, config["target"], env["STORAGE_ROOT"], - ], get_env()) + ], get_env(env)) -def backup_set_custom(target, target_user, target_pass, min_age): - config = get_backup_config() +def backup_set_custom(env, target, target_user, target_pass, min_age): + config = get_backup_config(env) # min_age must be an int if isinstance(min_age, str): @@ -320,23 +313,29 @@ def backup_set_custom(target, target_user, target_pass, min_age): config["target_pass"] = target_pass config["min_age_in_days"] = min_age - write_backup_config(config) + write_backup_config(env, config) return "Updated backup config" -def get_backup_config(): - try: - config = rtyaml.load(open(os.path.join(backup_root, 'custom.yaml'))) - if not isinstance(config, dict): raise ValueError() # caught below - except: - return default_config +def get_backup_config(env): + backup_root = os.path.join(env["STORAGE_ROOT"], 'backup') - merged_config = default_config.copy() - merged_config.update(config) + config = { + "min_age_in_days": 3, + "target": "file://" + os.path.join(backup_root, 'encrypted'), + } + + try: + custom_config = rtyaml.load(open(os.path.join(backup_root, 'custom.yaml'))) + if not isinstance(custom_config, dict): raise ValueError() # caught below + config.update(custom_config) + except: + pass return config -def write_backup_config(newconfig): +def write_backup_config(env, newconfig): + backup_root = os.path.join(env["STORAGE_ROOT"], 'backup') with open(os.path.join(backup_root, 'custom.yaml'), "w") as f: f.write(rtyaml.dump(newconfig)) diff --git a/management/daemon.py b/management/daemon.py index 4b63f71b..355e0662 100755 --- a/management/daemon.py +++ b/management/daemon.py @@ -406,13 +406,13 @@ def backup_status(): @authorized_personnel_only def backup_get_custom(): from backup import get_backup_config - return json_response(get_backup_config()) + return json_response(get_backup_config(env)) @app.route('/system/backup/config', methods=["POST"]) @authorized_personnel_only def backup_set_custom(): from backup import backup_set_custom - return json_response(backup_set_custom( + return json_response(backup_set_custom(env, request.form.get('target', ''), request.form.get('target_user', ''), request.form.get('target_pass', ''), From c7f8ead496923f1ea2c522aa707bf120129b04ca Mon Sep 17 00:00:00 2001 From: Joshua Tauberer Date: Sun, 9 Aug 2015 17:52:24 +0000 Subject: [PATCH 14/18] clean up the new backup configuration panel --- management/backup.py | 17 ++++++--- management/templates/system-backup.html | 47 ++++++++++++++----------- 2 files changed, 38 insertions(+), 26 deletions(-) diff --git a/management/backup.py b/management/backup.py index 17cc54f6..8f51127d 100755 --- a/management/backup.py +++ b/management/backup.py @@ -25,7 +25,6 @@ def backup_status(env): now = datetime.datetime.now(dateutil.tz.tzlocal()) backups = { } - backup_dir = os.path.join(backup_root, 'encrypted') backup_cache_dir = os.path.join(backup_root, 'cache') def reldate(date, ref, clip): @@ -112,8 +111,6 @@ def backup_status(env): bak["deleted_in"] = deleted_in return { - "directory": backup_dir, - "encpwfile": os.path.join(backup_root, 'secret_key.txt'), "tz": now.tzname(), "backups": backups, } @@ -302,7 +299,7 @@ def run_duplicity_verification(): def backup_set_custom(env, target, target_user, target_pass, min_age): - config = get_backup_config(env) + config = get_backup_config(env, for_save=True) # min_age must be an int if isinstance(min_age, str): @@ -317,14 +314,16 @@ def backup_set_custom(env, target, target_user, target_pass, min_age): return "Updated backup config" -def get_backup_config(env): +def get_backup_config(env, for_save=False): backup_root = os.path.join(env["STORAGE_ROOT"], 'backup') + # Defaults. config = { "min_age_in_days": 3, "target": "file://" + os.path.join(backup_root, 'encrypted'), } + # Merge in anything written to custom.yaml. try: custom_config = rtyaml.load(open(os.path.join(backup_root, 'custom.yaml'))) if not isinstance(custom_config, dict): raise ValueError() # caught below @@ -332,6 +331,14 @@ def get_backup_config(env): except: pass + # When updating config.yaml, don't do any further processing on what we find. + if for_save: + return config + + # helper fields for the admin + config["file_target_directory"] = os.path.join(backup_root, 'encrypted') + config["enc_pw_file"] = os.path.join(backup_root, 'secret_key.txt') + return config def write_backup_config(env, newconfig): diff --git a/management/templates/system-backup.html b/management/templates/system-backup.html index 615ba980..ed9f26f3 100644 --- a/management/templates/system-backup.html +++ b/management/templates/system-backup.html @@ -5,44 +5,50 @@

Backup Status

-

Copying Backup Files

-

The box makes an incremental backup each night. By default the backup is stored on the machine itself, but you can also have it stored on Amazon S3

-

You can also use SFTP (FTP over SSH) to copy files from . These files are encrypted, so they are safe to store anywhere. Copy the encryption password from also but keep it in a safe location.

- -

Backup Configuration

+

Configuration

+
+
+
Backups are stored on this machine’s own hard disk. You are responsible for periodically using SFTP (FTP over SSH) to copy the backup files from to a safe location. These files are encrypted, so they are safe to store anywhere.
+
+
+
+
+
Backups are stored in an Amazon Web Services S3 bucket. You must have an AWS account already.
+
+
-
+
-
- +
+
-
- +
+
@@ -54,9 +60,11 @@
-

Current Backups

+

Copy the encryption password from to a safe and secure location. You will need this file to decrypt backup files.

-

The backup directory currently contains the backups listed below. The total size on disk of the backups is currently .

+

Available Backups

+ +

The backup location currently contains the backups listed below. The total size of the backups is currently .

@@ -72,11 +80,8 @@ function toggle_form() { var target_type = $("#target-type").val(); - if (target_type == 'file') { - $(".form-advanced").hide(); - } else { - $(".form-advanced").show(); - } + $(".backup-target-file, .backup-target-s3").hide(); + $(".backup-target-" + target_type).show(); } function nice_size(bytes) { @@ -104,9 +109,6 @@ function show_system_backup() { "GET", { }, function(r) { - $('#backup-location').text(r.directory); - $('#backup-encpassword-file').text(r.encpwfile); - $('#backup-status tbody').html(""); var total_disk_size = 0; @@ -137,6 +139,7 @@ function show_system_backup() { } function show_custom_backup() { + $(".backup-target-file, .backup-target-s3").hide(); api( "/system/backup/config", "GET", @@ -148,6 +151,8 @@ function show_custom_backup() { $("#target-user").val(r.target_user); $("#target-pass").val(r.target_pass); $("#min-age").val(r.min_age_in_days); + $('#backup-location').text(r.file_target_directory); + $('#backup-encpassword-file').text(r.enc_pw_file); toggle_form() }) } From 3b4b57c081bc768cc5856d90b6b229e219670679 Mon Sep 17 00:00:00 2001 From: Joshua Tauberer Date: Sun, 9 Aug 2015 18:25:26 +0000 Subject: [PATCH 15/18] switching between backup options in the admin wasn't working at all * going from s3 to file target wasn't working * use 'local' in the config instead of a file: url, for the local target, so it is not path-specific * break out the S3 fields since users can't be expected to know how to form a URL * use boto to generate a list of S3 hosts * use boto to validate that the user input for s3 is valid * fix lots of html errors in the backup admin --- management/backup.py | 50 +++++++++++++++++- management/daemon.py | 6 +++ management/templates/system-backup.html | 70 ++++++++++++++++--------- 3 files changed, 101 insertions(+), 25 deletions(-) diff --git a/management/backup.py b/management/backup.py index 8f51127d..c85d04c2 100755 --- a/management/backup.py +++ b/management/backup.py @@ -297,6 +297,42 @@ def run_duplicity_verification(): env["STORAGE_ROOT"], ], get_env(env)) +def validate_target(config): + import urllib.parse + try: + p = urllib.parse.urlparse(config["target"]) + except ValueError: + return "invalid target" + + if p.scheme == "s3": + import boto.s3 + from boto.exception import BotoServerError + + # match to a Region + for region in boto.s3.regions(): + if region.endpoint == p.hostname: + break + else: + raise ValueError("Invalid S3 region/host.") + + bucket = p.path[1:].split('/')[0] + path = '/'.join(p.path[1:].split('/')[1:]) + '/' + if bucket == "": + raise ValueError("Enter an S3 bucket name.") + + # connect to the region & bucket + try: + conn = region.connect(aws_access_key_id=config["target_user"], aws_secret_access_key=config["target_pass"]) + bucket = conn.get_bucket(bucket) + except BotoServerError as e: + if e.status == 403: + raise ValueError("Invalid S3 access key or secret access key.") + elif e.status == 404: + raise ValueError("Invalid S3 bucket name.") + elif e.status == 301: + raise ValueError("Incorrect region for this bucket.") + raise ValueError(e.reason) + def backup_set_custom(env, target, target_user, target_pass, min_age): config = get_backup_config(env, for_save=True) @@ -309,6 +345,15 @@ def backup_set_custom(env, target, target_user, target_pass, min_age): config["target_user"] = target_user config["target_pass"] = target_pass config["min_age_in_days"] = min_age + + # Validate. + try: + if config["target"] != "local": + # "local" isn't supported by the following function, which expects a full url in the target key, + # which is what is there except when loading the config prior to saving + validate_target(config) + except ValueError as e: + return str(e) write_backup_config(env, config) @@ -320,7 +365,7 @@ def get_backup_config(env, for_save=False): # Defaults. config = { "min_age_in_days": 3, - "target": "file://" + os.path.join(backup_root, 'encrypted'), + "target": "local", } # Merge in anything written to custom.yaml. @@ -338,6 +383,9 @@ def get_backup_config(env, for_save=False): # helper fields for the admin config["file_target_directory"] = os.path.join(backup_root, 'encrypted') config["enc_pw_file"] = os.path.join(backup_root, 'secret_key.txt') + if config["target"] == "local": + # Expand to the full URL. + config["target"] = "file://" + config["file_target_directory"] return config diff --git a/management/daemon.py b/management/daemon.py index 355e0662..932a967f 100755 --- a/management/daemon.py +++ b/management/daemon.py @@ -90,13 +90,19 @@ def json_response(data): def index(): # Render the control panel. This route does not require user authentication # so it must be safe! + no_users_exist = (len(get_mail_users(env)) == 0) no_admins_exist = (len(get_admins(env)) == 0) + + import boto.s3 + backup_s3_hosts = [(r.name, r.endpoint) for r in boto.s3.regions()] + return render_template('index.html', hostname=env['PRIMARY_HOSTNAME'], storage_root=env['STORAGE_ROOT'], no_users_exist=no_users_exist, no_admins_exist=no_admins_exist, + backup_s3_hosts=backup_s3_hosts, ) @app.route('/me') diff --git a/management/templates/system-backup.html b/management/templates/system-backup.html index ed9f26f3..03eeb2c2 100644 --- a/management/templates/system-backup.html +++ b/management/templates/system-backup.html @@ -11,15 +11,15 @@
- +
- +
-
+
Backups are stored on this machine’s own hard disk. You are responsible for periodically using SFTP (FTP over SSH) to copy the backup files from to a safe location. These files are encrypted, so they are safe to store anywhere.
@@ -30,27 +30,37 @@
- +
- +
- +
- +
- +
- +
- +
- + +
+
+
+ +
+
@@ -79,8 +89,8 @@