From 2e6c4103361e9924f90d6b22a11fdf3bb1c854b0 Mon Sep 17 00:00:00 2001 From: Leo Koppelkamm Date: Sun, 26 Jul 2015 18:25:52 +0200 Subject: [PATCH 01/14] Make backups more configurable Backup location and maximum age can now be configured in the admin panel. For now only S3 is supported, but adding other duplicity supported backends should be straightforward. --- management/backup.py | 182 +++++++++++++++++------- management/daemon.py | 19 ++- management/templates/system-backup.html | 101 ++++++++++++- setup/management.sh | 5 +- 4 files changed, 246 insertions(+), 61 deletions(-) diff --git a/management/backup.py b/management/backup.py index 40d70458..26426313 100755 --- a/management/backup.py +++ b/management/backup.py @@ -10,20 +10,29 @@ import os, os.path, shutil, glob, re, datetime import dateutil.parser, dateutil.relativedelta, dateutil.tz +import rtyaml from utils import exclusive_process, load_environment, shell, wait_for_service +# Root folder +backup_root = os.path.join(load_environment()["STORAGE_ROOT"], 'backup') + +# Default settings # Destroy backups when the most recent increment in the chain # that depends on it is this many days old. -keep_backups_for_days = 3 +default_config = { + "max_age_in_days": 3, + "target": "file://" + os.path.join(backup_root, 'encrypted'), + "target_type": "file" +} def backup_status(env): # What is the current status of backups? - # Loop through all of the files in STORAGE_ROOT/backup/encrypted to - # get a list of all of the backups taken and sum up file sizes to - # see how large the storage is. - + # Query duplicity to get a list of all backups. + # Use the number of volumes to estimate the size. + config = get_backup_config() now = datetime.datetime.now(dateutil.tz.tzlocal()) + def reldate(date, ref, clip): if ref < date: return clip rd = dateutil.relativedelta.relativedelta(ref, date) @@ -33,29 +42,42 @@ def backup_status(env): if rd.days > 1: return "%d days, %d hours" % (rd.days, rd.hours) if rd.days == 1: return "%d day, %d hours" % (rd.days, rd.hours) return "%d hours, %d minutes" % (rd.hours, rd.minutes) + + def parse_line(line): + keys = line.strip().split() + date = dateutil.parser.parse(keys[1]) + return { + "date": keys[1], + "date_str": date.strftime("%x %X"), + "date_delta": reldate(date, now, "the future?"), + "full": keys[0] == "full", + "size": int(keys[2]) * 250 * 1000000, + } + + # Write duplicity status to file + shell('check_call', [ + "/usr/bin/duplicity", + "collection-status", + "--log-file", os.path.join(backup_root, "duplicity_status"), + "--gpg-options", "--cipher-algo=AES256", + config["target"], + ], + get_env()) + backups = { } - backup_root = os.path.join(env["STORAGE_ROOT"], 'backup') backup_dir = os.path.join(backup_root, 'encrypted') - os.makedirs(backup_dir, exist_ok=True) # os.listdir fails if directory does not exist - for fn in os.listdir(backup_dir): - m = re.match(r"duplicity-(full|full-signatures|(inc|new-signatures)\.(?P\d+T\d+Z)\.to)\.(?P\d+T\d+Z)\.", fn) - if not m: raise ValueError(fn) - - key = m.group("date") - if key not in backups: - date = dateutil.parser.parse(m.group("date")) - backups[key] = { - "date": m.group("date"), - "date_str": date.strftime("%x %X"), - "date_delta": reldate(date, now, "the future?"), - "full": m.group("incbase") is None, - "previous": m.group("incbase"), - "size": 0, - } - - backups[key]["size"] += os.path.getsize(os.path.join(backup_dir, fn)) + + # Parse backup data from status file + with open(os.path.join(backup_root, "duplicity_status"),'r') as status_file: + for line in status_file: + if line.startswith(" full") or line.startswith(" inc"): + backup = parse_line(line) + backups[backup["date"]] = backup + # Remove status file + os.remove(os.path.join(backup_root, "duplicity_status")) + # Ensure the rows are sorted reverse chronologically. # This is relied on by should_force_full() and the next step. backups = sorted(backups.values(), key = lambda b : b["date"], reverse=True) @@ -79,11 +101,11 @@ def backup_status(env): # when the threshold is met. deleted_in = None if incremental_count > 0 and first_full_size is not None: - deleted_in = "approx. %d days" % round(keep_backups_for_days + (.5 * first_full_size - incremental_size) / (incremental_size/incremental_count) + .5) + deleted_in = "approx. %d days" % round(config["max_age_in_days"] + (.5 * first_full_size - incremental_size) / (incremental_size/incremental_count) + .5) # When will a backup be deleted? saw_full = False - days_ago = now - datetime.timedelta(days=keep_backups_for_days) + days_ago = now - datetime.timedelta(days=config["max_age_in_days"]) for bak in backups: if deleted_in: # Subsequent backups are deleted when the most recent increment @@ -124,12 +146,35 @@ def should_force_full(env): # (I love for/else blocks. Here it's just to show off.) return True +def get_passphrase(): + # Get the encryption passphrase. secret_key.txt is 2048 random + # bits base64-encoded and with line breaks every 65 characters. + # gpg will only take the first line of text, so sanity check that + # that line is long enough to be a reasonable passphrase. It + # only needs to be 43 base64-characters to match AES256's key + # length of 32 bytes. + with open(os.path.join(backup_root, 'secret_key.txt')) as f: + passphrase = f.readline().strip() + if len(passphrase) < 43: raise Exception("secret_key.txt's first line is too short!") + + return passphrase + +def get_env(): + config = get_backup_config() + + env = { "PASSPHRASE" : get_passphrase() } + + if config["target_type"] == 's3': + env["AWS_ACCESS_KEY_ID"] = config["target_user"] + env["AWS_SECRET_ACCESS_KEY"] = config["target_pass"] + + return env + def perform_backup(full_backup): env = load_environment() exclusive_process("backup") - - backup_root = os.path.join(env["STORAGE_ROOT"], 'backup') + config = get_backup_config() backup_cache_dir = os.path.join(backup_root, 'cache') backup_dir = os.path.join(backup_root, 'encrypted') @@ -169,17 +214,6 @@ def perform_backup(full_backup): shell('check_call', ["/usr/sbin/service", "dovecot", "stop"]) shell('check_call', ["/usr/sbin/service", "postfix", "stop"]) - # Get the encryption passphrase. secret_key.txt is 2048 random - # bits base64-encoded and with line breaks every 65 characters. - # gpg will only take the first line of text, so sanity check that - # that line is long enough to be a reasonable passphrase. It - # only needs to be 43 base64-characters to match AES256's key - # length of 32 bytes. - with open(os.path.join(backup_root, 'secret_key.txt')) as f: - passphrase = f.readline().strip() - if len(passphrase) < 43: raise Exception("secret_key.txt's first line is too short!") - env_with_passphrase = { "PASSPHRASE" : passphrase } - # Run a backup of STORAGE_ROOT (but excluding the backups themselves!). # --allow-source-mismatch is needed in case the box's hostname is changed # after the first backup. See #396. @@ -192,10 +226,10 @@ def perform_backup(full_backup): "--volsize", "250", "--gpg-options", "--cipher-algo=AES256", env["STORAGE_ROOT"], - "file://" + backup_dir, - "--allow-source-mismatch" + config["target"], + "--allow-source-mismatch" ], - env_with_passphrase) + get_env()) finally: # Start services again. shell('check_call', ["/usr/sbin/service", "dovecot", "start"]) @@ -210,12 +244,12 @@ def perform_backup(full_backup): shell('check_call', [ "/usr/bin/duplicity", "remove-older-than", - "%dD" % keep_backups_for_days, + "%dD" % config["max_age_in_days"], "--archive-dir", backup_cache_dir, "--force", - "file://" + backup_dir + config["target"] ], - env_with_passphrase) + get_env()) # From duplicity's manual: # "This should only be necessary after a duplicity session fails or is @@ -227,13 +261,14 @@ def perform_backup(full_backup): "cleanup", "--archive-dir", backup_cache_dir, "--force", - "file://" + backup_dir + config["target"] ], - env_with_passphrase) + get_env()) # Change ownership of backups to the user-data user, so that the after-bcakup # script can access them. - shell('check_call', ["/bin/chown", "-R", env["STORAGE_USER"], backup_dir]) + if config["target_type"] == 'file': + shell('check_call', ["/bin/chown", "-R", env["STORAGE_USER"], backup_dir]) # Execute a post-backup script that does the copying to a remote server. # Run as the STORAGE_USER user, not as root. Pass our settings in @@ -241,8 +276,8 @@ def perform_backup(full_backup): post_script = os.path.join(backup_root, 'after-backup') if os.path.exists(post_script): shell('check_call', - ['su', env['STORAGE_USER'], '-c', post_script], - env=env) + ['su', env['STORAGE_USER'], '-c', post_script, config["target"]], + env=get_env()) # Our nightly cron job executes system status checks immediately after this # backup. Since it checks that dovecot and postfix are running, block for a @@ -253,10 +288,10 @@ def perform_backup(full_backup): def run_duplicity_verification(): env = load_environment() - backup_root = os.path.join(env["STORAGE_ROOT"], 'backup') + config = get_backup_config() backup_cache_dir = os.path.join(backup_root, 'cache') backup_dir = os.path.join(backup_root, 'encrypted') - env_with_passphrase = { "PASSPHRASE" : open(os.path.join(backup_root, 'secret_key.txt')).read() } + shell('check_call', [ "/usr/bin/duplicity", "--verbosity", "info", @@ -264,9 +299,47 @@ def run_duplicity_verification(): "--compare-data", "--archive-dir", backup_cache_dir, "--exclude", backup_root, - "file://" + backup_dir, + config["target"], env["STORAGE_ROOT"], - ], env_with_passphrase) + ], get_env()) + + +def backup_set_custom(target, target_user, target_pass, target_type, max_age): + config = get_backup_config() + + # max_age must be an int + if isinstance(max_age, str): + max_age = int(max_age) + + config["target"] = target + config["target_user"] = target_user + config["target_pass"] = target_pass + config["target_type"] = target_type + config["max_age_in_days"] = max_age + + write_backup_config(config) + + return "Updated backup config" + +def get_backup_config(): + try: + config = rtyaml.load(open(os.path.join(backup_root, 'custom.yaml'))) + if not isinstance(config, dict): raise ValueError() # caught below + except: + return default_config + + merged_config = default_config.copy() + merged_config.update(config) + + # max_age must be an int + if isinstance(merged_config["max_age_in_days"], str): + merged_config["max_age_in_days"] = int(merged_config["max_age_in_days"]) + + return config + +def write_backup_config(newconfig): + with open(os.path.join(backup_root, 'custom.yaml'), "w") as f: + f.write(rtyaml.dump(newconfig)) if __name__ == "__main__": import sys @@ -274,6 +347,7 @@ if __name__ == "__main__": # Run duplicity's verification command to check a) the backup files # are readable, and b) report if they are up to date. run_duplicity_verification() + else: # Perform a backup. Add --full to force a full backup rather than # possibly performing an incremental backup. diff --git a/management/daemon.py b/management/daemon.py index af15b1c3..46ba9685 100755 --- a/management/daemon.py +++ b/management/daemon.py @@ -402,6 +402,24 @@ def backup_status(): from backup import backup_status return json_response(backup_status(env)) +@app.route('/system/backup/get-custom') +@authorized_personnel_only +def backup_get_custom(): + from backup import get_backup_config + return json_response(get_backup_config()) + +@app.route('/system/backup/set-custom', methods=["POST"]) +@authorized_personnel_only +def backup_set_custom(): + from backup import backup_set_custom + return json_response(backup_set_custom( + request.form.get('target', ''), + request.form.get('target_user', ''), + request.form.get('target_pass', ''), + request.form.get('target_type', ''), + request.form.get('max_age', '') + )) + # MUNIN @app.route('/munin/') @@ -432,4 +450,3 @@ if __name__ == '__main__': # Start the application server. Listens on 127.0.0.1 (IPv4 only). app.run(port=10222) - diff --git a/management/templates/system-backup.html b/management/templates/system-backup.html index 01682cf3..bf34759d 100644 --- a/management/templates/system-backup.html +++ b/management/templates/system-backup.html @@ -7,12 +7,53 @@

Copying Backup Files

-

The box makes an incremental backup each night. The backup is stored on the machine itself. You are responsible for copying the backup files off of the machine.

- -

Many cloud providers make this easy by allowing you to take snapshots of the machine's disk.

+

The box makes an incremental backup each night. By default the backup is stored on the machine itself, but you can also have it stored on Amazon S3

You can also use SFTP (FTP over SSH) to copy files from . These files are encrypted, so they are safe to store anywhere. Copy the encryption password from also but keep it in a safe location.

+

Backup Configuration

+ +
+
+ +
+ +
+
+
+ +
+ +
+
+
+ +
+ +
+
+
+ +
+ +
+
+
+ +
+ +
+
+
+
+ +
+
+
+

Current Backups

The backup directory currently contains the backups listed below. The total size on disk of the backups is currently .

@@ -27,8 +68,17 @@ - diff --git a/setup/management.sh b/setup/management.sh index 518a2ad6..817818fb 100755 --- a/setup/management.sh +++ b/setup/management.sh @@ -4,8 +4,9 @@ source setup/functions.sh # build-essential libssl-dev libffi-dev python3-dev: Required to pip install cryptography. apt_install python3-flask links duplicity libyaml-dev python3-dnspython python3-dateutil \ - build-essential libssl-dev libffi-dev python3-dev -hide_output pip3 install --upgrade rtyaml email_validator idna cryptography + build-essential libssl-dev libffi-dev python3-dev python-pip +hide_output pip3 install --upgrade rtyaml email_validator idna cryptography boto +hide_output pip install --upgrade boto # email_validator is repeated in setup/questions.sh # Create a backup directory and a random key for encrypting backups. From 96fb0f78f7b651a2f4e88db1e1c0217c0550de43 Mon Sep 17 00:00:00 2001 From: Leo Koppelkamm Date: Mon, 27 Jul 2015 21:56:08 +0200 Subject: [PATCH 02/14] Add comment regarding the use of pip instead of pip3 --- setup/management.sh | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/setup/management.sh b/setup/management.sh index 817818fb..3df2c72e 100755 --- a/setup/management.sh +++ b/setup/management.sh @@ -6,8 +6,11 @@ source setup/functions.sh apt_install python3-flask links duplicity libyaml-dev python3-dnspython python3-dateutil \ build-essential libssl-dev libffi-dev python3-dev python-pip hide_output pip3 install --upgrade rtyaml email_validator idna cryptography boto + +# duplicity uses python 2 so we need to use the python 2 package of boto hide_output pip install --upgrade boto - # email_validator is repeated in setup/questions.sh + +# email_validator is repeated in setup/questions.sh # Create a backup directory and a random key for encrypting backups. mkdir -p $STORAGE_ROOT/backup From 1e3e34f15f186b653053194ed9f2e89aa9e57c55 Mon Sep 17 00:00:00 2001 From: Leo Koppelkamm Date: Mon, 27 Jul 2015 22:00:36 +0200 Subject: [PATCH 03/14] Make backup API RESTful --- management/daemon.py | 4 ++-- management/templates/system-backup.html | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/management/daemon.py b/management/daemon.py index 46ba9685..c8d98eb1 100755 --- a/management/daemon.py +++ b/management/daemon.py @@ -402,13 +402,13 @@ def backup_status(): from backup import backup_status return json_response(backup_status(env)) -@app.route('/system/backup/get-custom') +@app.route('/system/backup/config', methods=["GET"]) @authorized_personnel_only def backup_get_custom(): from backup import get_backup_config return json_response(get_backup_config()) -@app.route('/system/backup/set-custom', methods=["POST"]) +@app.route('/system/backup/config', methods=["POST"]) @authorized_personnel_only def backup_set_custom(): from backup import backup_set_custom diff --git a/management/templates/system-backup.html b/management/templates/system-backup.html index bf34759d..fbad719f 100644 --- a/management/templates/system-backup.html +++ b/management/templates/system-backup.html @@ -138,7 +138,7 @@ function show_system_backup() { function show_custom_backup() { api( - "/system/backup/get-custom", + "/system/backup/custom", "GET", { }, function(r) { @@ -158,7 +158,7 @@ function set_custom_backup() { var target_pass = $("#target-pass").val(); var max_age = $("#max-age").val(); api( - "/system/backup/set-custom", + "/system/backup/custom", "POST", { target: target, From 91e4ea6e2fd4a112301adf64dc6f6fc7df1c64bd Mon Sep 17 00:00:00 2001 From: Leo Koppelkamm Date: Mon, 27 Jul 2015 22:09:58 +0200 Subject: [PATCH 04/14] Infer target_type from url --- management/backup.py | 16 +++++++++------- management/daemon.py | 1 - management/templates/system-backup.html | 4 ++-- 3 files changed, 11 insertions(+), 10 deletions(-) diff --git a/management/backup.py b/management/backup.py index 26426313..b974902b 100755 --- a/management/backup.py +++ b/management/backup.py @@ -22,8 +22,7 @@ backup_root = os.path.join(load_environment()["STORAGE_ROOT"], 'backup') # that depends on it is this many days old. default_config = { "max_age_in_days": 3, - "target": "file://" + os.path.join(backup_root, 'encrypted'), - "target_type": "file" + "target": "file://" + os.path.join(backup_root, 'encrypted') } def backup_status(env): @@ -164,12 +163,16 @@ def get_env(): env = { "PASSPHRASE" : get_passphrase() } - if config["target_type"] == 's3': + if get_target_type(config) == 's3': env["AWS_ACCESS_KEY_ID"] = config["target_user"] env["AWS_SECRET_ACCESS_KEY"] = config["target_pass"] return env - + +def get_target_type(config): + protocol = config["target"].split(":")[0] + return protocol + def perform_backup(full_backup): env = load_environment() @@ -267,7 +270,7 @@ def perform_backup(full_backup): # Change ownership of backups to the user-data user, so that the after-bcakup # script can access them. - if config["target_type"] == 'file': + if get_target_type(config) == 'file': shell('check_call', ["/bin/chown", "-R", env["STORAGE_USER"], backup_dir]) # Execute a post-backup script that does the copying to a remote server. @@ -304,7 +307,7 @@ def run_duplicity_verification(): ], get_env()) -def backup_set_custom(target, target_user, target_pass, target_type, max_age): +def backup_set_custom(target, target_user, target_pass, max_age): config = get_backup_config() # max_age must be an int @@ -314,7 +317,6 @@ def backup_set_custom(target, target_user, target_pass, target_type, max_age): config["target"] = target config["target_user"] = target_user config["target_pass"] = target_pass - config["target_type"] = target_type config["max_age_in_days"] = max_age write_backup_config(config) diff --git a/management/daemon.py b/management/daemon.py index c8d98eb1..3380e757 100755 --- a/management/daemon.py +++ b/management/daemon.py @@ -416,7 +416,6 @@ def backup_set_custom(): request.form.get('target', ''), request.form.get('target_user', ''), request.form.get('target_pass', ''), - request.form.get('target_type', ''), request.form.get('max_age', '') )) diff --git a/management/templates/system-backup.html b/management/templates/system-backup.html index fbad719f..7ff67d0d 100644 --- a/management/templates/system-backup.html +++ b/management/templates/system-backup.html @@ -142,8 +142,9 @@ function show_custom_backup() { "GET", { }, function(r) { + var target_type = r.target.split(':')[0] $("#target").val(r.target); - $("#target-type").val(r.target_type); + $("#target-type").val(target_type); $("#target-user").val(r.target_user); $("#target-pass").val(r.target_pass); $("#max-age").val(r.max_age_in_days); @@ -162,7 +163,6 @@ function set_custom_backup() { "POST", { target: target, - target_type: target_type, target_user: target_user, target_pass: target_pass, max_age: max_age From 43fb7fe635b1e7093bf3a927f663c1bd0de7df6c Mon Sep 17 00:00:00 2001 From: Leo Koppelkamm Date: Mon, 27 Jul 2015 22:11:43 +0200 Subject: [PATCH 05/14] Remove unused variable --- management/backup.py | 1 - 1 file changed, 1 deletion(-) diff --git a/management/backup.py b/management/backup.py index b974902b..b86b70f3 100755 --- a/management/backup.py +++ b/management/backup.py @@ -293,7 +293,6 @@ def run_duplicity_verification(): env = load_environment() config = get_backup_config() backup_cache_dir = os.path.join(backup_root, 'cache') - backup_dir = os.path.join(backup_root, 'encrypted') shell('check_call', [ "/usr/bin/duplicity", From fa0dd684da05a800da867782f44106292086f833 Mon Sep 17 00:00:00 2001 From: Leo Koppelkamm Date: Mon, 27 Jul 2015 22:13:28 +0200 Subject: [PATCH 06/14] Add archive-dir argument to collection-status --- management/backup.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/management/backup.py b/management/backup.py index b86b70f3..8448bbce 100755 --- a/management/backup.py +++ b/management/backup.py @@ -31,7 +31,11 @@ def backup_status(env): # Use the number of volumes to estimate the size. config = get_backup_config() now = datetime.datetime.now(dateutil.tz.tzlocal()) - + + backups = { } + backup_dir = os.path.join(backup_root, 'encrypted') + backup_cache_dir = os.path.join(backup_root, 'cache') + def reldate(date, ref, clip): if ref < date: return clip rd = dateutil.relativedelta.relativedelta(ref, date) @@ -57,16 +61,13 @@ def backup_status(env): shell('check_call', [ "/usr/bin/duplicity", "collection-status", + "--archive-dir", backup_cache_dir, "--log-file", os.path.join(backup_root, "duplicity_status"), "--gpg-options", "--cipher-algo=AES256", config["target"], ], get_env()) - - backups = { } - backup_dir = os.path.join(backup_root, 'encrypted') - # Parse backup data from status file with open(os.path.join(backup_root, "duplicity_status"),'r') as status_file: for line in status_file: From e693802091c04e88da0755d0d3d5ea6afc3996c2 Mon Sep 17 00:00:00 2001 From: Leo Koppelkamm Date: Mon, 27 Jul 2015 22:18:19 +0200 Subject: [PATCH 07/14] Rename max_age to min_age Also clarify a comment and remove an unneeded type check --- management/backup.py | 29 +++++++++++-------------- management/templates/system-backup.html | 8 +++---- 2 files changed, 17 insertions(+), 20 deletions(-) diff --git a/management/backup.py b/management/backup.py index 8448bbce..3cccf1ff 100755 --- a/management/backup.py +++ b/management/backup.py @@ -18,10 +18,11 @@ from utils import exclusive_process, load_environment, shell, wait_for_service backup_root = os.path.join(load_environment()["STORAGE_ROOT"], 'backup') # Default settings -# Destroy backups when the most recent increment in the chain -# that depends on it is this many days old. +# min_age_in_days is the minimum amount of days a backup will be kept before +# it is eligble to be removed. Backups might be kept much longer if there's no +# new full backup yet. default_config = { - "max_age_in_days": 3, + "min_age_in_days": 3, "target": "file://" + os.path.join(backup_root, 'encrypted') } @@ -101,11 +102,11 @@ def backup_status(env): # when the threshold is met. deleted_in = None if incremental_count > 0 and first_full_size is not None: - deleted_in = "approx. %d days" % round(config["max_age_in_days"] + (.5 * first_full_size - incremental_size) / (incremental_size/incremental_count) + .5) + deleted_in = "approx. %d days" % round(config["min_age_in_days"] + (.5 * first_full_size - incremental_size) / (incremental_size/incremental_count) + .5) # When will a backup be deleted? saw_full = False - days_ago = now - datetime.timedelta(days=config["max_age_in_days"]) + days_ago = now - datetime.timedelta(days=config["min_age_in_days"]) for bak in backups: if deleted_in: # Subsequent backups are deleted when the most recent increment @@ -248,7 +249,7 @@ def perform_backup(full_backup): shell('check_call', [ "/usr/bin/duplicity", "remove-older-than", - "%dD" % config["max_age_in_days"], + "%dD" % config["min_age_in_days"], "--archive-dir", backup_cache_dir, "--force", config["target"] @@ -307,17 +308,17 @@ def run_duplicity_verification(): ], get_env()) -def backup_set_custom(target, target_user, target_pass, max_age): +def backup_set_custom(target, target_user, target_pass, min_age): config = get_backup_config() - # max_age must be an int - if isinstance(max_age, str): - max_age = int(max_age) + # min_age must be an int + if isinstance(min_age, str): + min_age = int(min_age) config["target"] = target config["target_user"] = target_user config["target_pass"] = target_pass - config["max_age_in_days"] = max_age + config["min_age_in_days"] = min_age write_backup_config(config) @@ -332,11 +333,7 @@ def get_backup_config(): merged_config = default_config.copy() merged_config.update(config) - - # max_age must be an int - if isinstance(merged_config["max_age_in_days"], str): - merged_config["max_age_in_days"] = int(merged_config["max_age_in_days"]) - + return config def write_backup_config(newconfig): diff --git a/management/templates/system-backup.html b/management/templates/system-backup.html index 7ff67d0d..ed344281 100644 --- a/management/templates/system-backup.html +++ b/management/templates/system-backup.html @@ -26,7 +26,7 @@
- +
@@ -147,7 +147,7 @@ function show_custom_backup() { $("#target-type").val(target_type); $("#target-user").val(r.target_user); $("#target-pass").val(r.target_pass); - $("#max-age").val(r.max_age_in_days); + $("#min-age").val(r.min_age_in_days); toggle_form() }) } @@ -157,7 +157,7 @@ function set_custom_backup() { var target_type = $("#target-type").val(); var target_user = $("#target-user").val(); var target_pass = $("#target-pass").val(); - var max_age = $("#max-age").val(); + var min_age = $("#min-age").val(); api( "/system/backup/custom", "POST", @@ -165,7 +165,7 @@ function set_custom_backup() { target: target, target_user: target_user, target_pass: target_pass, - max_age: max_age + min_age: min_age }, function(r) { // Responses are multiple lines of pre-formatted text. From ba9065cada64d23963403400fd9f57ceebe99d2d Mon Sep 17 00:00:00 2001 From: Leo Koppelkamm Date: Mon, 27 Jul 2015 22:30:22 +0200 Subject: [PATCH 08/14] Don't write collection_status output to file but parse it directly --- management/backup.py | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/management/backup.py b/management/backup.py index 3cccf1ff..3a7a29cc 100755 --- a/management/backup.py +++ b/management/backup.py @@ -46,7 +46,7 @@ def backup_status(env): if rd.days > 1: return "%d days, %d hours" % (rd.days, rd.hours) if rd.days == 1: return "%d day, %d hours" % (rd.days, rd.hours) return "%d hours, %d minutes" % (rd.hours, rd.minutes) - + def parse_line(line): keys = line.strip().split() date = dateutil.parser.parse(keys[1]) @@ -57,28 +57,28 @@ def backup_status(env): "full": keys[0] == "full", "size": int(keys[2]) * 250 * 1000000, } - - # Write duplicity status to file - shell('check_call', [ + + # Get duplicity collection status + collection_status = shell('check_output', [ "/usr/bin/duplicity", "collection-status", "--archive-dir", backup_cache_dir, "--log-file", os.path.join(backup_root, "duplicity_status"), "--gpg-options", "--cipher-algo=AES256", + "--log-fd", "1", config["target"], ], get_env()) - # Parse backup data from status file - with open(os.path.join(backup_root, "duplicity_status"),'r') as status_file: - for line in status_file: - if line.startswith(" full") or line.startswith(" inc"): - backup = parse_line(line) - backups[backup["date"]] = backup + # Split multi line string into list + collection_status = collection_status.split('\n') + + # Parse backup data from status file + for line in collection_status: + if line.startswith(" full") or line.startswith(" inc"): + backup = parse_line(line) + backups[backup["date"]] = backup - # Remove status file - os.remove(os.path.join(backup_root, "duplicity_status")) - # Ensure the rows are sorted reverse chronologically. # This is relied on by should_force_full() and the next step. backups = sorted(backups.values(), key = lambda b : b["date"], reverse=True) From 606cf6a941eb0f50513f185a34e26482a5ec45d4 Mon Sep 17 00:00:00 2001 From: Leo Koppelkamm Date: Tue, 28 Jul 2015 00:34:11 +0200 Subject: [PATCH 09/14] Fix API typo --- management/templates/system-backup.html | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/management/templates/system-backup.html b/management/templates/system-backup.html index ed344281..59c7357b 100644 --- a/management/templates/system-backup.html +++ b/management/templates/system-backup.html @@ -138,7 +138,7 @@ function show_system_backup() { function show_custom_backup() { api( - "/system/backup/custom", + "/system/backup/config", "GET", { }, function(r) { @@ -159,7 +159,7 @@ function set_custom_backup() { var target_pass = $("#target-pass").val(); var min_age = $("#min-age").val(); api( - "/system/backup/custom", + "/system/backup/config", "POST", { target: target, From 0d8a4099c1e4ee88339661f73936dbdc67698be5 Mon Sep 17 00:00:00 2001 From: Leo Koppelkamm Date: Tue, 28 Jul 2015 00:37:43 +0200 Subject: [PATCH 10/14] Add placeholder attribute; use input instead of textarea --- management/templates/system-backup.html | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/management/templates/system-backup.html b/management/templates/system-backup.html index 59c7357b..8542f100 100644 --- a/management/templates/system-backup.html +++ b/management/templates/system-backup.html @@ -32,19 +32,19 @@
- +
- +
- +
From 77099b3bce80b39c98b11d69b07f7796d8e66b0e Mon Sep 17 00:00:00 2001 From: Leo Koppelkamm Date: Tue, 28 Jul 2015 00:42:00 +0200 Subject: [PATCH 11/14] Reword backup min_time label --- management/templates/system-backup.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/management/templates/system-backup.html b/management/templates/system-backup.html index 8542f100..615ba980 100644 --- a/management/templates/system-backup.html +++ b/management/templates/system-backup.html @@ -24,7 +24,7 @@
- +
From 1cdd205eb763dc829ac10e3bdd08d4be5d1e5aca Mon Sep 17 00:00:00 2001 From: Leo Koppelkamm Date: Tue, 28 Jul 2015 20:58:39 +0200 Subject: [PATCH 12/14] Missed one max_age --- management/daemon.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/management/daemon.py b/management/daemon.py index 3380e757..4b63f71b 100755 --- a/management/daemon.py +++ b/management/daemon.py @@ -416,7 +416,7 @@ def backup_set_custom(): request.form.get('target', ''), request.form.get('target_user', ''), request.form.get('target_pass', ''), - request.form.get('max_age', '') + request.form.get('min_age', '') )) # MUNIN From df8de717ce26d7627779feed4dc20d174f0c5f7a Mon Sep 17 00:00:00 2001 From: Leo Koppelkamm Date: Mon, 3 Aug 2015 08:51:39 +0200 Subject: [PATCH 13/14] asynchronous uploads --- management/backup.py | 1 + 1 file changed, 1 insertion(+) diff --git a/management/backup.py b/management/backup.py index 3a7a29cc..71fa859e 100755 --- a/management/backup.py +++ b/management/backup.py @@ -227,6 +227,7 @@ def perform_backup(full_backup): "/usr/bin/duplicity", "full" if full_backup else "incr", "--archive-dir", backup_cache_dir, + "--asynchronous-upload", "--exclude", backup_root, "--volsize", "250", "--gpg-options", "--cipher-algo=AES256", From a2098b1ace0291adbec983d1115b08fd8e791be2 Mon Sep 17 00:00:00 2001 From: Leo Koppelkamm Date: Sat, 8 Aug 2015 11:27:25 +0200 Subject: [PATCH 14/14] Add logging facility --- management/backup.py | 58 +++++++++++++++++-------- management/daemon.py | 8 ++-- management/templates/system-backup.html | 21 ++++++--- 3 files changed, 60 insertions(+), 27 deletions(-) diff --git a/management/backup.py b/management/backup.py index 71fa859e..a14c872b 100755 --- a/management/backup.py +++ b/management/backup.py @@ -11,12 +11,17 @@ import os, os.path, shutil, glob, re, datetime import dateutil.parser, dateutil.relativedelta, dateutil.tz import rtyaml +import logging from utils import exclusive_process, load_environment, shell, wait_for_service # Root folder backup_root = os.path.join(load_environment()["STORAGE_ROOT"], 'backup') +# Setup logging +log_path = os.path.join(backup_root, 'backup.log') +logging.basicConfig(filename=log_path, format='%(levelname)s at %(asctime)s: %(message)s') + # Default settings # min_age_in_days is the minimum amount of days a backup will be kept before # it is eligble to be removed. Backups might be kept much longer if there's no @@ -236,6 +241,9 @@ def perform_backup(full_backup): "--allow-source-mismatch" ], get_env()) + logging.info("Backup successful") + except Exception as e: + logging.warn("Backup failed: {0}".format(e)) finally: # Start services again. shell('check_call', ["/usr/sbin/service", "dovecot", "start"]) @@ -247,29 +255,35 @@ def perform_backup(full_backup): # Remove old backups. This deletes all backup data no longer needed # from more than 3 days ago. - shell('check_call', [ - "/usr/bin/duplicity", - "remove-older-than", - "%dD" % config["min_age_in_days"], - "--archive-dir", backup_cache_dir, - "--force", - config["target"] - ], - get_env()) + try: + shell('check_call', [ + "/usr/bin/duplicity", + "remove-older-than", + "%dD" % config["min_age_in_days"], + "--archive-dir", backup_cache_dir, + "--force", + config["target"] + ], + get_env()) + except Exception as e: + logging.warn("Removal of old backups failed: {0}".format(e)) # From duplicity's manual: # "This should only be necessary after a duplicity session fails or is # aborted prematurely." # That may be unlikely here but we may as well ensure we tidy up if # that does happen - it might just have been a poorly timed reboot. - shell('check_call', [ - "/usr/bin/duplicity", - "cleanup", - "--archive-dir", backup_cache_dir, - "--force", - config["target"] - ], - get_env()) + try: + shell('check_call', [ + "/usr/bin/duplicity", + "cleanup", + "--archive-dir", backup_cache_dir, + "--force", + config["target"] + ], + get_env()) + except Exception as e: + logging.warn("Cleanup of backups failed: {0}".format(e)) # Change ownership of backups to the user-data user, so that the after-bcakup # script can access them. @@ -336,6 +350,16 @@ def get_backup_config(): merged_config.update(config) return config + +def get_backup_log(): + try: + fileHandle = open(log_path, 'r') + log = fileHandle.read() + fileHandle.close() + except: + log = "" + + return log def write_backup_config(newconfig): with open(os.path.join(backup_root, 'custom.yaml'), "w") as f: diff --git a/management/daemon.py b/management/daemon.py index 4b63f71b..b011823a 100755 --- a/management/daemon.py +++ b/management/daemon.py @@ -4,7 +4,7 @@ import os, os.path, re, json from functools import wraps -from flask import Flask, request, render_template, abort, Response, send_from_directory +from flask import Flask, request, render_template, abort, Response, send_from_directory, jsonify import auth, utils from mailconfig import get_mail_users, get_mail_users_ex, get_admins, add_mail_user, set_mail_password, remove_mail_user @@ -399,14 +399,14 @@ def do_updates(): @app.route('/system/backup/status') @authorized_personnel_only def backup_status(): - from backup import backup_status - return json_response(backup_status(env)) + from backup import backup_status, get_backup_log + return jsonify(backups=backup_status(env), log=get_backup_log()) @app.route('/system/backup/config', methods=["GET"]) @authorized_personnel_only def backup_get_custom(): from backup import get_backup_config - return json_response(get_backup_config()) + return jsonify(get_backup_config()) @app.route('/system/backup/config', methods=["POST"]) @authorized_personnel_only diff --git a/management/templates/system-backup.html b/management/templates/system-backup.html index 615ba980..24fa5f57 100644 --- a/management/templates/system-backup.html +++ b/management/templates/system-backup.html @@ -68,6 +68,11 @@ + +

Backup logs

+

+
+