From 42322455463e56685ba05bc2341451830c50a3d5 Mon Sep 17 00:00:00 2001 From: David Piggott Date: Thu, 26 Mar 2015 12:27:26 +0000 Subject: [PATCH 01/30] Use built in duplicity encryption (GPG) for backups, closes #362, closes #363 [Josh merged some subsequent commits:] * Guard via idempotency against termination between migration operations * Final corrections and tweaks * Pass passphrase through to all duplicity calls Empirical evidence (a failed cron job) shows that cleanup requires the passphrase (so it presumably needs to decrypt metadata), and though remove-older-than has been working fine without it, it won't do any harm to set it in case that changes or there are any special cases. * Add back the archive-dir override but locate it at STORAGE_ROOT/backup/cache --- management/backup.py | 163 ++++++++++++++---------- management/templates/system-backup.html | 5 +- 2 files changed, 98 insertions(+), 70 deletions(-) diff --git a/management/backup.py b/management/backup.py index fdcc5248..b0f41a81 100755 --- a/management/backup.py +++ b/management/backup.py @@ -2,11 +2,10 @@ # This script performs a backup of all user data: # 1) System services are stopped while a copy of user data is made. -# 2) An incremental backup is made using duplicity into the -# directory STORAGE_ROOT/backup/duplicity. +# 2) An incremental encrypted backup is made using duplicity into the +# directory STORAGE_ROOT/backup/encrypted. The password used for +# encryption is stored in backup/secret_key.txt. # 3) The stopped services are restarted. -# 4) The backup files are encrypted with a long password (stored in -# backup/secret_key.txt) to STORAGE_ROOT/backup/encrypted. # 5) STORAGE_ROOT/backup/after-backup is executd if it exists. import os, os.path, shutil, glob, re, datetime @@ -14,13 +13,13 @@ import dateutil.parser, dateutil.relativedelta, dateutil.tz from utils import exclusive_process, load_environment, shell -# destroy backups when the most recent increment in the chain +# Destroy backups when the most recent increment in the chain # that depends on it is this many days old. keep_backups_for_days = 3 def backup_status(env): # What is the current status of backups? - # Loop through all of the files in STORAGE_ROOT/backup/duplicity to + # Loop through all of the files in STORAGE_ROOT/backup/encrypted to # get a list of all of the backups taken and sum up file sizes to # see how large the storage is. @@ -36,10 +35,10 @@ def backup_status(env): return "%d hours, %d minutes" % (rd.hours, rd.minutes) backups = { } - basedir = os.path.join(env['STORAGE_ROOT'], 'backup/duplicity/') - encdir = os.path.join(env['STORAGE_ROOT'], 'backup/encrypted/') - os.makedirs(basedir, exist_ok=True) # os.listdir fails if directory does not exist - for fn in os.listdir(basedir): + backup_dir = os.path.join(env["STORAGE_ROOT"], 'backup') + backup_encrypted_dir = os.path.join(backup_dir, 'encrypted') + os.makedirs(backup_encrypted_dir, exist_ok=True) # os.listdir fails if directory does not exist + for fn in os.listdir(backup_encrypted_dir): m = re.match(r"duplicity-(full|full-signatures|(inc|new-signatures)\.(?P\d+T\d+Z)\.to)\.(?P\d+T\d+Z)\.", fn) if not m: raise ValueError(fn) @@ -53,15 +52,9 @@ def backup_status(env): "full": m.group("incbase") is None, "previous": m.group("incbase"), "size": 0, - "encsize": 0, } - backups[key]["size"] += os.path.getsize(os.path.join(basedir, fn)) - - # Also check encrypted size. - encfn = os.path.join(encdir, fn + ".enc") - if os.path.exists(encfn): - backups[key]["encsize"] += os.path.getsize(encfn) + backups[key]["size"] += os.path.getsize(os.path.join(backup_encrypted_dir, fn)) # Ensure the rows are sorted reverse chronologically. # This is relied on by should_force_full() and the next step. @@ -78,7 +71,7 @@ def backup_status(env): break incremental_count += 1 incremental_size += bak["size"] - + # Predict how many more increments until the next full backup, # and add to that the time we hold onto backups, to predict # how long the most recent full backup+increments will be held @@ -106,9 +99,8 @@ def backup_status(env): bak["deleted_in"] = deleted_in return { - "directory": basedir, - "encpwfile": os.path.join(env['STORAGE_ROOT'], 'backup/secret_key.txt'), - "encdirectory": encdir, + "directory": backup_encrypted_dir, + "encpwfile": os.path.join(backup_dir, 'secret_key.txt'), "tz": now.tzname(), "backups": backups, } @@ -137,10 +129,62 @@ def perform_backup(full_backup): exclusive_process("backup") - # Ensure the backup directory exists. backup_dir = os.path.join(env["STORAGE_ROOT"], 'backup') + backup_cache_dir = os.path.join(backup_dir, 'cache') + backup_encrypted_dir = os.path.join(backup_dir, 'encrypted') + + # In an older version of this script, duplicity was called + # such that it did not encrypt the backups it created (in + # backup/duplicity), and instead openssl was called separately + # after each backup run, creating AES256 encrypted copies of + # each file created by duplicity in backup/encrypted. + # + # We detect the transition by the presence of backup/duplicity + # and handle it by 'dupliception': we move all the old *un*encrypted + # duplicity files up out of the backup/duplicity directory (as + # backup/ is excluded from duplicity runs) in order that it is + # included in the next run, and we delete backup/encrypted (which + # duplicity will output files directly to, post-transition). + # + # This achieves two things: + # 1. It preserves the pre-transition unencrypted backup files + # within the encrypted backups we will immediately create, so + # that they are kept until the next full backup is triggered. + # (it is because those backups will be encrypted that we take + # the old *un*encrypted backups, not the duplicates encrypted + # with openssl). + # 2. It results in backup_status() being called on a non-existant + # backup/encrypted directory, which will trigger a full backup + # (though duplicity ought to do one anyway as it ought not + # recognise the old openssl encrypted .enc files, if we *had* + # left them there). More to the point it clears out those .enc + # files which are now redundant, thereby gaining disk space and + # preventing backup_status() getting terribly confused by their + # presence. + # + # A note about disk use: + # At no point in the transition will we use more disk space than + # was used pre-transition, because by deleting the openssl + # encrypted duplicates we decrease by more* than half the disk + # space used, while the addition by 'dupliception' of the old + # *un*encrypted backups takes less space than we gained from + # dropping the openssl encrypted duplicates. + # + # A note about the status page post-upgrade but pre-transition: + # Between the point that the new code is deployed and when the first + # daily backup is run, there will be a subtle change in the + # behaviour of the web control panel's backup status page, in that + # it will only sum the size of the encrypted backups when reporting + # the total size on disk i.e. it will not consider the unencrypted + # originals. + # + # *the openssl encrypted duplicates were base64 encrypted, hence + # accounting for more than half of the space used. backup_duplicity_dir = os.path.join(backup_dir, 'duplicity') - os.makedirs(backup_duplicity_dir, exist_ok=True) + migrated_unencrypted_backup_dir = os.path.join(env["STORAGE_ROOT"], "migrated_unencrypted_backup") + if os.path.isdir(backup_duplicity_dir): + shutil.rmtree(backup_encrypted_dir) + shutil.move(backup_duplicity_dir, migrated_unencrypted_backup_dir) # On the first run, always do a full backup. Incremental # will fail. Otherwise do a full backup when the size of @@ -152,70 +196,55 @@ def perform_backup(full_backup): shell('check_call', ["/usr/sbin/service", "dovecot", "stop"]) shell('check_call', ["/usr/sbin/service", "postfix", "stop"]) + env_with_passphrase = { "PASSPHRASE" : + open(os.path.join(backup_dir, 'secret_key.txt')).read() + } # Update the backup mirror directory which mirrors the current # STORAGE_ROOT (but excluding the backups themselves!). try: shell('check_call', [ "/usr/bin/duplicity", "full" if full_backup else "incr", - "--no-encryption", - "--archive-dir", "/tmp/duplicity-archive-dir", - "--name", "mailinabox", + "--archive-dir", backup_cache_dir, "--exclude", backup_dir, - "--volsize", "100", - "--verbosity", "warning", + "--volsize", "250", env["STORAGE_ROOT"], - "file://" + backup_duplicity_dir - ]) + "file://" + backup_encrypted_dir + ], + env_with_passphrase) finally: # Start services again. shell('check_call', ["/usr/sbin/service", "dovecot", "start"]) shell('check_call', ["/usr/sbin/service", "postfix", "start"]) + if os.path.isdir(migrated_unencrypted_backup_dir): + shutil.rmtree(migrated_unencrypted_backup_dir) + # Remove old backups. This deletes all backup data no longer needed - # from more than 31 days ago. Must do this before destroying the - # cache directory or else this command will re-create it. + # from more than 3 days ago. shell('check_call', [ "/usr/bin/duplicity", "remove-older-than", "%dD" % keep_backups_for_days, - "--archive-dir", "/tmp/duplicity-archive-dir", - "--name", "mailinabox", + "--archive-dir", backup_cache_dir, "--force", - "--verbosity", "warning", - "file://" + backup_duplicity_dir - ]) + "file://" + backup_encrypted_dir + ], + env_with_passphrase) - # Remove duplicity's cache directory because it's redundant with our backup directory. - shutil.rmtree("/tmp/duplicity-archive-dir") - - # Encrypt all of the new files. - backup_encrypted_dir = os.path.join(backup_dir, 'encrypted') - os.makedirs(backup_encrypted_dir, exist_ok=True) - for fn in os.listdir(backup_duplicity_dir): - fn2 = os.path.join(backup_encrypted_dir, fn) + ".enc" - if os.path.exists(fn2): continue - - # Encrypt the backup using the backup private key. - shell('check_call', [ - "/usr/bin/openssl", - "enc", - "-aes-256-cbc", - "-a", - "-salt", - "-in", os.path.join(backup_duplicity_dir, fn), - "-out", fn2, - "-pass", "file:%s" % os.path.join(backup_dir, "secret_key.txt"), - ]) - - # The backup can be decrypted with: - # openssl enc -d -aes-256-cbc -a -in latest.tgz.enc -out /dev/stdout -pass file:secret_key.txt | tar -z - - # Remove encrypted backups that are no longer needed. - for fn in os.listdir(backup_encrypted_dir): - fn2 = os.path.join(backup_duplicity_dir, fn.replace(".enc", "")) - if os.path.exists(fn2): continue - os.unlink(os.path.join(backup_encrypted_dir, fn)) + # From duplicity's manual: + # "This should only be necessary after a duplicity session fails or is + # aborted prematurely." + # That may be unlikely here but we may as well ensure we tidy up if + # that does happen - it might just have been a poorly timed reboot. + shell('check_call', [ + "/usr/bin/duplicity", + "cleanup", + "--archive-dir", backup_cache_dir, + "--force", + "file://" + backup_encrypted_dir + ], + env_with_passphrase) # Execute a post-backup script that does the copying to a remote server. # Run as the STORAGE_USER user, not as root. Pass our settings in diff --git a/management/templates/system-backup.html b/management/templates/system-backup.html index 7ff1a868..fffe8680 100644 --- a/management/templates/system-backup.html +++ b/management/templates/system-backup.html @@ -54,7 +54,7 @@ function show_system_backup() { "GET", { }, function(r) { - $('#backup-location').text(r.encdirectory); + $('#backup-location').text(r.directory); $('#backup-encpassword-file').text(r.encpwfile); $('#backup-status tbody').html(""); @@ -72,7 +72,7 @@ function show_system_backup() { tr.append( $('').text(b.date_str + " " + r.tz) ); tr.append( $('').text(b.date_delta + " ago") ); tr.append( $('').text(b.full ? "full" : "increment") ); - tr.append( $('').text( nice_size(b.encsize)) ); + tr.append( $('').text( nice_size(b.size)) ); if (b.deleted_in) tr.append( $('').text(b.deleted_in) ); else @@ -80,7 +80,6 @@ function show_system_backup() { $('#backup-status tbody').append(tr); total_disk_size += b.size; - total_disk_size += b.encsize; } $('#backup-total-size').text(nice_size(total_disk_size)); From d8279c48ac84243045fe2cf78eee1908bde9fd6d Mon Sep 17 00:00:00 2001 From: Joshua Tauberer Date: Sat, 11 Apr 2015 17:43:16 +0000 Subject: [PATCH 02/30] new backup method tweaks * use the AES256 cipher, be explicit that only the first line of secret_key.txt is used, and sanity check that the passphrase is long enough * change overship of the encrypted files to the user-data user * simplify variable names in management/backup.py * although I appreciate long comments I am trimming the commentary about the backup migration * revise the control panel template to not refer to the old unencrypted files * add CHANGELOG entry --- CHANGELOG.md | 2 + management/backup.py | 101 +++++++++++------------- management/templates/system-backup.html | 2 - 3 files changed, 46 insertions(+), 59 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 921995eb..29eb77e1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,8 @@ CHANGELOG ========= +* Backups now use duplicity's built-in gpg symmetric AES256 encryption rather than my home-brewed encryption. Old backups will be incorporated inside the first backup after this update but then deleted from disk (i.e. your backups from the previous few days will be backed up). + In Development... ----------------- diff --git a/management/backup.py b/management/backup.py index b0f41a81..7470e604 100755 --- a/management/backup.py +++ b/management/backup.py @@ -35,10 +35,10 @@ def backup_status(env): return "%d hours, %d minutes" % (rd.hours, rd.minutes) backups = { } - backup_dir = os.path.join(env["STORAGE_ROOT"], 'backup') - backup_encrypted_dir = os.path.join(backup_dir, 'encrypted') - os.makedirs(backup_encrypted_dir, exist_ok=True) # os.listdir fails if directory does not exist - for fn in os.listdir(backup_encrypted_dir): + backup_root = os.path.join(env["STORAGE_ROOT"], 'backup') + backup_dir = os.path.join(backup_root, 'encrypted') + os.makedirs(backup_dir, exist_ok=True) # os.listdir fails if directory does not exist + for fn in os.listdir(backup_dir): m = re.match(r"duplicity-(full|full-signatures|(inc|new-signatures)\.(?P\d+T\d+Z)\.to)\.(?P\d+T\d+Z)\.", fn) if not m: raise ValueError(fn) @@ -54,7 +54,7 @@ def backup_status(env): "size": 0, } - backups[key]["size"] += os.path.getsize(os.path.join(backup_encrypted_dir, fn)) + backups[key]["size"] += os.path.getsize(os.path.join(backup_dir, fn)) # Ensure the rows are sorted reverse chronologically. # This is relied on by should_force_full() and the next step. @@ -99,8 +99,8 @@ def backup_status(env): bak["deleted_in"] = deleted_in return { - "directory": backup_encrypted_dir, - "encpwfile": os.path.join(backup_dir, 'secret_key.txt'), + "directory": backup_dir, + "encpwfile": os.path.join(backup_root, 'secret_key.txt'), "tz": now.tzname(), "backups": backups, } @@ -129,9 +129,9 @@ def perform_backup(full_backup): exclusive_process("backup") - backup_dir = os.path.join(env["STORAGE_ROOT"], 'backup') - backup_cache_dir = os.path.join(backup_dir, 'cache') - backup_encrypted_dir = os.path.join(backup_dir, 'encrypted') + backup_root = os.path.join(env["STORAGE_ROOT"], 'backup') + backup_cache_dir = os.path.join(backup_root, 'cache') + backup_dir = os.path.join(backup_root, 'encrypted') # In an older version of this script, duplicity was called # such that it did not encrypt the backups it created (in @@ -145,46 +145,19 @@ def perform_backup(full_backup): # backup/ is excluded from duplicity runs) in order that it is # included in the next run, and we delete backup/encrypted (which # duplicity will output files directly to, post-transition). - # - # This achieves two things: - # 1. It preserves the pre-transition unencrypted backup files - # within the encrypted backups we will immediately create, so - # that they are kept until the next full backup is triggered. - # (it is because those backups will be encrypted that we take - # the old *un*encrypted backups, not the duplicates encrypted - # with openssl). - # 2. It results in backup_status() being called on a non-existant - # backup/encrypted directory, which will trigger a full backup - # (though duplicity ought to do one anyway as it ought not - # recognise the old openssl encrypted .enc files, if we *had* - # left them there). More to the point it clears out those .enc - # files which are now redundant, thereby gaining disk space and - # preventing backup_status() getting terribly confused by their - # presence. - # - # A note about disk use: - # At no point in the transition will we use more disk space than - # was used pre-transition, because by deleting the openssl - # encrypted duplicates we decrease by more* than half the disk - # space used, while the addition by 'dupliception' of the old - # *un*encrypted backups takes less space than we gained from - # dropping the openssl encrypted duplicates. - # - # A note about the status page post-upgrade but pre-transition: - # Between the point that the new code is deployed and when the first - # daily backup is run, there will be a subtle change in the - # behaviour of the web control panel's backup status page, in that - # it will only sum the size of the encrypted backups when reporting - # the total size on disk i.e. it will not consider the unencrypted - # originals. - # - # *the openssl encrypted duplicates were base64 encrypted, hence - # accounting for more than half of the space used. - backup_duplicity_dir = os.path.join(backup_dir, 'duplicity') + old_backup_dir = os.path.join(backup_root, 'duplicity') migrated_unencrypted_backup_dir = os.path.join(env["STORAGE_ROOT"], "migrated_unencrypted_backup") - if os.path.isdir(backup_duplicity_dir): - shutil.rmtree(backup_encrypted_dir) - shutil.move(backup_duplicity_dir, migrated_unencrypted_backup_dir) + if os.path.isdir(old_backup_dir): + # Move the old unencrpyted files to a new location outside of + # the backup root so they get included in the next (new) backup. + # Then we'll delete them. Also so that they do not get in the + # way of duplicity doing a full backup on the first run after + # we take care of this. + shutil.move(old_backup_dir, migrated_unencrypted_backup_dir) + + # The backup_dir (backup/encrypted) now has a new purpose. + # Clear it out. + shutil.rmtree(backup_dir) # On the first run, always do a full backup. Incremental # will fail. Otherwise do a full backup when the size of @@ -196,9 +169,17 @@ def perform_backup(full_backup): shell('check_call', ["/usr/sbin/service", "dovecot", "stop"]) shell('check_call', ["/usr/sbin/service", "postfix", "stop"]) - env_with_passphrase = { "PASSPHRASE" : - open(os.path.join(backup_dir, 'secret_key.txt')).read() - } + # Get the encryption passphrase. secret_key.txt is 2048 random + # bits base64-encoded and with line breaks every 65 characters. + # gpg will only take the first line of text, so sanity check that + # that line is long enough to be a reasonable passphrase. It + # only needs to be 43 base64-characters to match AES256's key + # length of 32 bytes. + with open(os.path.join(backup_root, 'secret_key.txt')) as f: + passphrase = f.readline().strip() + if len(passphrase) < 43: raise Exception("secret_key.txt's first line is too short!") + env_with_passphrase = { "PASSPHRASE" : passphrase } + # Update the backup mirror directory which mirrors the current # STORAGE_ROOT (but excluding the backups themselves!). try: @@ -206,10 +187,11 @@ def perform_backup(full_backup): "/usr/bin/duplicity", "full" if full_backup else "incr", "--archive-dir", backup_cache_dir, - "--exclude", backup_dir, + "--exclude", backup_root, "--volsize", "250", + "--gpg-options", "--cipher-algo=AES256", env["STORAGE_ROOT"], - "file://" + backup_encrypted_dir + "file://" + backup_dir ], env_with_passphrase) finally: @@ -217,6 +199,7 @@ def perform_backup(full_backup): shell('check_call', ["/usr/sbin/service", "dovecot", "start"]) shell('check_call', ["/usr/sbin/service", "postfix", "start"]) + # Once the migrated backup is included in a new backup, it can be deleted. if os.path.isdir(migrated_unencrypted_backup_dir): shutil.rmtree(migrated_unencrypted_backup_dir) @@ -228,7 +211,7 @@ def perform_backup(full_backup): "%dD" % keep_backups_for_days, "--archive-dir", backup_cache_dir, "--force", - "file://" + backup_encrypted_dir + "file://" + backup_dir ], env_with_passphrase) @@ -242,14 +225,18 @@ def perform_backup(full_backup): "cleanup", "--archive-dir", backup_cache_dir, "--force", - "file://" + backup_encrypted_dir + "file://" + backup_dir ], env_with_passphrase) + # Change ownership of backups to the user-data user, so that the after-bcakup + # script can access them. + shell('check_call', ["/bin/chown", "-R", env["STORAGE_USER"], backup_dir]) + # Execute a post-backup script that does the copying to a remote server. # Run as the STORAGE_USER user, not as root. Pass our settings in # environment variables so the script has access to STORAGE_ROOT. - post_script = os.path.join(backup_dir, 'after-backup') + post_script = os.path.join(backup_root, 'after-backup') if os.path.exists(post_script): shell('check_call', ['su', env['STORAGE_USER'], '-c', post_script], diff --git a/management/templates/system-backup.html b/management/templates/system-backup.html index fffe8680..ced1ec7f 100644 --- a/management/templates/system-backup.html +++ b/management/templates/system-backup.html @@ -28,8 +28,6 @@ -

The size column in the table indicates the size of the encrypted backup, but the total size on disk shown above includes storage for unencrypted intermediate files.

- - + +