From 5490142df5da780e9f072d365d6529ad32a7da77 Mon Sep 17 00:00:00 2001 From: Joshua Tauberer Date: Mon, 9 Jun 2014 09:34:52 -0400 Subject: [PATCH] re-do the backup script to use the duplicity program Duplicity will manage the process of creating incremental backups for us. Although duplicity can both encrypt & copy files to a remote host, I really don't like PGP and so I don't want to use that. Instead, we'll back up to a local directory unencrypted, then manually encrypt the full & incremental backup files. Synchronizing the encrypted backup directory to a remote host is a TODO. --- management/backup.py | 107 ++++++++++++++++++++++++++++++------------- setup/management.sh | 2 +- 2 files changed, 76 insertions(+), 33 deletions(-) diff --git a/management/backup.py b/management/backup.py index e1cd3cde..2553da35 100755 --- a/management/backup.py +++ b/management/backup.py @@ -10,57 +10,100 @@ # 4) The backup directory is compressed into a single file using tar. # 5) That file is encrypted with a long password stored in backup/secret_key.txt. -import os, os.path, subprocess +import sys, os, os.path, shutil from utils import exclusive_process, load_environment, shell +# settings +full_backup = "--full" in sys.argv +keep_backups_for = "31D" # destroy backups older than 31 days + env = load_environment() exclusive_process("backup") # Ensure the backup directory exists. backup_dir = os.path.join(env["STORAGE_ROOT"], 'backup') -rdiff_backup_dir = os.path.join(backup_dir, 'rdiff-history') +backup_duplicity_dir = os.path.join(backup_dir, 'duplicity') os.makedirs(backup_dir, exist_ok=True) # Stop services. shell('check_call', ["/usr/sbin/service", "dovecot", "stop"]) shell('check_call', ["/usr/sbin/service", "postfix", "stop"]) -# Update the backup directory which stores increments. +# Update the backup mirror directory which mirrors the current +# STORAGE_ROOT (but excluding the backups themselves!). try: shell('check_call', [ - "/usr/bin/rdiff-backup", + "/usr/bin/duplicity", + "full" if full_backup else "incr", + "--no-encryption", + "--archive-dir", "/tmp/duplicity-archive-dir", + "--name", "mailinabox", "--exclude", backup_dir, - env["STORAGE_ROOT"], - rdiff_backup_dir]) -except subprocess.CalledProcessError: - # Trap the error so we restart services again. - pass + "--verbosity", "warning", + env["STORAGE_ROOT"], + "file://" + backup_duplicity_dir + ]) +finally: + # Start services again. + shell('check_call', ["/usr/sbin/service", "dovecot", "start"]) + shell('check_call', ["/usr/sbin/service", "postfix", "start"]) -# Start services. -shell('check_call', ["/usr/sbin/service", "dovecot", "start"]) -shell('check_call', ["/usr/sbin/service", "postfix", "start"]) - -# Tar the rdiff-backup directory into a single file. +# Remove old backups. This deletes all backup data no longer needed +# from more than 31 days ago. Must do this before destroying the +# cache directory or else this command will re-create it. shell('check_call', [ - "/bin/tar", - "-zc", - "-f", os.path.join(backup_dir, "latest.tgz"), - "-C", rdiff_backup_dir, - "."]) - -# Encrypt the backup using the backup private key. -shell('check_call', [ - "/usr/bin/openssl", - "enc", - "-aes-256-cbc", - "-a", - "-salt", - "-in", os.path.join(backup_dir, "latest.tgz"), - "-out", os.path.join(backup_dir, "latest.tgz.enc"), - "-pass", "file:%s" % os.path.join(backup_dir, "secret_key.txt"), + "/usr/bin/duplicity", + "remove-older-than", + keep_backups_for, + "--archive-dir", "/tmp/duplicity-archive-dir", + "--name", "mailinabox", + "--force", + "--verbosity", "warning", + "file://" + backup_duplicity_dir ]) -# The backup can be decrypted with: -# openssl enc -d -aes-256-cbc -a -in latest.tgz.enc -out /dev/stdout -pass file:secret_key.txt | tar -z +# Remove old increments. This deletes incremental data obsoleted by +# any subsequent full backups. +shell('check_call', [ + "/usr/bin/duplicity", + "remove-all-inc-of-but-n-full", + "1", + "--archive-dir", "/tmp/duplicity-archive-dir", + "--name", "mailinabox", + "--force", + "--verbosity", "warning", + "file://" + backup_duplicity_dir + ]) + +# Remove duplicity's cache directory because it's redundant with our backup directory. +shutil.rmtree("/tmp/duplicity-archive-dir") + +# Encrypt all of the new files. +backup_encrypted_dir = os.path.join(backup_dir, 'encrypted') +os.makedirs(backup_encrypted_dir, exist_ok=True) +for fn in os.listdir(backup_duplicity_dir): + fn2 = os.path.join(backup_encrypted_dir, fn) + ".enc" + if os.path.exists(fn2): continue + + # Encrypt the backup using the backup private key. + shell('check_call', [ + "/usr/bin/openssl", + "enc", + "-aes-256-cbc", + "-a", + "-salt", + "-in", os.path.join(backup_duplicity_dir, fn), + "-out", fn2, + "-pass", "file:%s" % os.path.join(backup_dir, "secret_key.txt"), + ]) + + # The backup can be decrypted with: + # openssl enc -d -aes-256-cbc -a -in latest.tgz.enc -out /dev/stdout -pass file:secret_key.txt | tar -z + +# Remove encrypted backups that are no longer needed. +for fn in os.listdir(backup_encrypted_dir): + fn2 = os.path.join(backup_duplicity_dir, fn.replace(".enc", "")) + if os.path.exists(fn2): continue + os.unlink(os.path.join(backup_encrypted_dir, fn)) diff --git a/setup/management.sh b/setup/management.sh index 2196903f..d1db6d62 100755 --- a/setup/management.sh +++ b/setup/management.sh @@ -2,7 +2,7 @@ source setup/functions.sh -apt_install python3-flask links rdiff-backup +apt_install python3-flask links duplicity # Create a backup directory and a random key for encrypting backups. mkdir -p $STORAGE_ROOT/backup