re-do the backup script to use the duplicity program

Duplicity will manage the process of creating incremental backups for us.
Although duplicity can both encrypt & copy files to a remote host, I really
don't like PGP and so I don't want to use that.

Instead, we'll back up to a local directory unencrypted, then manually
encrypt the full & incremental backup files. Synchronizing the encrypted
backup directory to a remote host is a TODO.

management/backup.py

@@ -10,57 +10,100 @@
 # 4) The backup directory is compressed into a single file using tar.
 # 5) That file is encrypted with a long password stored in backup/secret_key.txt.
 
-import os, os.path, subprocess
+import sys, os, os.path, shutil
 
 from utils import exclusive_process, load_environment, shell
 
+# settings
+full_backup = "--full" in sys.argv
+keep_backups_for = "31D" # destroy backups older than 31 days
+
 env = load_environment()
 
 exclusive_process("backup")
 
 # Ensure the backup directory exists.
 backup_dir = os.path.join(env["STORAGE_ROOT"], 'backup')
-rdiff_backup_dir = os.path.join(backup_dir, 'rdiff-history')
+backup_duplicity_dir = os.path.join(backup_dir, 'duplicity')
 os.makedirs(backup_dir, exist_ok=True)
 
 # Stop services.
 shell('check_call', ["/usr/sbin/service", "dovecot", "stop"])
 shell('check_call', ["/usr/sbin/service", "postfix", "stop"])
 
-# Update the backup directory which stores increments.
+# Update the backup mirror directory which mirrors the current
+# STORAGE_ROOT (but excluding the backups themselves!).
 try:
 	shell('check_call', [
-		"/usr/bin/rdiff-backup",
+		"/usr/bin/duplicity",
+		"full" if full_backup else "incr",
+		"--no-encryption",
+		"--archive-dir", "/tmp/duplicity-archive-dir",
+		"--name", "mailinabox",
 		"--exclude", backup_dir,
+		"--verbosity", "warning",
 		env["STORAGE_ROOT"],
-	 	rdiff_backup_dir])
-except subprocess.CalledProcessError:
-	# Trap the error so we restart services again.
-	pass
+		"file://" + backup_duplicity_dir
+		])
+finally:
+	# Start services again.
+	shell('check_call', ["/usr/sbin/service", "dovecot", "start"])
+	shell('check_call', ["/usr/sbin/service", "postfix", "start"])
 
-# Start services.
-shell('check_call', ["/usr/sbin/service", "dovecot", "start"])
-shell('check_call', ["/usr/sbin/service", "postfix", "start"])
-
-# Tar the rdiff-backup directory into a single file.
+# Remove old backups. This deletes all backup data no longer needed
+# from more than 31 days ago. Must do this before destroying the
+# cache directory or else this command will re-create it.
 shell('check_call', [
-	"/bin/tar",
-	"-zc",
-	"-f", os.path.join(backup_dir, "latest.tgz"),
-	"-C", rdiff_backup_dir,
-	"."])
+	"/usr/bin/duplicity",
+	"remove-older-than",
+	keep_backups_for,
+	"--archive-dir", "/tmp/duplicity-archive-dir",
+	"--name", "mailinabox",
+	"--force",
+	"--verbosity", "warning",
+	"file://" + backup_duplicity_dir
+	])
 
-# Encrypt the backup using the backup private key.
+# Remove old increments. This deletes incremental data obsoleted by
+# any subsequent full backups.
 shell('check_call', [
+	"/usr/bin/duplicity",
+	"remove-all-inc-of-but-n-full",
+	"1",
+	"--archive-dir", "/tmp/duplicity-archive-dir",
+	"--name", "mailinabox",
+	"--force",
+	"--verbosity", "warning",
+	"file://" + backup_duplicity_dir
+	])
+
+# Remove duplicity's cache directory because it's redundant with our backup directory.
+shutil.rmtree("/tmp/duplicity-archive-dir")
+
+# Encrypt all of the new files.
+backup_encrypted_dir = os.path.join(backup_dir, 'encrypted')
+os.makedirs(backup_encrypted_dir, exist_ok=True)
+for fn in os.listdir(backup_duplicity_dir):
+	fn2 = os.path.join(backup_encrypted_dir, fn) + ".enc"
+	if os.path.exists(fn2): continue
+
+	# Encrypt the backup using the backup private key.
+	shell('check_call', [
 		"/usr/bin/openssl",
 		"enc",
 		"-aes-256-cbc",
 		"-a",
 		"-salt",
-	"-in", os.path.join(backup_dir, "latest.tgz"),
-	"-out", os.path.join(backup_dir, "latest.tgz.enc"),
+		"-in", os.path.join(backup_duplicity_dir, fn),
+		"-out", fn2,
 		"-pass", "file:%s" % os.path.join(backup_dir, "secret_key.txt"),
 		])
 
-# The backup can be decrypted with:
-# openssl enc -d -aes-256-cbc -a -in latest.tgz.enc -out /dev/stdout -pass file:secret_key.txt | tar -z
+	# The backup can be decrypted with:
+	# openssl enc -d -aes-256-cbc -a -in latest.tgz.enc -out /dev/stdout -pass file:secret_key.txt | tar -z
+
+# Remove encrypted backups that are no longer needed.
+for fn in os.listdir(backup_encrypted_dir):
+	fn2 = os.path.join(backup_duplicity_dir, fn.replace(".enc", ""))
+	if os.path.exists(fn2): continue
+	os.unlink(os.path.join(backup_encrypted_dir, fn))

setup/management.sh

@@ -2,7 +2,7 @@
 
 source setup/functions.sh
 
-apt_install python3-flask links rdiff-backup
+apt_install python3-flask links duplicity
 
 # Create a backup directory and a random key for encrypting backups.
 mkdir -p $STORAGE_ROOT/backup