mirror of
https://github.com/mail-in-a-box/mailinabox.git
synced 2026-03-04 15:54:48 +01:00
backups now use duplicity's built-in gpg symmetric encryption
Merge branch 'dhpiggott-gpg-encrypt-backups'
This commit is contained in:
Joshua Tauberer
@@ -2,11 +2,10 @@
|
||||
|
||||
# This script performs a backup of all user data:
|
||||
# 1) System services are stopped while a copy of user data is made.
|
||||
# 2) An incremental backup is made using duplicity into the
|
||||
# directory STORAGE_ROOT/backup/duplicity.
|
||||
# 2) An incremental encrypted backup is made using duplicity into the
|
||||
# directory STORAGE_ROOT/backup/encrypted. The password used for
|
||||
# encryption is stored in backup/secret_key.txt.
|
||||
# 3) The stopped services are restarted.
|
||||
# 4) The backup files are encrypted with a long password (stored in
|
||||
# backup/secret_key.txt) to STORAGE_ROOT/backup/encrypted.
|
||||
# 5) STORAGE_ROOT/backup/after-backup is executd if it exists.
|
||||
|
||||
import os, os.path, shutil, glob, re, datetime
|
||||
@@ -14,13 +13,13 @@ import dateutil.parser, dateutil.relativedelta, dateutil.tz
|
||||
|
||||
from utils import exclusive_process, load_environment, shell
|
||||
|
||||
# destroy backups when the most recent increment in the chain
|
||||
# Destroy backups when the most recent increment in the chain
|
||||
# that depends on it is this many days old.
|
||||
keep_backups_for_days = 3
|
||||
|
||||
def backup_status(env):
|
||||
# What is the current status of backups?
|
||||
# Loop through all of the files in STORAGE_ROOT/backup/duplicity to
|
||||
# Loop through all of the files in STORAGE_ROOT/backup/encrypted to
|
||||
# get a list of all of the backups taken and sum up file sizes to
|
||||
# see how large the storage is.
|
||||
|
||||
@@ -36,10 +35,10 @@ def backup_status(env):
|
||||
return "%d hours, %d minutes" % (rd.hours, rd.minutes)
|
||||
|
||||
backups = { }
|
||||
basedir = os.path.join(env['STORAGE_ROOT'], 'backup/duplicity/')
|
||||
encdir = os.path.join(env['STORAGE_ROOT'], 'backup/encrypted/')
|
||||
os.makedirs(basedir, exist_ok=True) # os.listdir fails if directory does not exist
|
||||
for fn in os.listdir(basedir):
|
||||
backup_root = os.path.join(env["STORAGE_ROOT"], 'backup')
|
||||
backup_dir = os.path.join(backup_root, 'encrypted')
|
||||
os.makedirs(backup_dir, exist_ok=True) # os.listdir fails if directory does not exist
|
||||
for fn in os.listdir(backup_dir):
|
||||
m = re.match(r"duplicity-(full|full-signatures|(inc|new-signatures)\.(?P<incbase>\d+T\d+Z)\.to)\.(?P<date>\d+T\d+Z)\.", fn)
|
||||
if not m: raise ValueError(fn)
|
||||
|
||||
@@ -53,15 +52,9 @@ def backup_status(env):
|
||||
"full": m.group("incbase") is None,
|
||||
"previous": m.group("incbase"),
|
||||
"size": 0,
|
||||
"encsize": 0,
|
||||
}
|
||||
|
||||
backups[key]["size"] += os.path.getsize(os.path.join(basedir, fn))
|
||||
|
||||
# Also check encrypted size.
|
||||
encfn = os.path.join(encdir, fn + ".enc")
|
||||
if os.path.exists(encfn):
|
||||
backups[key]["encsize"] += os.path.getsize(encfn)
|
||||
backups[key]["size"] += os.path.getsize(os.path.join(backup_dir, fn))
|
||||
|
||||
# Ensure the rows are sorted reverse chronologically.
|
||||
# This is relied on by should_force_full() and the next step.
|
||||
@@ -78,7 +71,7 @@ def backup_status(env):
|
||||
break
|
||||
incremental_count += 1
|
||||
incremental_size += bak["size"]
|
||||
|
||||
|
||||
# Predict how many more increments until the next full backup,
|
||||
# and add to that the time we hold onto backups, to predict
|
||||
# how long the most recent full backup+increments will be held
|
||||
@@ -106,9 +99,8 @@ def backup_status(env):
|
||||
bak["deleted_in"] = deleted_in
|
||||
|
||||
return {
|
||||
"directory": basedir,
|
||||
"encpwfile": os.path.join(env['STORAGE_ROOT'], 'backup/secret_key.txt'),
|
||||
"encdirectory": encdir,
|
||||
"directory": backup_dir,
|
||||
"encpwfile": os.path.join(backup_root, 'secret_key.txt'),
|
||||
"tz": now.tzname(),
|
||||
"backups": backups,
|
||||
}
|
||||
@@ -137,10 +129,35 @@ def perform_backup(full_backup):
|
||||
|
||||
exclusive_process("backup")
|
||||
|
||||
# Ensure the backup directory exists.
|
||||
backup_dir = os.path.join(env["STORAGE_ROOT"], 'backup')
|
||||
backup_duplicity_dir = os.path.join(backup_dir, 'duplicity')
|
||||
os.makedirs(backup_duplicity_dir, exist_ok=True)
|
||||
backup_root = os.path.join(env["STORAGE_ROOT"], 'backup')
|
||||
backup_cache_dir = os.path.join(backup_root, 'cache')
|
||||
backup_dir = os.path.join(backup_root, 'encrypted')
|
||||
|
||||
# In an older version of this script, duplicity was called
|
||||
# such that it did not encrypt the backups it created (in
|
||||
# backup/duplicity), and instead openssl was called separately
|
||||
# after each backup run, creating AES256 encrypted copies of
|
||||
# each file created by duplicity in backup/encrypted.
|
||||
#
|
||||
# We detect the transition by the presence of backup/duplicity
|
||||
# and handle it by 'dupliception': we move all the old *un*encrypted
|
||||
# duplicity files up out of the backup/duplicity directory (as
|
||||
# backup/ is excluded from duplicity runs) in order that it is
|
||||
# included in the next run, and we delete backup/encrypted (which
|
||||
# duplicity will output files directly to, post-transition).
|
||||
old_backup_dir = os.path.join(backup_root, 'duplicity')
|
||||
migrated_unencrypted_backup_dir = os.path.join(env["STORAGE_ROOT"], "migrated_unencrypted_backup")
|
||||
if os.path.isdir(old_backup_dir):
|
||||
# Move the old unencrpyted files to a new location outside of
|
||||
# the backup root so they get included in the next (new) backup.
|
||||
# Then we'll delete them. Also so that they do not get in the
|
||||
# way of duplicity doing a full backup on the first run after
|
||||
# we take care of this.
|
||||
shutil.move(old_backup_dir, migrated_unencrypted_backup_dir)
|
||||
|
||||
# The backup_dir (backup/encrypted) now has a new purpose.
|
||||
# Clear it out.
|
||||
shutil.rmtree(backup_dir)
|
||||
|
||||
# On the first run, always do a full backup. Incremental
|
||||
# will fail. Otherwise do a full backup when the size of
|
||||
@@ -152,75 +169,74 @@ def perform_backup(full_backup):
|
||||
shell('check_call', ["/usr/sbin/service", "dovecot", "stop"])
|
||||
shell('check_call', ["/usr/sbin/service", "postfix", "stop"])
|
||||
|
||||
# Get the encryption passphrase. secret_key.txt is 2048 random
|
||||
# bits base64-encoded and with line breaks every 65 characters.
|
||||
# gpg will only take the first line of text, so sanity check that
|
||||
# that line is long enough to be a reasonable passphrase. It
|
||||
# only needs to be 43 base64-characters to match AES256's key
|
||||
# length of 32 bytes.
|
||||
with open(os.path.join(backup_root, 'secret_key.txt')) as f:
|
||||
passphrase = f.readline().strip()
|
||||
if len(passphrase) < 43: raise Exception("secret_key.txt's first line is too short!")
|
||||
env_with_passphrase = { "PASSPHRASE" : passphrase }
|
||||
|
||||
# Update the backup mirror directory which mirrors the current
|
||||
# STORAGE_ROOT (but excluding the backups themselves!).
|
||||
try:
|
||||
shell('check_call', [
|
||||
"/usr/bin/duplicity",
|
||||
"full" if full_backup else "incr",
|
||||
"--no-encryption",
|
||||
"--archive-dir", "/tmp/duplicity-archive-dir",
|
||||
"--name", "mailinabox",
|
||||
"--exclude", backup_dir,
|
||||
"--volsize", "100",
|
||||
"--verbosity", "warning",
|
||||
"--archive-dir", backup_cache_dir,
|
||||
"--exclude", backup_root,
|
||||
"--volsize", "250",
|
||||
"--gpg-options", "--cipher-algo=AES256",
|
||||
env["STORAGE_ROOT"],
|
||||
"file://" + backup_duplicity_dir
|
||||
])
|
||||
"file://" + backup_dir
|
||||
],
|
||||
env_with_passphrase)
|
||||
finally:
|
||||
# Start services again.
|
||||
shell('check_call', ["/usr/sbin/service", "dovecot", "start"])
|
||||
shell('check_call', ["/usr/sbin/service", "postfix", "start"])
|
||||
|
||||
# Once the migrated backup is included in a new backup, it can be deleted.
|
||||
if os.path.isdir(migrated_unencrypted_backup_dir):
|
||||
shutil.rmtree(migrated_unencrypted_backup_dir)
|
||||
|
||||
# Remove old backups. This deletes all backup data no longer needed
|
||||
# from more than 31 days ago. Must do this before destroying the
|
||||
# cache directory or else this command will re-create it.
|
||||
# from more than 3 days ago.
|
||||
shell('check_call', [
|
||||
"/usr/bin/duplicity",
|
||||
"remove-older-than",
|
||||
"%dD" % keep_backups_for_days,
|
||||
"--archive-dir", "/tmp/duplicity-archive-dir",
|
||||
"--name", "mailinabox",
|
||||
"--archive-dir", backup_cache_dir,
|
||||
"--force",
|
||||
"--verbosity", "warning",
|
||||
"file://" + backup_duplicity_dir
|
||||
])
|
||||
"file://" + backup_dir
|
||||
],
|
||||
env_with_passphrase)
|
||||
|
||||
# Remove duplicity's cache directory because it's redundant with our backup directory.
|
||||
shutil.rmtree("/tmp/duplicity-archive-dir")
|
||||
# From duplicity's manual:
|
||||
# "This should only be necessary after a duplicity session fails or is
|
||||
# aborted prematurely."
|
||||
# That may be unlikely here but we may as well ensure we tidy up if
|
||||
# that does happen - it might just have been a poorly timed reboot.
|
||||
shell('check_call', [
|
||||
"/usr/bin/duplicity",
|
||||
"cleanup",
|
||||
"--archive-dir", backup_cache_dir,
|
||||
"--force",
|
||||
"file://" + backup_dir
|
||||
],
|
||||
env_with_passphrase)
|
||||
|
||||
# Encrypt all of the new files.
|
||||
backup_encrypted_dir = os.path.join(backup_dir, 'encrypted')
|
||||
os.makedirs(backup_encrypted_dir, exist_ok=True)
|
||||
for fn in os.listdir(backup_duplicity_dir):
|
||||
fn2 = os.path.join(backup_encrypted_dir, fn) + ".enc"
|
||||
if os.path.exists(fn2): continue
|
||||
|
||||
# Encrypt the backup using the backup private key.
|
||||
shell('check_call', [
|
||||
"/usr/bin/openssl",
|
||||
"enc",
|
||||
"-aes-256-cbc",
|
||||
"-a",
|
||||
"-salt",
|
||||
"-in", os.path.join(backup_duplicity_dir, fn),
|
||||
"-out", fn2,
|
||||
"-pass", "file:%s" % os.path.join(backup_dir, "secret_key.txt"),
|
||||
])
|
||||
|
||||
# The backup can be decrypted with:
|
||||
# openssl enc -d -aes-256-cbc -a -in latest.tgz.enc -out /dev/stdout -pass file:secret_key.txt | tar -z
|
||||
|
||||
# Remove encrypted backups that are no longer needed.
|
||||
for fn in os.listdir(backup_encrypted_dir):
|
||||
fn2 = os.path.join(backup_duplicity_dir, fn.replace(".enc", ""))
|
||||
if os.path.exists(fn2): continue
|
||||
os.unlink(os.path.join(backup_encrypted_dir, fn))
|
||||
# Change ownership of backups to the user-data user, so that the after-bcakup
|
||||
# script can access them.
|
||||
shell('check_call', ["/bin/chown", "-R", env["STORAGE_USER"], backup_dir])
|
||||
|
||||
# Execute a post-backup script that does the copying to a remote server.
|
||||
# Run as the STORAGE_USER user, not as root. Pass our settings in
|
||||
# environment variables so the script has access to STORAGE_ROOT.
|
||||
post_script = os.path.join(backup_dir, 'after-backup')
|
||||
post_script = os.path.join(backup_root, 'after-backup')
|
||||
if os.path.exists(post_script):
|
||||
shell('check_call',
|
||||
['su', env['STORAGE_USER'], '-c', post_script],
|
||||
|
||||
@@ -28,8 +28,6 @@
|
||||
</tbody>
|
||||
</table>
|
||||
|
||||
<p style="margin-top: 2em"><small>The size column in the table indicates the size of the encrypted backup, but the total size on disk shown above includes storage for unencrypted intermediate files.</small></p>
|
||||
|
||||
<script>
|
||||
function nice_size(bytes) {
|
||||
var powers = ['bytes', 'KB', 'MB', 'GB', 'TB'];
|
||||
@@ -54,7 +52,7 @@ function show_system_backup() {
|
||||
"GET",
|
||||
{ },
|
||||
function(r) {
|
||||
$('#backup-location').text(r.encdirectory);
|
||||
$('#backup-location').text(r.directory);
|
||||
$('#backup-encpassword-file').text(r.encpwfile);
|
||||
|
||||
$('#backup-status tbody').html("");
|
||||
@@ -72,7 +70,7 @@ function show_system_backup() {
|
||||
tr.append( $('<td/>').text(b.date_str + " " + r.tz) );
|
||||
tr.append( $('<td/>').text(b.date_delta + " ago") );
|
||||
tr.append( $('<td/>').text(b.full ? "full" : "increment") );
|
||||
tr.append( $('<td style="text-align: right"/>').text( nice_size(b.encsize)) );
|
||||
tr.append( $('<td style="text-align: right"/>').text( nice_size(b.size)) );
|
||||
if (b.deleted_in)
|
||||
tr.append( $('<td/>').text(b.deleted_in) );
|
||||
else
|
||||
@@ -80,7 +78,6 @@ function show_system_backup() {
|
||||
$('#backup-status tbody').append(tr);
|
||||
|
||||
total_disk_size += b.size;
|
||||
total_disk_size += b.encsize;
|
||||
}
|
||||
|
||||
$('#backup-total-size').text(nice_size(total_disk_size));
|
||||
|
||||
Reference in New Issue
Block a user