From c034b0f7899c62e085894ba4ac961519181f0161 Mon Sep 17 00:00:00 2001 From: Darren Sanders Date: Tue, 29 Aug 2023 13:37:25 -0700 Subject: [PATCH 1/3] Fix how the value is being passed for the gpg-options parameter Duplicity v2.1.0 backups are failing with the error: "... --gpg-options expected one argument". The issue is that duplicity v2.1.0 began using the argparse Python library and the parse_known_args function. This function interprets the argument being passed, "--cipher-algo=AES256", as an argument name (because of the leading '-') and not as an argument value. Because of that it exits with an error and reports that the --gpg-options arg is missing its value. Adding an extra set of quotes around this string causes parse_known_args to interpret the string as an argument value. --- management/backup.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/management/backup.py b/management/backup.py index 06285ba5..b7102e5b 100755 --- a/management/backup.py +++ b/management/backup.py @@ -57,7 +57,7 @@ def backup_status(env): "/usr/bin/duplicity", "collection-status", "--archive-dir", backup_cache_dir, - "--gpg-options", "--cipher-algo=AES256", + "--gpg-options", "'--cipher-algo=AES256'", "--log-fd", "1", get_duplicity_target_url(config), ] + get_duplicity_additional_args(env), @@ -321,7 +321,7 @@ def perform_backup(full_backup): "--archive-dir", backup_cache_dir, "--exclude", backup_root, "--volsize", "250", - "--gpg-options", "--cipher-algo=AES256", + "--gpg-options", "'--cipher-algo=AES256'", env["STORAGE_ROOT"], get_duplicity_target_url(config), "--allow-source-mismatch" From 7be687e601e39a2dd7ceb78b0e2c3f630050e5d5 Mon Sep 17 00:00:00 2001 From: Jeff Volkenant Date: Wed, 30 Aug 2023 21:48:51 -0700 Subject: [PATCH 2/3] Move source and target positional arguments to the end, required for Duplicity 2.1.0 (Modified by JT.) --- management/backup.py | 30 ++++++++++++++++++------------ 1 file changed, 18 insertions(+), 12 deletions(-) diff --git a/management/backup.py b/management/backup.py index b7102e5b..9bb472b6 100755 --- a/management/backup.py +++ b/management/backup.py @@ -59,8 +59,9 @@ def backup_status(env): "--archive-dir", backup_cache_dir, "--gpg-options", "'--cipher-algo=AES256'", "--log-fd", "1", - get_duplicity_target_url(config), - ] + get_duplicity_additional_args(env), + ] + get_duplicity_additional_args(env) + [ + get_duplicity_target_url(config) + ], get_duplicity_env_vars(env), trap=True) if code != 0: @@ -227,8 +228,8 @@ def get_duplicity_additional_args(env): port = 22 return [ - f"--ssh-options= -i /root/.ssh/id_rsa_miab -p {port}", - f"--rsync-options= -e \"/usr/bin/ssh -oStrictHostKeyChecking=no -oBatchMode=yes -p {port} -i /root/.ssh/id_rsa_miab\"", + f"--ssh-options='-i /root/.ssh/id_rsa_miab -p {port}'", + f"--rsync-options='-e \"/usr/bin/ssh -oStrictHostKeyChecking=no -oBatchMode=yes -p {port} -i /root/.ssh/id_rsa_miab\"'", ] elif get_target_type(config) == 's3': # See note about hostname in get_duplicity_target_url. @@ -322,10 +323,11 @@ def perform_backup(full_backup): "--exclude", backup_root, "--volsize", "250", "--gpg-options", "'--cipher-algo=AES256'", + "--allow-source-mismatch" + ] + get_duplicity_additional_args(env) + [ env["STORAGE_ROOT"], get_duplicity_target_url(config), - "--allow-source-mismatch" - ] + get_duplicity_additional_args(env), + ], get_duplicity_env_vars(env)) finally: # Start services again. @@ -343,8 +345,9 @@ def perform_backup(full_backup): "--verbosity", "error", "--archive-dir", backup_cache_dir, "--force", + ] + get_duplicity_additional_args(env) + [ get_duplicity_target_url(config) - ] + get_duplicity_additional_args(env), + ], get_duplicity_env_vars(env)) # From duplicity's manual: @@ -358,8 +361,9 @@ def perform_backup(full_backup): "--verbosity", "error", "--archive-dir", backup_cache_dir, "--force", + ] + get_duplicity_additional_args(env) + [ get_duplicity_target_url(config) - ] + get_duplicity_additional_args(env), + ], get_duplicity_env_vars(env)) # Change ownership of backups to the user-data user, so that the after-bcakup @@ -396,9 +400,10 @@ def run_duplicity_verification(): "--compare-data", "--archive-dir", backup_cache_dir, "--exclude", backup_root, + ] + get_duplicity_additional_args(env) + [ get_duplicity_target_url(config), env["STORAGE_ROOT"], - ] + get_duplicity_additional_args(env), get_duplicity_env_vars(env)) + ], get_duplicity_env_vars(env)) def run_duplicity_restore(args): env = load_environment() @@ -408,9 +413,10 @@ def run_duplicity_restore(args): "/usr/bin/duplicity", "restore", "--archive-dir", backup_cache_dir, - get_duplicity_target_url(config), - ] + get_duplicity_additional_args(env) + args, - get_duplicity_env_vars(env)) + ] + get_duplicity_additional_args(env) + [ + get_duplicity_target_url(config) + ] + args, + get_duplicity_env_vars(env)) def list_target_files(config): import urllib.parse From 08defb12be94582513707c3901a6b939d005e0e0 Mon Sep 17 00:00:00 2001 From: Joshua Tauberer Date: Sat, 2 Sep 2023 07:49:41 -0400 Subject: [PATCH 3/3] Add a new backup.py command to print the duplicity command to the console to help debugging --- management/backup.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/management/backup.py b/management/backup.py index 9bb472b6..3ad68cc7 100755 --- a/management/backup.py +++ b/management/backup.py @@ -418,6 +418,19 @@ def run_duplicity_restore(args): ] + args, get_duplicity_env_vars(env)) +def print_duplicity_command(): + import shlex + env = load_environment() + config = get_backup_config(env) + backup_cache_dir = os.path.join(env["STORAGE_ROOT"], 'backup', 'cache') + for k, v in get_duplicity_env_vars(env).items(): + print(f"export {k}={shlex.quote(v)}") + print("duplicity", "{command}", shlex.join([ + "--archive-dir", backup_cache_dir, + ] + get_duplicity_additional_args(env) + [ + get_duplicity_target_url(config) + ])) + def list_target_files(config): import urllib.parse try: @@ -624,6 +637,9 @@ if __name__ == "__main__": # to duplicity. The restore path should be specified. run_duplicity_restore(sys.argv[2:]) + elif sys.argv[-1] == "--duplicity-command": + print_duplicity_command() + else: # Perform a backup. Add --full to force a full backup rather than # possibly performing an incremental backup.