external/mailinabox
1
0
Fork 0
mirror of https://github.com/mail-in-a-box/mailinabox.git synced 2026-03-09 16:37:23 +01:00
Code Issues Releases Wiki Activity

Support for rsync+ssh backup target (#678)

Browse Source 
* Added support for backup to a remote server using rsync

* updated web interface to get data from user
* added way to list files from server

It’s not using the “username” field of the yaml configuration
file to minimise the amount of patches needed. So the username
is actually sorted within the rsync URL.

Signed-off-by: Bernard `Guyzmo` Pratz <guyzmo+github@m0g.net>

* Added ssh key generation upon installation for root user.

Signed-off-by: Bernard `Guyzmo` Pratz <guyzmo+github@m0g.net>

* Removed stale blank lines, and fixed typo

Signed-off-by: Bernard `Guyzmo` Pratz <guyzmo+github@m0g.net>

* fix backup-location lines, by switching it from id to class

* Various web UI fixes

- fixed user field being shadowed ;
- fixed settings reading comparaison ;
- fixed forgotten min-age field.

Signed-off-by: Bernard `Guyzmo` Pratz <guyzmo+github@m0g.net>

* Added SSH Public Key shown on the web interface UI

Signed-off-by: Bernard `Guyzmo` Pratz <guyzmo+github@m0g.net>

* trailing spaces.

Signed-off-by: Bernard `Guyzmo` Pratz <guyzmo+github@m0g.net>

* fixed the extraneous environment

Signed-off-by: Bernard `Guyzmo` Pratz <guyzmo+github@m0g.net>

* Updated key setup

- made key lower in bits, but stronger (using -a option),
- made ssh-keygen run in background using nohup,
- added independent key file, as id_rsa_miab,
- added ssh-options to all duplicity calls to use the id_rsa_miab keyfile,
- changed path to the public key display

Signed-off-by: Bernard `Guyzmo` Pratz <guyzmo+github@m0g.net>

* added rsync options for ssh identity support

Signed-off-by: Bernard `Guyzmo` Pratz <guyzmo+github@m0g.net>

* removed strict host checking for all backup operations

Signed-off-by: Bernard `Guyzmo` Pratz <guyzmo+github@m0g.net>

* Remove nohup from ssh-keygen so errors aren't hidden. Also only generate a key if none exists yet

* Add trailing slash when checking a remote backup. Also check if we actually can read the remote size

* Factorisation of the repeated rsync/ssh options

cf https://github.com/mail-in-a-box/mailinabox/pull/678#discussion_r81478919

* Updated message SSH key creation

https://github.com/mail-in-a-box/mailinabox/pull/678#discussion_r81478886
This commit is contained in:
guyzmo
2016-11-12 15:28:55 +01:00
committed by Joshua Tauberer
parent 3b78a8d9d6
commit 041b5f883f
3 changed files with 131 additions and 28 deletions
Download Patch File Download Diff File Expand all files Collapse all files

69
management/backup.py
View File

@@ -30,6 +30,11 @@ def backup_status(env):
backups = { }
backup_cache_dir = os.path.join(backup_root, 'cache')
rsync_ssh_options = [
"--ssh-options='-i /root/.ssh/id_rsa_miab'",
"--rsync-options=-e \"/usr/bin/ssh -oStrictHostKeyChecking=no -oBatchMode=yes -p 22 -i /root/.ssh/id_rsa_miab\"",
]
def reldate(date, ref, clip):
if ref < date: return clip
rd = dateutil.relativedelta.relativedelta(ref, date)
@@ -52,6 +57,7 @@ def backup_status(env):
"size": 0, # collection-status doesn't give us the size
"volumes": keys[2], # number of archive volumes for this backup (not really helpful)
}
code, collection_status = shell('check_output', [
"/usr/bin/duplicity",
"collection-status",
@@ -59,7 +65,7 @@ def backup_status(env):
"--gpg-options", "--cipher-algo=AES256",
"--log-fd", "1",
config["target"],
],
] + rsync_ssh_options,
get_env(env),
trap=True)
if code != 0:
@@ -177,24 +183,24 @@ def get_passphrase(env):
with open(os.path.join(backup_root, 'secret_key.txt')) as f:
passphrase = f.readline().strip()
if len(passphrase) < 43: raise Exception("secret_key.txt's first line is too short!")
return passphrase
def get_env(env):
config = get_backup_config(env)
env = { "PASSPHRASE" : get_passphrase(env) }
if get_target_type(config) == 's3':
env["AWS_ACCESS_KEY_ID"] = config["target_user"]
env["AWS_SECRET_ACCESS_KEY"] = config["target_pass"]
return env
def get_target_type(config):
protocol = config["target"].split(":")[0]
return protocol
def perform_backup(full_backup):
env = load_environment()
@@ -204,7 +210,7 @@ def perform_backup(full_backup):
backup_cache_dir = os.path.join(backup_root, 'cache')
backup_dir = os.path.join(backup_root, 'encrypted')
# Are backups dissbled?
# Are backups disabled?
if config["target"] == "off":
return
@@ -283,7 +289,7 @@ def perform_backup(full_backup):
env["STORAGE_ROOT"],
config["target"],
"--allow-source-mismatch"
],
] + rsync_ssh_options,
get_env(env))
finally:
# Start services again.
@@ -305,7 +311,7 @@ def perform_backup(full_backup):
"--archive-dir", backup_cache_dir,
"--force",
config["target"]
],
] + rsync_ssh_options,
get_env(env))
# From duplicity's manual:
@@ -320,7 +326,7 @@ def perform_backup(full_backup):
"--archive-dir", backup_cache_dir,
"--force",
config["target"]
],
] + rsync_ssh_options,
get_env(env))
# Change ownership of backups to the user-data user, so that the after-bcakup
@@ -359,7 +365,7 @@ def run_duplicity_verification():
"--exclude", backup_root,
config["target"],
env["STORAGE_ROOT"],
], get_env(env))
] + rsync_ssh_options, get_env(env))
def run_duplicity_restore(args):
env = load_environment()
@@ -370,7 +376,7 @@ def run_duplicity_restore(args):
"restore",
"--archive-dir", backup_cache_dir,
config["target"],
] + args,
] + rsync_ssh_options + args,
get_env(env))
def list_target_files(config):
@@ -383,6 +389,34 @@ def list_target_files(config):
if p.scheme == "file":
return [(fn, os.path.getsize(os.path.join(p.path, fn))) for fn in os.listdir(p.path)]
elif p.scheme == "rsync":
rsync_fn_size_re = re.compile(r'.* ([^ ]*) [^ ]* [^ ]* (.*)')
rsync_target = '{host}:{path}'
_, target_host, target_path = config['target'].split('//')
target_path = '/' + target_path
if not target_path.endswith('/'):
target_path += '/'
rsync_command = [ 'rsync',
'-e',
'/usr/bin/ssh -i /root/.ssh/id_rsa_miab -oStrictHostKeyChecking=no -oBatchMode=yes',
'--list-only',
'-r',
rsync_target.format(
host=target_host,
path=target_path)
]
code, listing = shell('check_output', rsync_command, trap=True)
if code == 0:
for l in listing.split('\n'):
match = rsync_fn_size_re.match(l)
if match:
yield (match.groups()[1], int(match.groups()[0].replace(',','')))
else:
raise ValueError("Connection to rsync host failed")
elif p.scheme == "s3":
# match to a Region
fix_boto() # must call prior to importing boto
@@ -425,7 +459,7 @@ def list_target_files(config):
def backup_set_custom(env, target, target_user, target_pass, min_age):
config = get_backup_config(env, for_save=True)
# min_age must be an int
if isinstance(min_age, str):
min_age = int(min_age)
@@ -443,11 +477,11 @@ def backup_set_custom(env, target, target_user, target_pass, min_age):
list_target_files(config)
except ValueError as e:
return str(e)
write_backup_config(env, config)
return "OK"
def get_backup_config(env, for_save=False, for_ui=False):
backup_root = os.path.join(env["STORAGE_ROOT"], 'backup')
@@ -482,6 +516,9 @@ def get_backup_config(env, for_save=False, for_ui=False):
if config["target"] == "local":
# Expand to the full URL.
config["target"] = "file://" + config["file_target_directory"]
ssh_pub_key = os.path.join('/root', '.ssh', 'id_rsa_miab.pub')
if os.path.exists(ssh_pub_key):
config["ssh_pub_key"] = open(ssh_pub_key, 'r').read()
return config