Allow to backup on any S3-compatible service

Starting from the versions subsequent to the v57a, the default backend
of duplicity for backups to S3-compatible services is boto3
which requires the indication of the region (--S3-region-name)
for most cases in which the S3 service is not provided by AWS.
Thanks to this intervention, the region of the S3 service
is provided as a parameter to duplicity and is stored in the
/home/user-data/backup/custom.yaml file, like any other backup
configuration parameters.
Moreover, the list of the regions shown to the user in the drop-down
menu in the backup configuration panel is replaced with
a normal text field, since the list included the AWS regions only
and therefore was not usable in all other cases.

See issue #2200.

api/mailinabox.yml

@@ -468,6 +468,7 @@ paths:
                   target: s3://s3.eu-central-1.amazonaws.com/box-example-com
                   target_user: ACCESS_KEY
                   target_pass: SECRET_ACCESS_KEY
+                  target_region: eu-central-1
                   minAge: 3
               local:
                 summary: Local backup
@@ -475,6 +476,7 @@ paths:
                   target: local
                   target_user: ''
                   target_pass: ''
+                  target_region: ''
                   minAge: 3
               rsync:
                 summary: Rsync backup
@@ -482,6 +484,7 @@ paths:
                   target: rsync://username@box.example.com//backups/box.example.com
                   target_user: ''
                   target_pass: ''
+                  target_region: ''
                   minAge: 3
               off:
                 summary: Disable backups
@@ -489,6 +492,7 @@ paths:
                   target: 'off'
                   target_user: ''
                   target_pass: ''
+                  target_region: ''
                   minAge: 0
       x-codeSamples:
         - lang: curl
@@ -497,6 +501,7 @@ paths:
               -d "target=<hostname>" \
               -d "target_user=<string>" \
               -d "target_pass=<password>" \
+              -d "target_region=<region>" \
               -d "min_age=<integer>" \
               -u "<email>:<password>"
       responses:
@@ -2460,6 +2465,7 @@ components:
         - target
         - target_user
         - target_pass
+        - target_region
         - min_age
       properties:
         target:
@@ -2473,6 +2479,9 @@ components:
           type: string
           example: password
           format: password
+        target_region:
+          type: string
+          example: eu-central-1
         min_age:
           type: integer
           format: int32
@@ -2514,6 +2523,9 @@ components:
           type: string
         target_pass:
           type: string
+        target_region:
+          type: string
+          example: eu-central-1
       description: Backup config response.
     SystemBackupStatusResponse:
       type: object

management/backup.py

@@ -222,7 +222,8 @@ def get_duplicity_additional_args(env):
 		from urllib.parse import urlsplit, urlunsplit
 		target = urlsplit(config["target"])
 		endpoint_url = urlunsplit(("https", target.netloc, '', '', ''))
-		return ["--s3-endpoint-url",  endpoint_url]
+		region = config["target_region"]
+		return ["--s3-endpoint-url",  endpoint_url, "--s3-region-name", region]
 
 	return []
 
@@ -495,7 +496,7 @@ def list_target_files(config):
 		raise ValueError(config["target"])
 
 
-def backup_set_custom(env, target, target_user, target_pass, min_age):
+def backup_set_custom(env, target, target_user, target_pass, target_region, min_age):
 	config = get_backup_config(env, for_save=True)
 
 	# min_age must be an int
@@ -505,6 +506,7 @@ def backup_set_custom(env, target, target_user, target_pass, min_age):
 	config["target"] = target
 	config["target_user"] = target_user
 	config["target_pass"] = target_pass
+	config["target_region"] = target_region
 	config["min_age_in_days"] = min_age
 
 	# Validate.

management/daemon.py

@@ -121,10 +121,6 @@ def index():
 	no_users_exist = (len(get_mail_users(env)) == 0)
 	no_admins_exist = (len(get_admins(env)) == 0)
 
-	import boto3.s3
-	backup_s3_hosts = [(r, f"s3.{r}.amazonaws.com") for r in boto3.session.Session().get_available_regions('s3')]
-
-
 	return render_template('index.html',
 		hostname=env['PRIMARY_HOSTNAME'],
 		storage_root=env['STORAGE_ROOT'],
@@ -132,7 +128,6 @@ def index():
 		no_users_exist=no_users_exist,
 		no_admins_exist=no_admins_exist,
 
-		backup_s3_hosts=backup_s3_hosts,
 		csr_country_codes=csr_country_codes,
 	)
 
@@ -636,6 +631,7 @@ def backup_set_custom():
 		request.form.get('target', ''),
 		request.form.get('target_user', ''),
 		request.form.get('target_pass', ''),
+		request.form.get('target_region', ''),
 		request.form.get('min_age', '')
 	))
 

management/templates/system-backup.html

@@ -77,23 +77,18 @@
       <p>You MUST manually copy the encryption password from <tt class="backup-encpassword-file"></tt> to a safe and secure location. You will need this file to decrypt backup files. It is <b>NOT</b> stored in your S3 bucket.</p>
     </div>
   </div>
-  <div class="form-group backup-target-s3">
-    <label for="backup-target-s3-host-select" class="col-sm-2 control-label">S3 Region</label>
-    <div class="col-sm-8">
-      <select class="form-control" rows="1" id="backup-target-s3-host-select">
-        {% for name, host in backup_s3_hosts %}
-          <option value="{{host}}">{{name}}</option>
-        {% endfor %}
-        <option value="other">Other (non AWS)</option>
-      </select>
-    </div>
-  </div>
   <div class="form-group backup-target-s3">
     <label for="backup-target-s3-host" class="col-sm-2 control-label">S3 Host / Endpoint</label>
     <div class="col-sm-8">
       <input type="text" placeholder="Endpoint" class="form-control" rows="1" id="backup-target-s3-host">
     </div>
   </div>
+  <div class="form-group backup-target-s3">
+    <label for="backup-target-s3-region" class="col-sm-2 control-label">S3 Region</label>
+    <div class="col-sm-8">
+      <input type="text" placeholder="Region" class="form-control" rows="1" id="backup-target-s3-region">
+    </div>
+  </div>
   <div class="form-group backup-target-s3">
     <label for="backup-target-s3-path" class="col-sm-2 control-label">S3 Path</label>
     <div class="col-sm-8">
@@ -172,8 +167,6 @@ function toggle_form() {
   var target_type = $("#backup-target-type").val();
   $(".backup-target-local, .backup-target-rsync, .backup-target-s3, .backup-target-b2").hide();
   $(".backup-target-" + target_type).show();
-
-  init_inputs(target_type);
 }
 
 function nice_size(bytes) {
@@ -253,6 +246,7 @@ function show_custom_backup() {
         $(".backup-location").text(r.file_target_directory);
         $(".backup-encpassword-file").text(r.enc_pw_file);
         $("#ssh-pub-key").val(r.ssh_pub_key);
+        $("#backup-target-s3-region").val(r.target_region);
 
         if (r.target == "file://" + r.file_target_directory) {
           $("#backup-target-type").val("local");
@@ -269,7 +263,6 @@ function show_custom_backup() {
           $("#backup-target-type").val("s3");
           var hostpath = r.target.substring(5).split('/');
           var host = hostpath.shift();
-          $("#backup-target-s3-host-select").val(host);
           $("#backup-target-s3-host").val(host);
           $("#backup-target-s3-path").val(hostpath.join('/'));
         } else if (r.target.substring(0, 5) == "b2://") {
@@ -308,6 +301,8 @@ function set_custom_backup() {
   }
 
 
+  var target_region = $("#backup-target-s3-region").val();
+
   var min_age = $("#min-age").val();
   api(
     "/system/backup/config",
@@ -316,7 +311,8 @@ function set_custom_backup() {
       target: target,
       target_user: target_user,
       target_pass: target_pass,
-      min_age: min_age
+      min_age: min_age,
+      target_region: target_region
     },
     function(r) {
       // use .text() --- it's a text response, not html
@@ -328,20 +324,4 @@ function set_custom_backup() {
     });
   return false;
 }
-
-function init_inputs(target_type) {
-  function set_host(host) {
-    if(host !== 'other') {
-      $("#backup-target-s3-host").val(host);
-    } else {
-      $("#backup-target-s3-host").val('');
-    }
-  }
-  if (target_type == "s3") {
-    $('#backup-target-s3-host-select').off('change').on('change', function() {
-      set_host($('#backup-target-s3-host-select').val());
-    });
-    set_host($('#backup-target-s3-host-select').val());
-  }
-}
 </script>