switching between backup options in the admin wasn't working at all

* going from s3 to file target wasn't working
* use 'local' in the config instead of a file: url, for the local target, so it is not path-specific
* break out the S3 fields since users can't be expected to know how to form a URL
* use boto to generate a list of S3 hosts
* use boto to validate that the user input for s3 is valid
* fix lots of html errors in the backup admin

management/backup.py

@@ -297,6 +297,42 @@ def run_duplicity_verification():
 		env["STORAGE_ROOT"],
 	], get_env(env))
 
+def validate_target(config):
+	import urllib.parse
+	try:
+		p = urllib.parse.urlparse(config["target"])
+	except ValueError:
+		return "invalid target"
+
+	if p.scheme == "s3":
+		import boto.s3
+		from boto.exception import BotoServerError
+
+		# match to a Region
+		for region in boto.s3.regions():
+			if region.endpoint == p.hostname:
+				break
+		else:
+			raise ValueError("Invalid S3 region/host.")
+
+		bucket = p.path[1:].split('/')[0]
+		path = '/'.join(p.path[1:].split('/')[1:]) + '/'
+		if bucket == "":
+			raise ValueError("Enter an S3 bucket name.")
+
+		# connect to the region & bucket
+		try:
+			conn = region.connect(aws_access_key_id=config["target_user"], aws_secret_access_key=config["target_pass"])
+			bucket = conn.get_bucket(bucket)
+		except BotoServerError as e:
+			if e.status == 403:
+				raise ValueError("Invalid S3 access key or secret access key.")
+			elif e.status == 404:
+				raise ValueError("Invalid S3 bucket name.")
+			elif e.status == 301:
+				raise ValueError("Incorrect region for this bucket.")
+			raise ValueError(e.reason)
+
 
 def backup_set_custom(env, target, target_user, target_pass, min_age):
 	config = get_backup_config(env, for_save=True)
@@ -309,6 +345,15 @@ def backup_set_custom(env, target, target_user, target_pass, min_age):
 	config["target_user"] = target_user
 	config["target_pass"] = target_pass
 	config["min_age_in_days"] = min_age
+
+	# Validate.
+	try:
+		if config["target"] != "local":
+			# "local" isn't supported by the following function, which expects a full url in the target key,
+			# which is what is there except when loading the config prior to saving
+			validate_target(config)
+	except ValueError as e:
+		return str(e)
 	
 	write_backup_config(env, config)
 
@@ -320,7 +365,7 @@ def get_backup_config(env, for_save=False):
 	# Defaults.
 	config = {
 		"min_age_in_days": 3,
-		"target": "file://" + os.path.join(backup_root, 'encrypted'),
+		"target": "local",
 	}
 
 	# Merge in anything written to custom.yaml.
@@ -338,6 +383,9 @@ def get_backup_config(env, for_save=False):
 	# helper fields for the admin
 	config["file_target_directory"] = os.path.join(backup_root, 'encrypted')
 	config["enc_pw_file"] = os.path.join(backup_root, 'secret_key.txt')
+	if config["target"] == "local":
+		# Expand to the full URL.
+		config["target"] = "file://" + config["file_target_directory"]
 
 	return config
 

management/daemon.py

@@ -90,13 +90,19 @@ def json_response(data):
 def index():
 	# Render the control panel. This route does not require user authentication
 	# so it must be safe!
+
 	no_users_exist = (len(get_mail_users(env)) == 0)
 	no_admins_exist = (len(get_admins(env)) == 0)
+
+	import boto.s3
+	backup_s3_hosts = [(r.name, r.endpoint) for r in boto.s3.regions()]
+
 	return render_template('index.html',
 		hostname=env['PRIMARY_HOSTNAME'],
 		storage_root=env['STORAGE_ROOT'],
 		no_users_exist=no_users_exist,
 		no_admins_exist=no_admins_exist,
+		backup_s3_hosts=backup_s3_hosts,
 	)
 
 @app.route('/me')

management/templates/system-backup.html

@@ -11,15 +11,15 @@
 
 <form class="form-horizontal" role="form" onsubmit="set_custom_backup(); return false;">
   <div class="form-group">
-    <label for="target" class="col-sm-2 control-label">Backup target</label>
+    <label for="backup-target-type" class="col-sm-2 control-label">Backup to:</label>
     <div class="col-sm-2">
-      <select class="form-control" rows="1" id="target-type" onchange="toggle_form()">
+      <select class="form-control" rows="1" id="backup-target-type" onchange="toggle_form()">
-        <option value="file">{{hostname}}</option>
+        <option value="local">{{hostname}}</option>
         <option value="s3">Amazon S3</option>
       </select>
     </div>
   </div>
-  <div class="form-group backup-target-file">
+  <div class="form-group backup-target-local">
     <div class="col-sm-10 col-sm-offset-2">
       <div>Backups are stored on this machine&rsquo;s own hard disk. You are responsible for periodically using SFTP (FTP over SSH) to copy the backup files from <tt id="backup-location"></tt> to a safe location. These files are encrypted, so they are safe to store anywhere.</div>
     </div>
@@ -30,27 +30,37 @@
     </div>
   </div>
   <div class="form-group">
-    <label for="target" class="col-sm-2 control-label">How many days should backups be kept?</label>
+    <label for="min-age" class="col-sm-2 control-label">How many days should backups be kept?</label>
     <div class="col-sm-8">
-      <input type="number" class="form-control" rows="1" id="min-age"></input>
+      <input type="number" class="form-control" rows="1" id="min-age">
     </div>
   </div>
   <div class="form-group backup-target-s3">
-    <label for="target" class="col-sm-2 control-label">S3 URL</label>
+    <label for="backup-target-s3-host" class="col-sm-2 control-label">S3 Region</label>
     <div class="col-sm-8">
-      <input type="text" placeholder="s3://s3-eu-central-1.amazonaws.com/bucket-name" class="form-control" rows="1" id="target"></textarea>
+      <select class="form-control" rows="1" id="backup-target-s3-host">
+        {% for name, host in backup_s3_hosts %}
+          <option value="{{host}}">{{name}}</option>
+        {% endfor %}
+      </select>
     </div>
   </div>
   <div class="form-group backup-target-s3">
-    <label for="target-user" class="col-sm-2 control-label">S3 Access Key</label>
+    <label for="backup-target-s3-path" class="col-sm-2 control-label">S3 Path</label>
     <div class="col-sm-8">
-      <input type="text" class="form-control" rows="1" id="target-user"></input>
+      <input type="text" placeholder="your-bucket-name/backup-directory" class="form-control" rows="1" id="backup-target-s3-path">
     </div>
   </div>
   <div class="form-group backup-target-s3">
-    <label for="target-pass" class="col-sm-2 control-label">S3 Secret Access Key</label>
+    <label for="backup-target-user" class="col-sm-2 control-label">S3 Access Key</label>
     <div class="col-sm-8">
-      <input type="text" class="form-control" rows="1" id="target-pass"></input>
+      <input type="text" class="form-control" rows="1" id="backup-target-user">
+    </div>
+  </div>
+  <div class="form-group backup-target-s3">
+    <label for="backup-target-pass" class="col-sm-2 control-label">S3 Secret Access Key</label>
+    <div class="col-sm-8">
+      <input type="text" class="form-control" rows="1" id="backup-target-pass">
     </div>
   </div>
   <div class="form-group">
@@ -79,8 +89,8 @@
 <script>
 
 function toggle_form() {
-  var target_type = $("#target-type").val();
+  var target_type = $("#backup-target-type").val();
-  $(".backup-target-file, .backup-target-s3").hide();
+  $(".backup-target-local, .backup-target-s3").hide();
   $(".backup-target-" + target_type).show();
 }
 
@@ -139,17 +149,23 @@ function show_system_backup() {
 }
 
 function show_custom_backup() {
-    $(".backup-target-file, .backup-target-s3").hide();
+    $(".backup-target-local, .backup-target-s3").hide();
     api(
       "/system/backup/config",
       "GET",
       { },
       function(r) {
-        var target_type = r.target.split(':')[0]
+        if (r.target == "file://" + r.file_target_directory) {
-        $("#target").val(r.target);
+          $("#backup-target-type").val("local");
-        $("#target-type").val(target_type);
+        } else if (r.target.substring(0, 5) == "s3://") {
-        $("#target-user").val(r.target_user);
+          $("#backup-target-type").val("s3");
-        $("#target-pass").val(r.target_pass);
+	  var hostpath = r.target.substring(5).split('/');
+          var host = hostpath.shift();
+          $("#backup-target-s3-host").val(host);
+          $("#backup-target-s3-path").val(hostpath.join('/'));
+        }
+        $("#backup-target-user").val(r.target_user);
+        $("#backup-target-pass").val(r.target_pass);
         $("#min-age").val(r.min_age_in_days);
         $('#backup-location').text(r.file_target_directory);
         $('#backup-encpassword-file').text(r.enc_pw_file);
@@ -158,10 +174,16 @@ function show_custom_backup() {
 }
 
 function set_custom_backup() {
-  var target = $("#target").val();
+  var target_type = $("#backup-target-type").val();
-  var target_type = $("#target-type").val();
+  var target_user = $("#backup-target-user").val();
-  var target_user = $("#target-user").val();
+  var target_pass = $("#backup-target-pass").val();
-  var target_pass = $("#target-pass").val();
+  
+  var target;
+  if (target_type == "local")
+    target = target_type;
+  else if (target_type == "s3")
+    target = "s3://" + $("#backup-target-s3-host").val() + "/" + $("#backup-target-s3-path").val();
+
   var min_age = $("#min-age").val();
   api(
     "/system/backup/config",