diff --git a/CHANGELOG.md b/CHANGELOG.md index 5b25747f..4c360c62 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,12 +8,31 @@ Mail: * Spam checking is now performed on messages larger than the previous limit of 64KB. * POP3S is now enabled (port 995). -* In order to guard against misconfiguration that can lead to domain control validation hijacking, email addresses that begin with admin, administrator, postmaster, hostmaster, and webmaster can no longer be used for (new) mail user accounts, and aliases for these addresses may direct mail only to the box's administrator(s). +* Roundcube updated to version 1.1.1. +* More mail headers with user agent info are anonymized. -System: +ownCloud: + +* Downloading files you uploaded to ownCloud broke because of a change in ownCloud 8. + +DNS: * Internationalized Domain Names (IDNs) should now work in email. If you had custom DNS or custom web settings for internationalized domains, check that they are still working. +* It is now possible to set multiple TXT and other types of records on the same domain in the control panel. +* The custom DNS API was completely rewritten to support setting multiple records of the same type on a domain. Any existing client code using the DNS API will have to be rewritten. (Existing code will just get 404s back.) +System / Control Panel: + +* In order to guard against misconfiguration that can lead to domain control validation hijacking, email addresses that begin with admin, administrator, postmaster, hostmaster, and webmaster can no longer be used for (new) mail user accounts, and aliases for these addresses may direct mail only to the box's administrator(s). +* Backups now use duplicity's built-in gpg symmetric AES256 encryption rather than my home-brewed encryption. Old backups will be incorporated inside the first backup after this update but then deleted from disk (i.e. your backups from the previous few days will be backed up). +* There was a race condition between backups and the new nightly status checks. +* The control panel would sometimes lock up with an unnecessary loading indicator. +* You can no longer delete your own account from the control panel. + +Setup: + +* All Mail-in-a-Box release tags are now signed on github, instructions for verifying the signature are added to the README, and the integrity of some packages downloaded during setup is now verified against a SHA1 hash stored in the tag itself. +* Bugs in first user account creation were fixed. v0.08 (April 1, 2015) --------------------- diff --git a/conf/nginx-primaryonly.conf b/conf/nginx-primaryonly.conf index b87e45f0..2ad5d7d3 100644 --- a/conf/nginx-primaryonly.conf +++ b/conf/nginx-primaryonly.conf @@ -28,6 +28,7 @@ fastcgi_param SCRIPT_FILENAME /usr/local/lib/owncloud/$2; fastcgi_param SCRIPT_NAME $1$2; fastcgi_param PATH_INFO $3; + # TODO: see the dispreferred "method 2" for xaccel at https://doc.owncloud.org/server/8.1/admin_manual/configuration_files/serving_static_files_configuration.html fastcgi_param MOD_X_ACCEL_REDIRECT_ENABLED on; fastcgi_read_timeout 630; fastcgi_pass php-fpm; @@ -36,10 +37,13 @@ client_max_body_size 1G; fastcgi_buffers 64 4K; } - location ^~ /cloud/data { - # In order to support MOD_X_ACCEL_REDIRECT_ENABLED, we need to expose - # the data directory but only allow 'internal' redirects within nginx - # so that this is not exposed to the world. + location ^~ $STORAGE_ROOT/owncloud { + # This directory is for MOD_X_ACCEL_REDIRECT_ENABLED. It works a little weird. + # The full path on disk of the file is passed as the URL path. ownCloud 8 totally + # busted the sane way this worked in ownCloud 7. There's a pending change using + # a new parameter to make this make more sense. + # We need to only allow 'internal' redirects within nginx so that the filesystem + # is not exposed to the world. internal; alias $STORAGE_ROOT/owncloud; } diff --git a/conf/postfix_outgoing_mail_header_filters b/conf/postfix_outgoing_mail_header_filters index 4848c014..a15d4afa 100644 --- a/conf/postfix_outgoing_mail_header_filters +++ b/conf/postfix_outgoing_mail_header_filters @@ -8,3 +8,7 @@ /^\s*X-Enigmail:/ IGNORE /^\s*X-Mailer:/ IGNORE /^\s*X-Originating-IP:/ IGNORE +/^\s*X-Pgp-Agent:/ IGNORE + +# The Mime-Version header can leak the user agent too, e.g. in Mime-Version: 1.0 (Mac OS X Mail 8.1 \(2010.6\)). +/^\s*(Mime-Version:\s*[0-9\.]+)\s.+/ REPLACE $1 diff --git a/management/auth.py b/management/auth.py index 1ae46d1e..04e605c3 100644 --- a/management/auth.py +++ b/management/auth.py @@ -111,12 +111,12 @@ class KeyAuthService: # Login failed. raise ValueError("Invalid password.") - # Get privileges for authorization. - - # (This call should never fail on a valid user. But if it did fail, it would - # return a tuple of an error message and an HTTP status code.) + # Get privileges for authorization. This call should never fail on a valid user, + # but if the caller passed a user-specific API key then the user may no longer + # exist --- in that case, get_mail_user_privileges will return a tuple of an + # error message and an HTTP status code. privs = get_mail_user_privileges(email, env) - if isinstance(privs, tuple): raise Exception("Error getting privileges.") + if isinstance(privs, tuple): raise ValueError(privs[0]) # Return a list of privileges. return privs diff --git a/management/backup.py b/management/backup.py index fdcc5248..e0349dd6 100755 --- a/management/backup.py +++ b/management/backup.py @@ -2,25 +2,24 @@ # This script performs a backup of all user data: # 1) System services are stopped while a copy of user data is made. -# 2) An incremental backup is made using duplicity into the -# directory STORAGE_ROOT/backup/duplicity. +# 2) An incremental encrypted backup is made using duplicity into the +# directory STORAGE_ROOT/backup/encrypted. The password used for +# encryption is stored in backup/secret_key.txt. # 3) The stopped services are restarted. -# 4) The backup files are encrypted with a long password (stored in -# backup/secret_key.txt) to STORAGE_ROOT/backup/encrypted. # 5) STORAGE_ROOT/backup/after-backup is executd if it exists. import os, os.path, shutil, glob, re, datetime import dateutil.parser, dateutil.relativedelta, dateutil.tz -from utils import exclusive_process, load_environment, shell +from utils import exclusive_process, load_environment, shell, wait_for_service -# destroy backups when the most recent increment in the chain +# Destroy backups when the most recent increment in the chain # that depends on it is this many days old. keep_backups_for_days = 3 def backup_status(env): # What is the current status of backups? - # Loop through all of the files in STORAGE_ROOT/backup/duplicity to + # Loop through all of the files in STORAGE_ROOT/backup/encrypted to # get a list of all of the backups taken and sum up file sizes to # see how large the storage is. @@ -36,10 +35,10 @@ def backup_status(env): return "%d hours, %d minutes" % (rd.hours, rd.minutes) backups = { } - basedir = os.path.join(env['STORAGE_ROOT'], 'backup/duplicity/') - encdir = os.path.join(env['STORAGE_ROOT'], 'backup/encrypted/') - os.makedirs(basedir, exist_ok=True) # os.listdir fails if directory does not exist - for fn in os.listdir(basedir): + backup_root = os.path.join(env["STORAGE_ROOT"], 'backup') + backup_dir = os.path.join(backup_root, 'encrypted') + os.makedirs(backup_dir, exist_ok=True) # os.listdir fails if directory does not exist + for fn in os.listdir(backup_dir): m = re.match(r"duplicity-(full|full-signatures|(inc|new-signatures)\.(?P\d+T\d+Z)\.to)\.(?P\d+T\d+Z)\.", fn) if not m: raise ValueError(fn) @@ -53,15 +52,9 @@ def backup_status(env): "full": m.group("incbase") is None, "previous": m.group("incbase"), "size": 0, - "encsize": 0, } - backups[key]["size"] += os.path.getsize(os.path.join(basedir, fn)) - - # Also check encrypted size. - encfn = os.path.join(encdir, fn + ".enc") - if os.path.exists(encfn): - backups[key]["encsize"] += os.path.getsize(encfn) + backups[key]["size"] += os.path.getsize(os.path.join(backup_dir, fn)) # Ensure the rows are sorted reverse chronologically. # This is relied on by should_force_full() and the next step. @@ -78,7 +71,7 @@ def backup_status(env): break incremental_count += 1 incremental_size += bak["size"] - + # Predict how many more increments until the next full backup, # and add to that the time we hold onto backups, to predict # how long the most recent full backup+increments will be held @@ -106,9 +99,8 @@ def backup_status(env): bak["deleted_in"] = deleted_in return { - "directory": basedir, - "encpwfile": os.path.join(env['STORAGE_ROOT'], 'backup/secret_key.txt'), - "encdirectory": encdir, + "directory": backup_dir, + "encpwfile": os.path.join(backup_root, 'secret_key.txt'), "tz": now.tzname(), "backups": backups, } @@ -137,10 +129,35 @@ def perform_backup(full_backup): exclusive_process("backup") - # Ensure the backup directory exists. - backup_dir = os.path.join(env["STORAGE_ROOT"], 'backup') - backup_duplicity_dir = os.path.join(backup_dir, 'duplicity') - os.makedirs(backup_duplicity_dir, exist_ok=True) + backup_root = os.path.join(env["STORAGE_ROOT"], 'backup') + backup_cache_dir = os.path.join(backup_root, 'cache') + backup_dir = os.path.join(backup_root, 'encrypted') + + # In an older version of this script, duplicity was called + # such that it did not encrypt the backups it created (in + # backup/duplicity), and instead openssl was called separately + # after each backup run, creating AES256 encrypted copies of + # each file created by duplicity in backup/encrypted. + # + # We detect the transition by the presence of backup/duplicity + # and handle it by 'dupliception': we move all the old *un*encrypted + # duplicity files up out of the backup/duplicity directory (as + # backup/ is excluded from duplicity runs) in order that it is + # included in the next run, and we delete backup/encrypted (which + # duplicity will output files directly to, post-transition). + old_backup_dir = os.path.join(backup_root, 'duplicity') + migrated_unencrypted_backup_dir = os.path.join(env["STORAGE_ROOT"], "migrated_unencrypted_backup") + if os.path.isdir(old_backup_dir): + # Move the old unencrpyted files to a new location outside of + # the backup root so they get included in the next (new) backup. + # Then we'll delete them. Also so that they do not get in the + # way of duplicity doing a full backup on the first run after + # we take care of this. + shutil.move(old_backup_dir, migrated_unencrypted_backup_dir) + + # The backup_dir (backup/encrypted) now has a new purpose. + # Clear it out. + shutil.rmtree(backup_dir) # On the first run, always do a full backup. Incremental # will fail. Otherwise do a full backup when the size of @@ -152,81 +169,111 @@ def perform_backup(full_backup): shell('check_call', ["/usr/sbin/service", "dovecot", "stop"]) shell('check_call', ["/usr/sbin/service", "postfix", "stop"]) + # Get the encryption passphrase. secret_key.txt is 2048 random + # bits base64-encoded and with line breaks every 65 characters. + # gpg will only take the first line of text, so sanity check that + # that line is long enough to be a reasonable passphrase. It + # only needs to be 43 base64-characters to match AES256's key + # length of 32 bytes. + with open(os.path.join(backup_root, 'secret_key.txt')) as f: + passphrase = f.readline().strip() + if len(passphrase) < 43: raise Exception("secret_key.txt's first line is too short!") + env_with_passphrase = { "PASSPHRASE" : passphrase } + # Update the backup mirror directory which mirrors the current # STORAGE_ROOT (but excluding the backups themselves!). try: shell('check_call', [ "/usr/bin/duplicity", "full" if full_backup else "incr", - "--no-encryption", - "--archive-dir", "/tmp/duplicity-archive-dir", - "--name", "mailinabox", - "--exclude", backup_dir, - "--volsize", "100", - "--verbosity", "warning", + "--archive-dir", backup_cache_dir, + "--exclude", backup_root, + "--volsize", "250", + "--gpg-options", "--cipher-algo=AES256", env["STORAGE_ROOT"], - "file://" + backup_duplicity_dir - ]) + "file://" + backup_dir + ], + env_with_passphrase) finally: # Start services again. shell('check_call', ["/usr/sbin/service", "dovecot", "start"]) shell('check_call', ["/usr/sbin/service", "postfix", "start"]) + # Once the migrated backup is included in a new backup, it can be deleted. + if os.path.isdir(migrated_unencrypted_backup_dir): + shutil.rmtree(migrated_unencrypted_backup_dir) + # Remove old backups. This deletes all backup data no longer needed - # from more than 31 days ago. Must do this before destroying the - # cache directory or else this command will re-create it. + # from more than 3 days ago. shell('check_call', [ "/usr/bin/duplicity", "remove-older-than", "%dD" % keep_backups_for_days, - "--archive-dir", "/tmp/duplicity-archive-dir", - "--name", "mailinabox", + "--archive-dir", backup_cache_dir, "--force", - "--verbosity", "warning", - "file://" + backup_duplicity_dir - ]) + "file://" + backup_dir + ], + env_with_passphrase) - # Remove duplicity's cache directory because it's redundant with our backup directory. - shutil.rmtree("/tmp/duplicity-archive-dir") + # From duplicity's manual: + # "This should only be necessary after a duplicity session fails or is + # aborted prematurely." + # That may be unlikely here but we may as well ensure we tidy up if + # that does happen - it might just have been a poorly timed reboot. + shell('check_call', [ + "/usr/bin/duplicity", + "cleanup", + "--archive-dir", backup_cache_dir, + "--force", + "file://" + backup_dir + ], + env_with_passphrase) - # Encrypt all of the new files. - backup_encrypted_dir = os.path.join(backup_dir, 'encrypted') - os.makedirs(backup_encrypted_dir, exist_ok=True) - for fn in os.listdir(backup_duplicity_dir): - fn2 = os.path.join(backup_encrypted_dir, fn) + ".enc" - if os.path.exists(fn2): continue - - # Encrypt the backup using the backup private key. - shell('check_call', [ - "/usr/bin/openssl", - "enc", - "-aes-256-cbc", - "-a", - "-salt", - "-in", os.path.join(backup_duplicity_dir, fn), - "-out", fn2, - "-pass", "file:%s" % os.path.join(backup_dir, "secret_key.txt"), - ]) - - # The backup can be decrypted with: - # openssl enc -d -aes-256-cbc -a -in latest.tgz.enc -out /dev/stdout -pass file:secret_key.txt | tar -z - - # Remove encrypted backups that are no longer needed. - for fn in os.listdir(backup_encrypted_dir): - fn2 = os.path.join(backup_duplicity_dir, fn.replace(".enc", "")) - if os.path.exists(fn2): continue - os.unlink(os.path.join(backup_encrypted_dir, fn)) + # Change ownership of backups to the user-data user, so that the after-bcakup + # script can access them. + shell('check_call', ["/bin/chown", "-R", env["STORAGE_USER"], backup_dir]) # Execute a post-backup script that does the copying to a remote server. # Run as the STORAGE_USER user, not as root. Pass our settings in # environment variables so the script has access to STORAGE_ROOT. - post_script = os.path.join(backup_dir, 'after-backup') + post_script = os.path.join(backup_root, 'after-backup') if os.path.exists(post_script): shell('check_call', ['su', env['STORAGE_USER'], '-c', post_script], env=env) + # Our nightly cron job executes system status checks immediately after this + # backup. Since it checks that dovecot and postfix are running, block for a + # bit (maximum of 10 seconds each) to give each a chance to finish restarting + # before the status checks might catch them down. See #381. + wait_for_service(25, True, env, 10) + wait_for_service(993, True, env, 10) + +def run_duplicity_verification(): + env = load_environment() + backup_root = os.path.join(env["STORAGE_ROOT"], 'backup') + backup_cache_dir = os.path.join(backup_root, 'cache') + backup_dir = os.path.join(backup_root, 'encrypted') + env_with_passphrase = { "PASSPHRASE" : open(os.path.join(backup_root, 'secret_key.txt')).read() } + shell('check_call', [ + "/usr/bin/duplicity", + "--verbosity", "info", + "verify", + "--compare-data", + "--archive-dir", backup_cache_dir, + "--exclude", backup_root, + "file://" + backup_dir, + env["STORAGE_ROOT"], + ], env_with_passphrase) + if __name__ == "__main__": import sys - full_backup = "--full" in sys.argv - perform_backup(full_backup) + if sys.argv[-1] == "--verify": + # Run duplicity's verification command to check a) the backup files + # are readable, and b) report if they are up to date. + run_duplicity_verification() + else: + # Perform a backup. Add --full to force a full backup rather than + # possibly performing an incremental backup. + full_backup = "--full" in sys.argv + perform_backup(full_backup) diff --git a/management/daemon.py b/management/daemon.py index 3f179fc8..71159672 100755 --- a/management/daemon.py +++ b/management/daemon.py @@ -80,7 +80,7 @@ def unauthorized(error): return auth_service.make_unauthorized_response() def json_response(data): - return Response(json.dumps(data), status=200, mimetype='application/json') + return Response(json.dumps(data, indent=2, sort_keys=True)+'\n', status=200, mimetype='application/json') ################################### @@ -90,10 +90,12 @@ def json_response(data): def index(): # Render the control panel. This route does not require user authentication # so it must be safe! + no_users_exist = (len(get_mail_users(env)) == 0) no_admins_exist = (len(get_admins(env)) == 0) return render_template('index.html', hostname=env['PRIMARY_HOSTNAME'], storage_root=env['STORAGE_ROOT'], + no_users_exist=no_users_exist, no_admins_exist=no_admins_exist, ) @@ -219,8 +221,8 @@ def dns_update(): @app.route('/dns/secondary-nameserver') @authorized_personnel_only def dns_get_secondary_nameserver(): - from dns_update import get_custom_dns_config - return json_response({ "hostname": get_custom_dns_config(env).get("_secondary_nameserver") }) + from dns_update import get_custom_dns_config, get_secondary_dns + return json_response({ "hostname": get_secondary_dns(get_custom_dns_config(env)) }) @app.route('/dns/secondary-nameserver', methods=['POST']) @authorized_personnel_only @@ -231,38 +233,70 @@ def dns_set_secondary_nameserver(): except ValueError as e: return (str(e), 400) -@app.route('/dns/set') +@app.route('/dns/custom') @authorized_personnel_only -def dns_get_records(): - from dns_update import get_custom_dns_config, get_custom_records - additional_records = get_custom_dns_config(env) - records = get_custom_records(None, additional_records, env) - return json_response([{ +def dns_get_records(qname=None, rtype=None): + from dns_update import get_custom_dns_config + return json_response([ + { "qname": r[0], "rtype": r[1], "value": r[2], - } for r in records]) + } + for r in get_custom_dns_config(env) + if r[0] != "_secondary_nameserver" + and (not qname or r[0] == qname) + and (not rtype or r[1] == rtype) ]) -@app.route('/dns/set/', methods=['POST']) -@app.route('/dns/set//', methods=['POST']) -@app.route('/dns/set///', methods=['POST']) +@app.route('/dns/custom/', methods=['GET', 'POST', 'PUT', 'DELETE']) +@app.route('/dns/custom//', methods=['GET', 'POST', 'PUT', 'DELETE']) @authorized_personnel_only -def dns_set_record(qname, rtype="A", value=None): +def dns_set_record(qname, rtype="A"): from dns_update import do_dns_update, set_custom_dns_record try: - # Get the value from the URL, then the POST parameters, or if it is not set then - # use the remote IP address of the request --- makes dynamic DNS easy. To clear a - # value, '' must be explicitly passed. - if value is None: - value = request.form.get("value") - if value is None: - value = request.environ.get("HTTP_X_FORWARDED_FOR") # normally REMOTE_ADDR but we're behind nginx as a reverse proxy - if value == '' or value == '__delete__': - # request deletion - value = None - if set_custom_dns_record(qname, rtype, value, env): - return do_dns_update(env) + # Normalize. + rtype = rtype.upper() + + # Read the record value from the request BODY, which must be + # ASCII-only. Not used with GET. + value = request.stream.read().decode("ascii", "ignore").strip() + + if request.method == "GET": + # Get the existing records matching the qname and rtype. + return dns_get_records(qname, rtype) + + elif request.method in ("POST", "PUT"): + # There is a default value for A/AAAA records. + if rtype in ("A", "AAAA") and value == "": + value = request.environ.get("HTTP_X_FORWARDED_FOR") # normally REMOTE_ADDR but we're behind nginx as a reverse proxy + + # Cannot add empty records. + if value == '': + return ("No value for the record provided.", 400) + + if request.method == "POST": + # Add a new record (in addition to any existing records + # for this qname-rtype pair). + action = "add" + elif request.method == "PUT": + # In REST, PUT is supposed to be idempotent, so we'll + # make this action set (replace all records for this + # qname-rtype pair) rather than add (add a new record). + action = "set" + + elif request.method == "DELETE": + if value == '': + # Delete all records for this qname-type pair. + value = None + else: + # Delete just the qname-rtype-value record exactly. + pass + action = "remove" + + if set_custom_dns_record(qname, rtype, value, action, env): + return do_dns_update(env) or "Something isn't right." return "OK" + except ValueError as e: return (str(e), 400) diff --git a/management/dns_update.py b/management/dns_update.py index 9043224e..b4178257 100755 --- a/management/dns_update.py +++ b/management/dns_update.py @@ -4,7 +4,7 @@ # and mail aliases and restarts nsd. ######################################################################## -import os, os.path, urllib.parse, datetime, re, hashlib, base64 +import sys, os, os.path, urllib.parse, datetime, re, hashlib, base64 import ipaddress import rtyaml import dns.resolver @@ -50,24 +50,13 @@ def get_dns_zones(env): return zonefiles -def get_custom_dns_config(env): - try: - return rtyaml.load(open(os.path.join(env['STORAGE_ROOT'], 'dns/custom.yaml'))) - except: - return { } - -def write_custom_dns_config(config, env): - config_yaml = rtyaml.dump(config) - with open(os.path.join(env['STORAGE_ROOT'], 'dns/custom.yaml'), "w") as f: - f.write(config_yaml) - def do_dns_update(env, force=False): # What domains (and their zone filenames) should we build? domains = get_dns_domains(env) zonefiles = get_dns_zones(env) # Custom records to add to zones. - additional_records = get_custom_dns_config(env) + additional_records = list(get_custom_dns_config(env)) # Write zone files. os.makedirs('/etc/nsd/zones', exist_ok=True) @@ -153,7 +142,7 @@ def build_zone(domain, all_domains, additional_records, env, is_zone=True): records.append((None, "NS", "ns1.%s." % env["PRIMARY_HOSTNAME"], False)) # Define ns2.PRIMARY_HOSTNAME or whatever the user overrides. - secondary_ns = additional_records.get("_secondary_nameserver", "ns2." + env["PRIMARY_HOSTNAME"]) + secondary_ns = get_secondary_dns(additional_records) or ("ns2." + env["PRIMARY_HOSTNAME"]) records.append((None, "NS", secondary_ns+'.', False)) @@ -196,20 +185,34 @@ def build_zone(domain, all_domains, additional_records, env, is_zone=True): child_qname += "." + subdomain_qname records.append((child_qname, child_rtype, child_value, child_explanation)) + has_rec_base = list(records) # clone current state def has_rec(qname, rtype, prefix=None): - for rec in records: + for rec in has_rec_base: if rec[0] == qname and rec[1] == rtype and (prefix is None or rec[2].startswith(prefix)): return True return False # The user may set other records that don't conflict with our settings. # Don't put any TXT records above this line, or it'll prevent any custom TXT records. - for qname, rtype, value in get_custom_records(domain, additional_records, env): + for qname, rtype, value in filter_custom_records(domain, additional_records): + # Don't allow custom records for record types that override anything above. + # But allow multiple custom records for the same rtype --- see how has_rec_base is used. if has_rec(qname, rtype): continue + + # The "local" keyword on A/AAAA records are short-hand for our own IP. + # This also flags for web configuration that the user wants a website here. + if rtype == "A" and value == "local": + value = env["PUBLIC_IP"] + if rtype == "AAAA" and value == "local": + if "PUBLIC_IPV6" in env: + value = env["PUBLIC_IPV6"] + else: + continue records.append((qname, rtype, value, "(Set by user.)")) # Add defaults if not overridden by the user's custom settings (and not otherwise configured). # Any "CNAME" record on the qname overrides A and AAAA. + has_rec_base = records defaults = [ (None, "A", env["PUBLIC_IP"], "Required. May have a different value. Sets the IP address that %s resolves to for web hosting and other services besides mail. The A record must be present but its value does not affect mail delivery." % domain), ("www", "A", env["PUBLIC_IP"], "Optional. Sets the IP address that www.%s resolves to, e.g. for web hosting." % domain), @@ -263,52 +266,6 @@ def build_zone(domain, all_domains, additional_records, env, is_zone=True): ######################################################################## -def get_custom_records(domain, additional_records, env): - for qname, value in additional_records.items(): - # We don't count the secondary nameserver config (if present) as a record - that would just be - # confusing to users. Instead it is accessed/manipulated directly via (get/set)_custom_dns_config. - if qname == "_secondary_nameserver": continue - - # Is this record for the domain or one of its subdomains? - # If `domain` is None, return records for all domains. - if domain is not None and qname != domain and not qname.endswith("." + domain): continue - - # Turn the fully qualified domain name in the YAML file into - # our short form (None => domain, or a relative QNAME) if - # domain is not None. - if domain is not None: - if qname == domain: - qname = None - else: - qname = qname[0:len(qname)-len("." + domain)] - - # Short form. Mapping a domain name to a string is short-hand - # for creating A records. - if isinstance(value, str): - values = [("A", value)] - if value == "local" and env.get("PUBLIC_IPV6"): - values.append( ("AAAA", value) ) - - # A mapping creates multiple records. - elif isinstance(value, dict): - values = value.items() - - # No other type of data is allowed. - else: - raise ValueError() - - for rtype, value2 in values: - # The "local" keyword on A/AAAA records are short-hand for our own IP. - # This also flags for web configuration that the user wants a website here. - if rtype == "A" and value2 == "local": - value2 = env["PUBLIC_IP"] - if rtype == "AAAA" and value2 == "local": - if "PUBLIC_IPV6" not in env: continue # no IPv6 address is available so don't set anything - value2 = env["PUBLIC_IPV6"] - yield (qname, rtype, value2) - -######################################################################## - def build_tlsa_record(env): # A DANE TLSA record in DNS specifies that connections on a port # must use TLS and the certificate must match a particular certificate. @@ -475,26 +432,10 @@ $TTL 1800 ; default time to live ######################################################################## def write_nsd_conf(zonefiles, additional_records, env): - # Basic header. - nsdconf = """ -server: - hide-version: yes - - # identify the server (CH TXT ID.SERVER entry). - identity: "" - - # The directory for zonefile: files. - zonesdir: "/etc/nsd/zones" -""" + # Write the list of zones to a configuration file. + nsd_conf_file = "/etc/nsd/zones.conf" + nsdconf = "" - # Since we have bind9 listening on localhost for locally-generated - # DNS queries that require a recursive nameserver, and the system - # might have other network interfaces for e.g. tunnelling, we have - # to be specific about the network interfaces that nsd binds to. - for ipaddr in (env.get("PRIVATE_IP", "") + " " + env.get("PRIVATE_IPV6", "")).split(" "): - if ipaddr == "": continue - nsdconf += " ip-address: %s\n" % ipaddr - # Append the zones. for domain, zonefile in zonefiles: nsdconf += """ @@ -505,9 +446,9 @@ zone: # If a custom secondary nameserver has been set, allow zone transfers # and notifies to that nameserver. - if additional_records.get("_secondary_nameserver"): + if get_secondary_dns(additional_records): # Get the IP address of the nameserver by resolving it. - hostname = additional_records.get("_secondary_nameserver") + hostname = get_secondary_dns(additional_records) resolver = dns.resolver.get_default_resolver() response = dns.resolver.query(hostname+'.', "A") ipaddr = str(response[0]) @@ -515,16 +456,17 @@ zone: provide-xfr: %s NOKEY """ % (ipaddr, ipaddr) - - # Check if the nsd.conf is changing. If it isn't changing, + # Check if the file is changing. If it isn't changing, # return False to flag that no change was made. - with open("/etc/nsd/nsd.conf") as f: - if f.read() == nsdconf: - return False + if os.path.exists(nsd_conf_file): + with open(nsd_conf_file) as f: + if f.read() == nsdconf: + return False - with open("/etc/nsd/nsd.conf", "w") as f: + # Write out new contents and return True to signal that + # configuration changed. + with open(nsd_conf_file, "w") as f: f.write(nsdconf) - return True ######################################################################## @@ -668,7 +610,94 @@ def write_opendkim_tables(domains, env): ######################################################################## -def set_custom_dns_record(qname, rtype, value, env): +def get_custom_dns_config(env): + try: + custom_dns = rtyaml.load(open(os.path.join(env['STORAGE_ROOT'], 'dns/custom.yaml'))) + if not isinstance(custom_dns, dict): raise ValueError() # caught below + except: + return [ ] + + for qname, value in custom_dns.items(): + # Short form. Mapping a domain name to a string is short-hand + # for creating A records. + if isinstance(value, str): + values = [("A", value)] + + # A mapping creates multiple records. + elif isinstance(value, dict): + values = value.items() + + # No other type of data is allowed. + else: + raise ValueError() + + for rtype, value2 in values: + if isinstance(value2, str): + yield (qname, rtype, value2) + elif isinstance(value2, list): + for value3 in value2: + yield (qname, rtype, value3) + # No other type of data is allowed. + else: + raise ValueError() + +def filter_custom_records(domain, custom_dns_iter): + for qname, rtype, value in custom_dns_iter: + # We don't count the secondary nameserver config (if present) as a record - that would just be + # confusing to users. Instead it is accessed/manipulated directly via (get/set)_custom_dns_config. + if qname == "_secondary_nameserver": continue + + # Is this record for the domain or one of its subdomains? + # If `domain` is None, return records for all domains. + if domain is not None and qname != domain and not qname.endswith("." + domain): continue + + # Turn the fully qualified domain name in the YAML file into + # our short form (None => domain, or a relative QNAME) if + # domain is not None. + if domain is not None: + if qname == domain: + qname = None + else: + qname = qname[0:len(qname)-len("." + domain)] + + yield (qname, rtype, value) + +def write_custom_dns_config(config, env): + # We get a list of (qname, rtype, value) triples. Convert this into a + # nice dictionary format for storage on disk. + from collections import OrderedDict + config = list(config) + dns = OrderedDict() + seen_qnames = set() + + # Process the qnames in the order we see them. + for qname in [rec[0] for rec in config]: + if qname in seen_qnames: continue + seen_qnames.add(qname) + + records = [(rec[1], rec[2]) for rec in config if rec[0] == qname] + if len(records) == 1 and records[0][0] == "A": + dns[qname] = records[0][1] + else: + dns[qname] = OrderedDict() + seen_rtypes = set() + + # Process the rtypes in the order we see them. + for rtype in [rec[0] for rec in records]: + if rtype in seen_rtypes: continue + seen_rtypes.add(rtype) + + values = [rec[1] for rec in records if rec[0] == rtype] + if len(values) == 1: + values = values[0] + dns[qname][rtype] = values + + # Write. + config_yaml = rtyaml.dump(dns) + with open(os.path.join(env['STORAGE_ROOT'], 'dns/custom.yaml'), "w") as f: + f.write(config_yaml) + +def set_custom_dns_record(qname, rtype, value, action, env): # validate qname for zone, fn in get_dns_zones(env): # It must match a zone apex or be a subdomain of a zone @@ -677,15 +706,17 @@ def set_custom_dns_record(qname, rtype, value, env): break else: # No match. - raise ValueError("%s is not a domain name or a subdomain of a domain name managed by this box." % qname) + if qname != "_secondary_nameserver": + raise ValueError("%s is not a domain name or a subdomain of a domain name managed by this box." % qname) # validate rtype rtype = rtype.upper() - if value is not None: + if value is not None and qname != "_secondary_nameserver": if rtype in ("A", "AAAA"): - v = ipaddress.ip_address(value) - if rtype == "A" and not isinstance(v, ipaddress.IPv4Address): raise ValueError("That's an IPv6 address.") - if rtype == "AAAA" and not isinstance(v, ipaddress.IPv6Address): raise ValueError("That's an IPv4 address.") + if value != "local": # "local" is a special flag for us + v = ipaddress.ip_address(value) # raises a ValueError if there's a problem + if rtype == "A" and not isinstance(v, ipaddress.IPv4Address): raise ValueError("That's an IPv6 address.") + if rtype == "AAAA" and not isinstance(v, ipaddress.IPv6Address): raise ValueError("That's an IPv4 address.") elif rtype in ("CNAME", "TXT", "SRV", "MX"): # anything goes pass @@ -693,69 +724,65 @@ def set_custom_dns_record(qname, rtype, value, env): raise ValueError("Unknown record type '%s'." % rtype) # load existing config - config = get_custom_dns_config(env) + config = list(get_custom_dns_config(env)) # update - if qname not in config: - if value is None: - # Is asking to delete a record that does not exist. - return False - elif rtype == "A": - # Add this record using the short form 'qname: value'. - config[qname] = value - else: - # Add this record. This is the qname's first record. - config[qname] = { rtype: value } - else: - if isinstance(config[qname], str): - # This is a short-form 'qname: value' implicit-A record. - if value is None and rtype != "A": - # Is asking to delete a record that doesn't exist. + newconfig = [] + made_change = False + needs_add = True + for _qname, _rtype, _value in config: + if action == "add": + if (_qname, _rtype, _value) == (qname, rtype, value): + # Record already exists. Bail. return False - elif value is None and rtype == "A": - # Delete record. - del config[qname] - elif rtype == "A": - # Update, keeping short form. - if config[qname] == "value": - # No change. - return False - config[qname] = value - else: - # Expand short form so we can add a new record type. - config[qname] = { "A": config[qname], rtype: value } - else: - # This is the qname: { ... } (dict) format. - if value is None: - if rtype not in config[qname]: - # Is asking to delete a record that doesn't exist. - return False + elif action == "set": + if (_qname, _rtype) == (qname, rtype): + if _value == value: + # Flag that the record already exists, don't + # need to add it. + needs_add = False else: - # Delete the record. If it's the last record, delete the domain. - del config[qname][rtype] - if len(config[qname]) == 0: - del config[qname] - else: - # Update the record. - if config[qname].get(rtype) == "value": - # No change. - return False - config[qname][rtype] = value + # Drop any other values for this (qname, rtype). + made_change = True + continue + elif action == "remove": + if (_qname, _rtype, _value) == (qname, rtype, value): + # Drop this record. + made_change = True + continue + if value == None and (_qname, _rtype) == (qname, rtype): + # Drop all qname-rtype records. + made_change = True + continue + else: + raise ValueError("Invalid action: " + action) - # serialize & save - write_custom_dns_config(config, env) + # Preserve this record. + newconfig.append((_qname, _rtype, _value)) - return True + if action in ("add", "set") and needs_add and value is not None: + newconfig.append((qname, rtype, value)) + made_change = True + + if made_change: + # serialize & save + write_custom_dns_config(newconfig, env) + + return made_change ######################################################################## +def get_secondary_dns(custom_dns): + for qname, rtype, value in custom_dns: + if qname == "_secondary_nameserver": + return value + return None + def set_secondary_dns(hostname, env): - config = get_custom_dns_config(env) if hostname in (None, ""): # Clear. - if "_secondary_nameserver" in config: - del config["_secondary_nameserver"] + set_custom_dns_record("_secondary_nameserver", "A", None, "set", env) else: # Validate. hostname = hostname.strip().lower() @@ -766,10 +793,9 @@ def set_secondary_dns(hostname, env): raise ValueError("Could not resolve the IP address of %s." % hostname) # Set. - config["_secondary_nameserver"] = hostname + set_custom_dns_record("_secondary_nameserver", "A", hostname, "set", env) - # Save and apply. - write_custom_dns_config(config, env) + # Apply. return do_dns_update(env) @@ -820,7 +846,7 @@ def build_recommended_dns(env): ret = [] domains = get_dns_domains(env) zonefiles = get_dns_zones(env) - additional_records = get_custom_dns_config(env) + additional_records = list(get_custom_dns_config(env)) for domain, zonefile in zonefiles: records = build_zone(domain, domains, additional_records, env) @@ -851,8 +877,11 @@ def build_recommended_dns(env): if __name__ == "__main__": from utils import load_environment env = load_environment() - for zone, records in build_recommended_dns(env): - for record in records: - print("; " + record['explanation']) - print(record['qname'], record['rtype'], record['value'], sep="\t") - print() + if sys.argv[-1] == "--lint": + write_custom_dns_config(get_custom_dns_config(env), env) + else: + for zone, records in build_recommended_dns(env): + for record in records: + print("; " + record['explanation']) + print(record['qname'], record['rtype'], record['value'], sep="\t") + print() diff --git a/management/mailconfig.py b/management/mailconfig.py index 093a8f71..90c3824b 100755 --- a/management/mailconfig.py +++ b/management/mailconfig.py @@ -2,6 +2,7 @@ import subprocess, shutil, os, sqlite3, re import utils +from email_validator import validate_email as validate_email_, EmailNotValidError def validate_email(email, mode=None): # Checks that an email address is syntactically valid. Returns True/False. @@ -15,50 +16,25 @@ def validate_email(email, mode=None): # When mode=="alias", we're allowing anything that can be in a Postfix # alias table, i.e. omitting the local part ("@domain.tld") is OK. - # Check that the address isn't absurdly long. - if len(email) > 255: return False + # Check the syntax of the address. + try: + validate_email_(email, + allow_smtputf8=False, + check_deliverability=False, + allow_empty_local=(mode=="alias") + ) + except EmailNotValidError: + return False if mode == 'user': # There are a lot of characters permitted in email addresses, but - # Dovecot's sqlite driver seems to get confused if there are any + # Dovecot's sqlite auth driver seems to get confused if there are any # unusual characters in the address. Bah. Also note that since # the mailbox path name is based on the email address, the address # shouldn't be absurdly long and must not have a forward slash. - ATEXT = r'[a-zA-Z0-9_\-]' - elif mode in (None, 'alias'): - # For aliases, we can allow any valid email address. - # Based on RFC 2822 and https://github.com/SyrusAkbary/validate_email/blob/master/validate_email.py, - # these characters are permitted in email addresses. - ATEXT = r'[a-zA-Z0-9_!#$%&\'\*\+\-/=\?\^`\{\|\}~]' # see 3.2.4 - else: - raise ValueError(mode) - - # per RFC 2822 3.2.4 - DOT_ATOM_TEXT_LOCAL = ATEXT + r'+(?:\.' + ATEXT + r'+)*' - if mode == 'alias': - # For aliases, Postfix accepts '@domain.tld' format for - # catch-all addresses on the source side and domain aliases - # on the destination side. Make the local part optional. - DOT_ATOM_TEXT_LOCAL = '(?:' + DOT_ATOM_TEXT_LOCAL + ')?' - - # We can require that the host part have at least one period in it, - # so use a "+" rather than a "*" at the end. - DOT_ATOM_TEXT_HOST = ATEXT + r'+(?:\.' + ATEXT + r'+)+' - - # per RFC 2822 3.4.1 - ADDR_SPEC = '^(%s)@(%s)$' % (DOT_ATOM_TEXT_LOCAL, DOT_ATOM_TEXT_HOST) - - # Check the regular expression. - m = re.match(ADDR_SPEC, email) - if not m: return False - - # Check that the domain part is valid IDNA. - localpart, domainpart = m.groups() - try: - domainpart.encode('ascii').decode("idna") - except: - # Domain is not valid IDNA. - return False + if len(email) > 255: return False + if re.search(r'[^\@\.a-zA-Z0-9_\-]+', email): + return False # Everything looks good. return True @@ -278,9 +254,10 @@ def add_mail_user(email, pw, privs, env): return ("Invalid email address.", 400) elif not validate_email(email, mode='user'): return ("User account email addresses may only use the ASCII letters A-Z, the digits 0-9, underscore (_), hyphen (-), and period (.).", 400) - elif is_dcv_address(email): + elif is_dcv_address(email) and len(get_mail_users(env)) > 0: # Make domain control validation hijacking a little harder to mess up by preventing the usual - # addresses used for DCV from being user accounts. + # addresses used for DCV from being user accounts. Except let it be the first account because + # during box setup the user won't know the rules. return ("You may not make a user account for that address because it is frequently used for domain control validation. Use an alias instead if necessary.", 400) # validate password diff --git a/management/status_checks.py b/management/status_checks.py index f7c566f3..cae2858d 100755 --- a/management/status_checks.py +++ b/management/status_checks.py @@ -11,7 +11,7 @@ import sys, os, os.path, re, subprocess, datetime, multiprocessing.pool import dns.reversename, dns.resolver import dateutil.parser, dateutil.tz -from dns_update import get_dns_zones, build_tlsa_record, get_custom_dns_config +from dns_update import get_dns_zones, build_tlsa_record, get_custom_dns_config, get_secondary_dns from web_update import get_web_domains, get_domain_ssl_files from mailconfig import get_mail_domains, get_mail_aliases @@ -357,11 +357,11 @@ def check_dns_zone(domain, env, output, dns_zonefiles): # the TLD, and so we're not actually checking the TLD. For that we'd need # to do a DNS trace. ip = query_dns(domain, "A") - custom_dns = get_custom_dns_config(env) + secondary_ns = get_secondary_dns(get_custom_dns_config(env)) or "ns2." + env['PRIMARY_HOSTNAME'] existing_ns = query_dns(domain, "NS") correct_ns = "; ".join(sorted([ "ns1." + env['PRIMARY_HOSTNAME'], - custom_dns.get("_secondary_nameserver", "ns2." + env['PRIMARY_HOSTNAME']), + secondary_ns, ])) if existing_ns.lower() == correct_ns.lower(): output.print_ok("Nameservers are set correctly at registrar. [%s]" % correct_ns) diff --git a/management/templates/custom-dns.html b/management/templates/custom-dns.html index 28e701ef..711bc384 100644 --- a/management/templates/custom-dns.html +++ b/management/templates/custom-dns.html @@ -93,44 +93,56 @@

Use your box’s DNS API to set custom DNS records on domains hosted here. For instance, you can create your own dynamic DNS service.

-

Send a POST request like this:

+

Usage:

-
curl -d "" --user {email}:{password} https://{{hostname}}/admin/dns/set/qname[/rtype[/value]]
+
curl -X VERB [-d "value"] --user {email}:{password} https://{{hostname}}/admin/dns/custom[/qname[/rtype]]
-

HTTP POST parameters

+

(Brackets denote an optional argument.)

+ +

Verbs

+ + + + + + + +
Verb Usage
GET Returns matching custom DNS records as a JSON array of objects. Each object has the keys qname, rtype, and value. The optional qname and rtype parameters in the request URL filter the records returned in the response. The request body (-d "...") must be omitted.
PUT Sets a custom DNS record replacing any existing records with the same qname and rtype. Use PUT (instead of POST) when you only have one value for a qname and rtype, such as typical A records (without round-robin).
POST Adds a new custom DNS record. Use POST when you have multiple TXT records or round-robin A records. (PUT would delete previously added records.)
DELETE Deletes custom DNS records. If the request body (-d "...") is empty or omitted, deletes all records matching the qname and rtype. If the request body is present, deletes only the record matching the qname, rtype and value.
+ +

Parameters

- - - + + +
Parameter Value
email The email address of any administrative user here.
password That user’s password.
qname The fully qualified domain name for the record you are trying to set.
rtype The resource type. A if omitted. Possible values: A (an IPv4 address), AAAA (an IPv6 address), TXT (a text string), or CNAME (an alias, which is a fully qualified domain name).
value The new record’s value. If omitted, the IPv4 address of the remote host is used. This is handy for dynamic DNS! To delete a record, use “__delete__”.
qname The fully qualified domain name for the record you are trying to set. It must be one of the domain names or a subdomain of one of the domain names hosted on this box. (Add mail users or aliases to add new domains.)
rtype The resource type. Defaults to A if omitted. Possible values: A (an IPv4 address), AAAA (an IPv6 address), TXT (a text string), CNAME (an alias, which is a fully qualified domain name — don’t forget the final period), MX, or SRV.
value For PUT, POST, and DELETE, the record’s value. If the rtype is A or AAAA and value is empty or omitted, the IPv4 or IPv6 address of the remote host is used (be sure to use the -4 or -6 options to curl). This is handy for dynamic DNS!
-

Note that -d "" is merely to ensure curl sends a POST request. You do not need to put anything inside the quotes. You can also pass the value using typical form encoding in the POST body.

-

Strict SPF and DMARC records will be added to all custom domains unless you override them.

Examples:

+

Try these examples. For simplicity the examples omit the --user me@mydomain.com:yourpassword command line argument which you must fill in with your email address and password.

+ 
# sets laptop.mydomain.com to point to the IP address of the machine you are executing curl on
-curl -d "" --user me@mydomain.com:###### https://{{hostname}}/admin/dns/set/laptop.mydomain.com
+curl -X PUT https://{{hostname}}/admin/dns/custom/laptop.mydomain.com
 
-# sets an alias
-curl -d "" --user me@mydomain.com:###### https://{{hostname}}/admin/dns/set/foo.mydomain.com/cname/bar.mydomain.com
+# deletes that record and all A records for that domain name
+curl -X DELETE https://{{hostname}}/admin/dns/custom/laptop.mydomain.com
 
-# clears the alias
-curl -d "" --user me@mydomain.com:###### https://{{hostname}}/admin/dns/set/bar.mydomain.com/cname/__delete__
+# sets a CNAME alias
+curl -X PUT -d "bar.mydomain.com." https://{{hostname}}/admin/dns/custom/foo.mydomain.com/cname
 
-# sets a TXT record using the alternate value syntax
-curl -d "value=something%20here" --user me@mydomain.com:###### https://{{hostname}}/admin/dns/set/foo.mydomain.com/txt
+# deletes that CNAME and all CNAME records for that domain name
+curl -X DELETE https://{{hostname}}/admin/dns/custom/foo.mydomain.com/cname
 
-# sets a SRV record for the "service" and "protocol" hosted on "target" server
-curl -d "" --user me@mydomain.com:###### https://{{hostname}}/admin/dns/set/_service._protocol.{{hostname}}/srv/"priority weight port target"
+# adds a TXT record using POST to preserve any previous TXT records
+curl -X POST -d "some text here" https://{{hostname}}/admin/dns/custom/foo.mydomain.com/txt
 
-# sets a SRV record using the value syntax
-curl -d "value=priority weight port target" --user me@mydomain.com:###### https://{{hostname}}/admin/dns/set/_service._protocol.host/srv
+# deletes that one TXT record while preserving other TXT records
+curl -X DELETE -d "some text here" https://{{hostname}}/admin/dns/custom/foo.mydomain.com/txt
- + +