internationalized domain names (DNS, web, CSRs, normalize to Unicode in database, prohibit non-ASCII characters in user account names)

* For non-ASCII domain names, we will keep the Unicode encoding in our users/aliases table. This is nice for the user and also simplifies things like sorting domain names (using Unicode lexicographic order is good, using ASCII lexicogrpahic order on IDNA is confusing).
* Write nsd config, nsd zone files, nginx config, and SSL CSRs with domains in IDNA-encoded ASCII.
* When checking SSL certificates, treat the CN and SANs as IDNA.
* Since Chrome has an interesting feature of converting Unicode to IDNA in <input type="email"> form fields, we'll also forcibly convert IDNA to Unicode in the domain part of email addresses before saving email addresses in the users/aliases tables so that the table is normalized to Unicode.
* Don't allow non-ASCII characters in user account email addresses. Dovecot gets confused when querying the Sqlite database (which we observed even for non-word ASCII characters too, so it may not be related to the character encoding).

CHANGELOG.md

@@ -4,19 +4,21 @@ CHANGELOG
 Development
 -----------
 
-DNS:
-
-* If a custom CNAME record is set, don't add a default A/AAAA record, e.g. for 'www', which end up preventing the CNAME record from working.
-
 Control panel:
 
 * Status checks now check that system services are actually running by pinging each port that should have something running on it.
+* If a custom CNAME record is set on a 'www' subdomain, the default A/AAAA records were preventing the CNAME from working.
 
 Setup:
 
 * Install cron if it isn't already installed.
 * Fix a units problem in the minimum memory check.
 
+Miscellaneous:
+
+* Internationalized domain names (IDNs) are now supported for DNS and web, but email is not yet tested.
+
+
 v0.06 (January 4, 2015)
 -----------------------
 

management/dns_update.py

@@ -382,17 +382,26 @@ $TTL 1800           ; default time to live
 """
 
 	# Replace replacement strings.
-	zone = zone.format(domain=domain, primary_domain=env["PRIMARY_HOSTNAME"])
+	zone = zone.format(domain=domain.encode("idna").decode("ascii"), primary_domain=env["PRIMARY_HOSTNAME"].encode("idna").decode("ascii"))
 
 	# Add records.
 	for subdomain, querytype, value, explanation in records:
 		if subdomain:
-			zone += subdomain
+			zone += subdomain.encode("idna").decode("ascii")
 		zone += "\tIN\t" + querytype + "\t"
 		if querytype == "TXT":
+			# Quote and escape.
 			value = value.replace('\\', '\\\\') # escape backslashes
 			value = value.replace('"', '\\"') # escape quotes
 			value = '"' + value + '"' # wrap in quotes
+		elif querytype in ("NS", "CNAME"):
+			# These records must be IDNA-encoded.
+			value = value.encode("idna").decode("ascii")
+		elif querytype == "MX":
+			# Also IDNA-encoded, but must parse first.
+			priority, host = value.split(" ", 1)
+			host = host.encode("idna").decode("ascii")
+			value = priority + " " + host
 		zone += value + "\n"
 
 	# DNSSEC requires re-signing a zone periodically. That requires
@@ -486,7 +495,7 @@ server:
 zone:
 	name: %s
 	zonefile: %s
-""" % (domain, zonefile)
+""" % (domain.encode("idna").decode("ascii"), zonefile)
 
 		# If a custom secondary nameserver has been set, allow zone transfers
 		# and notifies to that nameserver.
@@ -531,6 +540,9 @@ def sign_zone(domain, zonefile, env):
 	algo = dnssec_choose_algo(domain, env)
 	dnssec_keys = load_env_vars_from_file(os.path.join(env['STORAGE_ROOT'], 'dns/dnssec/%s.conf' % algo))
 
+	# From here, use the IDNA encoding of the domain name.
+	domain = domain.encode("idna").decode("ascii")
+
 	# In order to use the same keys for all domains, we have to generate
 	# a new .key file with a DNSSEC record for the specific domain. We
 	# can reuse the same key, but it won't validate without a DNSSEC

management/mailconfig.py

@@ -14,7 +14,7 @@ def validate_email(email, mode=None):
 
 	if mode == 'user':
 		# For Dovecot's benefit, only allow basic characters.
-		ATEXT = r'[\w\-]'
+		ATEXT = r'[a-zA-Z0-9_\-]'
 	elif mode in (None, 'alias'):
 		# For aliases, we can allow any valid email address.
 		# Based on RFC 2822 and https://github.com/SyrusAkbary/validate_email/blob/master/validate_email.py,
@@ -36,9 +36,34 @@ def validate_email(email, mode=None):
 	DOT_ATOM_TEXT_HOST = ATEXT + r'+(?:\.' + ATEXT + r'+)+'
 
 	# per RFC 2822 3.4.1
-	ADDR_SPEC = '^%s@%s$' % (DOT_ATOM_TEXT_LOCAL, DOT_ATOM_TEXT_HOST)
+	ADDR_SPEC = '^(%s)@(%s)$' % (DOT_ATOM_TEXT_LOCAL, DOT_ATOM_TEXT_HOST)
 
-	return re.match(ADDR_SPEC, email)
+	# Check the regular expression.
+	m = re.match(ADDR_SPEC, email)
+	if not m: return False
+
+	# Check that the domain part is IDNA-encodable.
+	localpart, domainpart = m.groups()
+	try:
+		domainpart.encode("idna")
+	except:
+		return False
+
+	return True
+
+def sanitize_idn_email_address(email):
+	# Convert an IDNA-encoded email address (domain part) into Unicode
+	# before storing in our database. Chrome may IDNA-ize <input type="email">
+	# values before POSTing, so we want to normalize before putting
+	# values into the database.
+	try:
+		localpart, domainpart = email.split("@")
+		domainpart = domainpart.encode("ascii").decode("idna")
+		return localpart + "@" + domainpart
+	except:
+		# Domain part is already Unicode or not IDNA-valid, so
+		# leave unchanged.
+		return email
 
 def open_database(env, with_connection=False):
 	conn = sqlite3.connect(env["STORAGE_ROOT"] + "/mail/users.sqlite")
@@ -230,6 +255,9 @@ def get_mail_domains(env, filter_aliases=lambda alias : True):
 		 )
 
 def add_mail_user(email, pw, privs, env):
+	# accept IDNA domain names but normalize to Unicode before going into database
+	email = sanitize_idn_email_address(email)
+
 	# validate email
 	if email.strip() == "":
 		return ("No email address provided.", 400)
@@ -284,6 +312,10 @@ def add_mail_user(email, pw, privs, env):
 	return kick(env, "mail user added")
 
 def set_mail_password(email, pw, env):
+	# accept IDNA domain names but normalize to Unicode before going into database
+	email = sanitize_idn_email_address(email)
+
+	# validate that password is acceptable
 	validate_password(pw)
 	
 	# hash the password
@@ -298,6 +330,10 @@ def set_mail_password(email, pw, env):
 	return "OK"
 
 def remove_mail_user(email, env):
+	# accept IDNA domain names but normalize to Unicode before going into database
+	email = sanitize_idn_email_address(email)
+
+	# remove
 	conn, c = open_database(env, with_connection=True)
 	c.execute("DELETE FROM users WHERE email=?", (email,))
 	if c.rowcount != 1:
@@ -311,6 +347,10 @@ def parse_privs(value):
 	return [p for p in value.split("\n") if p.strip() != ""]
 
 def get_mail_user_privileges(email, env):
+	# accept IDNA domain names but normalize to Unicode before going into database
+	email = sanitize_idn_email_address(email)
+
+	# get privs
 	c = open_database(env)
 	c.execute('SELECT privileges FROM users WHERE email=?', (email,))
 	rows = c.fetchall()
@@ -324,6 +364,9 @@ def validate_privilege(priv):
 	return None
 
 def add_remove_mail_user_privilege(email, priv, action, env):
+	# accept IDNA domain names but normalize to Unicode before going into database
+	email = sanitize_idn_email_address(email)
+
 	# validate
 	validation = validate_privilege(priv)
 	if validation: return validation
@@ -351,6 +394,9 @@ def add_remove_mail_user_privilege(email, priv, action, env):
 	return "OK"
 
 def add_mail_alias(source, destination, env, update_if_exists=False, do_kick=True):
+	# accept IDNA domain names but normalize to Unicode before going into database
+	source = sanitize_idn_email_address(source)
+
 	# validate source
 	if source.strip() == "":
 		return ("No incoming email address provided.", 400)
@@ -363,13 +409,14 @@ def add_mail_alias(source, destination, env, update_if_exists=False, do_kick=Tru
 	if validate_email(destination, mode='alias'):
 		# Oostfix allows a single @domain.tld as the destination, which means
 		# the local part on the address is preserved in the rewrite.
-		dests.append(destination)
+		dests.append(sanitize_idn_email_address(destination))
 	else:
 		# Parse comma and \n-separated destination emails & validate. In this
 		# case, the recipients must be complete email addresses.
 		for line in destination.split("\n"):
 			for email in line.split(","):
 				email = email.strip()
+				email = sanitize_idn_email_address(email) # Unicode => IDNA
 				if email == "": continue
 				if not validate_email(email):
 					return ("Invalid destination email address (%s)." % email, 400)
@@ -397,6 +444,10 @@ def add_mail_alias(source, destination, env, update_if_exists=False, do_kick=Tru
 		return kick(env, return_status)
 
 def remove_mail_alias(source, env, do_kick=True):
+	# accept IDNA domain names but normalize to Unicode before going into database
+	source = sanitize_idn_email_address(source)
+
+	# remove
 	conn, c = open_database(env, with_connection=True)
 	c.execute("DELETE FROM aliases WHERE source=?", (source,))
 	if c.rowcount != 1:

management/status_checks.py

@@ -552,6 +552,7 @@ def check_certificate(domain, ssl_certificate, ssl_private_key):
 		if m:
 			cert_expiration_date = dateutil.parser.parse(m.group(1))
 
+	domain = domain.encode("idna").decode("ascii")
 	wildcard_domain = re.sub("^[^\.]+", "*", domain)
 	if domain is not None and domain not in certificate_names and wildcard_domain not in certificate_names:
 		return ("The certificate is for the wrong domain name. It is for %s."

management/templates/aliases.html

@@ -27,6 +27,7 @@
     <label for="addaliasEmail" class="col-sm-1 control-label">Alias</label>
     <div class="col-sm-10">
       <input type="email" class="form-control" id="addaliasEmail">
+      <div style="margin-top: 3px; padding-left: 3px; font-size: 90%" class="text-muted">You may use international (non-ASCII) characters, but this has not yet been well tested.</div>
     </div>
   </div>
   <div class="form-group">

management/templates/users.html

@@ -12,7 +12,7 @@
 
 <h3>Add a mail user</h3>
 
-<p>Add an email address to this system. This will create a new login username/password. (Use <a href="javascript:show_panel('aliases')">aliases</a> to create email addresses that forward to existing accounts.)</p>
+<p>Add an email address to this system. This will create a new login username/password.</p>
 
 <form class="form-inline" role="form" onsubmit="return do_add_user(); return false;">
   <div class="form-group">
@@ -31,10 +31,12 @@
   </div>
   <button type="submit" class="btn btn-primary">Add User</button>
 </form>
-<p style="margin-top: .5em"><small>
+<ul style="margin-top: 1em; padding-left: 1.5em; font-size: 90%;">
-  Passwords must be at least four characters and may not contain spaces.
+  <li>Passwords must be at least four characters and may not contain spaces.</li>
-  Administrators get access to this control panel.
+  <li>Use <a href="javascript:show_panel('aliases')">aliases</a> to create email addresses that forward to existing accounts.</li>
-</small></p>
+  <li>Administrators get access to this control panel.</li>
+  <li>User accounts cannot contain any international (non-ASCII) characters, but <a href="javascript:show_panel('aliases')">aliases</a> can.</li>
+</ul>
 
 <h3>Existing mail users</h3>
 <table id="user_table" class="table" style="width: auto">

management/web_update.py

@@ -89,7 +89,7 @@ def make_domain_config(domain, template, template_for_primaryhost, env):
 
 	# Replace substitution strings in the template & return.
 	nginx_conf = nginx_conf.replace("$STORAGE_ROOT", env['STORAGE_ROOT'])
-	nginx_conf = nginx_conf.replace("$HOSTNAME", domain)
+	nginx_conf = nginx_conf.replace("$HOSTNAME", domain.encode("idna").decode("ascii"))
 	nginx_conf = nginx_conf.replace("$ROOT", root)
 	nginx_conf = nginx_conf.replace("$SSL_KEY", ssl_key)
 	nginx_conf = nginx_conf.replace("$SSL_CERTIFICATE", ssl_certificate)
@@ -210,7 +210,7 @@ def create_csr(domain, ssl_key, env):
                 "-key", ssl_key,
                 "-out",  "/dev/stdout",
                 "-sha256",
-                "-subj", "/C=%s/ST=/L=/O=/CN=%s" % (env["CSR_COUNTRY"], domain)])
+                "-subj", "/C=%s/ST=/L=/O=/CN=%s" % (env["CSR_COUNTRY"], domain.encode("idna").decode("ascii"))])
 
 def install_cert(domain, ssl_cert, ssl_chain, env):
 	if domain not in get_web_domains(env):