re-do the custom DNS get/set routines so it is possible to store more than one record for a qname-rtype pair, like multiple TXT records

management/daemon.py

@@ -221,8 +221,8 @@ def dns_update():
 @app.route('/dns/secondary-nameserver')
 @authorized_personnel_only
 def dns_get_secondary_nameserver():
-	from dns_update import get_custom_dns_config
+	from dns_update import get_custom_dns_config, get_secondary_dns
-	return json_response({ "hostname": get_custom_dns_config(env).get("_secondary_nameserver") })
+	return json_response({ "hostname": get_secondary_dns(get_custom_dns_config(env)) })
 
 @app.route('/dns/secondary-nameserver', methods=['POST'])
 @authorized_personnel_only
@@ -236,14 +236,12 @@ def dns_set_secondary_nameserver():
 @app.route('/dns/set')
 @authorized_personnel_only
 def dns_get_records():
-	from dns_update import get_custom_dns_config, get_custom_records
+	from dns_update import get_custom_dns_config
-	additional_records = get_custom_dns_config(env)
-	records = get_custom_records(None, additional_records, env)
 	return json_response([{
 		"qname": r[0],
 		"rtype": r[1],
 		"value": r[2],
-		} for r in records])
+		} for r in get_custom_dns_config(env) if r[0] != "_secondary_nameserver"])
 
 @app.route('/dns/set/<qname>', methods=['POST'])
 @app.route('/dns/set/<qname>/<rtype>', methods=['POST'])
@@ -262,8 +260,8 @@ def dns_set_record(qname, rtype="A", value=None):
 		if value == '' or value == '__delete__':
 			# request deletion
 			value = None
-		if set_custom_dns_record(qname, rtype, value, env):
+		if set_custom_dns_record(qname, rtype, value, "set", env):
-			return do_dns_update(env)
+			return do_dns_update(env) or "No Change"
 		return "OK"
 	except ValueError as e:
 		return (str(e), 400)

management/dns_update.py

@@ -4,7 +4,7 @@
 # and mail aliases and restarts nsd.
 ########################################################################
 
-import os, os.path, urllib.parse, datetime, re, hashlib, base64
+import sys, os, os.path, urllib.parse, datetime, re, hashlib, base64
 import ipaddress
 import rtyaml
 import dns.resolver
@@ -50,24 +50,13 @@ def get_dns_zones(env):
 
 	return zonefiles
 	
-def get_custom_dns_config(env):
-	try:
-		return rtyaml.load(open(os.path.join(env['STORAGE_ROOT'], 'dns/custom.yaml')))
-	except:
-		return { }
-
-def write_custom_dns_config(config, env):
-	config_yaml = rtyaml.dump(config)
-	with open(os.path.join(env['STORAGE_ROOT'], 'dns/custom.yaml'), "w") as f:
-		f.write(config_yaml)
-
 def do_dns_update(env, force=False):
 	# What domains (and their zone filenames) should we build?
 	domains = get_dns_domains(env)
 	zonefiles = get_dns_zones(env)
 
 	# Custom records to add to zones.
-	additional_records = get_custom_dns_config(env)
+	additional_records = list(get_custom_dns_config(env))
 
 	# Write zone files.
 	os.makedirs('/etc/nsd/zones', exist_ok=True)
@@ -153,7 +142,7 @@ def build_zone(domain, all_domains, additional_records, env, is_zone=True):
 		records.append((None,  "NS",  "ns1.%s." % env["PRIMARY_HOSTNAME"], False))
 
 		# Define ns2.PRIMARY_HOSTNAME or whatever the user overrides.
-		secondary_ns = additional_records.get("_secondary_nameserver", "ns2." + env["PRIMARY_HOSTNAME"])
+		secondary_ns = get_secondary_dns(additional_records) or ("ns2." + env["PRIMARY_HOSTNAME"])
 		records.append((None,  "NS", secondary_ns+'.', False))
 
 
@@ -196,20 +185,34 @@ def build_zone(domain, all_domains, additional_records, env, is_zone=True):
 				child_qname += "." + subdomain_qname
 			records.append((child_qname, child_rtype, child_value, child_explanation))
 
+	has_rec_base = list(records) # clone current state
 	def has_rec(qname, rtype, prefix=None):
-		for rec in records:
+		for rec in has_rec_base:
 			if rec[0] == qname and rec[1] == rtype and (prefix is None or rec[2].startswith(prefix)):
 				return True
 		return False
 
 	# The user may set other records that don't conflict with our settings.
 	# Don't put any TXT records above this line, or it'll prevent any custom TXT records.
-	for qname, rtype, value in get_custom_records(domain, additional_records, env):
+	for qname, rtype, value in filter_custom_records(domain, additional_records):
+		# Don't allow custom records for record types that override anything above.
+		# But allow multiple custom records for the same rtype --- see how has_rec_base is used.
 		if has_rec(qname, rtype): continue
+
+		# The "local" keyword on A/AAAA records are short-hand for our own IP.
+		# This also flags for web configuration that the user wants a website here.
+		if rtype == "A" and value == "local":
+			value = env["PUBLIC_IP"]
+		if rtype == "AAAA" and value == "local":
+			if "PUBLIC_IPV6" in env:
+				value = env["PUBLIC_IPV6"]
+			else:
+				continue
 		records.append((qname, rtype, value, "(Set by user.)"))
 
 	# Add defaults if not overridden by the user's custom settings (and not otherwise configured).
 	# Any "CNAME" record on the qname overrides A and AAAA.
+	has_rec_base = records
 	defaults = [
 		(None,  "A",    env["PUBLIC_IP"],       "Required. May have a different value. Sets the IP address that %s resolves to for web hosting and other services besides mail. The A record must be present but its value does not affect mail delivery." % domain),
 		("www", "A",    env["PUBLIC_IP"],       "Optional. Sets the IP address that www.%s resolves to, e.g. for web hosting." % domain),
@@ -263,52 +266,6 @@ def build_zone(domain, all_domains, additional_records, env, is_zone=True):
 
 ########################################################################
 
-def get_custom_records(domain, additional_records, env):
-	for qname, value in additional_records.items():
-		# We don't count the secondary nameserver config (if present) as a record - that would just be
-		# confusing to users. Instead it is accessed/manipulated directly via (get/set)_custom_dns_config.
-		if qname == "_secondary_nameserver": continue
-
-		# Is this record for the domain or one of its subdomains?
-		# If `domain` is None, return records for all domains.
-		if domain is not None and qname != domain and not qname.endswith("." + domain): continue
-
-		# Turn the fully qualified domain name in the YAML file into
-		# our short form (None => domain, or a relative QNAME) if
-		# domain is not None.
-		if domain is not None:
-			if qname == domain:
-				qname = None
-			else:
-				qname = qname[0:len(qname)-len("." + domain)]
-
-		# Short form. Mapping a domain name to a string is short-hand
-		# for creating A records.
-		if isinstance(value, str):
-			values = [("A", value)]
-			if value == "local" and env.get("PUBLIC_IPV6"):
-				values.append( ("AAAA", value) )
-
-		# A mapping creates multiple records.
-		elif isinstance(value, dict):
-			values = value.items()
-
-		# No other type of data is allowed.
-		else:
-			raise ValueError()
-
-		for rtype, value2 in values:
-			# The "local" keyword on A/AAAA records are short-hand for our own IP.
-			# This also flags for web configuration that the user wants a website here.
-			if rtype == "A" and value2 == "local":
-				value2 = env["PUBLIC_IP"]
-			if rtype == "AAAA" and value2 == "local":
-				if "PUBLIC_IPV6" not in env: continue # no IPv6 address is available so don't set anything
-				value2 = env["PUBLIC_IPV6"]
-			yield (qname, rtype, value2)
-
-########################################################################
-
 def build_tlsa_record(env):
 	# A DANE TLSA record in DNS specifies that connections on a port
 	# must use TLS and the certificate must match a particular certificate.
@@ -505,9 +462,9 @@ zone:
 
 		# If a custom secondary nameserver has been set, allow zone transfers
 		# and notifies to that nameserver.
-		if additional_records.get("_secondary_nameserver"):
+		if get_secondary_dns(additional_records):
 			# Get the IP address of the nameserver by resolving it.
-			hostname = additional_records.get("_secondary_nameserver")
+			hostname = get_secondary_dns(additional_records)
 			resolver = dns.resolver.get_default_resolver()
 			response = dns.resolver.query(hostname+'.', "A")
 			ipaddr = str(response[0])
@@ -668,7 +625,94 @@ def write_opendkim_tables(domains, env):
 
 ########################################################################
 
-def set_custom_dns_record(qname, rtype, value, env):
+def get_custom_dns_config(env):
+	try:
+		custom_dns = rtyaml.load(open(os.path.join(env['STORAGE_ROOT'], 'dns/custom.yaml')))
+		if not isinstance(custom_dns, dict): raise ValueError() # caught below
+	except:
+		return [ ]
+
+	for qname, value in custom_dns.items():
+		# Short form. Mapping a domain name to a string is short-hand
+		# for creating A records.
+		if isinstance(value, str):
+			values = [("A", value)]
+
+		# A mapping creates multiple records.
+		elif isinstance(value, dict):
+			values = value.items()
+
+		# No other type of data is allowed.
+		else:
+			raise ValueError()
+
+		for rtype, value2 in values:
+			if isinstance(value2, str):
+				yield (qname, rtype, value2)
+			elif isinstance(value2, list):
+				for value3 in value2:
+					yield (qname, rtype, value3)
+			# No other type of data is allowed.
+			else:
+				raise ValueError()
+
+def filter_custom_records(domain, custom_dns_iter):
+	for qname, rtype, value in custom_dns_iter:
+		# We don't count the secondary nameserver config (if present) as a record - that would just be
+		# confusing to users. Instead it is accessed/manipulated directly via (get/set)_custom_dns_config.
+		if qname == "_secondary_nameserver": continue
+
+		# Is this record for the domain or one of its subdomains?
+		# If `domain` is None, return records for all domains.
+		if domain is not None and qname != domain and not qname.endswith("." + domain): continue
+
+		# Turn the fully qualified domain name in the YAML file into
+		# our short form (None => domain, or a relative QNAME) if
+		# domain is not None.
+		if domain is not None:
+			if qname == domain:
+				qname = None
+			else:
+				qname = qname[0:len(qname)-len("." + domain)]
+
+		yield (qname, rtype, value)
+
+def write_custom_dns_config(config, env):
+	# We get a list of (qname, rtype, value) triples. Convert this into a
+	# nice dictionary format for storage on disk.
+	from collections import OrderedDict
+	config = list(config)
+	dns = OrderedDict()
+	seen_qnames = set()
+
+	# Process the qnames in the order we see them.
+	for qname in [rec[0] for rec in config]:
+		if qname in seen_qnames: continue
+		seen_qnames.add(qname)
+
+		records = [(rec[1], rec[2]) for rec in config if rec[0] == qname]
+		if len(records) == 1 and records[0][0] == "A":
+			dns[qname] = records[0][1]
+		else:
+			dns[qname] = OrderedDict()
+			seen_rtypes = set()
+
+			# Process the rtypes in the order we see them.
+			for rtype in [rec[0] for rec in records]:
+				if rtype in seen_rtypes: continue
+				seen_rtypes.add(rtype)
+
+				values = [rec[1] for rec in records if rec[0] == rtype]
+				if len(values) == 1:
+					values = values[0]
+				dns[qname][rtype] = values
+	
+	# Write.	
+	config_yaml = rtyaml.dump(dns)
+	with open(os.path.join(env['STORAGE_ROOT'], 'dns/custom.yaml'), "w") as f:
+		f.write(config_yaml)
+
+def set_custom_dns_record(qname, rtype, value, action, env):
 	# validate qname
 	for zone, fn in get_dns_zones(env):
 		# It must match a zone apex or be a subdomain of a zone
@@ -677,15 +721,17 @@ def set_custom_dns_record(qname, rtype, value, env):
 			break
 	else:
 		# No match.
-		raise ValueError("%s is not a domain name or a subdomain of a domain name managed by this box." % qname)
+		if qname != "_secondary_nameserver":
+			raise ValueError("%s is not a domain name or a subdomain of a domain name managed by this box." % qname)
 
 	# validate rtype
 	rtype = rtype.upper()
-	if value is not None:
+	if value is not None and qname != "_secondary_nameserver":
 		if rtype in ("A", "AAAA"):
-			v = ipaddress.ip_address(value)
+			if value != "local": # "local" is a special flag for us
-			if rtype == "A" and not isinstance(v, ipaddress.IPv4Address): raise ValueError("That's an IPv6 address.")
+				v = ipaddress.ip_address(value) # raises a ValueError if there's a problem
-			if rtype == "AAAA" and not isinstance(v, ipaddress.IPv6Address): raise ValueError("That's an IPv4 address.")
+				if rtype == "A" and not isinstance(v, ipaddress.IPv4Address): raise ValueError("That's an IPv6 address.")
+				if rtype == "AAAA" and not isinstance(v, ipaddress.IPv6Address): raise ValueError("That's an IPv4 address.")
 		elif rtype in ("CNAME", "TXT", "SRV", "MX"):
 			# anything goes
 			pass
@@ -693,69 +739,65 @@ def set_custom_dns_record(qname, rtype, value, env):
 			raise ValueError("Unknown record type '%s'." % rtype)
 
 	# load existing config
-	config = get_custom_dns_config(env)
+	config = list(get_custom_dns_config(env))
 
 	# update
-	if qname not in config:
+	newconfig = []
-		if value is None:
+	made_change = False
-			# Is asking to delete a record that does not exist.
+	needs_add = True
-			return False
+	for _qname, _rtype, _value in config:
-		elif rtype == "A":
+		if action == "add":
-			# Add this record using the short form 'qname: value'.
+			if (_qname, _rtype, _value) == (qname, rtype, value):
-			config[qname] = value
+				# Record already exists. Bail.
-		else:
-			# Add this record. This is the qname's first record.
-			config[qname] = { rtype: value }
-	else:
-		if isinstance(config[qname], str):
-			# This is a short-form 'qname: value' implicit-A record.
-			if value is None and rtype != "A":
-				# Is asking to delete a record that doesn't exist.
 				return False
-			elif value is None and rtype == "A":
+		elif action == "set":
-				# Delete record.
+			if (_qname, _rtype) == (qname, rtype):
-				del config[qname]
+				if _value == value:
-			elif rtype == "A":
+					# Flag that the record already exists, don't
-				# Update, keeping short form.
+					# need to add it.
-				if config[qname] == "value":
+					needs_add = False
-					# No change.
-					return False
-				config[qname] = value
-			else:
-				# Expand short form so we can add a new record type.
-				config[qname] = { "A": config[qname], rtype: value }
-		else:
-			# This is the qname: { ... } (dict) format.
-			if value is None:
-				if rtype not in config[qname]:
-					# Is asking to delete a record that doesn't exist.
-					return False
 				else:
-					# Delete the record. If it's the last record, delete the domain.
+					# Drop any other values for this (qname, rtype).
-					del config[qname][rtype]
+					made_change = True
-					if len(config[qname]) == 0:
+					continue
-						del config[qname]
+		elif action == "remove":
-			else:
+			if (_qname, _rtype, _value) == (qname, rtype, value):
-				# Update the record.
+				# Drop this record.
-				if config[qname].get(rtype) == "value":
+				made_change = True
-					# No change.
+				continue
-					return False
+			if value == None and (_qname, _rtype) == (qname, rtype):
-				config[qname][rtype] = value
+				# Drop all qname-rtype records.
+				made_change = True
+				continue
+		else:
+			raise ValueError("Invalid action: " + action)
 
-	# serialize & save
+		# Preserve this record.
-	write_custom_dns_config(config, env)
+		newconfig.append((_qname, _rtype, _value))
 
-	return True
+	if action in ("add", "set") and needs_add and value is not None:
+		newconfig.append((qname, rtype, value))
+		made_change = True
+
+	if made_change:
+		# serialize & save
+		write_custom_dns_config(newconfig, env)
+
+	return made_change
 
 ########################################################################
 
+def get_secondary_dns(custom_dns):
+	for qname, rtype, value in custom_dns:
+		if qname == "_secondary_nameserver":
+			return value
+	return None
+
 def set_secondary_dns(hostname, env):
-	config = get_custom_dns_config(env)
 
 	if hostname in (None, ""):
 		# Clear.
-		if "_secondary_nameserver" in config:
+		set_custom_dns_record("_secondary_nameserver", "A", None, "set", env)
-			del config["_secondary_nameserver"]
 	else:
 		# Validate.
 		hostname = hostname.strip().lower()
@@ -766,10 +808,9 @@ def set_secondary_dns(hostname, env):
 			raise ValueError("Could not resolve the IP address of %s." % hostname)
 
 		# Set.
-		config["_secondary_nameserver"] = hostname
+		set_custom_dns_record("_secondary_nameserver", "A", hostname, "set", env)
 
-	# Save and apply.
+	# Apply.
-	write_custom_dns_config(config, env)
 	return do_dns_update(env)
 
 
@@ -820,7 +861,7 @@ def build_recommended_dns(env):
 	ret = []
 	domains = get_dns_domains(env)
 	zonefiles = get_dns_zones(env)
-	additional_records = get_custom_dns_config(env)
+	additional_records = list(get_custom_dns_config(env))
 	for domain, zonefile in zonefiles:
 		records = build_zone(domain, domains, additional_records, env)
 
@@ -851,8 +892,11 @@ def build_recommended_dns(env):
 if __name__ == "__main__":
 	from utils import load_environment
 	env = load_environment()
-	for zone, records in build_recommended_dns(env):
+	if sys.argv[-1] == "--lint":
-		for record in records:
+		write_custom_dns_config(get_custom_dns_config(env), env)
-			print("; " + record['explanation'])
+	else:
-			print(record['qname'], record['rtype'], record['value'], sep="\t")
+		for zone, records in build_recommended_dns(env):
-			print()
+			for record in records:
+				print("; " + record['explanation'])
+				print(record['qname'], record['rtype'], record['value'], sep="\t")
+				print()

management/status_checks.py

@@ -11,7 +11,7 @@ import sys, os, os.path, re, subprocess, datetime, multiprocessing.pool
 import dns.reversename, dns.resolver
 import dateutil.parser, dateutil.tz
 
-from dns_update import get_dns_zones, build_tlsa_record, get_custom_dns_config
+from dns_update import get_dns_zones, build_tlsa_record, get_custom_dns_config, get_secondary_dns
 from web_update import get_web_domains, get_domain_ssl_files
 from mailconfig import get_mail_domains, get_mail_aliases
 
@@ -357,11 +357,11 @@ def check_dns_zone(domain, env, output, dns_zonefiles):
 	# the TLD, and so we're not actually checking the TLD. For that we'd need
 	# to do a DNS trace.
 	ip = query_dns(domain, "A")
-	custom_dns = get_custom_dns_config(env)
+	secondary_ns = get_secondary_dns(get_custom_dns_config(env)) or "ns2." + env['PRIMARY_HOSTNAME']
 	existing_ns = query_dns(domain, "NS")
 	correct_ns = "; ".join(sorted([
 		"ns1." + env['PRIMARY_HOSTNAME'],
-		custom_dns.get("_secondary_nameserver", "ns2." + env['PRIMARY_HOSTNAME']),
+		secondary_ns,
 		]))
 	if existing_ns.lower() == correct_ns.lower():
 		output.print_ok("Nameservers are set correctly at registrar. [%s]" % correct_ns)

management/templates/custom-dns.html

@@ -230,7 +230,7 @@ function do_set_custom_dns(qname, rtype, value) {
       show_current_custom_dns();
     },
     function(err) {
-      show_modal_error("Custom DNS", $("<pre/>").text(err));
+      show_modal_error("Custom DNS (Error)", $("<pre/>").text(err));
     });
 }
 

management/web_update.py

@@ -24,12 +24,9 @@ def get_web_domains(env):
 	# ...Unless the domain has an A/AAAA record that maps it to a different
 	# IP address than this box. Remove those domains from our list.
 	dns = get_custom_dns_config(env)
-	for domain, value in dns.items():
+	for domain, rtype, value in dns:
 		if domain not in domains: continue
-		if (isinstance(value, str) and (value != "local")) \
+		if rtype == "CNAME" or (rtype in ("A", "AAAA") and value != "local"):
-		  or (isinstance(value, dict) and ("CNAME" in value)) \
-		  or (isinstance(value, dict) and ("A" in value) and (value["A"] != "local")) \
-		  or (isinstance(value, dict) and ("AAAA" in value) and (value["AAAA"] != "local")):
 			domains.remove(domain)
 
 	# Sort the list. Put PRIMARY_HOSTNAME first so it becomes the