Merge changes from upstream v0.54

management/dns_update.py

@@ -9,7 +9,6 @@ import ipaddress
 import rtyaml
 import dns.resolver
 
-from mailconfig import get_mail_domains, get_mail_aliases
 from utils import shell, load_env_vars_from_file, safe_domain_name, sort_domains
 from ssl_certificates import get_ssl_certificates, check_certificate
 
@@ -20,10 +19,14 @@ from ssl_certificates import get_ssl_certificates, check_certificate
 DOMAIN_RE = "^(?!\-)(?:[*][.])?(?:[a-zA-Z\d\-_]{0,62}[a-zA-Z\d_]\.){1,126}(?!\d+)[a-zA-Z\d_]{1,63}(\.?)$"
 
 def get_dns_domains(env):
-	# Add all domain names in use by email users and mail aliases and ensure
-	# PRIMARY_HOSTNAME is in the list.
+	# Add all domain names in use by email users and mail aliases, any
+	# domains we serve web for (except www redirects because that would
+	# lead to infinite recursion here) and ensure PRIMARY_HOSTNAME is in the list.
+	from mailconfig import get_mail_domains
+	from web_update import get_web_domains
 	domains = set()
-	domains |= get_mail_domains(env)
+	domains |= set(get_mail_domains(env))
+	domains |= set(get_web_domains(env, include_www_redirects=False))
 	domains.add(env['PRIMARY_HOSTNAME'])
 	return domains
 
@@ -97,7 +100,8 @@ def do_dns_update(env, force=False):
 	if len(updated_domains) > 0:
 		shell('check_call', ["/usr/sbin/service", "nsd", "restart"])
 
-	# Write the OpenDKIM configuration tables for all of the domains.
+	# Write the OpenDKIM configuration tables for all of the mail domains.
+	from mailconfig import get_mail_domains
 	if write_opendkim_tables(get_mail_domains(env), env):
 		# Settings changed. Kick opendkim.
 		shell('check_call', ["/usr/sbin/service", "opendkim", "restart"])
@@ -122,18 +126,47 @@ def build_zones(env):
 	domains = get_dns_domains(env)
 	zonefiles = get_dns_zones(env)
 
-	# Custom records to add to zones.
-	additional_records = list(get_custom_dns_config(env))
+	# Create a dictionary of domains to a set of attributes for each
+	# domain, such as whether there are mail users at the domain.
+	from mailconfig import get_mail_domains
 	from web_update import get_web_domains
-	www_redirect_domains = set(get_web_domains(env)) - set(get_web_domains(env, include_www_redirects=False))
+	mail_domains = set(get_mail_domains(env))
+	mail_user_domains = set(get_mail_domains(env, users_only=True)) # i.e. will log in for mail, Nextcloud
+	web_domains = set(get_web_domains(env))
+	auto_domains = web_domains - set(get_web_domains(env, include_auto=False))
+	domains |= auto_domains # www redirects not included in the initial list, see above
+
+	# Add ns1/ns2+PRIMARY_HOSTNAME which must also have A/AAAA records
+	# when the box is acting as authoritative DNS server for its domains.
+	for ns in ("ns1", "ns2"):
+		d = ns + "." + env["PRIMARY_HOSTNAME"]
+		domains.add(d)
+		auto_domains.add(d)
+
+	domains = {
+		domain: {
+			"user": domain in mail_user_domains,
+			"mail": domain in mail_domains,
+			"web": domain in web_domains,
+			"auto": domain in auto_domains,
+		}
+		for domain in domains
+	}
+
+	# For MTA-STS, we'll need to check if the PRIMARY_HOSTNAME certificate is
+	# singned and valid. Check that now rather than repeatedly for each domain.
+	domains[env["PRIMARY_HOSTNAME"]]["certificate-is-valid"] = is_domain_cert_signed_and_valid(env["PRIMARY_HOSTNAME"], env)
+
+	# Load custom records to add to zones.
+	additional_records = list(get_custom_dns_config(env))
 
 	# Build DNS records for each zone.
 	for domain, zonefile in zonefiles:
 		# Build the records to put in the zone.
-		records = build_zone(domain, domains, additional_records, www_redirect_domains, env)
+		records = build_zone(domain, domains, additional_records, env)
 		yield (domain, zonefile, records)
 
-def build_zone(domain, all_domains, additional_records, www_redirect_domains, env, is_zone=True):
+def build_zone(domain, domain_properties, additional_records, env, is_zone=True):
 	records = []
 
 	# For top-level zones, define the authoritative name servers.
@@ -170,15 +203,6 @@ def build_zone(domain, all_domains, additional_records, www_redirect_domains, en
 
 	# In PRIMARY_HOSTNAME...
 	if domain == env["PRIMARY_HOSTNAME"]:
-		# Define ns1 and ns2.
-		# 'False' in the tuple indicates these records would not be used if the zone
-		# is managed outside of the box.
-		records.append(("ns1", "A", env["PUBLIC_IP"], False))
-		records.append(("ns2", "A", env["PUBLIC_IP"], False))
-		if env.get('PUBLIC_IPV6'):
-			records.append(("ns1", "AAAA", env["PUBLIC_IPV6"], False))
-			records.append(("ns2", "AAAA", env["PUBLIC_IPV6"], False))
-
 		# Set the A/AAAA records. Do this early for the PRIMARY_HOSTNAME so that the user cannot override them
 		# and we can provide different explanatory text.
 		records.append((None, "A", env["PUBLIC_IP"], "Required. Sets the IP address of the box."))
@@ -196,16 +220,17 @@ def build_zone(domain, all_domains, additional_records, www_redirect_domains, en
 
 	# Add DNS records for any subdomains of this domain. We should not have a zone for
 	# both a domain and one of its subdomains.
-	subdomains = [d for d in all_domains if d.endswith("." + domain)]
-	for subdomain in subdomains:
-		subdomain_qname = subdomain[0:-len("." + domain)]
-		subzone = build_zone(subdomain, [], additional_records, www_redirect_domains, env, is_zone=False)
-		for child_qname, child_rtype, child_value, child_explanation in subzone:
-			if child_qname == None:
-				child_qname = subdomain_qname
-			else:
-				child_qname += "." + subdomain_qname
-			records.append((child_qname, child_rtype, child_value, child_explanation))
+	if is_zone: # don't recurse when we're just loading data for a subdomain
+		subdomains = [d for d in domain_properties if d.endswith("." + domain)]
+		for subdomain in subdomains:
+			subdomain_qname = subdomain[0:-len("." + domain)]
+			subzone = build_zone(subdomain, domain_properties, additional_records, env, is_zone=False)
+			for child_qname, child_rtype, child_value, child_explanation in subzone:
+				if child_qname == None:
+					child_qname = subdomain_qname
+				else:
+					child_qname += "." + subdomain_qname
+				records.append((child_qname, child_rtype, child_value, child_explanation))
 
 	has_rec_base = list(records) # clone current state
 	def has_rec(qname, rtype, prefix=None):
@@ -232,21 +257,23 @@ def build_zone(domain, all_domains, additional_records, www_redirect_domains, en
 				continue
 		records.append((qname, rtype, value, "(Set by user.)"))
 
-	# Add defaults if not overridden by the user's custom settings (and not otherwise configured).
+	# Add A/AAAA defaults if not overridden by the user's custom settings (and not otherwise configured).
 	# Any CNAME or A record on the qname overrides A and AAAA. But when we set the default A record,
 	# we should not cause the default AAAA record to be skipped because it thinks a custom A record
 	# was set. So set has_rec_base to a clone of the current set of DNS settings, and don't update
 	# during this process.
 	has_rec_base = list(records)
+	a_expl = "Required. May have a different value. Sets the IP address that %s resolves to for web hosting and other services besides mail. The A record must be present but its value does not affect mail delivery." % domain
+	if domain_properties[domain]["auto"]:
+		if domain.startswith("ns1.") or domain.startswith("ns2."): a_expl = False # omit from 'External DNS' page since this only applies if box is its own DNS server
+		if domain.startswith("www."): a_expl = "Optional. Sets the IP address that %s resolves to so that the box can provide a redirect to the parent domain." % domain
+		if domain.startswith("mta-sts."): a_expl = "Optional. MTA-STS Policy Host serving /.well-known/mta-sts.txt."
+		if domain.startswith("autoconfig."): a_expl = "Provides email configuration autodiscovery support for Thunderbird Autoconfig."
+		if domain.startswith("autodiscover."): a_expl = "Provides email configuration autodiscovery support for Z-Push ActiveSync Autodiscover."
 	defaults = [
-		(None,  "A",    env["PUBLIC_IP"],       "Required. May have a different value. Sets the IP address that %s resolves to for web hosting and other services besides mail. The A record must be present but its value does not affect mail delivery." % domain),
+		(None,  "A",    env["PUBLIC_IP"], a_expl),
 		(None,  "AAAA", env.get('PUBLIC_IPV6'), "Optional. Sets the IPv6 address that %s resolves to, e.g. for web hosting. (It is not necessary for receiving mail on this domain.)" % domain),
 	]
-	if "www." + domain in www_redirect_domains:
-		defaults += [
-			("www", "A",    env["PUBLIC_IP"],       "Optional. Sets the IP address that www.%s resolves to so that the box can provide a redirect to the parent domain." % domain),
-			("www", "AAAA", env.get('PUBLIC_IPV6'), "Optional. Sets the IPv6 address that www.%s resolves to so that the box can provide a redirect to the parent domain." % domain),
-		]
 	for qname, rtype, value, explanation in defaults:
 		if value is None or value.strip() == "": continue # skip IPV6 if not set
 		if not is_zone and qname == "www": continue # don't create any default 'www' subdomains on what are themselves subdomains
@@ -260,63 +287,40 @@ def build_zone(domain, all_domains, additional_records, www_redirect_domains, en
 	# Don't pin the list of records that has_rec checks against anymore.
 	has_rec_base = records
 
-	# The MX record says where email for the domain should be delivered: Here!
-	if not has_rec(None, "MX", prefix="10 "):
-		records.append((None,  "MX",  "10 %s." % env["PRIMARY_HOSTNAME"], "Required. Specifies the hostname (and priority) of the machine that handles @%s mail." % domain))
+	if domain_properties[domain]["mail"]:
+		# The MX record says where email for the domain should be delivered: Here!
+		if not has_rec(None, "MX", prefix="10 "):
+			records.append((None,  "MX",  "10 %s." % env["PRIMARY_HOSTNAME"], "Required. Specifies the hostname (and priority) of the machine that handles @%s mail." % domain))
 
-	# SPF record: Permit the box ('mx', see above) to send mail on behalf of
-	# the domain, and no one else.
-	# Skip if the user has set a custom SPF record.
-	if not has_rec(None, "TXT", prefix="v=spf1 "):
-		records.append((None,  "TXT", 'v=spf1 mx -all', "Recommended. Specifies that only the box is permitted to send @%s mail." % domain))
+		# SPF record: Permit the box ('mx', see above) to send mail on behalf of
+		# the domain, and no one else.
+		# Skip if the user has set a custom SPF record.
+		if not has_rec(None, "TXT", prefix="v=spf1 "):
+			records.append((None,  "TXT", 'v=spf1 mx -all', "Recommended. Specifies that only the box is permitted to send @%s mail." % domain))
 
-	# Append the DKIM TXT record to the zone as generated by OpenDKIM.
-	# Skip if the user has set a DKIM record already.
-	opendkim_record_file = os.path.join(env['STORAGE_ROOT'], 'mail/dkim/mail.txt')
-	with open(opendkim_record_file) as orf:
-		m = re.match(r'(\S+)\s+IN\s+TXT\s+\( ((?:"[^"]+"\s+)+)\)', orf.read(), re.S)
-		val = "".join(re.findall(r'"([^"]+)"', m.group(2)))
-		if not has_rec(m.group(1), "TXT", prefix="v=DKIM1; "):
-			records.append((m.group(1), "TXT", val, "Recommended. Provides a way for recipients to verify that this machine sent @%s mail." % domain))
+		# Append the DKIM TXT record to the zone as generated by OpenDKIM.
+		# Skip if the user has set a DKIM record already.
+		opendkim_record_file = os.path.join(env['STORAGE_ROOT'], 'mail/dkim/mail.txt')
+		with open(opendkim_record_file) as orf:
+			m = re.match(r'(\S+)\s+IN\s+TXT\s+\( ((?:"[^"]+"\s+)+)\)', orf.read(), re.S)
+			val = "".join(re.findall(r'"([^"]+)"', m.group(2)))
+			if not has_rec(m.group(1), "TXT", prefix="v=DKIM1; "):
+				records.append((m.group(1), "TXT", val, "Recommended. Provides a way for recipients to verify that this machine sent @%s mail." % domain))
 
-	# Append a DMARC record.
-	# Skip if the user has set a DMARC record already.
-	if not has_rec("_dmarc", "TXT", prefix="v=DMARC1; "):
-		records.append(("_dmarc", "TXT", 'v=DMARC1; p=quarantine', "Recommended. Specifies that mail that does not originate from the box but claims to be from @%s or which does not have a valid DKIM signature is suspect and should be quarantined by the recipient's mail system." % domain))
+		# Append a DMARC record.
+		# Skip if the user has set a DMARC record already.
+		if not has_rec("_dmarc", "TXT", prefix="v=DMARC1; "):
+			records.append(("_dmarc", "TXT", 'v=DMARC1; p=quarantine', "Recommended. Specifies that mail that does not originate from the box but claims to be from @%s or which does not have a valid DKIM signature is suspect and should be quarantined by the recipient's mail system." % domain))
 
-	# For any subdomain with an A record but no SPF or DMARC record, add strict policy records.
-	all_resolvable_qnames = set(r[0] for r in records if r[1] in ("A", "AAAA"))
-	for qname in all_resolvable_qnames:
-		if not has_rec(qname, "TXT", prefix="v=spf1 "):
-			records.append((qname,  "TXT", 'v=spf1 -all', "Recommended. Prevents use of this domain name for outbound mail by specifying that no servers are valid sources for mail from @%s. If you do send email from this domain name you should either override this record such that the SPF rule does allow the originating server, or, take the recommended approach and have the box handle mail for this domain (simply add any receiving alias at this domain name to make this machine treat the domain name as one of its mail domains)." % (qname + "." + domain)))
-		dmarc_qname = "_dmarc" + ("" if qname is None else "." + qname)
-		if not has_rec(dmarc_qname, "TXT", prefix="v=DMARC1; "):
-			records.append((dmarc_qname, "TXT", 'v=DMARC1; p=reject', "Recommended. Prevents use of this domain name for outbound mail by specifying that the SPF rule should be honoured for mail from @%s." % (qname + "." + domain)))
-
-	# Add CardDAV/CalDAV SRV records on the non-primary hostname that points to the primary hostname
-	# for autoconfiguration of mail clients (so only domains hosting user accounts need it).
-	# The SRV record format is priority (0, whatever), weight (0, whatever), port, service provider hostname (w/ trailing dot).
-	if domain != env["PRIMARY_HOSTNAME"] and domain in get_mail_domains(env, users_only=True):
-		for dav in ("card", "cal"):
-			qname = "_" + dav + "davs._tcp"
-			if not has_rec(qname, "SRV"):
-				records.append((qname, "SRV", "0 0 443 " + env["PRIMARY_HOSTNAME"] + ".", "Recommended. Specifies the hostname of the server that handles CardDAV/CalDAV services for email addresses on this domain."))
-
-	# Adds autoconfiguration A records for all domains that there are user accounts at.
-	# This allows the following clients to automatically configure email addresses in the respective applications.
-	# autodiscover.* - Z-Push ActiveSync Autodiscover
-	# autoconfig.* - Thunderbird Autoconfig
-	if domain in get_mail_domains(env, users_only=True):
-		autodiscover_records = [
-			("autodiscover", "A", env["PUBLIC_IP"], "Provides email configuration autodiscovery support for Z-Push ActiveSync Autodiscover."),
-			("autodiscover", "AAAA", env["PUBLIC_IPV6"], "Provides email configuration autodiscovery support for Z-Push ActiveSync Autodiscover."),
-			("autoconfig", "A", env["PUBLIC_IP"], "Provides email configuration autodiscovery support for Thunderbird Autoconfig."),
-			("autoconfig", "AAAA", env["PUBLIC_IPV6"], "Provides email configuration autodiscovery support for Thunderbird Autoconfig.")
-		]
-		for qname, rtype, value, explanation in autodiscover_records:
-			if value is None or value.strip() == "": continue # skip IPV6 if not set
-			if not has_rec(qname, rtype):
-				records.append((qname, rtype, value, explanation))
+	if domain_properties[domain]["user"]:
+		# Add CardDAV/CalDAV SRV records on the non-primary hostname that points to the primary hostname
+		# for autoconfiguration of mail clients (so only domains hosting user accounts need it).
+		# The SRV record format is priority (0, whatever), weight (0, whatever), port, service provider hostname (w/ trailing dot).
+		if domain != env["PRIMARY_HOSTNAME"]:
+			for dav in ("card", "cal"):
+				qname = "_" + dav + "davs._tcp"
+				if not has_rec(qname, "SRV"):
+					records.append((qname, "SRV", "0 0 443 " + env["PRIMARY_HOSTNAME"] + ".", "Recommended. Specifies the hostname of the server that handles CardDAV/CalDAV services for email addresses on this domain."))
 
 	# If this is a domain name that there are email addresses configured for, i.e. "something@"
 	# this domain name, then the domain name is a MTA-STS (https://tools.ietf.org/html/rfc8461)
@@ -333,25 +337,12 @@ def build_zone(domain, all_domains, additional_records, www_redirect_domains, en
 	# subdomain must be valid certificate for that domain. Do not set an MTA-STS policy if either
 	# certificate in use is not valid (e.g. because it is self-signed and a valid certificate has not
 	# yet been provisioned). Since we cannot provision a certificate without A/AAAA records, we
-	# always set them --- only the TXT records depend on there being valid certificates.
-	mta_sts_enabled = False
-	mta_sts_records = [
-		("mta-sts", "A", env["PUBLIC_IP"], "Optional. MTA-STS Policy Host serving /.well-known/mta-sts.txt."),
-		("mta-sts", "AAAA", env.get('PUBLIC_IPV6'), "Optional. MTA-STS Policy Host serving /.well-known/mta-sts.txt."),
-	]
-	if domain in get_mail_domains(env):
-		# Check that PRIMARY_HOSTNAME and the mta_sts domain both have valid certificates.
-		for d in (env['PRIMARY_HOSTNAME'], "mta-sts." + domain):
-			cert = get_ssl_certificates(env).get(d)
-			if not cert:
-				break # no certificate provisioned for this domain
-			cert_status = check_certificate(d, cert['certificate'], cert['private-key'])
-			if cert_status[0] != 'OK':
-				break # certificate is not valid
-		else:
-			# 'break' was not encountered above, so both domains are good
-			mta_sts_enabled = True
-	if mta_sts_enabled:
+	# always set them (by including them in the www domains) --- only the TXT records depend on there
+	# being valid certificates.
+	mta_sts_records = [ ]
+	if domain_properties[domain]["mail"] \
+	  and domain_properties[env["PRIMARY_HOSTNAME"]]["certificate-is-valid"] \
+	  and is_domain_cert_signed_and_valid("mta-sts." + domain, env):
 		# Compute an up-to-32-character hash of the policy file. We'll take a SHA-1 hash of the policy
 		# file (20 bytes) and encode it as base-64 (28 bytes, using alphanumeric alternate characters
 		# instead of '+' and '/' which are not allowed in an MTA-STS policy id) but then just take its
@@ -368,15 +359,39 @@ def build_zone(domain, all_domains, additional_records, www_redirect_domains, en
 		if env.get("MTA_STS_TLSRPT_RUA") and not has_rec("_smtp._tls", "TXT", prefix="v=TLSRPTv1;"):
 			mta_sts_records.append(("_smtp._tls", "TXT", "v=TLSRPTv1; rua=" + env["MTA_STS_TLSRPT_RUA"], "Optional. Enables MTA-STS reporting."))
 	for qname, rtype, value, explanation in mta_sts_records:
-		if value is None or value.strip() == "": continue # skip IPV6 if not set
 		if not has_rec(qname, rtype):
 			records.append((qname, rtype, value, explanation))
 
+	# Add no-mail-here records for any qname that has an A or AAAA record
+	# but no MX record. This would include domain itself if domain is a
+	# non-mail domain and also may include qnames from custom DNS records.
+	# Do this once at the end of generating a zone.
+	if is_zone:
+		qnames_with_a = set(qname for (qname, rtype, value, explanation) in records if rtype in ("A", "AAAA"))
+		qnames_with_mx = set(qname for (qname, rtype, value, explanation) in records if rtype == "MX")
+		for qname in qnames_with_a - qnames_with_mx:
+			# Mark this domain as not sending mail with hard-fail SPF and DMARC records.
+			d = (qname+"." if qname else "") + domain
+			if not has_rec(qname, "TXT", prefix="v=spf1 "):
+				records.append((qname,  "TXT", 'v=spf1 -all', "Recommended. Prevents use of this domain name for outbound mail by specifying that no servers are valid sources for mail from @%s. If you do send email from this domain name you should either override this record such that the SPF rule does allow the originating server, or, take the recommended approach and have the box handle mail for this domain (simply add any receiving alias at this domain name to make this machine treat the domain name as one of its mail domains)." % d))
+			if not has_rec("_dmarc" + ("."+qname if qname else ""), "TXT", prefix="v=DMARC1; "):
+				records.append(("_dmarc" + ("."+qname if qname else ""), "TXT", 'v=DMARC1; p=reject', "Recommended. Prevents use of this domain name for outbound mail by specifying that the SPF rule should be honoured for mail from @%s." % d))
+
+			# And with a null MX record (https://explained-from-first-principles.com/email/#null-mx-record)
+			if not has_rec(qname, "MX"):
+				records.append((qname, "MX", '0 .', "Recommended. Prevents use of this domain name for incoming mail."))
+
 	# Sort the records. The None records *must* go first in the nsd zone file. Otherwise it doesn't matter.
 	records.sort(key = lambda rec : list(reversed(rec[0].split(".")) if rec[0] is not None else ""))
 
 	return records
 
+def is_domain_cert_signed_and_valid(domain, env):
+	cert = get_ssl_certificates(env).get(domain)
+	if not cert: return False # no certificate provisioned
+	cert_status = check_certificate(domain, cert['certificate'], cert['private-key'])
+	return cert_status[0] == 'OK'
+
 ########################################################################
 
 def build_tlsa_record(env):
@@ -441,6 +456,7 @@ def build_sshfp_records():
 	# to the zone file (that trigger bumping the serial number). However,
 	# if SSH has been configured to listen on a nonstandard port, we must
 	# specify that port to sshkeyscan.
+
 	port = 22
 	with open('/etc/ssh/sshd_config', 'r') as f:
 		for line in f:
@@ -451,8 +467,11 @@ def build_sshfp_records():
 				except ValueError:
 					pass
 				break
+
 	keys = shell("check_output", ["ssh-keyscan", "-t", "rsa,dsa,ecdsa,ed25519", "-p", str(port), "localhost"])
-	for key in sorted(keys.split("\n")):
+	keys = sorted(keys.split("\n"))
+
+	for key in keys:
 		if key.strip() == "" or key[0] == "#": continue
 		try:
 			host, keytype, pubkey = key.split(" ")
@@ -472,10 +491,10 @@ def write_nsd_zone(domain, zonefile, records, env, force):
 	# On the $ORIGIN line, there's typically a ';' comment at the end explaining
 	# what the $ORIGIN line does. Any further data after the domain confuses
 	# ldns-signzone, however. It used to say '; default zone domain'.
-
+	#
 	# The SOA contact address for all of the domains on this system is hostmaster
 	# @ the PRIMARY_HOSTNAME. Hopefully that's legit.
-
+	#
 	# For the refresh through TTL fields, a good reference is:
 	# http://www.peerwisdom.org/2013/05/15/dns-understanding-the-soa-record/
 	
@@ -493,6 +512,9 @@ def write_nsd_zone(domain, zonefile, records, env, force):
 	#  - Name the secondary DNS server as primary DNS in the SOA record
 	#  - Do not add NS records for the Mail-in-a-Box server
 
+	# A hash of the available DNSSEC keys are added in a comment so that when
+	# the keys change we force a re-generation of the zone which triggers
+	# re-signing it.
 
 	zone = """
 $ORIGIN {domain}.
@@ -563,6 +585,9 @@ $TTL {defttl}          ; default time to live
 			value = v2
 		zone += value + "\n"
 
+	# Append a stable hash of DNSSEC signing keys in a comment.
+	zone += "\n; DNSSEC signing keys hash: {}\n".format(hash_dnssec_keys(domain, env))
+
 	# DNSSEC requires re-signing a zone periodically. That requires
 	# bumping the serial number even if no other records have changed.
 	# We don't see the DNSSEC records yet, so we have to figure out
@@ -673,53 +698,77 @@ zone:
 
 ########################################################################
 
-def dnssec_choose_algo(domain, env):
-	if '.' in domain and domain.rsplit('.')[-1] in \
-		("email", "guide", "fund", "be", "lv"):
-		# At GoDaddy, RSASHA256 is the only algorithm supported
-		# for .email and .guide.
-		# A variety of algorithms are supported for .fund. This
-		# is preferred.
-		# Gandi tells me that .be does not support RSASHA1-NSEC3-SHA1
-		# Nic.lv does not support RSASHA1-NSEC3-SHA1 for .lv tld's
-		return "RSASHA256"
+def find_dnssec_signing_keys(domain, env):
+	# For key that we generated (one per algorithm)...
+	d = os.path.join(env['STORAGE_ROOT'], 'dns/dnssec')
+	keyconfs = [f for f in os.listdir(d) if f.endswith(".conf")]
+	for keyconf in keyconfs:
+		# Load the file holding the KSK and ZSK key filenames.
+		keyconf_fn = os.path.join(d, keyconf)
+		keyinfo = load_env_vars_from_file(keyconf_fn)
 
-	# For any domain we were able to sign before, don't change the algorithm
-	# on existing users. We'll probably want to migrate to SHA256 later.
-	return "RSASHA1-NSEC3-SHA1"
+		# Skip this key if the conf file has a setting named DOMAINS,
+		# holding a comma-separated list of domain names, and if this
+		# domain is not in the list. This allows easily disabling a
+		# key by setting "DOMAINS=" or "DOMAINS=none", other than
+		# deleting the key's .conf file, which might result in the key
+		# being regenerated next upgrade. Keys should be disabled if
+		# they are not needed to reduce the DNSSEC query response size.
+		if "DOMAINS" in keyinfo and domain not in [dd.strip() for dd in keyinfo["DOMAINS"].split(",")]:
+			continue
+
+		for keytype in ("KSK", "ZSK"):
+			yield keytype, keyinfo[keytype]
+
+def hash_dnssec_keys(domain, env):
+	# Create a stable (by sorting the items) hash of all of the private keys
+	# that will be used to sign this domain.
+	keydata = []
+	for keytype, keyfn in sorted(find_dnssec_signing_keys(domain, env)):
+		oldkeyfn = os.path.join(env['STORAGE_ROOT'], 'dns/dnssec', keyfn + ".private")
+		keydata.append(keytype)
+		keydata.append(keyfn)
+		with open(oldkeyfn, "r") as fr:
+			keydata.append( fr.read() )
+	keydata = "".join(keydata).encode("utf8")
+	return hashlib.sha1(keydata).hexdigest()
 
 def sign_zone(domain, zonefile, env):
-	algo = dnssec_choose_algo(domain, env)
-	dnssec_keys = load_env_vars_from_file(os.path.join(env['STORAGE_ROOT'], 'dns/dnssec/%s.conf' % algo))
+	# Sign the zone with all of the keys that were generated during
+	# setup so that the user can choose which to use in their DS record at
+	# their registrar, and also to support migration to newer algorithms.
 
-	# In order to use the same keys for all domains, we have to generate
-	# a new .key file with a DNSSEC record for the specific domain. We
-	# can reuse the same key, but it won't validate without a DNSSEC
-	# record specifically for the domain.
+	# In order to use the key files generated at setup which are for
+	# the domain _domain_, we have to re-write the files and place
+	# the actual domain name in it, so that ldns-signzone works.
 	#
-	# Copy the .key and .private files to /tmp to patch them up.
-	#
-	# Use os.umask and open().write() to securely create a copy that only
-	# we (root) can read.
-	files_to_kill = []
-	for key in ("KSK", "ZSK"):
-		if dnssec_keys.get(key, "").strip() == "": raise Exception("DNSSEC is not properly set up.")
-		oldkeyfn = os.path.join(env['STORAGE_ROOT'], 'dns/dnssec/' + dnssec_keys[key])
-		newkeyfn = '/tmp/' + dnssec_keys[key].replace("_domain_", domain)
-		dnssec_keys[key] = newkeyfn
+	# Patch each key, storing the patched version in /tmp for now.
+	# Each key has a .key and .private file. Collect a list of filenames
+	# for all of the keys (and separately just the key-signing keys).
+	all_keys = []
+	ksk_keys = []
+	for keytype, keyfn in find_dnssec_signing_keys(domain, env):
+		newkeyfn = '/tmp/' + keyfn.replace("_domain_", domain)
+
 		for ext in (".private", ".key"):
-			if not os.path.exists(oldkeyfn + ext): raise Exception("DNSSEC is not properly set up.")
-			with open(oldkeyfn + ext, "r") as fr:
+			# Copy the .key and .private files to /tmp to patch them up.
+			#
+			# Use os.umask and open().write() to securely create a copy that only
+			# we (root) can read.
+			oldkeyfn = os.path.join(env['STORAGE_ROOT'], 'dns/dnssec', keyfn + ext)
+			with open(oldkeyfn, "r") as fr:
 				keydata = fr.read()
-			keydata = keydata.replace("_domain_", domain) # trick ldns-signkey into letting our generic key be used by this zone
-			fn = newkeyfn + ext
+			keydata = keydata.replace("_domain_", domain)
 			prev_umask = os.umask(0o77) # ensure written file is not world-readable
 			try:
-				with open(fn, "w") as fw:
+				with open(newkeyfn + ext, "w") as fw:
 					fw.write(keydata)
 			finally:
 				os.umask(prev_umask) # other files we write should be world-readable
-			files_to_kill.append(fn)
+
+		# Put the patched key filename base (without extension) into the list of keys we'll sign with.
+		all_keys.append(newkeyfn)
+		if keytype == "KSK": ksk_keys.append(newkeyfn)
 
 	# Do the signing.
 	expiry_date = (datetime.datetime.now() + datetime.timedelta(days=30)).strftime("%Y%m%d")
@@ -732,32 +781,34 @@ def sign_zone(domain, zonefile, env):
 
 		# zonefile to sign
 		"/etc/nsd/zones/" + zonefile,
-
+	]
 		# keys to sign with (order doesn't matter -- it'll figure it out)
-		dnssec_keys["KSK"],
-		dnssec_keys["ZSK"],
-	])
+		+ all_keys
+	)
 
 	# Create a DS record based on the patched-up key files. The DS record is specific to the
 	# zone being signed, so we can't use the .ds files generated when we created the keys.
 	# The DS record points to the KSK only. Write this next to the zone file so we can
 	# get it later to give to the user with instructions on what to do with it.
 	#
-	# We want to be able to validate DS records too, but multiple forms may be valid depending
-	# on the digest type. So we'll write all (both) valid records. Only one DS record should
-	# actually be deployed. Preferebly the first.
+	# Generate a DS record for each key. There are also several possible hash algorithms that may
+	# be used, so we'll pre-generate all for each key. One DS record per line. Only one
+	# needs to actually be deployed at the registrar. We'll select the preferred one
+	# in the status checks.
 	with open("/etc/nsd/zones/" + zonefile + ".ds", "w") as f:
-		for digest_type in ('2', '1'):
-			rr_ds = shell('check_output', ["/usr/bin/ldns-key2ds",
-				"-n", # output to stdout
-				"-" + digest_type, # 1=SHA1, 2=SHA256
-				dnssec_keys["KSK"] + ".key"
-			])
-			f.write(rr_ds)
+		for key in ksk_keys:
+			for digest_type in ('1', '2', '4'):
+				rr_ds = shell('check_output', ["/usr/bin/ldns-key2ds",
+					"-n", # output to stdout
+					"-" + digest_type, # 1=SHA1, 2=SHA256, 4=SHA384
+					key + ".key"
+				])
+				f.write(rr_ds)
 
-	# Remove our temporary file.
-	for fn in files_to_kill:
-		os.unlink(fn)
+	# Remove the temporary patched key files.
+	for fn in all_keys:
+		os.unlink(fn + ".private")
+		os.unlink(fn + ".key")
 
 ########################################################################
 

management/status_checks.py

@@ -34,6 +34,7 @@ def get_services():
 		{ "name": "SSH Login (ssh)", "port": get_ssh_port(), "public": True, },
 		{ "name": "Public DNS (nsd4)", "port": 53, "public": True, },
 		{ "name": "Incoming Mail (SMTP/postfix)", "port": 25, "public": True, },
+		{ "name": "Outgoing Mail (SMTP 465/postfix)", "port": 465, "public": True, },
 		{ "name": "Outgoing Mail (SMTP 587/postfix)", "port": 587, "public": True, },
 		#{ "name": "Postfix/master", "port": 10587, "public": True, },
 		{ "name": "IMAPS (dovecot)", "port": 993, "public": True, },
@@ -42,7 +43,7 @@ def get_services():
 		{ "name": "HTTPS Web (nginx)", "port": 443, "public": True, },
 	]
 
-def run_checks(rounded_values, env, output, pool):
+def run_checks(rounded_values, env, output, pool, domains_to_check=None):
 	# run systems checks
 	output.add_heading("System")
 
@@ -63,7 +64,7 @@ def run_checks(rounded_values, env, output, pool):
 	# perform other checks asynchronously
 
 	run_network_checks(env, output)
-	run_domain_checks(rounded_values, env, output, pool)
+	run_domain_checks(rounded_values, env, output, pool, domains_to_check=domains_to_check)
 
 def get_ssh_port():
 	# Returns ssh port
@@ -302,7 +303,7 @@ def run_network_checks(env, output):
 			which may prevent recipients from receiving your email. See http://www.spamhaus.org/query/ip/%s."""
 			% (env['PUBLIC_IP'], zen, env['PUBLIC_IP']))
 
-def run_domain_checks(rounded_time, env, output, pool):
+def run_domain_checks(rounded_time, env, output, pool, domains_to_check=None):
 	# Get the list of domains we handle mail for.
 	mail_domains = get_mail_domains(env)
 
@@ -313,7 +314,8 @@ def run_domain_checks(rounded_time, env, output, pool):
 	# Get the list of domains we serve HTTPS for.
 	web_domains = set(get_web_domains(env))
 
-	domains_to_check = mail_domains | dns_domains | web_domains
+	if domains_to_check is None:
+		domains_to_check = mail_domains | dns_domains | web_domains
 
 	# Remove "www", "autoconfig", "autodiscover", and "mta-sts" subdomains, which we group with their parent,
 	# if their parent is in the domains to check list.
@@ -569,61 +571,103 @@ def check_dns_zone_suggestions(domain, env, output, dns_zonefiles, domains_with_
 
 
 def check_dnssec(domain, env, output, dns_zonefiles, is_checking_primary=False):
-	# See if the domain has a DS record set at the registrar. The DS record may have
-	# several forms. We have to be prepared to check for any valid record. We've
-	# pre-generated all of the valid digests --- read them in.
+	# See if the domain has a DS record set at the registrar. The DS record must
+	# match one of the keys that we've used to sign the zone. It may use one of
+	# several hashing algorithms. We've pre-generated all possible valid DS
+	# records, although some will be preferred.
+
+	alg_name_map = { '7': 'RSASHA1-NSEC3-SHA1', '8': 'RSASHA256', '13': 'ECDSAP256SHA256' }
+	digalg_name_map = { '1': 'SHA-1', '2': 'SHA-256', '4': 'SHA-384' }
+
+	# Read in the pre-generated DS records
+	expected_ds_records = { }
 	ds_file = '/etc/nsd/zones/' + dns_zonefiles[domain] + '.ds'
 	if not os.path.exists(ds_file): return # Domain is in our database but DNS has not yet been updated.
-	ds_correct = open(ds_file).read().strip().split("\n")
-	digests = { }
-	for rr_ds in ds_correct:
-		ds_keytag, ds_alg, ds_digalg, ds_digest = rr_ds.split("\t")[4].split(" ")
-		digests[ds_digalg] = ds_digest
+	with open(ds_file) as f:
+		for rr_ds in f:
+			rr_ds = rr_ds.rstrip()
+			ds_keytag, ds_alg, ds_digalg, ds_digest = rr_ds.split("\t")[4].split(" ")
 
-	# Some registrars may want the public key so they can compute the digest. The DS
-	# record that we suggest using is for the KSK (and that's how the DS records were generated).
-	alg_name_map = { '7': 'RSASHA1-NSEC3-SHA1', '8': 'RSASHA256' }
-	dnssec_keys = load_env_vars_from_file(os.path.join(env['STORAGE_ROOT'], 'dns/dnssec/%s.conf' % alg_name_map[ds_alg]))
-	dnsssec_pubkey = open(os.path.join(env['STORAGE_ROOT'], 'dns/dnssec/' + dnssec_keys['KSK'] + '.key')).read().split("\t")[3].split(" ")[3]
+			# Some registrars may want the public key so they can compute the digest. The DS
+			# record that we suggest using is for the KSK (and that's how the DS records were generated).
+			# We'll also give the nice name for the key algorithm.
+			dnssec_keys = load_env_vars_from_file(os.path.join(env['STORAGE_ROOT'], 'dns/dnssec/%s.conf' % alg_name_map[ds_alg]))
+			dnsssec_pubkey = open(os.path.join(env['STORAGE_ROOT'], 'dns/dnssec/' + dnssec_keys['KSK'] + '.key')).read().split("\t")[3].split(" ")[3]
+
+			expected_ds_records[ (ds_keytag, ds_alg, ds_digalg, ds_digest) ] = {
+				"record": rr_ds,
+				"keytag": ds_keytag,
+				"alg": ds_alg,
+				"alg_name": alg_name_map[ds_alg],
+				"digalg": ds_digalg,
+				"digalg_name": digalg_name_map[ds_digalg],
+				"digest": ds_digest,
+				"pubkey": dnsssec_pubkey,
+			}
 
 	# Query public DNS for the DS record at the registrar.
-	ds = query_dns(domain, "DS", nxdomain=None)
-	ds_looks_valid = ds and len(ds.split(" ")) == 4
-	if ds_looks_valid: ds = ds.split(" ")
-	if ds_looks_valid and ds[0] == ds_keytag and ds[1] == ds_alg and ds[3] == digests.get(ds[2]):
-		if is_checking_primary: return
-		output.print_ok("DNSSEC 'DS' record is set correctly at registrar.")
+	ds = query_dns(domain, "DS", nxdomain=None, as_list=True)
+	if ds is None or isinstance(ds, str): ds = []
+
+	# There may be more that one record, so we get the result as a list.
+	# Filter out records that don't look valid, just in case, and split
+	# each record on spaces.
+	ds = [tuple(str(rr).split(" ")) for rr in ds if len(str(rr).split(" ")) == 4]
+
+	if len(ds) == 0:
+		output.print_warning("""This domain's DNSSEC DS record is not set. The DS record is optional. The DS record activates DNSSEC. See below for instructions.""")
 	else:
-		if ds == None:
-			if is_checking_primary: return
-			output.print_warning("""This domain's DNSSEC DS record is not set. The DS record is optional. The DS record activates DNSSEC.
-				To set a DS record, you must follow the instructions provided by your domain name registrar and provide to them this information:""")
+		matched_ds = set(ds) & set(expected_ds_records)
+		if matched_ds:
+			# At least one DS record matches one that corresponds with one of the ways we signed
+			# the zone, so it is valid.
+			#
+			# But it may not be preferred. Only algorithm 13 is preferred. Warn if any of the
+			# matched zones uses a different algorithm.
+			if set(r[1] for r in matched_ds) == { '13' }: # all are alg 13
+				output.print_ok("DNSSEC 'DS' record is set correctly at registrar.")
+				return
+			elif '13' in set(r[1] for r in matched_ds): # some but not all are alg 13
+				output.print_ok("DNSSEC 'DS' record is set correctly at registrar. (Records using algorithm other than ECDSAP256SHA256 should be removed.)")
+				return
+			else: # no record uses alg 13
+				output.print_warning("DNSSEC 'DS' record set at registrar is valid but should be updated to ECDSAP256SHA256 (see below).")
 		else:
 			if is_checking_primary:
 				output.print_error("""The DNSSEC 'DS' record for %s is incorrect. See further details below.""" % domain)
 				return
 			output.print_error("""This domain's DNSSEC DS record is incorrect. The chain of trust is broken between the public DNS system
 				and this machine's DNS server. It may take several hours for public DNS to update after a change. If you did not recently
-				make a change, you must resolve this immediately by following the instructions provided by your domain name registrar and
-				provide to them this information:""")
+				make a change, you must resolve this immediately (see below).""")
+
+	output.print_line("""Follow the instructions provided by your domain name registrar to set a DS record.
+		Registrars support different sorts of DS records. Use the first option that works:""")
+	preferred_ds_order = [(7, 1), (7, 2), (8, 4), (13, 4), (8, 1), (8, 2), (13, 1), (13, 2)] # low to high
+	def preferred_ds_order_func(ds_suggestion):
+		k = (int(ds_suggestion['alg']), int(ds_suggestion['digalg']))
+		if k in preferred_ds_order:
+			return preferred_ds_order.index(k)
+		return -1 # index before first item
+	output.print_line("")
+	for i, ds_suggestion in enumerate(sorted(expected_ds_records.values(), key=preferred_ds_order_func, reverse=True)):
 		output.print_line("")
-		output.print_line("Key Tag: " + ds_keytag + ("" if not ds_looks_valid or ds[0] == ds_keytag else " (Got '%s')" % ds[0]))
+		output.print_line("Option " + str(i+1) + ":")
+		output.print_line("----------")
+		output.print_line("Key Tag: " + ds_suggestion['keytag'])
 		output.print_line("Key Flags: KSK")
-		output.print_line(
-			  ("Algorithm: %s / %s" % (ds_alg, alg_name_map[ds_alg]))
-			+ ("" if not ds_looks_valid or ds[1] == ds_alg else " (Got '%s')" % ds[1]))
-			# see http://www.iana.org/assignments/dns-sec-alg-numbers/dns-sec-alg-numbers.xhtml
-		output.print_line("Digest Type: 2 / SHA-256")
-			# http://www.ietf.org/assignments/ds-rr-types/ds-rr-types.xml
-		output.print_line("Digest: " + digests['2'])
-		if ds_looks_valid and ds[3] != digests.get(ds[2]):
-			output.print_line("(Got digest type %s and digest %s which do not match.)" % (ds[2], ds[3]))
+		output.print_line("Algorithm: %s / %s" % (ds_suggestion['alg'], ds_suggestion['alg_name']))
+		output.print_line("Digest Type: %s / %s" % (ds_suggestion['digalg'], ds_suggestion['digalg_name']))
+		output.print_line("Digest: " + ds_suggestion['digest'])
 		output.print_line("Public Key: ")
-		output.print_line(dnsssec_pubkey, monospace=True)
+		output.print_line(ds_suggestion['pubkey'], monospace=True)
 		output.print_line("")
 		output.print_line("Bulk/Record Format:")
-		output.print_line("" + ds_correct[0])
+		output.print_line(ds_suggestion['record'], monospace=True)
+	if len(ds) > 0:
 		output.print_line("")
+		output.print_line("The DS record is currently set to:")
+		for rr in ds:
+			output.print_line("Key Tag: {0}, Algorithm: {1}, Digest Type: {2}, Digest: {3}".format(*rr))
 
 def check_mail_domain(domain, env, output):
 	# Check the MX record.
@@ -633,6 +677,8 @@ def check_mail_domain(domain, env, output):
 
 	if mx is None:
 		mxhost = None
+	elif mx == "[timeout]":
+		mxhost = None
 	else:
 		# query_dns returns a semicolon-delimited list
 		# of priority-host pairs.
@@ -725,7 +771,7 @@ def check_web_domain(domain, rounded_time, ssl_certificates, env, output):
 	# website for also needs a signed certificate.
 	check_ssl_cert(domain, rounded_time, ssl_certificates, env, output)
 
-def query_dns(qname, rtype, nxdomain='[Not Set]', at=None):
+def query_dns(qname, rtype, nxdomain='[Not Set]', at=None, as_list=False):
 	# Make the qname absolute by appending a period. Without this, dns.resolver.query
 	# will fall back a failed lookup to a second query with this machine's hostname
 	# appended. This has been causing some false-positive Spamhaus reports. The
@@ -762,6 +808,9 @@ def query_dns(qname, rtype, nxdomain='[Not Set]', at=None):
 	if rtype in ("A", "AAAA"):
 		response = [normalize_ip(str(r)) for r in response]
 
+	if as_list:
+		return response
+
 	# There may be multiple answers; concatenate the response. Remove trailing
 	# periods from responses since that's how qnames are encoded in DNS but is
 	# confusing for us. The order of the answers doesn't matter, so sort so we
@@ -1066,3 +1115,7 @@ if __name__ == "__main__":
 
 	elif sys.argv[1] == "--version":
 		print(what_version_is_this(env))
+
+	elif sys.argv[1] == "--only":
+		with multiprocessing.pool.Pool(processes=10) as pool:
+			run_checks(False, env, ConsoleOutput(), pool, domains_to_check=sys.argv[2:])

management/templates/mail-guide.html

@@ -30,8 +30,8 @@
 					<tr><th>Mail server</th> <td>{{hostname}}</td>
 					<tr><th>IMAP Port</th> <td>993</td></tr>
 					<tr><th>IMAP Security</th> <td>SSL or TLS</td></tr>
-					<tr><th>SMTP Port</th> <td>587</td></tr>
-					<tr><th>SMTP Security</td> <td>STARTTLS <small>(&ldquo;always&rdquo; or &ldquo;required&rdquo;, if prompted)</small></td></tr>
+					<tr><th>SMTP Port</th> <td>465</td></tr>
+					<tr><th>SMTP Security</td> <td>SSL or TLS</td></tr>
 					<tr><th>Username:</th> <td>Your whole email address.</td></tr>
 					<tr><th>Password:</th> <td>Your mail password.</td></tr>
 					</table>

management/templates/system-backup.html

@@ -5,7 +5,7 @@
 
 <h2>Backup Status</h2>
 
-<p>The box makes an incremental backup each night. By default the backup is stored on the machine itself, but you can also have it stored on Amazon S3.</p>
+<p>The box makes an incremental backup each night. By default the backup is stored on the machine itself, but you can also store in on S3-compatible services like Amazon Web Services (AWS).</p>
 
 <h3>Configuration</h3>
 
@@ -17,7 +17,7 @@
         <option value="off">Nowhere (Disable Backups)</option>
         <option value="local">{{hostname}}</option>
         <option value="rsync">rsync</option>
-        <option value="s3">Amazon S3</option>
+        <option value="s3">S3 (Amazon or compatible) </option>
         <option value="b2">Backblaze B2</option>
       </select>
     </div>
@@ -73,8 +73,8 @@
   <!-- S3 BACKUP -->
   <div class="form-group backup-target-s3">
     <div class="col-sm-10 col-sm-offset-2">
-      <p>Backups are stored in an Amazon Web Services S3 bucket. You must have an AWS account already.</p>
-      <p>You MUST manually copy the encryption password from <tt class="backup-encpassword-file"></tt> to a safe and secure location. You will need this file to decrypt backup files. It is NOT stored in your Amazon S3 bucket.</p>
+      <p>Backups are stored in an S3-compatible bucket. You must have an AWS or other S3 service account already.</p>
+      <p>You MUST manually copy the encryption password from <tt class="backup-encpassword-file"></tt> to a safe and secure location. You will need this file to decrypt backup files. It is <b>NOT</b> stored in your S3 bucket.</p>
     </div>
   </div>
   <div class="form-group backup-target-s3">
@@ -84,7 +84,7 @@
         {% for name, host in backup_s3_hosts %}
           <option value="{{host}}">{{name}}</option>
         {% endfor %}
-        <option value="other">Other</option>
+        <option value="other">Other (non AWS)</option>
       </select>
     </div>
   </div>
@@ -343,4 +343,4 @@ function init_inputs(target_type) {
     set_host($('#backup-target-s3-host-select').val());
   }
 }
-</script>
+</script>

management/web_update.py

@@ -10,7 +10,7 @@ from ssl_certificates import get_ssl_certificates, get_domain_ssl_files, check_c
 from utils import shell, safe_domain_name, sort_domains, get_php_version
 from wwwconfig import get_www_domains
 
-def get_web_domains(env, include_www_redirects=True, exclude_dns_elsewhere=True):
+def get_web_domains(env, include_www_redirects=True, include_auto=True, exclude_dns_elsewhere=True):
 	# What domains should we serve HTTP(S) for?
 	domains = set()
 
@@ -22,21 +22,22 @@ def get_web_domains(env, include_www_redirects=True, exclude_dns_elsewhere=True)
 	# Add domains for which we only serve www
 	domains |= get_www_domains(domains)
 
-	if include_www_redirects:
+	if include_www_redirects and include_auto:
 		# Add 'www.' subdomains that we want to provide default redirects
 		# to the main domain for. We'll add 'www.' to any DNS zones, i.e.
 		# the topmost of each domain we serve.
 		domains |= set('www.' + zone for zone, zonefile in get_dns_zones(env))
 		domains |= set('www.' + wwwdomain for wwwdomain in get_www_domains(get_mail_domains(env)))
 
-	# Add Autoconfiguration domains for domains that there are user accounts at:
-	# 'autoconfig.' for Mozilla Thunderbird auto setup.
-	# 'autodiscover.' for Activesync autodiscovery.
-	domains |= set('autoconfig.' + maildomain for maildomain in get_mail_domains(env, users_only=True))
-	domains |= set('autodiscover.' + maildomain for maildomain in get_mail_domains(env, users_only=True))
+	if include_auto:
+		# Add Autoconfiguration domains for domains that there are user accounts at:
+		# 'autoconfig.' for Mozilla Thunderbird auto setup.
+		# 'autodiscover.' for ActiveSync autodiscovery (Z-Push).
+		domains |= set('autoconfig.' + maildomain for maildomain in get_mail_domains(env, users_only=True))
+		domains |= set('autodiscover.' + maildomain for maildomain in get_mail_domains(env, users_only=True))
 
-	# 'mta-sts.' for MTA-STS support for all domains that have email addresses.
-	domains |= set('mta-sts.' + maildomain for maildomain in get_mail_domains(env))
+		# 'mta-sts.' for MTA-STS support for all domains that have email addresses.
+		domains |= set('mta-sts.' + maildomain for maildomain in get_mail_domains(env))
 
 	if exclude_dns_elsewhere:
 		# ...Unless the domain has an A/AAAA record that maps it to a different
@@ -172,17 +173,27 @@ def make_domain_config(domain, templates, ssl_certificates, env):
 			for path, url in yaml.get("proxies", {}).items():
 				# Parse some flags in the fragment of the URL.
 				pass_http_host_header = False
+				proxy_redirect_off = False
+				frame_options_header_sameorigin = False
 				m = re.search("#(.*)$", url)
 				if m:
 					for flag in m.group(1).split(","):
 						if flag == "pass-http-host":
 							pass_http_host_header = True
+						elif flag == "no-proxy-redirect":
+							proxy_redirect_off = True
+						elif flag == "frame-options-sameorigin":
+							frame_options_header_sameorigin = True
 					url = re.sub("#(.*)$", "", url)
 
 				nginx_conf_extra += "\tlocation %s {" % path
 				nginx_conf_extra += "\n\t\tproxy_pass %s;" % url
+				if proxy_redirect_off:
+					nginx_conf_extra += "\n\t\tproxy_redirect off;"
 				if pass_http_host_header:
 					nginx_conf_extra += "\n\t\tproxy_set_header Host $http_host;"
+				if frame_options_header_sameorigin:
+					nginx_conf_extra += "\n\t\tproxy_set_header X-Frame-Options SAMEORIGIN;"
 				nginx_conf_extra += "\n\t\tproxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;"
 				nginx_conf_extra += "\n\t\tproxy_set_header X-Forwarded-Host $http_host;"
 				nginx_conf_extra += "\n\t\tproxy_set_header X-Forwarded-Proto $scheme;"
@@ -267,3 +278,4 @@ def get_web_domains_info(env):
 		}
 		for domain in get_web_domains(env)
 	]
+