Merge branch 'master' into ldap

management/dns_update.py

@@ -9,8 +9,9 @@ import ipaddress
 import rtyaml
 import dns.resolver
 
-from mailconfig import get_mail_domains
+from mailconfig import get_mail_domains, get_mail_aliases
 from utils import shell, load_env_vars_from_file, safe_domain_name, sort_domains
+from ssl_certificates import get_ssl_certificates, check_certificate
 
 # From https://stackoverflow.com/questions/3026957/how-to-validate-a-domain-name-using-regex-php/16491074#16491074
 # This regular expression matches domain names according to RFCs, it also accepts fqdn with an leading dot,
@@ -304,6 +305,52 @@ def build_zone(domain, all_domains, additional_records, www_redirect_domains, en
 		if not has_rec(qname, rtype):
 			records.append((qname, rtype, value, explanation))
 
+	# If this is a domain name that there are email addresses configured for, i.e. "something@"
+	# this domain name, then the domain name is a MTA-STS (https://tools.ietf.org/html/rfc8461)
+	# Policy Domain.
+	#
+	# A "_mta-sts" TXT record signals the presence of a MTA-STS policy, and an effectively random policy
+	# ID is used to signal that a new policy may (or may not) be deployed any time the DNS is
+	# updated.
+	#
+	# The policy itself is served at the "mta-sts" (no underscore) subdomain over HTTPS. Therefore
+	# the TLS certificate used by Postfix for STARTTLS must be a valid certificate for the MX
+	# domain name (PRIMARY_HOSTNAME) *and* the TLS certificate used by nginx for HTTPS on the mta-sts
+	# subdomain must be valid certificate for that domain. Do not set an MTA-STS policy if either
+	# certificate in use is not valid (e.g. because it is self-signed and a valid certificate has not
+	# yet been provisioned).
+	mta_sts_enabled = False
+	if domain in get_mail_domains(env):
+		# Check that PRIMARY_HOSTNAME and the mta_sts domain both have valid certificates.
+		for d in (env['PRIMARY_HOSTNAME'], "mta-sts." + domain):
+			cert = get_ssl_certificates(env).get(d)
+			if not cert:
+				break # no certificate provisioned for this domain
+			cert_status = check_certificate(d, cert['certificate'], cert['private-key'])
+			if cert_status[0] != 'OK':
+				break # certificate is not valid
+		else:
+			# 'break' was not encountered above, so both domains are good
+			mta_sts_enabled = True
+	if mta_sts_enabled:
+		mta_sts_records = [
+			("mta-sts", "A", env["PUBLIC_IP"], "Optional. MTA-STS Policy Host serving /.well-known/mta-sts.txt."),
+			("mta-sts", "AAAA", env.get('PUBLIC_IPV6'), "Optional. MTA-STS Policy Host serving /.well-known/mta-sts.txt."),
+			("_mta-sts", "TXT", "v=STSv1; id=%sZ" % datetime.datetime.now().strftime("%Y%m%d%H%M%S"), "Optional. Part of the MTA-STS policy for incoming mail. If set, a MTA-STS policy must also be published.")
+		]
+		# Rules can be custom configured accoring to https://tools.ietf.org/html/rfc8460.
+		# Skip if the rules below if the user has set a custom _smtp._tls record.
+		if not has_rec("_smtp._tls", "TXT", prefix="v=TLSRPTv1;"):
+			tls_rpt_string = ""
+			tls_rpt_email = env.get("MTA_STS_TLSRPT_EMAIL", "postmaster@%s" % env['PRIMARY_HOSTNAME'])
+			if tls_rpt_email: # if a reporting address is not cleared
+				tls_rpt_string = " rua=mailto:%s" % tls_rpt_email
+			mta_sts_records.append(("_smtp._tls", "TXT", "v=TLSRPTv1;%s" % tls_rpt_string, "Optional. Enables MTA-STS reporting."))
+		for qname, rtype, value, explanation in mta_sts_records:
+			if value is None or value.strip() == "": continue # skip IPV6 if not set
+			if not has_rec(qname, rtype):
+				records.append((qname, rtype, value, explanation))
+
 	# Sort the records. The None records *must* go first in the nsd zone file. Otherwise it doesn't matter.
 	records.sort(key = lambda rec : list(reversed(rec[0].split(".")) if rec[0] is not None else ""))
 

management/ssl_certificates.py

@@ -183,7 +183,7 @@ def get_certificates_to_provision(env, limit_domains=None, show_valid_certs=True
 	# for and subtract:
 	# * domains not in limit_domains if limit_domains is not empty
 	# * domains with custom "A" records, i.e. they are hosted elsewhere
-	# * domains with actual "A" records that point elsewhere
+	# * domains with actual "A" records that point elsewhere (misconfiguration)
 	# * domains that already have certificates that will be valid for a while
 
 	from web_update import get_web_domains
@@ -259,15 +259,41 @@ def provision_certificates(env, limit_domains):
 			"result": "skipped",
 		})
 
+	# Break into groups by DNS zone: Group every domain with its parent domain, if
+	# its parent domain is in the list of domains to request a certificate for.
+	# Start with the zones so that if the zone doesn't need a certificate itself,
+	# its children will still be grouped together. Sort the provision domains to
+	# put parents ahead of children.
+	# Since Let's Encrypt requests are limited to 100 domains at a time,
+	# we'll create a list of lists of domains where the inner lists have
+	# at most 100 items. By sorting we also get the DNS zone domain as the first
+	# entry in each list (unless we overflow beyond 100) which ends up as the
+	# primary domain listed in each certificate.
+	from dns_update import get_dns_zones
+	certs = { }
+	for zone, zonefile in get_dns_zones(env):
+		certs[zone] = [[]]
+	for domain in sort_domains(domains, env):
+		# Does the domain end with any domain we've seen so far.
+		for parent in certs.keys():
+			if domain.endswith("." + parent):
+				# Add this to the parent's list of domains.
+				# Start a new group if the list already has
+				# 100 items.
+				if len(certs[parent][-1]) == 100:
+					certs[parent].append([])
+				certs[parent][-1].append(domain)
+				break
+		else:
+			# This domain is not a child of any domain we've seen yet, so
+			# start a new group. This shouldn't happen since every zone
+			# was already added.
+			certs[domain] = [[domain]]
 
-	# Break into groups of up to 100 certificates at a time, which is Let's Encrypt's
-	# limit for a single certificate. We'll sort to put related domains together.
-	max_domains_per_group = 100
-	domains = sort_domains(domains, env)
-	certs = []
-	while len(domains) > 0:
-		certs.append( domains[:max_domains_per_group] )
-		domains = domains[max_domains_per_group:]
+	# Flatten to a list of lists of domains (from a mapping). Remove empty
+	# lists (zones with no domains that need certs).
+	certs = sum(certs.values(), [])
+	certs = [_ for _ in certs if len(_) > 0]
 
 	# Prepare to provision.
 

management/status_checks.py

@@ -5,11 +5,13 @@
 # what to do next.
 
 import sys, os, os.path, re, subprocess, datetime, multiprocessing.pool
+import asyncio
 
 import dns.reversename, dns.resolver
 import dateutil.parser, dateutil.tz
 import idna
 import psutil
+import postfix_mta_sts_resolver.resolver
 
 from dns_update import get_dns_zones, build_tlsa_record, get_custom_dns_config, get_secondary_dns, get_custom_dns_records
 from web_update import get_web_domains, get_domains_with_a_records
@@ -309,6 +311,17 @@ def run_domain_checks(rounded_time, env, output, pool):
 
 	domains_to_check = mail_domains | dns_domains | web_domains
 
+	# Remove "www", "autoconfig", "autodiscover", and "mta-sts" subdomains, which we group with their parent,
+	# if their parent is in the domains to check list.
+	domains_to_check = [
+		d for d in domains_to_check
+		if not (
+		   d.split(".", 1)[0] in ("www", "autoconfig", "autodiscover", "mta-sts")
+		   and len(d.split(".", 1)) == 2
+		   and d.split(".", 1)[1] in domains_to_check
+		)
+	]
+
 	# Get the list of domains that we don't serve web for because of a custom CNAME/A record.
 	domains_with_a_records = get_domains_with_a_records(env)
 
@@ -327,6 +340,11 @@ def run_domain_checks(rounded_time, env, output, pool):
 def run_domain_checks_on_domain(domain, rounded_time, env, dns_domains, dns_zonefiles, mail_domains, web_domains, domains_with_a_records):
 	output = BufferedOutput()
 
+	# When running inside Flask, the worker threads don't get a thread pool automatically.
+	# Also this method is called in a forked worker pool, so creating a new loop is probably
+	# a good idea.
+	asyncio.set_event_loop(asyncio.new_event_loop())
+
 	# we'd move this up, but this returns non-pickleable values
 	ssl_certificates = get_ssl_certificates(env)
 
@@ -354,6 +372,26 @@ def run_domain_checks_on_domain(domain, rounded_time, env, dns_domains, dns_zone
 	if domain in dns_domains:
 		check_dns_zone_suggestions(domain, env, output, dns_zonefiles, domains_with_a_records)
 
+	# Check auto-configured subdomains. See run_domain_checks.
+	# Skip mta-sts because we check the policy directly.
+	for label in ("www", "autoconfig", "autodiscover"):
+		subdomain = label + "." + domain
+		if subdomain in web_domains or subdomain in mail_domains:
+			# Run checks.
+			subdomain_output = run_domain_checks_on_domain(subdomain, rounded_time, env, dns_domains, dns_zonefiles, mail_domains, web_domains, domains_with_a_records)
+
+			# Prepend the domain name to the start of each check line, and then add to the
+			# checks for this domain.
+			for attr, args, kwargs in subdomain_output[1].buf:
+				if attr == "add_heading":
+					# Drop the heading, but use its text as the subdomain name in
+					# each line since it is in Unicode form.
+					subdomain = args[0]
+					continue
+				if len(args) == 1 and isinstance(args[0], str):
+					args = [ subdomain + ": " + args[0] ]
+				getattr(output, attr)(*args, **kwargs)
+
 	return (domain, output)
 
 def check_primary_hostname_dns(domain, env, output, dns_domains, dns_zonefiles):
@@ -611,6 +649,19 @@ def check_mail_domain(domain, env, output):
 		if mx != recommended_mx:
 			good_news += "  This configuration is non-standard.  The recommended configuration is '%s'." % (recommended_mx,)
 		output.print_ok(good_news)
+
+		# Check MTA-STS policy.
+		loop = asyncio.get_event_loop()
+		sts_resolver = postfix_mta_sts_resolver.resolver.STSResolver(loop=loop)
+		valid, policy = loop.run_until_complete(sts_resolver.resolve(domain))
+		if valid == postfix_mta_sts_resolver.resolver.STSFetchResult.VALID:
+			if policy[1].get("mx") == [env['PRIMARY_HOSTNAME']] and policy[1].get("mode") == "enforce": # policy[0] is the policyid
+				output.print_ok("MTA-STS policy is present.")
+			else:
+				output.print_error("MTA-STS policy is present but has unexpected settings. [{}]".format(policy[1]))
+		else:
+			output.print_error("MTA-STS policy is missing: {}".format(valid))
+
 	else:
 		output.print_error("""This domain's DNS MX record is incorrect. It is currently set to '%s' but should be '%s'. Mail will not
 			be delivered to this box. It may take several hours for public DNS to update after a change. This problem may result from

management/web_update.py

@@ -33,6 +33,9 @@ def get_web_domains(env, include_www_redirects=True, exclude_dns_elsewhere=True,
 		domains |= set('autoconfig.' + maildomain for maildomain in get_mail_domains(env, category='mail'))
 		domains |= set('autodiscover.' + maildomain for maildomain in get_mail_domains(env, category='mail'))
 
+	# 'mta-sts.' for MTA-STS support.
+	domains |= set('mta-sts.' + maildomain for maildomain in get_mail_domains(env))
+
 	if exclude_dns_elsewhere:
 		# ...Unless the domain has an A/AAAA record that maps it to a different
 		# IP address than this box. Remove those domains from our list.