1
0
mirror of https://github.com/mail-in-a-box/mailinabox.git synced 2026-03-12 17:07:23 +01:00
v0.50 (September 25, 2020)
--------------------------

Setup:

* When upgrading from versions before v0.40, setup will now warn that ownCloud/Nextcloud data cannot be migrated rather than failing the installation.

Mail:

* An MTA-STS policy for incoming mail is now published (in DNS and over HTTPS) when the primary hostname and email address domain both have a signed TLS certificate installed, allowing senders to know that an encrypted connection should be enforced.
* The per-IP connection limit to the IMAP server has been doubled to allow more devices to connect at once, especially with multiple users behind a NAT.

DNS:

* autoconfig and autodiscover subdomains and CalDAV/CardDAV SRV records are no longer generated for domains that don't have user accounts since they are unnecessary.
* IPv6 addresses can now be specified for secondary DNS nameservers in the control panel.

TLS:

* TLS certificates are now provisioned in groups by parent domain to limit easy domain enumeration and make provisioning more resilient to errors for particular domains.

Control Panel:

* The control panel API is now fully documented at https://mailinabox.email/api-docs.html.
* User passwords can now have spaces.
* Status checks for automatic subdomains have been moved into the section for the parent domain.
* Typo fixed.

Web:

* The default web page served on fresh installations now adds the `noindex` meta tag.
* The HSTS header is revised to also be sent on non-success responses.

# gpg verification failed.

# Conflicts:
#	.gitignore
#	setup/bootstrap.sh
This commit is contained in:
John Supplee
2020-10-11 18:16:36 +02:00
26 changed files with 2886 additions and 75 deletions

View File

@@ -460,9 +460,8 @@ def system_status():
self.items[-1]["extra"].append({ "text": message, "monospace": monospace })
output = WebOutput()
# Create a temporary pool of processes for the status checks
pool = multiprocessing.pool.Pool(processes=5)
run_checks(False, env, output, pool)
pool.terminate()
with multiprocessing.pool.Pool(processes=5) as pool:
run_checks(False, env, output, pool)
return json_response(output.items)
@app.route('/system/updates')

View File

@@ -16,10 +16,10 @@ if [ `date "+%u"` -eq 1 ]; then
fi
# Take a backup.
management/backup.py | management/email_administrator.py "Backup Status"
management/backup.py 2>&1 | management/email_administrator.py "Backup Status"
# Provision any new certificates for new domains or domains with expiring certificates.
management/ssl_certificates.py -q | management/email_administrator.py "TLS Certificate Provisioning Result"
management/ssl_certificates.py -q 2>&1 | management/email_administrator.py "TLS Certificate Provisioning Result"
# Run status checks and email the administrator if anything changed.
management/status_checks.py --show-changes | management/email_administrator.py "Status Checks Change Notice"
management/status_checks.py --show-changes 2>&1 | management/email_administrator.py "Status Checks Change Notice"

View File

@@ -9,8 +9,9 @@ import ipaddress
import rtyaml
import dns.resolver
from mailconfig import get_mail_domains
from mailconfig import get_mail_domains, get_mail_aliases
from utils import shell, load_env_vars_from_file, safe_domain_name, sort_domains
from ssl_certificates import get_ssl_certificates, check_certificate
# From https://stackoverflow.com/questions/3026957/how-to-validate-a-domain-name-using-regex-php/16491074#16491074
# This regular expression matches domain names according to RFCs, it also accepts fqdn with an leading dot,
@@ -280,25 +281,81 @@ def build_zone(domain, all_domains, additional_records, www_redirect_domains, en
if not has_rec(dmarc_qname, "TXT", prefix="v=DMARC1; "):
records.append((dmarc_qname, "TXT", 'v=DMARC1; p=reject', "Recommended. Prevents use of this domain name for outbound mail by specifying that the SPF rule should be honoured for mail from @%s." % (qname + "." + domain)))
# Add CardDAV/CalDAV SRV records on the non-primary hostname that points to the primary hostname.
# Add CardDAV/CalDAV SRV records on the non-primary hostname that points to the primary hostname
# for autoconfiguration of mail clients (so only domains hosting user accounts need it).
# The SRV record format is priority (0, whatever), weight (0, whatever), port, service provider hostname (w/ trailing dot).
if domain != env["PRIMARY_HOSTNAME"]:
if domain != env["PRIMARY_HOSTNAME"] and domain in get_mail_domains(env, users_only=True):
for dav in ("card", "cal"):
qname = "_" + dav + "davs._tcp"
if not has_rec(qname, "SRV"):
records.append((qname, "SRV", "0 0 443 " + env["PRIMARY_HOSTNAME"] + ".", "Recommended. Specifies the hostname of the server that handles CardDAV/CalDAV services for email addresses on this domain."))
# Adds autoconfiguration A records for all domains.
# Adds autoconfiguration A records for all domains that there are user accounts at.
# This allows the following clients to automatically configure email addresses in the respective applications.
# autodiscover.* - Z-Push ActiveSync Autodiscover
# autoconfig.* - Thunderbird Autoconfig
autodiscover_records = [
("autodiscover", "A", env["PUBLIC_IP"], "Provides email configuration autodiscovery support for Z-Push ActiveSync Autodiscover."),
("autodiscover", "AAAA", env["PUBLIC_IPV6"], "Provides email configuration autodiscovery support for Z-Push ActiveSync Autodiscover."),
("autoconfig", "A", env["PUBLIC_IP"], "Provides email configuration autodiscovery support for Thunderbird Autoconfig."),
("autoconfig", "AAAA", env["PUBLIC_IPV6"], "Provides email configuration autodiscovery support for Thunderbird Autoconfig.")
if domain in get_mail_domains(env, users_only=True):
autodiscover_records = [
("autodiscover", "A", env["PUBLIC_IP"], "Provides email configuration autodiscovery support for Z-Push ActiveSync Autodiscover."),
("autodiscover", "AAAA", env["PUBLIC_IPV6"], "Provides email configuration autodiscovery support for Z-Push ActiveSync Autodiscover."),
("autoconfig", "A", env["PUBLIC_IP"], "Provides email configuration autodiscovery support for Thunderbird Autoconfig."),
("autoconfig", "AAAA", env["PUBLIC_IPV6"], "Provides email configuration autodiscovery support for Thunderbird Autoconfig.")
]
for qname, rtype, value, explanation in autodiscover_records:
if value is None or value.strip() == "": continue # skip IPV6 if not set
if not has_rec(qname, rtype):
records.append((qname, rtype, value, explanation))
# If this is a domain name that there are email addresses configured for, i.e. "something@"
# this domain name, then the domain name is a MTA-STS (https://tools.ietf.org/html/rfc8461)
# Policy Domain.
#
# A "_mta-sts" TXT record signals the presence of a MTA-STS policy. The id field helps clients
# cache the policy. It should be stable so we don't update DNS unnecessarily but change when
# the policy changes. It must be at most 32 letters and numbers, so we compute a hash of the
# policy file.
#
# The policy itself is served at the "mta-sts" (no underscore) subdomain over HTTPS. Therefore
# the TLS certificate used by Postfix for STARTTLS must be a valid certificate for the MX
# domain name (PRIMARY_HOSTNAME) *and* the TLS certificate used by nginx for HTTPS on the mta-sts
# subdomain must be valid certificate for that domain. Do not set an MTA-STS policy if either
# certificate in use is not valid (e.g. because it is self-signed and a valid certificate has not
# yet been provisioned). Since we cannot provision a certificate without A/AAAA records, we
# always set them --- only the TXT records depend on there being valid certificates.
mta_sts_enabled = False
mta_sts_records = [
("mta-sts", "A", env["PUBLIC_IP"], "Optional. MTA-STS Policy Host serving /.well-known/mta-sts.txt."),
("mta-sts", "AAAA", env.get('PUBLIC_IPV6'), "Optional. MTA-STS Policy Host serving /.well-known/mta-sts.txt."),
]
for qname, rtype, value, explanation in autodiscover_records:
if domain in get_mail_domains(env):
# Check that PRIMARY_HOSTNAME and the mta_sts domain both have valid certificates.
for d in (env['PRIMARY_HOSTNAME'], "mta-sts." + domain):
cert = get_ssl_certificates(env).get(d)
if not cert:
break # no certificate provisioned for this domain
cert_status = check_certificate(d, cert['certificate'], cert['private-key'])
if cert_status[0] != 'OK':
break # certificate is not valid
else:
# 'break' was not encountered above, so both domains are good
mta_sts_enabled = True
if mta_sts_enabled:
# Compute an up-to-32-character hash of the policy file. We'll take a SHA-1 hash of the policy
# file (20 bytes) and encode it as base-64 (28 bytes, using alphanumeric alternate characters
# instead of '+' and '/' which are not allowed in an MTA-STS policy id) but then just take its
# first 20 characters, which is more than sufficient to change whenever the policy file changes
# (and ensures any '=' padding at the end of the base64 encoding is dropped).
with open("/var/lib/mailinabox/mta-sts.txt", "rb") as f:
mta_sts_policy_id = base64.b64encode(hashlib.sha1(f.read()).digest(), altchars=b"AA").decode("ascii")[0:20]
mta_sts_records.extend([
("_mta-sts", "TXT", "v=STSv1; id=" + mta_sts_policy_id, "Optional. Part of the MTA-STS policy for incoming mail. If set, a MTA-STS policy must also be published.")
])
# Enable SMTP TLS reporting (https://tools.ietf.org/html/rfc8460) if the user has set a config option.
# Skip if the rules below if the user has set a custom _smtp._tls record.
if env.get("MTA_STS_TLSRPT_RUA") and not has_rec("_smtp._tls", "TXT", prefix="v=TLSRPTv1;"):
mta_sts_records.append(("_smtp._tls", "TXT", "v=TLSRPTv1; rua=" + env["MTA_STS_TLSRPT_RUA"], "Optional. Enables MTA-STS reporting."))
for qname, rtype, value, explanation in mta_sts_records:
if value is None or value.strip() == "": continue # skip IPV6 if not set
if not has_rec(qname, rtype):
records.append((qname, rtype, value, explanation))
@@ -906,18 +963,19 @@ def set_secondary_dns(hostnames, env):
try:
response = resolver.query(item, "A")
except (dns.resolver.NoNameservers, dns.resolver.NXDOMAIN, dns.resolver.NoAnswer):
raise ValueError("Could not resolve the IP address of %s." % item)
try:
response = resolver.query(item, "AAAA")
except (dns.resolver.NoNameservers, dns.resolver.NXDOMAIN, dns.resolver.NoAnswer):
raise ValueError("Could not resolve the IP address of %s." % item)
else:
# Validate IP address.
try:
if "/" in item[4:]:
v = ipaddress.ip_network(item[4:]) # raises a ValueError if there's a problem
if not isinstance(v, ipaddress.IPv4Network): raise ValueError("That's an IPv6 subnet.")
else:
v = ipaddress.ip_address(item[4:]) # raises a ValueError if there's a problem
if not isinstance(v, ipaddress.IPv4Address): raise ValueError("That's an IPv6 address.")
except ValueError:
raise ValueError("'%s' is not an IPv4 address or subnet." % item[4:])
raise ValueError("'%s' is not an IPv4 or IPv6 address or subnet." % item[4:])
# Set.
set_custom_dns_record("_secondary_nameserver", "A", " ".join(hostnames), "set", env)

View File

@@ -307,13 +307,15 @@ def get_domain(emailaddr, as_unicode=True):
pass
return ret
def get_mail_domains(env, filter_aliases=lambda alias : True):
def get_mail_domains(env, filter_aliases=lambda alias : True, users_only=False):
# Returns the domain names (IDNA-encoded) of all of the email addresses
# configured on the system.
return set(
[get_domain(login, as_unicode=False) for login in get_mail_users(env)]
+ [get_domain(address, as_unicode=False) for address, *_ in get_mail_aliases(env) if filter_aliases(address) ]
)
# configured on the system. If users_only is True, only return domains
# with email addresses that correspond to user accounts.
domains = []
domains.extend([get_domain(login, as_unicode=False) for login in get_mail_users(env)])
if not users_only:
domains.extend([get_domain(address, as_unicode=False) for address, *_ in get_mail_aliases(env) if filter_aliases(address) ])
return set(domains)
def add_mail_user(email, pw, privs, quota, env):
# validate email
@@ -715,8 +717,6 @@ def validate_password(pw):
# validate password
if pw.strip() == "":
raise ValueError("No password provided.")
if re.search(r"[\s]", pw):
raise ValueError("Passwords cannot contain spaces.")
if len(pw) < 8:
raise ValueError("Passwords must be at least eight characters.")

View File

@@ -180,7 +180,7 @@ def get_certificates_to_provision(env, limit_domains=None, show_valid_certs=True
# for and subtract:
# * domains not in limit_domains if limit_domains is not empty
# * domains with custom "A" records, i.e. they are hosted elsewhere
# * domains with actual "A" records that point elsewhere
# * domains with actual "A" records that point elsewhere (misconfiguration)
# * domains that already have certificates that will be valid for a while
from web_update import get_web_domains
@@ -256,15 +256,41 @@ def provision_certificates(env, limit_domains):
"result": "skipped",
})
# Break into groups by DNS zone: Group every domain with its parent domain, if
# its parent domain is in the list of domains to request a certificate for.
# Start with the zones so that if the zone doesn't need a certificate itself,
# its children will still be grouped together. Sort the provision domains to
# put parents ahead of children.
# Since Let's Encrypt requests are limited to 100 domains at a time,
# we'll create a list of lists of domains where the inner lists have
# at most 100 items. By sorting we also get the DNS zone domain as the first
# entry in each list (unless we overflow beyond 100) which ends up as the
# primary domain listed in each certificate.
from dns_update import get_dns_zones
certs = { }
for zone, zonefile in get_dns_zones(env):
certs[zone] = [[]]
for domain in sort_domains(domains, env):
# Does the domain end with any domain we've seen so far.
for parent in certs.keys():
if domain.endswith("." + parent):
# Add this to the parent's list of domains.
# Start a new group if the list already has
# 100 items.
if len(certs[parent][-1]) == 100:
certs[parent].append([])
certs[parent][-1].append(domain)
break
else:
# This domain is not a child of any domain we've seen yet, so
# start a new group. This shouldn't happen since every zone
# was already added.
certs[domain] = [[domain]]
# Break into groups of up to 100 certificates at a time, which is Let's Encrypt's
# limit for a single certificate. We'll sort to put related domains together.
max_domains_per_group = 100
domains = sort_domains(domains, env)
certs = []
while len(domains) > 0:
certs.append( domains[:max_domains_per_group] )
domains = domains[max_domains_per_group:]
# Flatten to a list of lists of domains (from a mapping). Remove empty
# lists (zones with no domains that need certs).
certs = sum(certs.values(), [])
certs = [_ for _ in certs if len(_) > 0]
# Prepare to provision.

View File

@@ -5,11 +5,13 @@
# what to do next.
import sys, os, os.path, re, subprocess, datetime, multiprocessing.pool
import asyncio
import dns.reversename, dns.resolver
import dateutil.parser, dateutil.tz
import idna
import psutil
import postfix_mta_sts_resolver.resolver
from dns_update import get_dns_zones, build_tlsa_record, get_custom_dns_config, get_secondary_dns, get_custom_dns_records
from web_update import get_web_domains, get_domains_with_a_records
@@ -309,6 +311,17 @@ def run_domain_checks(rounded_time, env, output, pool):
domains_to_check = mail_domains | dns_domains | web_domains
# Remove "www", "autoconfig", "autodiscover", and "mta-sts" subdomains, which we group with their parent,
# if their parent is in the domains to check list.
domains_to_check = [
d for d in domains_to_check
if not (
d.split(".", 1)[0] in ("www", "autoconfig", "autodiscover", "mta-sts")
and len(d.split(".", 1)) == 2
and d.split(".", 1)[1] in domains_to_check
)
]
# Get the list of domains that we don't serve web for because of a custom CNAME/A record.
domains_with_a_records = get_domains_with_a_records(env)
@@ -327,6 +340,11 @@ def run_domain_checks(rounded_time, env, output, pool):
def run_domain_checks_on_domain(domain, rounded_time, env, dns_domains, dns_zonefiles, mail_domains, web_domains, domains_with_a_records):
output = BufferedOutput()
# When running inside Flask, the worker threads don't get a thread pool automatically.
# Also this method is called in a forked worker pool, so creating a new loop is probably
# a good idea.
asyncio.set_event_loop(asyncio.new_event_loop())
# we'd move this up, but this returns non-pickleable values
ssl_certificates = get_ssl_certificates(env)
@@ -354,6 +372,26 @@ def run_domain_checks_on_domain(domain, rounded_time, env, dns_domains, dns_zone
if domain in dns_domains:
check_dns_zone_suggestions(domain, env, output, dns_zonefiles, domains_with_a_records)
# Check auto-configured subdomains. See run_domain_checks.
# Skip mta-sts because we check the policy directly.
for label in ("www", "autoconfig", "autodiscover"):
subdomain = label + "." + domain
if subdomain in web_domains or subdomain in mail_domains:
# Run checks.
subdomain_output = run_domain_checks_on_domain(subdomain, rounded_time, env, dns_domains, dns_zonefiles, mail_domains, web_domains, domains_with_a_records)
# Prepend the domain name to the start of each check line, and then add to the
# checks for this domain.
for attr, args, kwargs in subdomain_output[1].buf:
if attr == "add_heading":
# Drop the heading, but use its text as the subdomain name in
# each line since it is in Unicode form.
subdomain = args[0]
continue
if len(args) == 1 and isinstance(args[0], str):
args = [ subdomain + ": " + args[0] ]
getattr(output, attr)(*args, **kwargs)
return (domain, output)
def check_primary_hostname_dns(domain, env, output, dns_domains, dns_zonefiles):
@@ -611,6 +649,19 @@ def check_mail_domain(domain, env, output):
if mx != recommended_mx:
good_news += " This configuration is non-standard. The recommended configuration is '%s'." % (recommended_mx,)
output.print_ok(good_news)
# Check MTA-STS policy.
loop = asyncio.get_event_loop()
sts_resolver = postfix_mta_sts_resolver.resolver.STSResolver(loop=loop)
valid, policy = loop.run_until_complete(sts_resolver.resolve(domain))
if valid == postfix_mta_sts_resolver.resolver.STSFetchResult.VALID:
if policy[1].get("mx") == [env['PRIMARY_HOSTNAME']] and policy[1].get("mode") == "enforce": # policy[0] is the policyid
output.print_ok("MTA-STS policy is present.")
else:
output.print_error("MTA-STS policy is present but has unexpected settings. [{}]".format(policy[1]))
else:
output.print_error("MTA-STS policy is missing: {}".format(valid))
else:
output.print_error("""This domain's DNS MX record is incorrect. It is currently set to '%s' but should be '%s'. Mail will not
be delivered to this box. It may take several hours for public DNS to update after a change. This problem may result from
@@ -970,13 +1021,14 @@ if __name__ == "__main__":
from utils import load_environment
env = load_environment()
pool = multiprocessing.pool.Pool(processes=10)
if len(sys.argv) == 1:
run_checks(False, env, ConsoleOutput(), pool)
with multiprocessing.pool.Pool(processes=10) as pool:
run_checks(False, env, ConsoleOutput(), pool)
elif sys.argv[1] == "--show-changes":
run_and_output_changes(env, pool)
with multiprocessing.pool.Pool(processes=10) as pool:
run_and_output_changes(env, pool)
elif sys.argv[1] == "--check-primary-hostname":
# See if the primary hostname appears resolvable and has a signed certificate.

View File

@@ -288,7 +288,7 @@ function aliases_remove(elem) {
},
function(r) {
// Responses are multiple lines of pre-formatted text.
show_modal_error("Remove User", $("<pre/>").text(r));
show_modal_error("Remove Alias", $("<pre/>").text(r));
show_aliases();
});
});

View File

@@ -90,7 +90,7 @@
<div class="col-sm-offset-1 col-sm-11">
<p class="small">
Multiple secondary servers can be separated with commas or spaces (i.e., <code>ns2.hostingcompany.com ns3.hostingcompany.com</code>).
To enable zone transfers to additional servers without listing them as secondary nameservers, add an IP address or subnet using <code>xfr:10.20.30.40</code> or <code>xfr:10.20.30.40/24</code>.
To enable zone transfers to additional servers without listing them as secondary nameservers, add an IP address or subnet using <code>xfr:10.20.30.40</code> or <code>xfr:10.0.0.0/8</code>.
</p>
<p id="secondarydns-clear-instructions" style="display: none" class="small">
Clear the input field above and click Update to use this machine itself as secondary DNS, which is the default/normal setup.

View File

@@ -24,11 +24,14 @@ def get_web_domains(env, include_www_redirects=True, exclude_dns_elsewhere=True)
# the topmost of each domain we serve.
domains |= set('www.' + zone for zone, zonefile in get_dns_zones(env))
# Add Autoconfiguration domains, allowing us to serve correct SSL certs.
# Add Autoconfiguration domains for domains that there are user accounts at:
# 'autoconfig.' for Mozilla Thunderbird auto setup.
# 'autodiscover.' for Activesync autodiscovery.
domains |= set('autoconfig.' + maildomain for maildomain in get_mail_domains(env))
domains |= set('autodiscover.' + maildomain for maildomain in get_mail_domains(env))
domains |= set('autoconfig.' + maildomain for maildomain in get_mail_domains(env, users_only=True))
domains |= set('autodiscover.' + maildomain for maildomain in get_mail_domains(env, users_only=True))
# 'mta-sts.' for MTA-STS support for all domains that have email addresses.
domains |= set('mta-sts.' + maildomain for maildomain in get_mail_domains(env))
if exclude_dns_elsewhere:
# ...Unless the domain has an A/AAAA record that maps it to a different
@@ -155,9 +158,23 @@ def make_domain_config(domain, templates, ssl_certificates, env):
# any proxy or redirect here?
for path, url in yaml.get("proxies", {}).items():
# Parse some flags in the fragment of the URL.
pass_http_host_header = False
m = re.search("#(.*)$", url)
if m:
for flag in m.group(1).split(","):
if flag == "pass-http-host":
pass_http_host_header = True
url = re.sub("#(.*)$", "", url)
nginx_conf_extra += "\tlocation %s {" % path
nginx_conf_extra += "\n\t\tproxy_pass %s;" % url
if pass_http_host_header:
nginx_conf_extra += "\n\t\tproxy_set_header Host $http_host;"
nginx_conf_extra += "\n\t\tproxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;"
nginx_conf_extra += "\n\t\tproxy_set_header X-Forwarded-Host $http_host;"
nginx_conf_extra += "\n\t\tproxy_set_header X-Forwarded-Proto $scheme;"
nginx_conf_extra += "\n\t\tproxy_set_header X-Real-IP $remote_addr;"
nginx_conf_extra += "\n\t}\n"
for path, alias in yaml.get("aliases", {}).items():
nginx_conf_extra += "\tlocation %s {" % path
@@ -171,9 +188,9 @@ def make_domain_config(domain, templates, ssl_certificates, env):
# Add the HSTS header.
if hsts == "yes":
nginx_conf_extra += "add_header Strict-Transport-Security max-age=15768000;\n"
nginx_conf_extra += "add_header Strict-Transport-Security \"max-age=15768000\" always;\n"
elif hsts == "preload":
nginx_conf_extra += "add_header Strict-Transport-Security \"max-age=15768000; includeSubDomains; preload\";\n"
nginx_conf_extra += "add_header Strict-Transport-Security \"max-age=15768000; includeSubDomains; preload\" always;\n"
# Add in any user customizations in the includes/ folder.
nginx_conf_custom_include = os.path.join(env["STORAGE_ROOT"], "www", safe_domain_name(domain) + ".conf")