From b6933a73fa233a837b40aa8c7ebcb24d74254eca Mon Sep 17 00:00:00 2001 From: Joshua Tauberer Date: Sat, 10 Oct 2015 22:03:55 +0000 Subject: [PATCH 1/9] provision and install free SSL certificates from Let's Encrypt --- conf/nginx.conf | 21 +- management/daily_tasks.sh | 3 + management/ssl_certificates.py | 388 ++++++++++++++++++++++++++++++--- management/status_checks.py | 42 ++-- management/web_update.py | 23 +- setup/management.sh | 7 +- setup/start.sh | 3 + 7 files changed, 408 insertions(+), 79 deletions(-) mode change 100644 => 100755 management/ssl_certificates.py diff --git a/conf/nginx.conf b/conf/nginx.conf index 34117efc..538f0743 100644 --- a/conf/nginx.conf +++ b/conf/nginx.conf @@ -1,6 +1,8 @@ ## $HOSTNAME -# Redirect all HTTP to HTTPS. +# Redirect all HTTP to HTTPS *except* the ACME challenges (Let's Encrypt SSL certificate +# domain validation challenges) path, which must be served over HTTP per the ACME spec +# (due to some Apache vulnerability). server { listen 80; listen [::]:80; @@ -12,10 +14,19 @@ server { # error pages and in the "Server" HTTP-Header. server_tokens off; - # Redirect using the 'return' directive and the built-in - # variable '$request_uri' to avoid any capturing, matching - # or evaluation of regular expressions. - return 301 https://$HOSTNAME$request_uri; + location / { + # Redirect using the 'return' directive and the built-in + # variable '$request_uri' to avoid any capturing, matching + # or evaluation of regular expressions. + return 301 https://$HOSTNAME$request_uri; + } + + location /.well-known/acme-challenge/ { + # This path must be served over HTTP for ACME domain validation. + # We map this to a special path where our SSL cert provisioning + # tool knows to store challenge response files. + alias $STORAGE_ROOT/ssl/lets_encrypt/acme_challenges/; + } } # The secure HTTPS server. diff --git a/management/daily_tasks.sh b/management/daily_tasks.sh index 69d14c71..52bf4602 100755 --- a/management/daily_tasks.sh +++ b/management/daily_tasks.sh @@ -4,5 +4,8 @@ # Take a backup. management/backup.py | management/email_administrator.py "Backup Status" +# Provision any new certificates for new domains or domains with expiring certificates. +management/ssl_certificates.py --headless | management/email_administrator.py "Error Provisioning TLS Certificate" + # Run status checks and email the administrator if anything changed. management/status_checks.py --show-changes | management/email_administrator.py "Status Checks Change Notice" diff --git a/management/ssl_certificates.py b/management/ssl_certificates.py old mode 100644 new mode 100755 index 1e9a9ca8..7edd2108 --- a/management/ssl_certificates.py +++ b/management/ssl_certificates.py @@ -1,8 +1,11 @@ +#!/usr/bin/python3 # Utilities for installing and selecting SSL certificates. import os, os.path, re, shutil -from utils import shell, safe_domain_name +from utils import shell, safe_domain_name, sort_domains + +# SELECTING SSL CERTIFICATES FOR USE IN WEB def get_ssl_certificates(env): # Scan all of the installed SSL certificates and map every domain @@ -17,6 +20,8 @@ def get_ssl_certificates(env): # List all of the files in the SSL directory and one level deep. def get_file_list(): + if not os.path.exists(ssl_root): + return for fn in os.listdir(ssl_root): fn = os.path.join(ssl_root, fn) if os.path.isfile(fn): @@ -82,10 +87,27 @@ def get_ssl_certificates(env): # prefer one that is not self-signed cert.issuer != cert.subject, + ########################################################### + # The above lines ensure that valid certificates are chosen + # over invalid certificates. The lines below choose between + # multiple valid certificates available for this domain. + ########################################################### + # prefer one with the expiration furthest into the future so # that we can easily rotate to new certs as we get them cert.not_valid_after, + ########################################################### + # We always choose the certificate that is good for the + # longest period of time. This is important for how we + # provision certificates for Let's Encrypt. To ensure that + # we don't re-provision every night, we have to ensure that + # if we choose to provison a certificate that it will + # *actually* be used so the provisioning logic knows it + # doesn't still need to provision a certificate for the + # domain. + ########################################################### + # in case a certificate is installed in multiple paths, # prefer the... lexicographically last one? cert._filename, @@ -96,46 +118,348 @@ def get_ssl_certificates(env): "private-key": cert._private_key._filename, "certificate": cert._filename, "primary-domain": cert._primary_domain, + "certificate_object": cert, } return ret -def get_domain_ssl_files(domain, ssl_certificates, env, allow_missing_cert=False): - # Get the default paths. +def get_domain_ssl_files(domain, ssl_certificates, env, allow_missing_cert=False, raw=False): + # Get the system certificate info. ssl_private_key = os.path.join(os.path.join(env["STORAGE_ROOT"], 'ssl', 'ssl_private_key.pem')) ssl_certificate = os.path.join(os.path.join(env["STORAGE_ROOT"], 'ssl', 'ssl_certificate.pem')) + system_certificate = { + "private-key": ssl_private_key, + "certificate": ssl_certificate, + "primary-domain": env['PRIMARY_HOSTNAME'], + "certificate_object": load_pem(load_cert_chain(ssl_certificate)[0]), + } if domain == env['PRIMARY_HOSTNAME']: # The primary domain must use the server certificate because # it is hard-coded in some service configuration files. - return ssl_private_key, ssl_certificate, None + return system_certificate wildcard_domain = re.sub("^[^\.]+", "*", domain) - if domain in ssl_certificates: - cert_info = ssl_certificates[domain] - cert_type = "multi-domain" + return ssl_certificates[domain] elif wildcard_domain in ssl_certificates: - cert_info = ssl_certificates[wildcard_domain] - cert_type = "wildcard" + return ssl_certificates[wildcard_domain] elif not allow_missing_cert: - # No certificate is available for this domain! Return default files. - ssl_via = "Using certificate for %s." % env['PRIMARY_HOSTNAME'] - return ssl_private_key, ssl_certificate, ssl_via + # No valid certificate is available for this domain! Return default files. + return system_certificate else: - # No certificate is available - and warn appropriately. + # No valid certificate is available for this domain. return None - # 'via' is a hint to the user about which certificate is in use for the domain - if cert_info['certificate'] == os.path.join(env["STORAGE_ROOT"], 'ssl', 'ssl_certificate.pem'): - # Using the server certificate. - via = "Using same %s certificate as for %s." % (cert_type, env['PRIMARY_HOSTNAME']) - elif cert_info['primary-domain'] != domain and cert_info['primary-domain'] in ssl_certificates and cert_info == ssl_certificates[cert_info['primary-domain']]: - via = "Using same %s certificate as for %s." % (cert_type, cert_info['primary-domain']) - else: - via = None # don't show a hint - show expiration info instead - return cert_info['private-key'], cert_info['certificate'], via +# PROVISIONING CERTIFICATES FROM LETSENCRYPT + +def get_certificates_to_provision(env): + # Get a list of domain names that we should now provision certificates + # for. Provision if a domain name has no valid certificate or if any + # certificate is expiring in 14 days. If provisioning anything, also + # provision certificates expiring within 30 days. The period between + # 14 and 30 days allows us to consolidate domains into multi-domain + # certificates for domains expiring around the same time. + + from web_update import get_web_domains + + import datetime + now = datetime.datetime.utcnow() + + # Get domains with missing & expiring certificates. + certs = get_ssl_certificates(env) + domains = set() + domains_if_any = set() + for domain in get_web_domains(env): + try: + cert = get_domain_ssl_files(domain, certs, env, allow_missing_cert=True) + except FileNotFoundError: + # system certificate is not present + continue + if cert is None: + # No valid certificate available. + domains.add(domain) + else: + cert = cert["certificate_object"] + if cert.issuer == cert.subject: + # This is self-signed. Get a real one. + domains.add(domain) + + # Valid certificate today, but is it expiring soon? + elif cert.not_valid_after-now < datetime.timedelta(days=14): + domains.add(domain) + elif cert.not_valid_after-now < datetime.timedelta(days=30): + domains_if_any.add(domain) + + # Filter out domains that don't have correct DNS, because then the CA + # won't be able to do DNS validation. + def is_domain_dns_correct(domain): + # Must make qname absolute to prevent a fall-back lookup with a + # search domain appended. + import dns.resolver + try: + response = dns.resolver.query(domain + ".", "A") + except: + return False + return len(response) == 1 and str(response[0]) == env["PUBLIC_IP"] + domains = set(d for d in domains if is_domain_dns_correct(d)) + domains_if_any = set(d for d in domains_if_any if is_domain_dns_correct(d)) + + # If there are any domains we definitely will provision for, add in + # additional domains to do at this time. + if len(domains) > 0: + domains |= domains_if_any + + # Sort, just to keep related domain names together in the next step. + domains = sort_domains(domains, env) + + # Break into groups of up to 100 certificates at a time, which is Let's Encrypt's + # limit for a single certificate. + cert_domains = [] + while len(domains) > 0: + cert_domains.append( domains[0:100] ) + domains = domains[100:] + + # Return a list of lists of domain names. + return cert_domains + +def provision_certificates(env, agree_to_tos_url=None, logger=None): + import requests.exceptions + import acme.messages + + from free_tls_certificates import client + + # What domains to provision certificates for? This is a list of + # lists of domains. + certs = get_certificates_to_provision(env) + if len(certs) == 0: + return { + "requests": [], + } + + # Prepare to provision. + + # Where should we put our Let's Encrypt account info and state cache. + account_path = os.path.join(env['STORAGE_ROOT'], 'ssl/lets_encrypt') + if not os.path.exists(account_path): + os.mkdir(account_path) + + # Where should we put ACME challenge files. This is mapped to /.well-known/acme_challenge + # by the nginx configuration. + challenges_path = os.path.join(account_path, 'acme_challenges') + if not os.path.exists(challenges_path): + os.mkdir(challenges_path) + + # Read in the private key that we use for all TLS certificates. We'll need that + # to generate a CSR (done by free_tls_certificates). + with open(os.path.join(env['STORAGE_ROOT'], 'ssl/ssl_private_key.pem'), 'rb') as f: + private_key = f.read() + + # Provision certificates. + + ret = [] + for domain_list in certs: + # For return. + ret_item = { + "domains": domain_list, + "log": [], + } + ret.append(ret_item) + + # Logging for free_tls_certificates. + def my_logger(message): + if logger: logger(message) + ret_item["log"].append(message) + + # Attempt to provision a certificate. + try: + try: + cert = client.issue_certificate( + domain_list, + account_path, + agree_to_tos_url=agree_to_tos_url, + private_key=private_key, + logger=my_logger) + + except client.NeedToTakeAction as e: + # Write out the ACME challenge files. + + for action in e.actions: + if isinstance(action, client.NeedToInstallFile): + fn = os.path.join(challenges_path, action.file_name) + with open(fn, 'w') as f: + f.write(action.contents) + else: + raise ValueError(str(action)) + + # Try to provision now that the challenge files are installed. + + cert = client.issue_certificate( + domain_list, + account_path, + private_key=private_key, + logger=my_logger) + + except client.NeedToAgreeToTOS as e: + # The user must agree to the Let's Encrypt terms of service agreement + # before any further action can be taken. + ret_item.update({ + "result": "agree-to-tos", + "url": e.url, + }) + + except client.WaitABit as e: + # We need to hold on for a bit before querying again to see if we can + # acquire a provisioned certificate. + import time, datetime + ret_item.update({ + "result": "wait", + "until": e.until_when, #.isoformat(), + "seconds": (e.until_when - datetime.datetime.now()).total_seconds() + }) + + except client.AccountDataIsCorrupt as e: + # This is an extremely rare condition. + ret_item.update({ + "result": "error", + "message": "Something unexpected went wrong. It looks like your local Let's Encrypt account data is corrupted. There was a problem with the file " + e.account_file_path + ".", + }) + + except (client.InvalidDomainName, client.NeedToTakeAction, acme.messages.Error, requests.exceptions.RequestException) as e: + ret_item.update({ + "result": "error", + "message": "Something unexpected went wrong: " + str(e), + }) + + else: + # A certificate was issued. + + install_status = install_cert(domain_list[0], cert['cert'].decode("ascii"), b"\n".join(cert['chain']).decode("ascii"), env, raw=True) + + # str indicates the certificate was not installed. + if isinstance(install_status, str): + ret_item.update({ + "result": "error", + "message": "Something unexpected was wrong with the provisioned certificate: " + install_status, + }) + else: + # A list indicates success and what happened next. + ret_item["log"].extend(install_status) + ret_item.update({ + "result": "installed", + }) + + # Return what happened with each certificate request. + return { + "requests": ret + } + +def provision_certificates_cmdline(): + import sys + from utils import load_environment, exclusive_process + + exclusive_process("update_tls_certificates") + env = load_environment() + + agree_to_tos_url = None + while True: + # Run the provisioning script. This installs certificates. If there are + # a very large number of domains on this box, it issues separate + # certificates for groups of domains. We have to check the result for + # each group. + def my_logger(message): + if "-v" in sys.argv: + print(">", message) + status = provision_certificates(env, agree_to_tos_url=agree_to_tos_url, logger=my_logger) + agree_to_tos_url = None # reset to prevent infinite looping + + if not status["requests"]: + # No domains need certificates. + if "--headless" not in sys.argv or "-v" in sys.argv: + print("No domains hosted on this box need a certificate at this time.") + sys.exit(0) + + # What happened? + wait_until = None + wait_domains = [] + for request in status["requests"]: + if request["result"] == "agree-to-tos": + # We may have asked already in a previous iteration. + if agree_to_tos_url is not None: + continue + + # Can't ask the user a question in this mode. + if "--headless" in sys.argv: + print("Can't issue TLS certficate until user has agreed to Let's Encrypt TOS.") + sys.exit(1) + + print(""" +I'm going to provision a TLS certificate (formerly called a SSL certificate) +for you from Let's Encrypt (letsencrypt.org). + +TLS certificates are cryptographic keys that ensure communication between +you and this box are secure when getting and sending mail and visiting +websites hosted on this box. Let's Encrypt is a free provider of TLS +certificates. + +Please open this document in your web browser: + +%s + +It is Let's Encrypt's terms of service agreement. If you agree, I can +provision that TLS certificate. If you don't agree, you will have an +opportunity to install your own TLS certificate from the Mail-in-a-Box +control panel. + +Do you agree to the agreement? Type Y or N and press : """ + % request["url"], end='', flush=True) + + if sys.stdin.readline().strip().upper() != "Y": + print("\nYou didn't agree. Quitting.") + sys.exit(1) + + # Okay, indicate agreement on next iteration. + agree_to_tos_url = request["url"] + + if request["result"] == "wait": + # Must wait. We'll record until when. The wait occurs below. + if wait_until is None: + wait_until = request["until"] + else: + wait_until = max(wait_until, request["until"]) + wait_domains += request["domains"] + + if request["result"] == "error": + print(", ".join(request["domains"]) + ":") + print(request["message"]) + + if request["result"] == "installed": + print("A TLS certificate was successfully installed for " + ", ".join(request["domains"]) + ".") + + if wait_until: + # Wait, then loop. + import time, datetime + print() + print("A TLS certificate was requested for: " + ", ".join(wait_domains) + ".") + first = True + while wait_until > datetime.datetime.now(): + if "--headless" not in sys.argv or first: + print ("We have to wait", int(round((wait_until - datetime.datetime.now()).total_seconds())), "seconds for the certificate to be issued...") + time.sleep(10) + first = False + + continue # Loop! + + if agree_to_tos_url: + # The user agrees to the TOS. Loop to try again by agreeing. + continue # Loop! + + # Unless we were instructed to wait, or we just agreed to the TOS, + # we're done for now. + break + +# INSTALLING A NEW CERTIFICATE FROM THE CONTROL PANEL def create_csr(domain, ssl_key, country_code, env): return shell("check_output", [ @@ -144,7 +468,7 @@ def create_csr(domain, ssl_key, country_code, env): "-sha256", "-subj", "/C=%s/ST=/L=/O=/CN=%s" % (country_code, domain)]) -def install_cert(domain, ssl_cert, ssl_chain, env): +def install_cert(domain, ssl_cert, ssl_chain, env, raw=False): # Write the combined cert+chain to a temporary path and validate that it is OK. # The certificate always goes above the chain. import tempfile @@ -203,8 +527,10 @@ def install_cert(domain, ssl_cert, ssl_chain, env): # Update the web configuration so nginx picks up the new certificate file. from web_update import do_web_update ret.append( do_web_update(env) ) + if raw: return ret return "\n".join(ret) +# VALIDATION OF CERTIFICATES def check_certificate(domain, ssl_certificate, ssl_private_key, warn_if_expiring_soon=True, rounded_time=False, just_check_domain=False): # Check that the ssl_certificate & ssl_private_key files are good @@ -305,16 +631,16 @@ def check_certificate(domain, ssl_certificate, ssl_private_key, warn_if_expiring # But is it expiring soon? cert_expiration_date = cert.not_valid_after ndays = (cert_expiration_date-now).days - if not rounded_time or ndays < 7: + if not rounded_time or ndays <= 10: + # Yikes better renew soon! expiry_info = "The certificate expires in %d days on %s." % (ndays, cert_expiration_date.strftime("%x")) - elif ndays <= 14: - expiry_info = "The certificate expires in less than two weeks, on %s." % cert_expiration_date.strftime("%x") - elif ndays <= 31: - expiry_info = "The certificate expires in less than a month, on %s." % cert_expiration_date.strftime("%x") else: + # We'll renew it with Lets Encrypt. expiry_info = "The certificate expires on %s." % cert_expiration_date.strftime("%x") - if ndays <= 31 and warn_if_expiring_soon: + if ndays <= 10 and warn_if_expiring_soon: + # Warn on day 10 to give 4 days for us to automatically renew the + # certificate, which occurs on day 14. return ("The certificate is expiring soon: " + expiry_info, None) # Return the special OK code. @@ -381,3 +707,7 @@ def get_certificate_domains(cert): pass return names, cn + +if __name__ == "__main__": + # Provision certificates. + provision_certificates_cmdline() diff --git a/management/status_checks.py b/management/status_checks.py index f7578dd2..4a62b091 100755 --- a/management/status_checks.py +++ b/management/status_checks.py @@ -278,23 +278,24 @@ def run_domain_checks(rounded_time, env, output, pool): # Get the list of domains that we don't serve web for because of a custom CNAME/A record. domains_with_a_records = get_domains_with_a_records(env) - ssl_certificates = get_ssl_certificates(env) - # Serial version: #for domain in sort_domains(domains_to_check, env): # run_domain_checks_on_domain(domain, rounded_time, env, dns_domains, dns_zonefiles, mail_domains, web_domains) # Parallelize the checks across a worker pool. - args = ((domain, rounded_time, env, dns_domains, dns_zonefiles, mail_domains, web_domains, domains_with_a_records, ssl_certificates) + args = ((domain, rounded_time, env, dns_domains, dns_zonefiles, mail_domains, web_domains, domains_with_a_records) for domain in domains_to_check) ret = pool.starmap(run_domain_checks_on_domain, args, chunksize=1) ret = dict(ret) # (domain, output) => { domain: output } for domain in sort_domains(ret, env): ret[domain].playback(output) -def run_domain_checks_on_domain(domain, rounded_time, env, dns_domains, dns_zonefiles, mail_domains, web_domains, domains_with_a_records, ssl_certificates): +def run_domain_checks_on_domain(domain, rounded_time, env, dns_domains, dns_zonefiles, mail_domains, web_domains, domains_with_a_records): output = BufferedOutput() + # we'd move this up, but this returns non-pickleable values + ssl_certificates = get_ssl_certificates(env) + # The domain is IDNA-encoded in the database, but for display use Unicode. try: domain_display = idna.decode(domain.encode('ascii')) @@ -656,45 +657,28 @@ def check_ssl_cert(domain, rounded_time, ssl_certificates, env, output): if query_dns(domain, "A", None) not in (env['PUBLIC_IP'], None): return # Where is the SSL stored? - x = get_domain_ssl_files(domain, ssl_certificates, env, allow_missing_cert=True) - - if x is None: + tls_cert = get_domain_ssl_files(domain, ssl_certificates, env, allow_missing_cert=True) + if tls_cert is None: output.print_warning("""No SSL certificate is installed for this domain. Visitors to a website on this domain will get a security warning. If you are not serving a website on this domain, you do not need to take any action. Use the SSL Certificates page in the control panel to install a SSL certificate.""") return - ssl_key, ssl_certificate, ssl_via = x - # Check that the certificate is good. - cert_status, cert_status_details = check_certificate(domain, ssl_certificate, ssl_key, rounded_time=rounded_time) + cert_status, cert_status_details = check_certificate(domain, tls_cert["certificate"], tls_cert["private-key"], rounded_time=rounded_time) if cert_status == "OK": # The certificate is ok. The details has expiry info. - output.print_ok("SSL certificate is signed & valid. %s %s" % (ssl_via if ssl_via else "", cert_status_details)) + output.print_ok("SSL certificate is signed & valid. " + cert_status_details) elif cert_status == "SELF-SIGNED": # Offer instructions for purchasing a signed certificate. - - fingerprint = shell('check_output', [ - "openssl", - "x509", - "-in", ssl_certificate, - "-noout", - "-fingerprint" - ]) - fingerprint = re.sub(".*Fingerprint=", "", fingerprint).strip() - if domain == env['PRIMARY_HOSTNAME']: output.print_error("""The SSL certificate for this domain is currently self-signed. You will get a security warning when you check or send email and when visiting this domain in a web browser (for webmail or - static site hosting). Use the SSL Certificates page in the control panel to install a signed SSL certificate. - You may choose to leave the self-signed certificate in place and confirm the security exception, but check that - the certificate fingerprint matches the following:""") - output.print_line("") - output.print_line(" " + fingerprint, monospace=True) + static site hosting).""") else: output.print_error("""The SSL certificate for this domain is self-signed.""") @@ -927,10 +911,10 @@ if __name__ == "__main__": if query_dns(domain, "A") != env['PUBLIC_IP']: sys.exit(1) ssl_certificates = get_ssl_certificates(env) - ssl_key, ssl_certificate, ssl_via = get_domain_ssl_files(domain, ssl_certificates, env) - if not os.path.exists(ssl_certificate): + tls_cert = get_domain_ssl_files(domain, ssl_certificates, env) + if not os.path.exists(tls_cert["certificate"]): sys.exit(1) - cert_status, cert_status_details = check_certificate(domain, ssl_certificate, ssl_key, warn_if_expiring_soon=False) + cert_status, cert_status_details = check_certificate(domain, tls_cert["certificate"], tls_cert["private-key"], warn_if_expiring_soon=False) if cert_status != "OK": sys.exit(1) sys.exit(0) diff --git a/management/web_update.py b/management/web_update.py index 9757dc84..5f5b6170 100644 --- a/management/web_update.py +++ b/management/web_update.py @@ -119,7 +119,7 @@ def make_domain_config(domain, templates, ssl_certificates, env): root = get_web_root(domain, env) # What private key and SSL certificate will we use for this domain? - ssl_key, ssl_certificate, ssl_via = get_domain_ssl_files(domain, ssl_certificates, env) + tls_cert = get_domain_ssl_files(domain, ssl_certificates, env) # ADDITIONAL DIRECTIVES. @@ -136,7 +136,7 @@ def make_domain_config(domain, templates, ssl_certificates, env): finally: f.close() return sha1.hexdigest() - nginx_conf_extra += "# ssl files sha1: %s / %s\n" % (hashfile(ssl_key), hashfile(ssl_certificate)) + nginx_conf_extra += "# ssl files sha1: %s / %s\n" % (hashfile(tls_cert["private-key"]), hashfile(tls_cert["certificate"])) # Add in any user customizations in YAML format. hsts = "yes" @@ -177,8 +177,8 @@ def make_domain_config(domain, templates, ssl_certificates, env): nginx_conf = nginx_conf.replace("$STORAGE_ROOT", env['STORAGE_ROOT']) nginx_conf = nginx_conf.replace("$HOSTNAME", domain) nginx_conf = nginx_conf.replace("$ROOT", root) - nginx_conf = nginx_conf.replace("$SSL_KEY", ssl_key) - nginx_conf = nginx_conf.replace("$SSL_CERTIFICATE", ssl_certificate) + nginx_conf = nginx_conf.replace("$SSL_KEY", tls_cert["private-key"]) + nginx_conf = nginx_conf.replace("$SSL_CERTIFICATE", tls_cert["certificate"]) nginx_conf = nginx_conf.replace("$REDIRECT_DOMAIN", re.sub(r"^www\.", "", domain)) # for default www redirects to parent domain return nginx_conf @@ -193,20 +193,15 @@ def get_web_root(domain, env, test_exists=True): def get_web_domains_info(env): www_redirects = set(get_web_domains(env)) - set(get_web_domains(env, include_www_redirects=False)) has_root_proxy_or_redirect = set(get_web_domains_with_root_overrides(env)) + ssl_certificates = get_ssl_certificates(env) # for the SSL config panel, get cert status def check_cert(domain): - ssl_certificates = get_ssl_certificates(env) - x = get_domain_ssl_files(domain, ssl_certificates, env, allow_missing_cert=True) - if x is None: return ("danger", "No Certificate Installed") - ssl_key, ssl_certificate, ssl_via = x - cert_status, cert_status_details = check_certificate(domain, ssl_certificate, ssl_key) + tls_cert = get_domain_ssl_files(domain, ssl_certificates, env, allow_missing_cert=True) + if tls_cert is None: return ("danger", "No Certificate Installed") + cert_status, cert_status_details = check_certificate(domain, tls_cert["certificate"], tls_cert["private-key"]) if cert_status == "OK": - if not ssl_via: - return ("success", "Signed & valid. " + cert_status_details) - else: - # This is an alternate domain but using the same cert as the primary domain. - return ("success", "Signed & valid. " + ssl_via) + return ("success", "Signed & valid. " + cert_status_details) elif cert_status == "SELF-SIGNED": return ("warning", "Self-signed. Get a signed certificate to stop warnings.") else: diff --git a/setup/management.sh b/setup/management.sh index 35e59c41..d4d22638 100755 --- a/setup/management.sh +++ b/setup/management.sh @@ -11,8 +11,11 @@ if [ -f /usr/local/lib/python2.7/dist-packages/boto/__init__.py ]; then hide_out # build-essential libssl-dev libffi-dev python3-dev: Required to pip install cryptography. apt_install python3-flask links duplicity python-boto libyaml-dev python3-dnspython python3-dateutil \ build-essential libssl-dev libffi-dev python3-dev python-pip -hide_output pip3 install --upgrade rtyaml "email_validator>=1.0.0" "idna>=2.0.0" "cryptography>=1.0.2" boto psutil +# Install other Python packages. The first line is the packages that Josh maintains himself! +hide_output pip3 install --upgrade \ + rtyaml "email_validator>=1.0.0" free_tls_certificates \ + "idna>=2.0.0" "cryptography>=1.0.2" boto psutil # email_validator is repeated in setup/questions.sh # Create a backup directory and a random key for encrypting backups. @@ -44,5 +47,5 @@ cat > /etc/cron.d/mailinabox-nightly << EOF; 0 3 * * * root (cd `pwd` && management/daily_tasks.sh) EOF -# Start it. +# Start the management server. restart_service mailinabox diff --git a/setup/start.sh b/setup/start.sh index 7f1e989c..8c174109 100755 --- a/setup/start.sh +++ b/setup/start.sh @@ -116,6 +116,9 @@ done tools/dns_update tools/web_update +# If DNS is already working, try to provision TLS certficates from Let's Encrypt. +management/ssl_certificates.py + # If there aren't any mail users yet, create one. source setup/firstuser.sh From b1b57f9bfdcf7abedebe50a657f48e5feb79f989 Mon Sep 17 00:00:00 2001 From: Joshua Tauberer Date: Fri, 1 Jan 2016 12:10:32 -0500 Subject: [PATCH 2/9] don't try to get certs for IDNA domains and report all reasons for not fetching a certificate fixes #646 --- management/ssl_certificates.py | 107 +++++++++++++++++++++++---------- 1 file changed, 74 insertions(+), 33 deletions(-) diff --git a/management/ssl_certificates.py b/management/ssl_certificates.py index 7edd2108..c71c6950 100755 --- a/management/ssl_certificates.py +++ b/management/ssl_certificates.py @@ -5,6 +5,8 @@ import os, os.path, re, shutil from utils import shell, safe_domain_name, sort_domains +import idna + # SELECTING SSL CERTIFICATES FOR USE IN WEB def get_ssl_certificates(env): @@ -155,7 +157,7 @@ def get_domain_ssl_files(domain, ssl_certificates, env, allow_missing_cert=False # PROVISIONING CERTIFICATES FROM LETSENCRYPT def get_certificates_to_provision(env): - # Get a list of domain names that we should now provision certificates + # Get a set of domain names that we should now provision certificates # for. Provision if a domain name has no valid certificate or if any # certificate is expiring in 14 days. If provisioning anything, also # provision certificates expiring within 30 days. The period between @@ -171,11 +173,13 @@ def get_certificates_to_provision(env): certs = get_ssl_certificates(env) domains = set() domains_if_any = set() + problems = { } for domain in get_web_domains(env): try: cert = get_domain_ssl_files(domain, certs, env, allow_missing_cert=True) - except FileNotFoundError: + except FileNotFoundError as e: # system certificate is not present + problems[domain] = "Error: " + str(e) continue if cert is None: # No valid certificate available. @@ -192,37 +196,49 @@ def get_certificates_to_provision(env): elif cert.not_valid_after-now < datetime.timedelta(days=30): domains_if_any.add(domain) - # Filter out domains that don't have correct DNS, because then the CA - # won't be able to do DNS validation. - def is_domain_dns_correct(domain): - # Must make qname absolute to prevent a fall-back lookup with a - # search domain appended. - import dns.resolver - try: - response = dns.resolver.query(domain + ".", "A") - except: + # It's valid. + problems[domain] = "The certificate is valid for at least another 30 days --- no need to replace." + + # Filter out domains that we can't provision a certificate for. + def can_provision_for_domain(domain): + # Let's Encrypt doesn't yet support IDNA domains. + # We store domains in IDNA (ASCII). To see if this domain is IDNA, + # we'll see if its IDNA-decoded form is different. + if idna.decode(domain.encode("ascii")) != domain: + problems[domain] = "Let's Encrypt does not yet support provisioning certificates for internationalized domains." return False - return len(response) == 1 and str(response[0]) == env["PUBLIC_IP"] - domains = set(d for d in domains if is_domain_dns_correct(d)) - domains_if_any = set(d for d in domains_if_any if is_domain_dns_correct(d)) + + # Does the domain resolve to this machine in public DNS? If not, + # we can't do domain control validation. For IPv6 is configured, + # make sure both IPv4 and IPv6 are correct because we don't know + # how Let's Encrypt will connect. + import dns.resolver + for rtype, value in [("A", env["PUBLIC_IP"]), ("AAAA", env.get("PUBLIC_IPV6"))]: + if not value: continue # IPv6 is not configured + try: + # Must make the qname absolute to prevent a fall-back lookup with a + # search domain appended, by adding a period to the end. + response = dns.resolver.query(domain + ".", rtype) + except (dns.resolver.NoNameservers, dns.resolver.NXDOMAIN, dns.resolver.NoAnswer) as e: + problems[domain] = "DNS isn't configured properly for this domain: DNS resolution failed (%s: %s)." % (rtype, str(e) or repr(e)) # NoAnswer's str is empty + return False + except Exception as e: + problems[domain] = "DNS isn't configured properly for this domain: DNS lookup had an error: %s." % str(e) + return False + if len(response) != 1 or str(response[0]) != value: + problems[domain] = "Domain control validation cannot be performed for this domain because DNS points the domain to another machine (%s %s)." % (rtype, ", ".join(str(r) for r in response)) + return False + + return True + + domains = set(filter(can_provision_for_domain, domains)) # If there are any domains we definitely will provision for, add in # additional domains to do at this time. if len(domains) > 0: - domains |= domains_if_any + domains |= set(filter(can_provision_for_domain, domains_if_any)) - # Sort, just to keep related domain names together in the next step. - domains = sort_domains(domains, env) - - # Break into groups of up to 100 certificates at a time, which is Let's Encrypt's - # limit for a single certificate. - cert_domains = [] - while len(domains) > 0: - cert_domains.append( domains[0:100] ) - domains = domains[100:] - - # Return a list of lists of domain names. - return cert_domains + return (domains, problems) def provision_certificates(env, agree_to_tos_url=None, logger=None): import requests.exceptions @@ -230,14 +246,25 @@ def provision_certificates(env, agree_to_tos_url=None, logger=None): from free_tls_certificates import client - # What domains to provision certificates for? This is a list of - # lists of domains. - certs = get_certificates_to_provision(env) - if len(certs) == 0: + # What domains should we provision certificates for? And what + # errors prevent provisioning for other domains. + domains, problems = get_certificates_to_provision(env) + + # Exit fast if there is nothing to do. + if len(domains) == 0: return { "requests": [], + "problems": problems, } + # Break into groups of up to 100 certificates at a time, which is Let's Encrypt's + # limit for a single certificate. We'll sort to put related domains together. + domains = sort_domains(domains, env) + certs = [] + while len(domains) > 0: + certs.append( domains[0:100] ) + domains = domains[100:] + # Prepare to provision. # Where should we put our Let's Encrypt account info and state cache. @@ -352,7 +379,8 @@ def provision_certificates(env, agree_to_tos_url=None, logger=None): # Return what happened with each certificate request. return { - "requests": ret + "requests": ret, + "problems": problems, } def provision_certificates_cmdline(): @@ -377,7 +405,14 @@ def provision_certificates_cmdline(): if not status["requests"]: # No domains need certificates. if "--headless" not in sys.argv or "-v" in sys.argv: - print("No domains hosted on this box need a certificate at this time.") + if len(status["problems"]) == 0: + print("No domains hosted on this box need a new TLS certificate at this time.") + elif len(status["problems"]) > 0: + print("No TLS certificates could be provisoned at this time:") + print() + for domain in sort_domains(status["problems"], env): + print("%s: %s" % (domain, status["problems"][domain])) + sys.exit(0) # What happened? @@ -459,6 +494,12 @@ Do you agree to the agreement? Type Y or N and press : """ # we're done for now. break + # And finally show the domains with problems. + if len(status["problems"]) > 0: + print("TLS certificates could not be provisoned for:") + for domain in sort_domains(status["problems"], env): + print("%s: %s" % (domain, status["problems"][domain])) + # INSTALLING A NEW CERTIFICATE FROM THE CONTROL PANEL def create_csr(domain, ssl_key, country_code, env): From 4b4f670adf6379718bbdbb9d8b1268e44b8d61a9 Mon Sep 17 00:00:00 2001 From: Joshua Tauberer Date: Sat, 2 Jan 2016 18:01:20 -0500 Subject: [PATCH 3/9] s/SSL/TLS/ in user-visible text throughout the project --- conf/nginx.conf | 4 ++-- management/ssl_certificates.py | 2 +- management/status_checks.py | 22 +++++++++++----------- management/templates/index.html | 4 ++-- management/templates/ssl.html | 24 ++++++++++++------------ security.md | 2 +- 6 files changed, 29 insertions(+), 29 deletions(-) diff --git a/conf/nginx.conf b/conf/nginx.conf index 538f0743..0a08439e 100644 --- a/conf/nginx.conf +++ b/conf/nginx.conf @@ -1,6 +1,6 @@ ## $HOSTNAME -# Redirect all HTTP to HTTPS *except* the ACME challenges (Let's Encrypt SSL certificate +# Redirect all HTTP to HTTPS *except* the ACME challenges (Let's Encrypt TLS certificate # domain validation challenges) path, which must be served over HTTP per the ACME spec # (due to some Apache vulnerability). server { @@ -23,7 +23,7 @@ server { location /.well-known/acme-challenge/ { # This path must be served over HTTP for ACME domain validation. - # We map this to a special path where our SSL cert provisioning + # We map this to a special path where our TLS cert provisioning # tool knows to store challenge response files. alias $STORAGE_ROOT/ssl/lets_encrypt/acme_challenges/; } diff --git a/management/ssl_certificates.py b/management/ssl_certificates.py index c71c6950..0f019796 100755 --- a/management/ssl_certificates.py +++ b/management/ssl_certificates.py @@ -663,7 +663,7 @@ def check_certificate(domain, ssl_certificate, ssl_private_key, warn_if_expiring return ("The certificate is missing an intermediate chain or the intermediate chain is incorrect or incomplete. (%s)" % verifyoutput, None) # There is some unknown problem. Return the `openssl verify` raw output. - return ("There is a problem with the SSL certificate.", verifyoutput.strip()) + return ("There is a problem with the certificate.", verifyoutput.strip()) else: # `openssl verify` returned a zero exit status so the cert is currently diff --git a/management/status_checks.py b/management/status_checks.py index 4a62b091..4e906533 100755 --- a/management/status_checks.py +++ b/management/status_checks.py @@ -1,7 +1,7 @@ #!/usr/bin/python3 # # Checks that the upstream DNS has been set correctly and that -# SSL certificates have been signed, etc., and if not tells the user +# TLS certificates have been signed, etc., and if not tells the user # what to do next. import sys, os, os.path, re, subprocess, datetime, multiprocessing.pool @@ -609,7 +609,7 @@ def check_web_domain(domain, rounded_time, ssl_certificates, env, output): webmail or a website on this domain. The domain currently resolves to %s in public DNS. It may take several hours for public DNS to update after a change. This problem may result from other issues listed here.""" % (env['PUBLIC_IP'], ip)) - # We need a SSL certificate for PRIMARY_HOSTNAME because that's where the + # We need a TLS certificate for PRIMARY_HOSTNAME because that's where the # user will log in with IMAP or webmail. Any other domain we serve a # website for also needs a signed certificate. check_ssl_cert(domain, rounded_time, ssl_certificates, env, output) @@ -651,18 +651,18 @@ def query_dns(qname, rtype, nxdomain='[Not Set]', at=None): return "; ".join(sorted(str(r).rstrip('.') for r in response)) def check_ssl_cert(domain, rounded_time, ssl_certificates, env, output): - # Check that SSL certificate is signed. + # Check that TLS certificate is signed. # Skip the check if the A record is not pointed here. if query_dns(domain, "A", None) not in (env['PUBLIC_IP'], None): return - # Where is the SSL stored? + # Where is the certificate file stored? tls_cert = get_domain_ssl_files(domain, ssl_certificates, env, allow_missing_cert=True) if tls_cert is None: - output.print_warning("""No SSL certificate is installed for this domain. Visitors to a website on + output.print_warning("""No TLS (SSL) certificate is installed for this domain. Visitors to a website on this domain will get a security warning. If you are not serving a website on this domain, you do - not need to take any action. Use the SSL Certificates page in the control panel to install a - SSL certificate.""") + not need to take any action. Use the TLS Certificates page in the control panel to install a + TLS certificate.""") return # Check that the certificate is good. @@ -671,19 +671,19 @@ def check_ssl_cert(domain, rounded_time, ssl_certificates, env, output): if cert_status == "OK": # The certificate is ok. The details has expiry info. - output.print_ok("SSL certificate is signed & valid. " + cert_status_details) + output.print_ok("TLS (SSL) certificate is signed & valid. " + cert_status_details) elif cert_status == "SELF-SIGNED": # Offer instructions for purchasing a signed certificate. if domain == env['PRIMARY_HOSTNAME']: - output.print_error("""The SSL certificate for this domain is currently self-signed. You will get a security + output.print_error("""The TLS (SSL) certificate for this domain is currently self-signed. You will get a security warning when you check or send email and when visiting this domain in a web browser (for webmail or static site hosting).""") else: - output.print_error("""The SSL certificate for this domain is self-signed.""") + output.print_error("""The TLS (SSL) certificate for this domain is self-signed.""") else: - output.print_error("The SSL certificate has a problem: " + cert_status) + output.print_error("The TLS (SSL) certificate has a problem: " + cert_status) if cert_status_details: output.print_line("") output.print_line(cert_status_details) diff --git a/management/templates/index.html b/management/templates/index.html index 2b59abea..6f2929ec 100644 --- a/management/templates/index.html +++ b/management/templates/index.html @@ -87,7 +87,7 @@ System
  • Status Checks
    • -
  • SSL Certificates
    • +
  • TLS (SSL) Certificates
  • Backup Status
  • Advanced Pages
    • @@ -155,7 +155,7 @@ {% include "web.html" %} -
    +
    {% include "ssl.html" %}
    diff --git a/management/templates/ssl.html b/management/templates/ssl.html index d411098c..48fc91c8 100644 --- a/management/templates/ssl.html +++ b/management/templates/ssl.html @@ -1,7 +1,7 @@ -

    SSL Certificates

    +

    TLS (SSL) Certificates

    Certificate Status

    @@ -20,15 +20,15 @@

    A multi-domain or wildcard certificate will be automatically applied to any domains it is valid for.

    -

    Install SSL Certificate

    +

    Install Certificate

    -

    There are many places where you can get a free or cheap SSL certificate. We recommend Namecheap’s $9 certificate, StartSSL’s free express lane or WoSign’s free SSL.

    +

    There are many places where you can get a free or cheap certificate. We recommend Namecheap’s $9 certificate, StartSSL’s free express lane or WoSign’s free TLS.

    -

    Which domain are you getting an SSL certificate for?

    +

    Which domain are you getting a certificate for?

    -

    What country are you in? This is required by some SSL certificate providers. You may leave this blank if you know your SSL certificate provider doesn't require it.

    +

    What country are you in? This is required by some TLS certificate providers. You may leave this blank if you know your TLS certificate provider doesn't require it.

    From 2b9fb9643d11a3e72e2b73f0126bd92276053024 Mon Sep 17 00:00:00 2001 From: Joshua Tauberer Date: Mon, 4 Jan 2016 18:41:12 -0500 Subject: [PATCH 9/9] changelog entries for lets encrypt --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 34f6442d..c90d4ace 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,11 +10,13 @@ Mail: Control Panel: +* The SSL (now "TLS") certificates page now supports provisioning free SSL certificates from Let's Encrypt. * Report free memory usage. System: * The daily backup will now email the administrator if there is a problem. +* Expiring TLS (SSL) certificates are now automatically renewed via Let's Encrypt. v0.15 (January 1, 2016) -----------------------