Merge branch 'master' into management-memory-usage

management/csr_country_codes.tsv

@@ -0,0 +1,253 @@
+# This list is derived from https://en.wikipedia.org/wiki/ISO_3166-1_alpha-2.
+# The columns are ISO_3166-1_alpha-2 code, display name, Wikipedia page name.
+# The top 21 countries by number of Internet users are grouped first, see
+# https://en.wikipedia.org/wiki/List_of_countries_by_number_of_Internet_users.
+CN	China	
+IN	India	
+US	United States	
+JP	Japan	
+BR	Brazil	
+RU	Russian Federation	Russia
+DE	Germany	
+NG	Nigeria	
+GB	United Kingdom	
+FR	France	
+MX	Mexico	
+EG	Egypt	
+KR	South Korea
+VN	Vietnam
+ID	Indonesia	
+PH	Philippines	
+TR	Turkey	
+IT	Italy	
+PK	Pakistan	
+ES	Spain	
+CA	Canada	
+AD	Andorra	
+AE	United Arab Emirates	
+AF	Afghanistan	
+AG	Antigua and Barbuda	
+AI	Anguilla	
+AL	Albania	
+AM	Armenia	
+AO	Angola	
+AQ	Antarctica	
+AR	Argentina	
+AS	American Samoa	
+AT	Austria	
+AU	Australia	
+AW	Aruba	
+AX	Åland Islands	
+AZ	Azerbaijan	
+BA	Bosnia and Herzegovina	
+BB	Barbados	
+BD	Bangladesh	
+BE	Belgium	
+BF	Burkina Faso	
+BG	Bulgaria	
+BH	Bahrain	
+BI	Burundi	
+BJ	Benin	
+BL	Saint Barthélemy	
+BM	Bermuda	
+BN	Brunei
+BO	Bolivia
+BQ	Bonaire, Sint Eustatius and Saba	Caribbean Netherlands
+BS	Bahamas	The Bahamas
+BT	Bhutan	
+BV	Bouvet Island	
+BW	Botswana	
+BY	Belarus	
+BZ	Belize	
+CC	Cocos (Keeling) Islands	
+CD	Congo, the Democratic Republic of the	Democratic Republic of the Congo
+CF	Central African Republic	
+CG	Congo	Republic of the Congo
+CH	Switzerland	
+CI	Côte d'Ivoire	
+CK	Cook Islands	
+CL	Chile	
+CM	Cameroon	
+CO	Colombia	
+CR	Costa Rica	
+CU	Cuba	
+CV	Cabo Verde	
+CW	Curaçao	
+CX	Christmas Island	
+CY	Cyprus	
+CZ	Czech Republic	
+DJ	Djibouti	
+DK	Denmark	
+DM	Dominica	
+DO	Dominican Republic	
+DZ	Algeria	
+EC	Ecuador	
+EE	Estonia	
+EH	Western Sahara	
+ER	Eritrea	
+ET	Ethiopia	
+FI	Finland	
+FJ	Fiji	
+FK	Falkland Islands (Malvinas)	Falkland Islands
+FM	Federated States of Micronesia
+FO	Faroe Islands	
+GA	Gabon	
+GD	Grenada	
+GE	Georgia	Georgia (country)
+GF	French Guiana	
+GG	Guernsey	
+GH	Ghana	
+GI	Gibraltar	
+GL	Greenland	
+GM	Gambia	The Gambia
+GN	Guinea	
+GP	Guadeloupe	
+GQ	Equatorial Guinea	
+GR	Greece	
+GS	South Georgia and the South Sandwich Islands	
+GT	Guatemala	
+GU	Guam	
+GW	Guinea-Bissau	
+GY	Guyana	
+HK	Hong Kong	
+HM	Heard Island and McDonald Islands	
+HN	Honduras	
+HR	Croatia	
+HT	Haiti	
+HU	Hungary	
+IE	Ireland	Republic of Ireland
+IL	Israel	
+IM	Isle of Man	
+IO	British Indian Ocean Territory	
+IQ	Iraq	
+IR	Iran
+IS	Iceland	
+JE	Jersey	
+JM	Jamaica	
+JO	Jordan	
+KE	Kenya	
+KG	Kyrgyzstan	
+KH	Cambodia	
+KI	Kiribati	
+KM	Comoros	
+KN	Saint Kitts and Nevis	
+KP	North Korea
+KW	Kuwait	
+KY	Cayman Islands	
+KZ	Kazakhstan	
+LA	Laos
+LB	Lebanon	
+LC	Saint Lucia	
+LI	Liechtenstein	
+LK	Sri Lanka	
+LR	Liberia	
+LS	Lesotho	
+LT	Lithuania	
+LU	Luxembourg	
+LV	Latvia	
+LY	Libya	
+MA	Morocco	
+MC	Monaco	
+MD	Moldova
+ME	Montenegro	
+MF	Saint Martin (French part)	Collectivity of Saint Martin
+MG	Madagascar	
+MH	Marshall Islands	
+MK	Macedonia	Republic of Macedonia
+ML	Mali	
+MM	Myanmar	
+MN	Mongolia	
+MO	Macao	Macau
+MP	Northern Mariana Islands	
+MQ	Martinique	
+MR	Mauritania	
+MS	Montserrat	
+MT	Malta	
+MU	Mauritius	
+MV	Maldives	
+MW	Malawi	
+MY	Malaysia	
+MZ	Mozambique	
+NA	Namibia	
+NC	New Caledonia	
+NE	Niger	
+NF	Norfolk Island	
+NI	Nicaragua	
+NL	Netherlands	
+NO	Norway	
+NP	Nepal	
+NR	Nauru	
+NU	Niue	
+NZ	New Zealand	
+OM	Oman	
+PA	Panama	
+PE	Peru	
+PF	French Polynesia	
+PG	Papua New Guinea	
+PL	Poland	
+PM	Saint Pierre and Miquelon	
+PN	Pitcairn	Pitcairn Islands
+PR	Puerto Rico	
+PS	Palestine	State of Palestine
+PT	Portugal	
+PW	Palau	
+PY	Paraguay	
+QA	Qatar	
+RE	Réunion	
+RO	Romania	
+RS	Serbia	
+RW	Rwanda	
+SA	Saudi Arabia	
+SB	Solomon Islands	
+SC	Seychelles	
+SD	Sudan	
+SE	Sweden	
+SG	Singapore	
+SH	Saint Helena, Ascension and Tristan da Cunha	
+SI	Slovenia	
+SJ	Svalbard and Jan Mayen	
+SK	Slovakia	
+SL	Sierra Leone	
+SM	San Marino	
+SN	Senegal	
+SO	Somalia	
+SR	Suriname	
+SS	South Sudan	
+ST	Sao Tome and Principe
+SV	El Salvador	
+SX	Sint Maarten (Dutch part)	Sint Maarten
+SY	Syria
+SZ	Swaziland	
+TC	Turks and Caicos Islands	
+TD	Chad	
+TF	French Southern Territories	French Southern and Antarctic Lands
+TG	Togo	
+TH	Thailand	
+TJ	Tajikistan	
+TK	Tokelau	
+TL	Timor-Leste	East Timor
+TM	Turkmenistan	
+TN	Tunisia	
+TO	Tonga	
+TT	Trinidad and Tobago	
+TV	Tuvalu	
+TW	Taiwan	
+TZ	Tanzania
+UA	Ukraine	
+UG	Uganda	
+UM	United States Minor Outlying Islands	
+UY	Uruguay	
+UZ	Uzbekistan	
+VA	Vatican City
+VC	Saint Vincent and the Grenadines	
+VE	Venezuela
+VG	Virgin Islands, British	British Virgin Islands
+VI	Virgin Islands, U.S.	United States Virgin Islands
+VU	Vanuatu	
+WF	Wallis and Futuna	
+WS	Samoa	
+YE	Yemen	
+YT	Mayotte	
+ZA	South Africa	
+ZM	Zambia	
+ZW	Zimbabwe	

management/daemon.py

@@ -28,6 +28,14 @@ try:
 except OSError:
 	pass
 
+# for generating CSRs we need a list of country codes
+csr_country_codes = []
+with open(os.path.join(os.path.dirname(me), "csr_country_codes.tsv")) as f:
+	for line in f:
+		if line.strip() == "" or line.startswith("#"): continue
+		code, name = line.strip().split("\t")[0:2]
+		csr_country_codes.append((code, name))
+
 app = Flask(__name__, template_folder=os.path.abspath(os.path.join(os.path.dirname(me), "templates")))
 
 # Decorator to protect views that require a user with 'admin' privileges.
@@ -101,9 +109,12 @@ def index():
 	return render_template('index.html',
 		hostname=env['PRIMARY_HOSTNAME'],
 		storage_root=env['STORAGE_ROOT'],
+
 		no_users_exist=no_users_exist,
 		no_admins_exist=no_admins_exist,
+
 		backup_s3_hosts=backup_s3_hosts,
+		csr_country_codes=csr_country_codes,
 	)
 
 @app.route('/me')
@@ -321,7 +332,7 @@ def dns_get_dump():
 def ssl_get_csr(domain):
 	from ssl_certificates import create_csr
 	ssl_private_key = os.path.join(os.path.join(env["STORAGE_ROOT"], 'ssl', 'ssl_private_key.pem'))
-	return create_csr(domain, ssl_private_key, env)
+	return create_csr(domain, ssl_private_key, request.form.get('countrycode', ''), env)
 
 @app.route('/ssl/install', methods=['POST'])
 @authorized_personnel_only

management/daily_tasks.sh

@@ -0,0 +1,8 @@
+#!/bin/bash
+# This script is run daily (at 3am each night).
+
+# Take a backup.
+management/backup.py
+
+# Run status checks and email the administrator if anything changed.
+management/status_checks.py --show-changes --smtp

management/dns_update.py

@@ -283,26 +283,40 @@ def build_zone(domain, all_domains, additional_records, www_redirect_domains, en
 
 def build_tlsa_record(env):
 	# A DANE TLSA record in DNS specifies that connections on a port
-	# must use TLS and the certificate must match a particular certificate.
+	# must use TLS and the certificate must match a particular criteria.
 	#
 	# Thanks to http://blog.huque.com/2012/10/dnssec-and-certificates.html
-	# for explaining all of this!
+	# and https://community.letsencrypt.org/t/please-avoid-3-0-1-and-3-0-2-dane-tlsa-records-with-le-certificates/7022
+	# for explaining all of this! Also see https://tools.ietf.org/html/rfc6698#section-2.1
+	# and https://github.com/mail-in-a-box/mailinabox/issues/268#issuecomment-167160243.
+	#
+	# There are several criteria. We used to use "3 0 1" criteria, which
+	# meant to pin a leaf (3) certificate (0) with SHA256 hash (1). But
+	# certificates change, and especially as we move to short-lived certs
+	# they change often. The TLSA record handily supports the criteria of
+	# a leaf certificate (3)'s subject public key (1) with SHA256 hash (1).
+	# The subject public key is the public key portion of the private key
+	# that generated the CSR that generated the certificate. Since we
+	# generate a private key once the first time Mail-in-a-Box is set up
+	# and reuse it for all subsequent certificates, the TLSA record will
+	# remain valid indefinitely.
 
-	# Get the hex SHA256 of the DER-encoded server certificate:
-	certder = shell("check_output", [
-		"/usr/bin/openssl",
-		"x509",
-		"-in", os.path.join(env["STORAGE_ROOT"], "ssl", "ssl_certificate.pem"),
-		"-outform", "DER"
-		],
-		return_bytes=True)
-	certhash = hashlib.sha256(certder).hexdigest()
+	from ssl_certificates import load_cert_chain, load_pem
+	from cryptography.hazmat.primitives.serialization import Encoding, PublicFormat
+
+	fn = os.path.join(env["STORAGE_ROOT"], "ssl", "ssl_certificate.pem")
+	cert = load_pem(load_cert_chain(fn)[0])
+
+	subject_public_key = cert.public_key().public_bytes(Encoding.DER, PublicFormat.SubjectPublicKeyInfo)
+	# We could have also loaded ssl_private_key.pem and called priv_key.public_key().public_bytes(...)
+
+	pk_hash = hashlib.sha256(subject_public_key).hexdigest()
 
 	# Specify the TLSA parameters:
-	# 3: This is the certificate that the client should trust. No CA is needed.
-	# 0: The whole certificate is matched.
-	# 1: The certificate is SHA256'd here.
-	return "3 0 1 " + certhash
+	# 3: Match the (leaf) certificate. (No CA, no trust path needed.)
+	# 1: Match its subject public key.
+	# 1: Use SHA256.
+	return "3 1 1 " + pk_hash
 
 def build_sshfp_records():
 	# The SSHFP record is a way for us to embed this server's SSH public

management/ssl_certificates.py

@@ -137,12 +137,12 @@ def get_domain_ssl_files(domain, ssl_certificates, env, allow_missing_cert=False
 
 	return cert_info['private-key'], cert_info['certificate'], via
 
-def create_csr(domain, ssl_key, env):
+def create_csr(domain, ssl_key, country_code, env):
 	return shell("check_output", [
                 "openssl", "req", "-new",
                 "-key", ssl_key,
                 "-sha256",
-                "-subj", "/C=%s/ST=/L=/O=/CN=%s" % (env["CSR_COUNTRY"], domain)])
+                "-subj", "/C=%s/ST=/L=/O=/CN=%s" % (country_code, domain)])
 
 def install_cert(domain, ssl_cert, ssl_chain, env):
 	# Write the combined cert+chain to a temporary path and validate that it is OK.
@@ -184,21 +184,22 @@ def install_cert(domain, ssl_cert, ssl_chain, env):
 
 	# When updating the cert for PRIMARY_HOSTNAME, symlink it from the system
 	# certificate path, which is hard-coded for various purposes, and then
-	# update DNS (because of the DANE TLSA record), postfix, and dovecot,
-	# which all use the file.
+	# restart postfix and dovecot.
 	if domain == env['PRIMARY_HOSTNAME']:
 		# Update symlink.
 		system_ssl_certificate = os.path.join(os.path.join(env["STORAGE_ROOT"], 'ssl', 'ssl_certificate.pem'))
 		os.unlink(system_ssl_certificate)
 		os.symlink(ssl_certificate, system_ssl_certificate)
 
-		# Update DNS & restart postfix and dovecot so they pick up the new file.
-		from dns_update import do_dns_update
-		ret.append( do_dns_update(env) )
+		# Restart postfix and dovecot so they pick up the new file.
 		shell('check_call', ["/usr/sbin/service", "postfix", "restart"])
 		shell('check_call', ["/usr/sbin/service", "dovecot", "restart"])
 		ret.append("mail services restarted")
 
+		# The DANE TLSA record will remain valid so long as the private key
+		# hasn't changed. We don't ever change the private key automatically.
+		# If the user does it, they must manually update DNS.
+
 	# Update the web configuration so nginx picks up the new certificate file.
 	from web_update import do_web_update
 	ret.append( do_web_update(env) )

management/templates/ssl.html

@@ -28,6 +28,15 @@
 
 <p><select id="ssldomain" onchange="show_csr()" class="form-control" style="width: auto"></select></p>
 
+<p>What country are you in? This is required by some SSL certificate providers. You may leave this blank if you know your SSL certificate provider doesn't require it.</p>
+
+<p><select id="sslcc" onchange="show_csr()" class="form-control" style="width: auto">
+	<option value="">(Select)</option>
+	{% for code, name in csr_country_codes %}
+		<option value="{{code}}">{{name}}</option>
+	{% endfor %}
+</select></p>
+
 <div id="csr_info" style="display: none">
   <p>You will need to provide the SSL certificate provider this Certificate Signing Request (CSR):</p>
 
@@ -94,6 +103,7 @@ function show_csr() {
     "/ssl/csr/" + $('#ssldomain').val(),
     "POST",
     {
+        countrycode: $('#sslcc').val()
     },
     function(data) {
       $('#ssl_csr').text(data);

management/templates/system-backup.html

@@ -32,12 +32,6 @@
       <p>You MUST manually copy the encryption password from <tt class="backup-encpassword-file"></tt> to a safe and secure location. You will need this file to decrypt backup files. It is NOT stored in your Amazon S3 bucket.</p>
     </div>
   </div>
-  <div class="form-group backup-target-local backup-target-s3">
-    <label for="min-age" class="col-sm-2 control-label">How many days should backups be kept?</label>
-    <div class="col-sm-8">
-      <input type="number" class="form-control" rows="1" id="min-age">
-    </div>
-  </div>
   <div class="form-group backup-target-s3">
     <label for="backup-target-s3-host" class="col-sm-2 control-label">S3 Region</label>
     <div class="col-sm-8">
@@ -66,6 +60,13 @@
       <input type="text" class="form-control" rows="1" id="backup-target-pass">
     </div>
   </div>
+  <div class="form-group backup-target-local backup-target-s3">
+    <label for="min-age" class="col-sm-2 control-label">Days:</label>
+    <div class="col-sm-8">
+      <input type="number" class="form-control" rows="1" id="min-age">
+      <div class="small" style="margin-top: 2px">This is the <i>minimum</i> number of days backup data is kept for. The box makes an incremental backup, so backup data is often kept much longer. An incremental backup file that is less than this number of days old requires that all previous increments back to the most recent full backup, plus that full backup, remain available.</div>
+    </div>
+  </div>
   <div class="form-group">
     <div class="col-sm-offset-2 col-sm-10">
       <button id="set-s3-backup-button" type="submit" class="btn btn-primary">Save</button>