Merge branch 'master' into postgrey-whitelist

CHANGELOG.md

@@ -1,8 +1,13 @@
 CHANGELOG
 =========
 
-In Development
+v0.53 (April 12, 2021)
---------------
+----------------------
+
+* Migrate to the ECDSAP256SHA256 DNSSEC algorithm. If a DS record is set for any of your domain names that have DNS hosted on your box, you will be prompted by status checks to update the DS record.
+
+v0.53 (April 12, 2021)
+----------------------
 
 Software updates:
 

management/daemon_reports.py

@@ -102,6 +102,18 @@ def add_reports(app, env, authorized_personnel_only):
 		finally:
 			db_conn_factory.close(conn)
 
+	@app.route('/reports/uidata/imap-details', methods=['POST'])
+	@authorized_personnel_only
+	@json_payload
+	def get_imap_details(payload):
+		conn = db_conn_factory.connect()
+		try:
+			return jsonify(uidata.imap_details(conn, payload))
+		except uidata.InvalidArgsError as e:
+			return ('invalid request', 400)
+		finally:
+			db_conn_factory.close(conn)
+
 	@app.route('/reports/uidata/flagged-connections', methods=['POST'])
 	@authorized_personnel_only
 	@json_payload

management/dns_update.py

@@ -128,6 +128,10 @@ def build_zones(env):
 	from web_update import get_web_domains
 	www_redirect_domains = set(get_web_domains(env)) - set(get_web_domains(env, include_www_redirects=False))
 
+	# For MTA-STS, we'll need to check if the PRIMARY_HOSTNAME certificate is
+	# singned and valid. Check that now rather than repeatedly for each domain.
+	env["-primary-hostname-certificate-is-valid"] = is_domain_cert_signed_and_valid(env["PRIMARY_HOSTNAME"], env)
+
 	# Build DNS records for each zone.
 	for domain, zonefile in zonefiles:
 		# Build the records to put in the zone.
@@ -323,24 +327,11 @@ def build_zone(domain, all_domains, additional_records, www_redirect_domains, en
 	# certificate in use is not valid (e.g. because it is self-signed and a valid certificate has not
 	# yet been provisioned). Since we cannot provision a certificate without A/AAAA records, we
 	# always set them --- only the TXT records depend on there being valid certificates.
-	mta_sts_enabled = False
 	mta_sts_records = [
 		("mta-sts", "A", env["PUBLIC_IP"], "Optional. MTA-STS Policy Host serving /.well-known/mta-sts.txt."),
 		("mta-sts", "AAAA", env.get('PUBLIC_IPV6'), "Optional. MTA-STS Policy Host serving /.well-known/mta-sts.txt."),
 	]
-	if domain in get_mail_domains(env):
+	if domain in get_mail_domains(env) and env["-primary-hostname-certificate-is-valid"] and is_domain_cert_signed_and_valid("mta-sts." + domain, env):
-		# Check that PRIMARY_HOSTNAME and the mta_sts domain both have valid certificates.
-		for d in (env['PRIMARY_HOSTNAME'], "mta-sts." + domain):
-			cert = get_ssl_certificates(env).get(d)
-			if not cert:
-				break # no certificate provisioned for this domain
-			cert_status = check_certificate(d, cert['certificate'], cert['private-key'])
-			if cert_status[0] != 'OK':
-				break # certificate is not valid
-		else:
-			# 'break' was not encountered above, so both domains are good
-			mta_sts_enabled = True
-	if mta_sts_enabled:
 		# Compute an up-to-32-character hash of the policy file. We'll take a SHA-1 hash of the policy
 		# file (20 bytes) and encode it as base-64 (28 bytes, using alphanumeric alternate characters
 		# instead of '+' and '/' which are not allowed in an MTA-STS policy id) but then just take its
@@ -366,6 +357,13 @@ def build_zone(domain, all_domains, additional_records, www_redirect_domains, en
 
 	return records
 
+def is_domain_cert_signed_and_valid(domain, env):
+	cert = get_ssl_certificates(env).get(domain)
+	if not cert: return False # no certificate provisioned
+	cert_status = check_certificate(domain, cert['certificate'], cert['private-key'])
+	print(domain, cert_status)
+	return cert_status[0] == 'OK'
+
 ########################################################################
 
 def build_tlsa_record(env):
@@ -430,6 +428,7 @@ def build_sshfp_records():
 	# to the zone file (that trigger bumping the serial number). However,
 	# if SSH has been configured to listen on a nonstandard port, we must
 	# specify that port to sshkeyscan.
+
 	port = 22
 	with open('/etc/ssh/sshd_config', 'r') as f:
 		for line in f:
@@ -440,8 +439,11 @@ def build_sshfp_records():
 				except ValueError:
 					pass
 				break
+
 	keys = shell("check_output", ["ssh-keyscan", "-t", "rsa,dsa,ecdsa,ed25519", "-p", str(port), "localhost"])
-	for key in sorted(keys.split("\n")):
+	keys = sorted(keys.split("\n"))
+
+	for key in keys:
 		if key.strip() == "" or key[0] == "#": continue
 		try:
 			host, keytype, pubkey = key.split(" ")
@@ -461,13 +463,16 @@ def write_nsd_zone(domain, zonefile, records, env, force):
 	# On the $ORIGIN line, there's typically a ';' comment at the end explaining
 	# what the $ORIGIN line does. Any further data after the domain confuses
 	# ldns-signzone, however. It used to say '; default zone domain'.
-
+	#
 	# The SOA contact address for all of the domains on this system is hostmaster
 	# @ the PRIMARY_HOSTNAME. Hopefully that's legit.
-
+	#
 	# For the refresh through TTL fields, a good reference is:
 	# http://www.peerwisdom.org/2013/05/15/dns-understanding-the-soa-record/
-
+	#
+	# A hash of the available DNSSEC keys are added in a comment so that when
+	# the keys change we force a re-generation of the zone which triggers
+	# re-signing it.
 
 	zone = """
 $ORIGIN {domain}.
@@ -503,6 +508,9 @@ $TTL 86400          ; default time to live
 			value = v2
 		zone += value + "\n"
 
+	# Append a stable hash of DNSSEC signing keys in a comment.
+	zone += "\n; DNSSEC signing keys hash: {}\n".format(hash_dnssec_keys(domain, env))
+
 	# DNSSEC requires re-signing a zone periodically. That requires
 	# bumping the serial number even if no other records have changed.
 	# We don't see the DNSSEC records yet, so we have to figure out
@@ -613,53 +621,77 @@ zone:
 
 ########################################################################
 
-def dnssec_choose_algo(domain, env):
+def find_dnssec_signing_keys(domain, env):
-	if '.' in domain and domain.rsplit('.')[-1] in \
+	# For key that we generated (one per algorithm)...
-		("email", "guide", "fund", "be", "lv"):
+	d = os.path.join(env['STORAGE_ROOT'], 'dns/dnssec')
-		# At GoDaddy, RSASHA256 is the only algorithm supported
+	keyconfs = [f for f in os.listdir(d) if f.endswith(".conf")]
-		# for .email and .guide.
+	for keyconf in keyconfs:
-		# A variety of algorithms are supported for .fund. This
+		# Load the file holding the KSK and ZSK key filenames.
-		# is preferred.
+		keyconf_fn = os.path.join(d, keyconf)
-		# Gandi tells me that .be does not support RSASHA1-NSEC3-SHA1
+		keyinfo = load_env_vars_from_file(keyconf_fn)
-        # Nic.lv does not support RSASHA1-NSEC3-SHA1 for .lv tld's
-		return "RSASHA256"
 
-	# For any domain we were able to sign before, don't change the algorithm
+		# Skip this key if the conf file has a setting named DOMAINS,
-	# on existing users. We'll probably want to migrate to SHA256 later.
+		# holding a comma-separated list of domain names, and if this
-	return "RSASHA1-NSEC3-SHA1"
+		# domain is not in the list. This allows easily disabling a
+		# key by setting "DOMAINS=" or "DOMAINS=none", other than
+		# deleting the key's .conf file, which might result in the key
+		# being regenerated next upgrade. Keys should be disabled if
+		# they are not needed to reduce the DNSSEC query response size.
+		if "DOMAINS" in keyinfo and domain not in [dd.strip() for dd in keyinfo["DOMAINS"].split(",")]:
+			continue
+
+		for keytype in ("KSK", "ZSK"):
+			yield keytype, keyinfo[keytype]
+
+def hash_dnssec_keys(domain, env):
+	# Create a stable (by sorting the items) hash of all of the private keys
+	# that will be used to sign this domain.
+	keydata = []
+	for keytype, keyfn in sorted(find_dnssec_signing_keys(domain, env)):
+		oldkeyfn = os.path.join(env['STORAGE_ROOT'], 'dns/dnssec', keyfn + ".private")
+		keydata.append(keytype)
+		keydata.append(keyfn)
+		with open(oldkeyfn, "r") as fr:
+			keydata.append( fr.read() )
+	keydata = "".join(keydata).encode("utf8")
+	return hashlib.sha1(keydata).hexdigest()
 
 def sign_zone(domain, zonefile, env):
-	algo = dnssec_choose_algo(domain, env)
+	# Sign the zone with all of the keys that were generated during
-	dnssec_keys = load_env_vars_from_file(os.path.join(env['STORAGE_ROOT'], 'dns/dnssec/%s.conf' % algo))
+	# setup so that the user can choose which to use in their DS record at
+	# their registrar, and also to support migration to newer algorithms.
 
-	# In order to use the same keys for all domains, we have to generate
+	# In order to use the key files generated at setup which are for
-	# a new .key file with a DNSSEC record for the specific domain. We
+	# the domain _domain_, we have to re-write the files and place
-	# can reuse the same key, but it won't validate without a DNSSEC
+	# the actual domain name in it, so that ldns-signzone works.
-	# record specifically for the domain.
 	#
-	# Copy the .key and .private files to /tmp to patch them up.
+	# Patch each key, storing the patched version in /tmp for now.
-	#
+	# Each key has a .key and .private file. Collect a list of filenames
-	# Use os.umask and open().write() to securely create a copy that only
+	# for all of the keys (and separately just the key-signing keys).
-	# we (root) can read.
+	all_keys = []
-	files_to_kill = []
+	ksk_keys = []
-	for key in ("KSK", "ZSK"):
+	for keytype, keyfn in find_dnssec_signing_keys(domain, env):
-		if dnssec_keys.get(key, "").strip() == "": raise Exception("DNSSEC is not properly set up.")
+		newkeyfn = '/tmp/' + keyfn.replace("_domain_", domain)
-		oldkeyfn = os.path.join(env['STORAGE_ROOT'], 'dns/dnssec/' + dnssec_keys[key])
+
-		newkeyfn = '/tmp/' + dnssec_keys[key].replace("_domain_", domain)
-		dnssec_keys[key] = newkeyfn
 		for ext in (".private", ".key"):
-			if not os.path.exists(oldkeyfn + ext): raise Exception("DNSSEC is not properly set up.")
+			# Copy the .key and .private files to /tmp to patch them up.
-			with open(oldkeyfn + ext, "r") as fr:
+			#
+			# Use os.umask and open().write() to securely create a copy that only
+			# we (root) can read.
+			oldkeyfn = os.path.join(env['STORAGE_ROOT'], 'dns/dnssec', keyfn + ext)
+			with open(oldkeyfn, "r") as fr:
 				keydata = fr.read()
-			keydata = keydata.replace("_domain_", domain) # trick ldns-signkey into letting our generic key be used by this zone
+			keydata = keydata.replace("_domain_", domain)
-			fn = newkeyfn + ext
 			prev_umask = os.umask(0o77) # ensure written file is not world-readable
 			try:
-				with open(fn, "w") as fw:
+				with open(newkeyfn + ext, "w") as fw:
 					fw.write(keydata)
 			finally:
 				os.umask(prev_umask) # other files we write should be world-readable
-			files_to_kill.append(fn)
+
+		# Put the patched key filename base (without extension) into the list of keys we'll sign with.
+		all_keys.append(newkeyfn)
+		if keytype == "KSK": ksk_keys.append(newkeyfn)
 
 	# Do the signing.
 	expiry_date = (datetime.datetime.now() + datetime.timedelta(days=30)).strftime("%Y%m%d")
@@ -672,32 +704,34 @@ def sign_zone(domain, zonefile, env):
 
 		# zonefile to sign
 		"/etc/nsd/zones/" + zonefile,
-
+	]
 		# keys to sign with (order doesn't matter -- it'll figure it out)
-		dnssec_keys["KSK"],
+		+ all_keys
-		dnssec_keys["ZSK"],
+	)
-	])
 
 	# Create a DS record based on the patched-up key files. The DS record is specific to the
 	# zone being signed, so we can't use the .ds files generated when we created the keys.
 	# The DS record points to the KSK only. Write this next to the zone file so we can
 	# get it later to give to the user with instructions on what to do with it.
 	#
-	# We want to be able to validate DS records too, but multiple forms may be valid depending
+	# Generate a DS record for each key. There are also several possible hash algorithms that may
-	# on the digest type. So we'll write all (both) valid records. Only one DS record should
+	# be used, so we'll pre-generate all for each key. One DS record per line. Only one
-	# actually be deployed. Preferebly the first.
+	# needs to actually be deployed at the registrar. We'll select the preferred one
+	# in the status checks.
 	with open("/etc/nsd/zones/" + zonefile + ".ds", "w") as f:
-		for digest_type in ('2', '1'):
+		for key in ksk_keys:
-			rr_ds = shell('check_output', ["/usr/bin/ldns-key2ds",
+			for digest_type in ('1', '2', '4'):
-				"-n", # output to stdout
+				rr_ds = shell('check_output', ["/usr/bin/ldns-key2ds",
-				"-" + digest_type, # 1=SHA1, 2=SHA256
+					"-n", # output to stdout
-				dnssec_keys["KSK"] + ".key"
+					"-" + digest_type, # 1=SHA1, 2=SHA256, 4=SHA384
-			])
+					key + ".key"
-			f.write(rr_ds)
+				])
+				f.write(rr_ds)
 
-	# Remove our temporary file.
+	# Remove the temporary patched key files.
-	for fn in files_to_kill:
+	for fn in all_keys:
-		os.unlink(fn)
+		os.unlink(fn + ".private")
+		os.unlink(fn + ".key")
 
 ########################################################################
 

management/reporting/ui/chart-table.js

@@ -2,7 +2,8 @@ export default Vue.component('chart-table', {
     props: {
         items: Array,
         fields: Array,
-        caption: String
+        caption: String,
+        small: { type:Boolean, default:true }
     },
 
     /* <b-table-lite striped small :fields="fields_x" :items="items" caption-top><template #table-caption><span class="text-nowrap">{{caption}}</span></template></b-table>*/
@@ -19,7 +20,7 @@ export default Vue.component('chart-table', {
         var table = ce('b-table-lite', {
             props: {
                 'striped': true,
-                'small': true,
+                'small': this.small,
                 'fields': this.fields_x,
                 'items': this.items,
                 'caption-top': true

management/reporting/ui/charting.js

@@ -493,6 +493,8 @@ export class BvTableField {
         }
         else if (ft.type == 'number') {
             if (ft.subtype == 'plain' ||
+                ft.subtype === null ||
+                ft.subtype === undefined ||
                 ft.subtype == 'decimal' && isNaN(ft.places)
                )
             {

management/reporting/ui/message_headers_view.js

@@ -0,0 +1,57 @@
+import { spinner } from "../../ui-common/page-header.js";
+
+export default Vue.component('message-headers-view', {
+    props: {
+        lmtp_id: String,
+        user_id: String
+    },
+    
+    template:
+    '<div>' +
+        '<div class="text-center" v-if="loading">{{loading_msg}} <spinner></spinner></div>' +
+        '<pre v-else>{{ message_headers }}</pre>' +
+        '</div>',
+
+    data: function() {
+        return {
+            loading: true,
+            loading_msg: '',
+            message_headers: ''
+        }
+    },
+
+    watch: {
+        lmtp_id: function() {
+            this.load(this.lmtp_id, this.user_id);
+        }
+    },
+
+    mounted: function() {
+        this.load(this.lmtp_id, this.user_id);
+    },
+
+    methods: {
+        load: function(lmtp_id, user_id) {
+            if (!lmtp_id || !user_id) {
+                this.message_headers = 'no data';
+                return;
+            }
+            this.loading = true;
+            this.loading_msg = 'Searching for message with LMTP ID ' + lmtp_id;
+            axios.post('reports/uidata/message-headers', {
+                lmtp_id,
+                user_id
+            }).then(response => {
+                this.message_headers = response.data;
+                if (this.message_headers == '')
+                    this.message_headers = `Message with LMTP "${lmtp_id}" not found. It may have been deleted.`;
+            }).catch(e => {
+                this.message_headers = '' + (e.response.data || e);
+            }).finally( () => {
+                this.loading = false;
+            });
+        }
+    }
+    
+});
+

management/reporting/ui/panel-user-activity.html

@@ -84,26 +84,30 @@
 
     <b-tab>
       <template #title>
-        IMAP Connections<sup v-if="imap_details.items.length >= get_row_limit()">*</sup> ({{imap_details.items.length}})
+        IMAP Connections
       </template>
+
       <b-table
-        class="sticky-table-header-0 bg-light"
+        tbody-tr-class="cursor-pointer"
-        small
+        selectable
+        select-mode="single"
         :filter="show_only_flagged_filter"
         :filter-function="table_filter_cb"
-        tbody-tr-class="cursor-pointer"
+        :items="imap_conn_summary.items"
-        details-td-class="cursor-default"
+        :fields="imap_conn_summary.fields"
-        @row-clicked="row_clicked"
+        @row-clicked="load_imap_details">
-        :items="imap_details.items"
+      </b-table>       
-        :fields="imap_details.fields">
+
-        <template #row-details="row">
+      <div v-if="imap_details" class="bg-white">
-          <b-card>
+        <div class="mt-3 text-center bg-info p-1">{{imap_details._desc}} ({{imap_details.items.length}} rows<sup v-if="imap_details.items.length >= get_row_limit()">*</sup>)</div>
-            <div><strong>Connection disposition</strong>: {{ disposition_formatter(row.item.disposition) }}</div>
+        <b-table
-            <div><strong>Connection security</strong> {{ row.item.connection_security }}</div>
+          class="sticky-table-header-0"
-            <div><strong>Disconnect reason</strong> {{ row.item.disconnect_reason }}</div>
+          small
-          </b-card>
+          :items="imap_details.items"
-        </template>
+          :fields="imap_details.fields">
-      </b-table>
+        </b-table>
+      </div>
+      
     </b-tab>
 
   </b-tabs>

management/reporting/ui/panel-user-activity.js

@@ -3,9 +3,10 @@
 */
 
 import wbr_text from "./wbr-text.js";
+import chart_table from "./chart-table.js";
 import message_headers_view from "./message_headers_view.js";
 import UserSettings from "./settings.js";
-import { MailBvTable, ConnectionDisposition } from "./charting.js";
+import { BvTable, MailBvTable, ConnectionDisposition } from "./charting.js";
 
 
 export default Vue.component('panel-user-activity', function(resolve, reject) {
@@ -19,7 +20,8 @@ export default Vue.component('panel-user-activity', function(resolve, reject) {
 
         components: {
             'wbr-text': wbr_text,
-            'message-headers-view': message_headers_view
+            'message-headers-view': message_headers_view,
+            'chart-table': chart_table,
         },
         
         data: function() {
@@ -36,6 +38,7 @@ export default Vue.component('panel-user-activity', function(resolve, reject) {
                 data_date_range: null, /* date range for active table data */
                 sent_mail: null,
                 received_mail: null,
+                imap_conn_summary: null,
                 imap_details: null,
                 lmtp_id: null, /* for message headers modal */
                 all_users: [],
@@ -169,11 +172,19 @@ export default Vue.component('panel-user-activity', function(resolve, reject) {
                 f.label = 'Envelope From (user)';
             },
 
+            combine_imap_conn_summary_fields: function() {
+                // remove 'first_conn_time'
+                this.imap_conn_summary.combine_fields('first_connection_time');
+                // clear the label for the 'total' column (pct)
+                const f_total = this.imap_conn_summary.get_field('total');
+                f_total.label = '';
+            },
+
             combine_imap_details_fields: function() {
                 // remove these fields
                 this.imap_details.combine_fields([
-                    'disconnect_reason',
+                    'remote_host',
-                    'connection_security',
+                    'disposition',
                 ]);
             },
 
@@ -272,16 +283,21 @@ export default Vue.component('panel-user-activity', function(resolve, reject) {
                         .get_field('connect_time')
                         .add_tdClass('text-nowrap');
 
-                    /* setup imap_details */
+                    
-                    this.imap_details = new MailBvTable(
+                    /* setup imap_conn_summary */
-                        response.data.imap_details, {
+                    this.imap_conn_summary = new MailBvTable(
-                            _showDetails: true
+                        response.data.imap_conn_summary
+                    );
+                    this.combine_imap_conn_summary_fields();
+                    this.imap_conn_summary.flag_fields();
+                    ['last_connection_time','count']
+                        .forEach(name => {
+                            const f = this.imap_conn_summary.get_field(name);
+                            f.add_cls('text-nowrap', 'tdClass');
                         });
-                    this.combine_imap_details_fields();
+
-                    this.imap_details
+                    /* clear imap_details */
-                        .flag_fields()
+                    this.imap_details = null;
-                        .get_field('connect_time')
-                        .add_tdClass('text-nowrap');
 
                 }).catch(error => {
                     this.$root.handleError(error);
@@ -304,6 +320,36 @@ export default Vue.component('panel-user-activity', function(resolve, reject) {
                 
                 // show the modal dialog
                 this.$refs.message_headers_modal.show();
+            },
+
+            load_imap_details: function(item, index, event) {
+                this.$emit('loading', 1);
+                this.imap_details = null;
+                const promise = axios.post('reports/uidata/imap-details', {
+                    row_limit: this.get_row_limit(),
+                    user_id: this.user_id.trim(),
+                    start_date: this.date_range[0],
+                    end_date: this.date_range[1],
+                    disposition: item.disposition,
+                    remote_host: item.remote_host
+                }).then(response => {
+                    this.imap_details = new MailBvTable(
+                        response.data.imap_details
+                    );
+                    this.combine_imap_details_fields();
+                    this.imap_details.get_field('connect_time')
+                        .add_tdClass('text-nowrap');
+                    this.imap_details.get_field('disconnect_time')
+                        .add_tdClass('text-nowrap');
+                    this.imap_details._desc =
+                        `${item.remote_host}/${item.disposition}`;
+                    
+                }).catch(error => {
+                    this.$root.handleError(error);
+                    
+                }).finally( () => {
+                    this.$emit('loading', -1);
+                });
             }
             
         }

management/reporting/uidata/Timeseries.py

@@ -22,7 +22,7 @@ class Timeseries(object):
         # parsefmt is a date parser string to be used to re-interpret
         # "bin" grouping dates (data.dates) to native dates. server
         # always returns utc dates
-        parsefmt = '%Y-%m-%d %H:%M:%S'
+        self.parsefmt = '%Y-%m-%d %H:%M:%S'
 
         self.dates = []   # dates must be "bin" date strings
         self.series = []
@@ -31,7 +31,7 @@ class Timeseries(object):
             'range': [ self.start, self.end ],
             'range_parse_format': '%Y-%m-%d %H:%M:%S',
             'binsize': self.binsize,
-            'date_parse_format': parsefmt,
+            'date_parse_format': self.parsefmt,
             'y': desc,
             'dates': self.dates,
             'series': self.series

management/reporting/uidata/__init__.py

@@ -3,6 +3,7 @@ from .select_list_suggestions import select_list_suggestions
 from .messages_sent import messages_sent
 from .messages_received import messages_received
 from .user_activity import user_activity
+from .imap_details import imap_details
 from .remote_sender_activity import remote_sender_activity
 from .flagged_connections import flagged_connections
 from .capture_db_stats import capture_db_stats

management/reporting/uidata/imap_details.1.sql

@@ -0,0 +1,25 @@
+--
+-- details on user imap connections
+--
+SELECT
+  connect_time,
+  disconnect_time,
+  CASE WHEN remote_host='unknown' THEN remote_ip ELSE remote_host END AS `remote_host`,
+  sasl_method,
+  disconnect_reason,
+  connection_security,
+  disposition,
+  in_bytes,
+  out_bytes
+FROM
+  imap_connection
+WHERE
+  sasl_username = :user_id AND
+  connect_time >= :start_date AND
+  connect_time < :end_date AND
+  (:remote_host IS NULL OR
+     remote_host = :remote_host OR remote_ip = :remote_host) AND
+  (:disposition IS NULL OR
+     disposition = :disposition)
+ORDER BY
+  connect_time

management/reporting/uidata/imap_details.py

@@ -0,0 +1,83 @@
+from .Timeseries import Timeseries
+from .exceptions import InvalidArgsError
+
+with open(__file__.replace('.py','.1.sql')) as fp:
+    select_1 = fp.read()
+
+
+def imap_details(conn, args):
+    '''
+    details on imap connections
+    '''    
+    try:
+        user_id = args['user_id']
+
+        # use Timeseries to get a normalized start/end range
+        ts = Timeseries(
+            'IMAP details',
+            args['start_date'],
+            args['end_date'],
+            0
+        )
+
+        # optional
+        remote_host = args.get('remote_host')
+        disposition = args.get('disposition')
+        
+    except KeyError:
+        raise InvalidArgsError()
+
+    # limit results
+    try:
+        limit = 'LIMIT ' + str(int(args.get('row_limit', 1000)));
+    except ValueError:
+        limit = 'LIMIT 1000'
+
+
+    c = conn.cursor()
+
+    imap_details = {
+        'start': ts.start,
+        'end': ts.end,
+        'y': 'IMAP Details',
+        'fields': [
+            'connect_time',
+            'disconnect_time',
+            'remote_host',
+            'sasl_method',
+            'disconnect_reason',
+            'connection_security',
+            'disposition',
+            'in_bytes',
+            'out_bytes'
+        ],
+        'field_types': [
+            { 'type':'datetime', 'format': ts.parsefmt }, # connect_time
+            { 'type':'datetime', 'format': ts.parsefmt }, # disconnect_time
+            'text/plain',    # remote_host
+            'text/plain',    # sasl_method
+            'text/plain',    # disconnect_reason
+            'text/plain',    # connection_security
+            'text/plain',    # disposition
+            'number/size',   # in_bytes,
+            'number/size',   # out_bytes,
+        ],
+        'items': []
+    }
+
+    for row in c.execute(select_1 + limit, {
+            'user_id': user_id,
+            'start_date': ts.start,
+            'end_date': ts.end,
+            'remote_host': remote_host,
+            'disposition': disposition
+    }):
+        v = []
+        for key in imap_details['fields']:
+            v.append(row[key])
+        imap_details['items'].append(v)
+
+    
+    return {
+        'imap_details': imap_details
+    }

management/reporting/uidata/user_activity.3.sql

@@ -1,20 +1,22 @@
 --
--- details on user imap connections
+-- imap connection summary
 --
 SELECT
-  connect_time,
+  count(*) as `count`,
-  CASE WHEN remote_host='unknown' THEN remote_ip ELSE remote_host END AS `remote_host`,
-  sasl_method,
-  disconnect_reason,
-  connection_security,
   disposition,
-  in_bytes,
+  CASE WHEN remote_host='unknown' THEN remote_ip ELSE remote_host END AS `remote_host`,
-  out_bytes
+  sum(in_bytes) as `in_bytes`,
+  sum(out_bytes) as `out_bytes`,
+  min(connect_time) as `first_connection_time`,
+  max(connect_time) as `last_connection_time`
 FROM
   imap_connection
 WHERE
   sasl_username = :user_id AND
   connect_time >= :start_date AND
   connect_time < :end_date
+GROUP BY
+  disposition,
+  CASE WHEN remote_host='unknown' THEN remote_ip ELSE remote_host END
 ORDER BY
-  connect_time
+  `count` DESC, disposition

management/reporting/uidata/user_activity.py

@@ -200,50 +200,64 @@ def user_activity(conn, args):
 
 
     #
-    # imap connections by user
+    # IMAP connections by disposition, by remote host
+    #   Disposition
+    #   Remote host
+    #   Count
+    #   In bytes (sum)
+    #   Out bytes (sum)
+    #   % of total
     #
 
-    imap_details = {
+    imap_conn_summary = {
         'start': ts.start,
         'end': ts.end,
-        'y': 'IMAP Details',
+        'y': 'IMAP connection summary by host and disposition',
         'fields': [
-            'connect_time',
+            'count',
+            'total',
             'remote_host',
-            'sasl_method',
-            'disconnect_reason',
-            'connection_security',
             'disposition',
+            'first_connection_time',
+            'last_connection_time',
             'in_bytes',
-            'out_bytes'
+            'out_bytes',
         ],
         'field_types': [
-            { 'type':'datetime', 'format': '%Y-%m-%d %H:%M:%S' },# connect_time
+            'number',        # count
+            { 'type': 'number/percent', 'places': 1 }, # total
             'text/plain',    # remote_host
-            'text/plain',    # sasl_method
-            'text/plain',    # disconnect_reason
-            'text/plain',    # connection_security
             'text/plain',    # disposition
+            { 'type':'datetime', 'format': ts.parsefmt }, # first_conn_time
+            { 'type':'datetime', 'format': ts.parsefmt }, # last_conn_time
             'number/size',   # in_bytes,
             'number/size',   # out_bytes,
         ],
         'items': []
     }
 
+    count_field_idx = 0
+    total_field_idx = 1
+    total = 0
     for row in c.execute(select_3 + limit, {
             'user_id': user_id,
             'start_date': ts.start,
             'end_date': ts.end
     }):
         v = []
-        for key in imap_details['fields']:
+        for key in imap_conn_summary['fields']:
-            v.append(row[key])
+            if key=='count':
-        imap_details['items'].append(v)
+                total += row[key]
+            if key!='total':
+                v.append(row[key])
 
+        imap_conn_summary['items'].append(v)
 
+    for v in imap_conn_summary['items']:
+        v.insert(total_field_idx, v[count_field_idx] / total)
         
     return {
         'sent_mail': sent_mail,
         'received_mail': received_mail,
-        'imap_details': imap_details
+        'imap_conn_summary': imap_conn_summary
     }

management/status_checks.py

@@ -42,7 +42,7 @@ def get_services():
 		{ "name": "HTTPS Web (nginx)", "port": 443, "public": True, },
 	]
 
-def run_checks(rounded_values, env, output, pool):
+def run_checks(rounded_values, env, output, pool, domains_to_check=None):
 	# run systems checks
 	output.add_heading("System")
 
@@ -63,7 +63,7 @@ def run_checks(rounded_values, env, output, pool):
 	# perform other checks asynchronously
 
 	run_network_checks(env, output)
-	run_domain_checks(rounded_values, env, output, pool)
+	run_domain_checks(rounded_values, env, output, pool, domains_to_check=domains_to_check)
 
 def get_ssh_port():
 	# Returns ssh port
@@ -300,7 +300,7 @@ def run_network_checks(env, output):
 			which may prevent recipients from receiving your email. See http://www.spamhaus.org/query/ip/%s."""
 			% (env['PUBLIC_IP'], zen, env['PUBLIC_IP']))
 
-def run_domain_checks(rounded_time, env, output, pool):
+def run_domain_checks(rounded_time, env, output, pool, domains_to_check=None):
 	# Get the list of domains we handle mail for.
 	mail_domains = get_mail_domains(env)
 
@@ -311,7 +311,8 @@ def run_domain_checks(rounded_time, env, output, pool):
 	# Get the list of domains we serve HTTPS for.
 	web_domains = set(get_web_domains(env))
 
-	domains_to_check = mail_domains | dns_domains | web_domains
+	if domains_to_check is None:
+		domains_to_check = mail_domains | dns_domains | web_domains
 
 	# Remove "www", "autoconfig", "autodiscover", and "mta-sts" subdomains, which we group with their parent,
 	# if their parent is in the domains to check list.
@@ -557,61 +558,103 @@ def check_dns_zone_suggestions(domain, env, output, dns_zonefiles, domains_with_
 
 
 def check_dnssec(domain, env, output, dns_zonefiles, is_checking_primary=False):
-	# See if the domain has a DS record set at the registrar. The DS record may have
+	# See if the domain has a DS record set at the registrar. The DS record must
-	# several forms. We have to be prepared to check for any valid record. We've
+	# match one of the keys that we've used to sign the zone. It may use one of
-	# pre-generated all of the valid digests --- read them in.
+	# several hashing algorithms. We've pre-generated all possible valid DS
+	# records, although some will be preferred.
+
+	alg_name_map = { '7': 'RSASHA1-NSEC3-SHA1', '8': 'RSASHA256', '13': 'ECDSAP256SHA256' }
+	digalg_name_map = { '1': 'SHA-1', '2': 'SHA-256', '4': 'SHA-384' }
+
+	# Read in the pre-generated DS records
+	expected_ds_records = { }
 	ds_file = '/etc/nsd/zones/' + dns_zonefiles[domain] + '.ds'
 	if not os.path.exists(ds_file): return # Domain is in our database but DNS has not yet been updated.
-	ds_correct = open(ds_file).read().strip().split("\n")
+	with open(ds_file) as f:
-	digests = { }
+		for rr_ds in f:
-	for rr_ds in ds_correct:
+			rr_ds = rr_ds.rstrip()
-		ds_keytag, ds_alg, ds_digalg, ds_digest = rr_ds.split("\t")[4].split(" ")
+			ds_keytag, ds_alg, ds_digalg, ds_digest = rr_ds.split("\t")[4].split(" ")
-		digests[ds_digalg] = ds_digest
 
-	# Some registrars may want the public key so they can compute the digest. The DS
+			# Some registrars may want the public key so they can compute the digest. The DS
-	# record that we suggest using is for the KSK (and that's how the DS records were generated).
+			# record that we suggest using is for the KSK (and that's how the DS records were generated).
-	alg_name_map = { '7': 'RSASHA1-NSEC3-SHA1', '8': 'RSASHA256' }
+			# We'll also give the nice name for the key algorithm.
-	dnssec_keys = load_env_vars_from_file(os.path.join(env['STORAGE_ROOT'], 'dns/dnssec/%s.conf' % alg_name_map[ds_alg]))
+			dnssec_keys = load_env_vars_from_file(os.path.join(env['STORAGE_ROOT'], 'dns/dnssec/%s.conf' % alg_name_map[ds_alg]))
-	dnsssec_pubkey = open(os.path.join(env['STORAGE_ROOT'], 'dns/dnssec/' + dnssec_keys['KSK'] + '.key')).read().split("\t")[3].split(" ")[3]
+			dnsssec_pubkey = open(os.path.join(env['STORAGE_ROOT'], 'dns/dnssec/' + dnssec_keys['KSK'] + '.key')).read().split("\t")[3].split(" ")[3]
+
+			expected_ds_records[ (ds_keytag, ds_alg, ds_digalg, ds_digest) ] = {
+				"record": rr_ds,
+				"keytag": ds_keytag,
+				"alg": ds_alg,
+				"alg_name": alg_name_map[ds_alg],
+				"digalg": ds_digalg,
+				"digalg_name": digalg_name_map[ds_digalg],
+				"digest": ds_digest,
+				"pubkey": dnsssec_pubkey,
+			}
 
 	# Query public DNS for the DS record at the registrar.
-	ds = query_dns(domain, "DS", nxdomain=None)
+	ds = query_dns(domain, "DS", nxdomain=None, as_list=True)
-	ds_looks_valid = ds and len(ds.split(" ")) == 4
+	if ds is None or isinstance(ds, str): ds = []
-	if ds_looks_valid: ds = ds.split(" ")
+
-	if ds_looks_valid and ds[0] == ds_keytag and ds[1] == ds_alg and ds[3] == digests.get(ds[2]):
+	# There may be more that one record, so we get the result as a list.
-		if is_checking_primary: return
+	# Filter out records that don't look valid, just in case, and split
-		output.print_ok("DNSSEC 'DS' record is set correctly at registrar.")
+	# each record on spaces.
+	ds = [tuple(str(rr).split(" ")) for rr in ds if len(str(rr).split(" ")) == 4]
+
+	if len(ds) == 0:
+		output.print_warning("""This domain's DNSSEC DS record is not set. The DS record is optional. The DS record activates DNSSEC. See below for instructions.""")
 	else:
-		if ds == None:
+		matched_ds = set(ds) & set(expected_ds_records)
-			if is_checking_primary: return
+		if matched_ds:
-			output.print_warning("""This domain's DNSSEC DS record is not set. The DS record is optional. The DS record activates DNSSEC.
+			# At least one DS record matches one that corresponds with one of the ways we signed
-				To set a DS record, you must follow the instructions provided by your domain name registrar and provide to them this information:""")
+			# the zone, so it is valid.
+			#
+			# But it may not be preferred. Only algorithm 13 is preferred. Warn if any of the
+			# matched zones uses a different algorithm.
+			if set(r[1] for r in matched_ds) == { '13' }: # all are alg 13
+				output.print_ok("DNSSEC 'DS' record is set correctly at registrar.")
+				return
+			elif '13' in set(r[1] for r in matched_ds): # some but not all are alg 13
+				output.print_ok("DNSSEC 'DS' record is set correctly at registrar. (Records using algorithm other than ECDSAP256SHA256 should be removed.)")
+				return
+			else: # no record uses alg 13
+				output.print_warning("DNSSEC 'DS' record set at registrar is valid but should be updated to ECDSAP256SHA256 (see below).")
 		else:
 			if is_checking_primary:
 				output.print_error("""The DNSSEC 'DS' record for %s is incorrect. See further details below.""" % domain)
 				return
 			output.print_error("""This domain's DNSSEC DS record is incorrect. The chain of trust is broken between the public DNS system
 				and this machine's DNS server. It may take several hours for public DNS to update after a change. If you did not recently
-				make a change, you must resolve this immediately by following the instructions provided by your domain name registrar and
+				make a change, you must resolve this immediately (see below).""")
-				provide to them this information:""")
+
+	output.print_line("""Follow the instructions provided by your domain name registrar to set a DS record.
+		Registrars support different sorts of DS records. Use the first option that works:""")
+	preferred_ds_order = [(7, 1), (7, 2), (8, 4), (13, 4), (8, 1), (8, 2), (13, 1), (13, 2)] # low to high
+	def preferred_ds_order_func(ds_suggestion):
+		k = (int(ds_suggestion['alg']), int(ds_suggestion['digalg']))
+		if k in preferred_ds_order:
+			return preferred_ds_order.index(k)
+		return -1 # index before first item
+	output.print_line("")
+	for i, ds_suggestion in enumerate(sorted(expected_ds_records.values(), key=preferred_ds_order_func, reverse=True)):
 		output.print_line("")
-		output.print_line("Key Tag: " + ds_keytag + ("" if not ds_looks_valid or ds[0] == ds_keytag else " (Got '%s')" % ds[0]))
+		output.print_line("Option " + str(i+1) + ":")
+		output.print_line("----------")
+		output.print_line("Key Tag: " + ds_suggestion['keytag'])
 		output.print_line("Key Flags: KSK")
-		output.print_line(
+		output.print_line("Algorithm: %s / %s" % (ds_suggestion['alg'], ds_suggestion['alg_name']))
-			  ("Algorithm: %s / %s" % (ds_alg, alg_name_map[ds_alg]))
+		output.print_line("Digest Type: %s / %s" % (ds_suggestion['digalg'], ds_suggestion['digalg_name']))
-			+ ("" if not ds_looks_valid or ds[1] == ds_alg else " (Got '%s')" % ds[1]))
+		output.print_line("Digest: " + ds_suggestion['digest'])
-			# see http://www.iana.org/assignments/dns-sec-alg-numbers/dns-sec-alg-numbers.xhtml
-		output.print_line("Digest Type: 2 / SHA-256")
-			# http://www.ietf.org/assignments/ds-rr-types/ds-rr-types.xml
-		output.print_line("Digest: " + digests['2'])
-		if ds_looks_valid and ds[3] != digests.get(ds[2]):
-			output.print_line("(Got digest type %s and digest %s which do not match.)" % (ds[2], ds[3]))
 		output.print_line("Public Key: ")
-		output.print_line(dnsssec_pubkey, monospace=True)
+		output.print_line(ds_suggestion['pubkey'], monospace=True)
 		output.print_line("")
 		output.print_line("Bulk/Record Format:")
-		output.print_line("" + ds_correct[0])
+		output.print_line(ds_suggestion['record'], monospace=True)
+	if len(ds) > 0:
 		output.print_line("")
+		output.print_line("The DS record is currently set to:")
+		for rr in ds:
+			output.print_line("Key Tag: {0}, Algorithm: {1}, Digest Type: {2}, Digest: {3}".format(*rr))
 
 def check_mail_domain(domain, env, output):
 	# Check the MX record.
@@ -713,7 +756,7 @@ def check_web_domain(domain, rounded_time, ssl_certificates, env, output):
 	# website for also needs a signed certificate.
 	check_ssl_cert(domain, rounded_time, ssl_certificates, env, output)
 
-def query_dns(qname, rtype, nxdomain='[Not Set]', at=None):
+def query_dns(qname, rtype, nxdomain='[Not Set]', at=None, as_list=False):
 	# Make the qname absolute by appending a period. Without this, dns.resolver.query
 	# will fall back a failed lookup to a second query with this machine's hostname
 	# appended. This has been causing some false-positive Spamhaus reports. The
@@ -750,6 +793,9 @@ def query_dns(qname, rtype, nxdomain='[Not Set]', at=None):
 	if rtype in ("A", "AAAA"):
 		response = [normalize_ip(str(r)) for r in response]
 
+	if as_list:
+		return response
+
 	# There may be multiple answers; concatenate the response. Remove trailing
 	# periods from responses since that's how qnames are encoded in DNS but is
 	# confusing for us. The order of the answers doesn't matter, so sort so we
@@ -1050,3 +1096,7 @@ if __name__ == "__main__":
 
 	elif sys.argv[1] == "--version":
 		print(what_version_is_this(env))
+
+	elif sys.argv[1] == "--only":
+		with multiprocessing.pool.Pool(processes=10) as pool:
+			run_checks(False, env, ConsoleOutput(), pool, domains_to_check=sys.argv[2:])

setup/bootstrap.sh

@@ -20,7 +20,7 @@ if [ -z "$TAG" ]; then
 	# want to display in status checks.
 	if [ "`lsb_release -d | sed 's/.*:\s*//' | sed 's/18\.04\.[0-9]/18.04/' `" == "Ubuntu 18.04 LTS" ]; then
 		# This machine is running Ubuntu 18.04.
-		TAG=v0.52
+		TAG=v0.53
 
 	elif [ "`lsb_release -d | sed 's/.*:\s*//' | sed 's/14\.04\.[0-9]/14.04/' `" == "Ubuntu 14.04 LTS" ]; then
 		# This machine is running Ubuntu 14.04.

setup/dns.sh

@@ -86,27 +86,15 @@ echo "include: /etc/nsd/zones.conf" >> /etc/nsd/nsd.conf;
 
 mkdir -p "$STORAGE_ROOT/dns/dnssec";
 
-# TLDs don't all support the same algorithms, so we'll generate keys using a few
+# TLDs, registrars, and validating nameservers don't all support the same algorithms,
-# different algorithms. RSASHA1-NSEC3-SHA1 was possibly the first widely used
+# so we'll generate keys using a few different algorithms so that dns_update.py can
-# algorithm that supported NSEC3, which is a security best practice. However TLDs
+# choose which algorithm to use when generating the zonefiles. See #1953 for recent
-# will probably be moving away from it to a a SHA256-based algorithm.
+# discussion. File for previously used algorithms (i.e. RSASHA1-NSEC3-SHA1) may still
-#
+# be in the output directory, and we'll continue to support signing zones with them
-# Supports `RSASHA1-NSEC3-SHA1` (didn't test with `RSASHA256`):
+# so that trust isn't broken with deployed DS records, but we won't generate those
-#
+# keys on new systems.
-#  * .info
-#  * .me
-#
-# Requires `RSASHA256`
-#
-#  * .email
-#  * .guide
-#
-# Supports `RSASHA256` (and defaulting to this)
-#
-#  * .fund
-
 FIRST=1 #NODOC
-for algo in RSASHA1-NSEC3-SHA1 RSASHA256; do
+for algo in RSASHA256 ECDSAP256SHA256; do
 if [ ! -f "$STORAGE_ROOT/dns/dnssec/$algo.conf" ]; then
 	if [ $FIRST == 1 ]; then
 		echo "Generating DNSSEC signing keys..."
@@ -115,7 +103,7 @@ if [ ! -f "$STORAGE_ROOT/dns/dnssec/$algo.conf" ]; then
 
 	# Create the Key-Signing Key (KSK) (with `-k`) which is the so-called
 	# Secure Entry Point. The domain name we provide ("_domain_") doesn't
-	#  matter -- we'll use the same keys for all our domains.
+	# matter -- we'll use the same keys for all our domains.
 	#
 	# `ldns-keygen` outputs the new key's filename to stdout, which
 	# we're capturing into the `KSK` variable.
@@ -123,17 +111,22 @@ if [ ! -f "$STORAGE_ROOT/dns/dnssec/$algo.conf" ]; then
 	# ldns-keygen uses /dev/random for generating random numbers by default.
 	# This is slow and unecessary if we ensure /dev/urandom is seeded properly,
 	# so we use /dev/urandom. See system.sh for an explanation. See #596, #115.
-	KSK=$(umask 077; cd $STORAGE_ROOT/dns/dnssec; ldns-keygen -r /dev/urandom -a $algo -b 2048 -k _domain_);
+	# (This previously used -b 2048 but it's unclear if this setting makes sense
+	# for non-RSA keys, so it's removed. The RSA-based keys are not recommended
+	# anymore anyway.)
+	KSK=$(umask 077; cd $STORAGE_ROOT/dns/dnssec; ldns-keygen -r /dev/urandom -a $algo -k _domain_);
 
 	# Now create a Zone-Signing Key (ZSK) which is expected to be
 	# rotated more often than a KSK, although we have no plans to
 	# rotate it (and doing so would be difficult to do without
-	# disturbing DNS availability.) Omit `-k` and use a shorter key length.
+	# disturbing DNS availability.) Omit `-k`.
-	ZSK=$(umask 077; cd $STORAGE_ROOT/dns/dnssec; ldns-keygen -r /dev/urandom -a $algo -b 1024 _domain_);
+	# (This previously used -b 1024 but it's unclear if this setting makes sense
+	# for non-RSA keys, so it's removed.)
+	ZSK=$(umask 077; cd $STORAGE_ROOT/dns/dnssec; ldns-keygen -r /dev/urandom -a $algo _domain_);
 
 	# These generate two sets of files like:
 	#
-	# * `K_domain_.+007+08882.ds`: DS record normally provided to domain name registrar (but it's actually invalid with `_domain_`)
+	# * `K_domain_.+007+08882.ds`: DS record normally provided to domain name registrar (but it's actually invalid with `_domain_` so we don't use this file)
 	# * `K_domain_.+007+08882.key`: public key
 	# * `K_domain_.+007+08882.private`: private key (secret!)
 

tests/vagrant/Vagrantfile

@@ -43,7 +43,7 @@ SH
 cd /mailinabox
 source tests/vagrant/globals.sh || exit 1
 export PRIMARY_HOSTNAME=qa3.abc.com
-export UPSTREAM_TAG=master
+export UPSTREAM_TAG=main
 tests/system-setup/upgrade-from-upstream.sh basic totpuser || exit 1
 tests/runner.sh upgrade-basic upgrade-totpuser default || exit 2
 SH