v0.27

changelog entry
Update Roundcube to 1.3.6 (#1376 )
2026-03-13 17:17:23 +01:00 · 2018-06-14 07:49:20 -04:00 · 2018-05-12 09:43:41 -04:00 · 2018-04-17 11:54:24 -04:00 · 2018-02-25 11:55:06 -05:00 · 2018-02-24 09:24:15 -05:00
10 changed files with 114 additions and 90 deletions
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,6 +1,13 @@
 CHANGELOG
 =========
 v0.27 (June 14, 2018)
 ---------------------
 * A report of box activity, including sent/received mail totals and logins by user, is now emailed to the box's administrator user each week.
 * Update Roundcube to version 1.3.6 and Z-Push to version 2.3.9.
 * The undocumented feature for proxying web requests to another server now sets X-Forwarded-For.
 v0.26c (February 13, 2018)
 --------------------------
--- a/README.md
+++ b/README.md
@@ -59,7 +59,7 @@ by me:
 	$ curl -s https://keybase.io/joshdata/key.asc | gpg --import
 	gpg: key C10BDD81: public key "Joshua Tauberer <jt@occams.info>" imported
-	$ git verify-tag v0.26c
+	$ git verify-tag v0.27
 	gpg: Signature made ..... using RSA key ID C10BDD81
 	gpg: Good signature from "Joshua Tauberer <jt@occams.info>"
 	gpg: WARNING: This key is not certified with a trusted signature!
@@ -72,7 +72,7 @@ and on my [personal homepage](https://razor.occams.info/). (Of course, if this r
 Checkout the tag corresponding to the most recent release:
-	$ git checkout v0.26c
+	$ git checkout v0.27
 Begin the installation.
--- a/management/daily_tasks.sh
+++ b/management/daily_tasks.sh
@@ -9,6 +9,12 @@ export LC_ALL=en_US.UTF-8
 export LANG=en_US.UTF-8
 export LC_TYPE=en_US.UTF-8
 # On Mondays, i.e. once a week, send the administrator a report of total emails
 # sent and received so the admin might notice server abuse.
 if [ `date "+%u"` -eq 1 ]; then
    management/mail_log.py -t week | management/email_administrator.py "Mail-in-a-Box Usage Report"
 fi
 # Take a backup.
 management/backup.py | management/email_administrator.py "Backup Status"
--- a/management/email_administrator.py
+++ b/management/email_administrator.py
@@ -4,8 +4,14 @@
 import sys
 import html
 import smtplib
-from email.message import Message
+
 from email.mime.multipart import MIMEMultipart
 from email.mime.text import MIMEText
 # In Python 3.6:
 #from email.message import Message
 from utils import load_environment
@@ -26,11 +32,23 @@ if content == "":
 	sys.exit(0)
 # create MIME message
-msg = Message()
+msg = MIMEMultipart('alternative')
 # In Python 3.6:
 #msg = Message()
 msg['From'] = "\"%s\" <%s>" % (env['PRIMARY_HOSTNAME'], admin_addr)
 msg['To'] = admin_addr
 msg['Subject'] = "[%s] %s" % (env['PRIMARY_HOSTNAME'], subject)
-msg.set_payload(content, "UTF-8")
+
 content_html = "<html><body><pre>{}</pre></body></html>".format(html.escape(content))
 msg.attach(MIMEText(content, 'plain'))
 msg.attach(MIMEText(content_html, 'html'))
 # In Python 3.6:
 #msg.set_content(content)
 #msg.add_alternative(content_html, "html")
 # send
 smtpclient = smtplib.SMTP('127.0.0.1', 25)
--- a/management/mail_log.py
+++ b/management/mail_log.py
@@ -53,10 +53,10 @@ VERBOSE = False
 # List of strings to filter users with
 FILTERS = None
-# What to show by default
+# What to show (with defaults)
 SCAN_OUT = True  # Outgoing email
 SCAN_IN = True  # Incoming email
-SCAN_CONN = False  # IMAP and POP3 logins
+SCAN_DOVECOT_LOGIN = True  # Dovecot Logins
 SCAN_GREY = False  # Greylisted email
 SCAN_BLOCKED = False  # Rejected email
@@ -76,7 +76,8 @@ def scan_files(collector):
            tmp_file = tempfile.NamedTemporaryFile()
            shutil.copyfileobj(gzip.open(fn), tmp_file)
-        print("Processing file", fn, "...")
+        if VERBOSE:
            print("Processing file", fn, "...")
        fn = tmp_file.name if tmp_file else fn
        for line in reverse_readline(fn):
@@ -105,7 +106,7 @@ def scan_mail_log(env):
        "scan_time": time.time(),  # The time in seconds the scan took
        "sent_mail": OrderedDict(),  # Data about email sent by users
        "received_mail": OrderedDict(),  # Data about email received by users
-        "dovecot": OrderedDict(),  # Data about Dovecot activity
+        "logins": OrderedDict(),  # Data about login activity
        "postgrey": {},  # Data about greylisting of email addresses
        "rejected": OrderedDict(),  # Emails that were blocked
        "known_addresses": None,  # Addresses handled by the Miab installation
@@ -119,8 +120,8 @@ def scan_mail_log(env):
    except ImportError:
        pass
-    print("Scanning from {:%Y-%m-%d %H:%M:%S} back to {:%Y-%m-%d %H:%M:%S}".format(
+    print("Scanning logs from {:%Y-%m-%d %H:%M:%S} to {:%Y-%m-%d %H:%M:%S}".format(
-        START_DATE, END_DATE)
+        END_DATE, START_DATE)
    )
    # Scan the lines in the log files until the date goes out of range
@@ -138,8 +139,8 @@ def scan_mail_log(env):
    # Print Sent Mail report
    if collector["sent_mail"]:
-        msg = "Sent email between {:%Y-%m-%d %H:%M:%S} and {:%Y-%m-%d %H:%M:%S}"
+        msg = "Sent email"
-        print_header(msg.format(END_DATE, START_DATE))
+        print_header(msg)
        data = OrderedDict(sorted(collector["sent_mail"].items(), key=email_sort))
@@ -173,8 +174,8 @@ def scan_mail_log(env):
    # Print Received Mail report
    if collector["received_mail"]:
-        msg = "Received email between {:%Y-%m-%d %H:%M:%S} and {:%Y-%m-%d %H:%M:%S}"
+        msg = "Received email"
-        print_header(msg.format(END_DATE, START_DATE))
+        print_header(msg)
        data = OrderedDict(sorted(collector["received_mail"].items(), key=email_sort))
@@ -199,43 +200,55 @@ def scan_mail_log(env):
            [accum]
        )
-    # Print Dovecot report
+    # Print login report
-    if collector["dovecot"]:
+    if collector["logins"]:
-        msg = "Email client logins between {:%Y-%m-%d %H:%M:%S} and {:%Y-%m-%d %H:%M:%S}"
+        msg = "User logins per hour"
-        print_header(msg.format(END_DATE, START_DATE))
+        print_header(msg)
-        data = OrderedDict(sorted(collector["dovecot"].items(), key=email_sort))
+        data = OrderedDict(sorted(collector["logins"].items(), key=email_sort))
        # Get a list of all of the protocols seen in the logs in reverse count order.
        all_protocols = defaultdict(int)
        for u in data.values():
            for protocol_name, count in u["totals_by_protocol"].items():
                all_protocols[protocol_name] += count
        all_protocols = [k for k, v in sorted(all_protocols.items(), key=lambda kv : -kv[1])]
        print_user_table(
            data.keys(),
            data=[
-                ("imap", [u["imap"] for u in data.values()]),
+                (protocol_name, [
-                ("pop3", [u["pop3"] for u in data.values()]),
+                    round(u["totals_by_protocol"][protocol_name] / (u["latest"]-u["earliest"]).total_seconds() * 60*60, 1)
                    if (u["latest"]-u["earliest"]).total_seconds() > 0
                    else 0 # prevent division by zero
                  for u in data.values()])
                for protocol_name in all_protocols
            ],
            sub_data=[
-                ("IMAP IP addresses", [[k + " (%d)" % v for k, v in u["imap-logins"].items()]
+                ("Protocol and Source", [[
-                                       for u in data.values()]),
+                    "{} {}: {} times".format(protocol_name, host, count)
-                ("POP3 IP addresses", [[k + " (%d)" % v for k, v in u["pop3-logins"].items()]
+                    for (protocol_name, host), count
-                                       for u in data.values()]),
+                    in sorted(u["totals_by_protocol_and_host"].items(), key=lambda kv:-kv[1])
                  ] for u in data.values()])
            ],
            activity=[
-                ("imap", [u["activity-by-hour"]["imap"] for u in data.values()]),
+                (protocol_name, [u["activity-by-hour"][protocol_name] for u in data.values()])
-                ("pop3", [u["activity-by-hour"]["pop3"] for u in data.values()]),
+                for protocol_name in all_protocols
            ],
            earliest=[u["earliest"] for u in data.values()],
            latest=[u["latest"] for u in data.values()],
            numstr=lambda n : str(round(n, 1)),
        )
-        accum = {"imap": defaultdict(int), "pop3": defaultdict(int), "both": defaultdict(int)}
+        accum = { protocol_name: defaultdict(int) for protocol_name in all_protocols }
        for h in range(24):
-            accum["imap"][h] = sum(d["activity-by-hour"]["imap"][h] for d in data.values())
+            for protocol_name in all_protocols:
-            accum["pop3"][h] = sum(d["activity-by-hour"]["pop3"][h] for d in data.values())
+              accum[protocol_name][h] = sum(d["activity-by-hour"][protocol_name][h] for d in data.values())
            accum["both"][h] = accum["imap"][h] + accum["pop3"][h]
        print_time_table(
-            ["imap", "pop3", "   +"],
+            all_protocols,
-            [accum["imap"], accum["pop3"], accum["both"]]
+            [accum[protocol_name] for protocol_name in all_protocols]
        )
    if collector["postgrey"]:
@@ -348,9 +361,9 @@ def scan_mail_log_line(line, collector):
    elif service == "postfix/lmtp":
        if SCAN_IN:
            scan_postfix_lmtp_line(date, log, collector)
-    elif service in ("imap-login", "pop3-login"):
+    elif service.endswith("-login"):
-        if SCAN_CONN:
+        if SCAN_DOVECOT_LOGIN:
-            scan_dovecot_line(date, log, collector, service[:4])
+            scan_dovecot_login_line(date, log, collector, service[:4])
    elif service == "postgrey":
        if SCAN_GREY:
            scan_postgrey_line(date, log, collector)
@@ -448,44 +461,43 @@ def scan_postfix_smtpd_line(date, log, collector):
                collector["rejected"][user] = data
-def scan_dovecot_line(date, log, collector, prot):
+def scan_dovecot_login_line(date, log, collector, protocol_name):
-    """ Scan a dovecot log line and extract interesting data """
+    """ Scan a dovecot login log line and extract interesting data """
    m = re.match("Info: Login: user=<(.*?)>, method=PLAIN, rip=(.*?),", log)
    if m:
        # TODO: CHECK DIT
-        user, rip = m.groups()
+        user, host = m.groups()
        if user_match(user):
            add_login(user, date, protocol_name, host, collector)
 def add_login(user, date, protocol_name, host, collector):
            # Get the user data, or create it if the user is new
-            data = collector["dovecot"].get(
+            data = collector["logins"].get(
                user,
                {
                    "imap": 0,
                    "pop3": 0,
                    "earliest": None,
                    "latest": None,
-                    "imap-logins": defaultdict(int),
+                    "totals_by_protocol": defaultdict(int),
-                    "pop3-logins": defaultdict(int),
+                    "totals_by_protocol_and_host": defaultdict(int),
-                    "activity-by-hour": {
+                    "activity-by-hour": defaultdict(lambda : defaultdict(int)),
                        "imap": defaultdict(int),
                        "pop3": defaultdict(int),
                    },
                }
            )
            data[prot] += 1
            data["activity-by-hour"][prot][date.hour] += 1
            if data["latest"] is None:
                data["latest"] = date
            data["earliest"] = date
-            if rip not in ("127.0.0.1", "::1") or True:
+            data["totals_by_protocol"][protocol_name] += 1
-                data["%s-logins" % prot][rip] += 1
+            data["totals_by_protocol_and_host"][(protocol_name, host)] += 1
-            collector["dovecot"][user] = data
+            if host not in ("127.0.0.1", "::1") or True:
                data["activity-by-hour"][protocol_name][date.hour] += 1
            collector["logins"][user] = data
 def scan_postfix_lmtp_line(date, log, collector):
@@ -561,6 +573,8 @@ def scan_postfix_submission_line(date, log, collector):
            collector["sent_mail"][user] = data
            # Also log this as a login.
            add_login(user, date, "smtp", client, collector)
 # Utility functions
@@ -640,7 +654,7 @@ def print_time_table(labels, data, do_print=True):
        for i, d in enumerate(data):
            lines[i] += base.format(d[h])
-    lines.insert(0, "┬")
+    lines.insert(0, "┬ totals by time of day:")
    lines.append("└" + (len(lines[-1]) - 2) * "─")
    if do_print:
@@ -650,7 +664,7 @@ def print_time_table(labels, data, do_print=True):
 def print_user_table(users, data=None, sub_data=None, activity=None, latest=None, earliest=None,
-                     delimit=False):
+                     delimit=False, numstr=str):
    str_temp = "{:<32} "
    lines = []
    data = data or []
@@ -764,7 +778,7 @@ def print_user_table(users, data=None, sub_data=None, activity=None, latest=None
    # Print totals
-    data_accum = [str(a) for a in data_accum]
+    data_accum = [numstr(a) for a in data_accum]
    footer = str_temp.format("Totals:" if do_accum else " ")
    for row, (l, _) in enumerate(data):
        temp = "{:>%d}" % max(5, len(l) + 1)
@@ -818,7 +832,7 @@ if __name__ == "__main__":
                        action="store_true")
    parser.add_argument("-s", "--sent", help="Scan for sent emails.",
                        action="store_true")
-    parser.add_argument("-l", "--logins", help="Scan for IMAP/POP logins.",
+    parser.add_argument("-l", "--logins", help="Scan for user logins to IMAP/POP3.",
                        action="store_true")
    parser.add_argument("-g", "--grey", help="Scan for greylisted emails.",
                        action="store_true")
@@ -863,8 +877,8 @@ if __name__ == "__main__":
        if not SCAN_OUT:
            print("Ignoring sent emails")
-        SCAN_CONN = args.logins
+        SCAN_DOVECOT_LOGIN = args.logins
-        if not SCAN_CONN:
+        if not SCAN_DOVECOT_LOGIN:
            print("Ignoring logins")
        SCAN_GREY = args.grey
--- a/management/web_update.py
+++ b/management/web_update.py
@@ -149,7 +149,10 @@ def make_domain_config(domain, templates, ssl_certificates, env):
 			# any proxy or redirect here?
 			for path, url in yaml.get("proxies", {}).items():
-				nginx_conf_extra += "\tlocation %s {\n\t\tproxy_pass %s;\n\t}\n" % (path, url)
+				nginx_conf_extra += "\tlocation %s {" % path
 				nginx_conf_extra += "\n\t\tproxy_pass %s;" % url
 				nginx_conf_extra += "\n\t\tproxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;"
 				nginx_conf_extra += "\n\t}\n"
 			for path, url in yaml.get("redirects", {}).items():
 				nginx_conf_extra += "\trewrite %s %s permanent;\n" % (path, url)
--- a/setup/bootstrap.sh
+++ b/setup/bootstrap.sh
@@ -7,7 +7,7 @@
 #########################################################
 if [ -z "$TAG" ]; then
-	TAG=v0.26c
+	TAG=v0.27
 fi
 # Are we running as root?
--- a/setup/webmail.sh
+++ b/setup/webmail.sh
@@ -35,8 +35,8 @@ apt-get purge -qq -y roundcube* #NODOC
 # Install Roundcube from source if it is not already present or if it is out of date.
 # Combine the Roundcube version number with the commit hash of plugins to track
 # whether we have the latest version of everything.
-VERSION=1.3.3
+VERSION=1.3.6
-HASH=903a4eb1bfc25e9a08d782a7f98502cddfa579de
+HASH=ece5cfc9c7af0cbe90c0065ef33e85ed42991830
 PERSISTENT_LOGIN_VERSION=dc5ca3d3f4415cc41edb2fde533c8a8628a94c76
 HTML5_NOTIFIER_VERSION=4b370e3cd60dabd2f428a26f45b677ad1b7118d5
 CARDDAV_VERSION=2.0.4
--- a/setup/zpush.sh
+++ b/setup/zpush.sh
@@ -22,7 +22,7 @@ apt_install \
 phpenmod -v php7.0 imap
 # Copy Z-Push into place.
-VERSION=2.3.8
+VERSION=2.3.9
 needs_update=0 #NODOC
 if [ ! -f /usr/local/lib/z-push/version ]; then
 	needs_update=1 #NODOC
--- a/tools/update-subresource-integrity.py
+++ b/tools/update-subresource-integrity.py
@@ -1,24 +0,0 @@
 #!/usr/bin/python3
 # Updates subresource integrity attributes in management/templates/index.html
 # to prevent CDN-hosted resources from being used as an attack vector. Run this
 # after updating the Bootstrap and jQuery <link> and <script> to compute the
 # appropriate hash and insert it into the template.
 import re, urllib.request, hashlib, base64
 fn = "management/templates/index.html"
 with open(fn, 'r') as f:
 	content = f.read()
 def make_integrity(url):
 	resource = urllib.request.urlopen(url).read()
 	return "sha256-" + base64.b64encode(hashlib.sha256(resource).digest()).decode('ascii')
 content = re.sub(
 	r'<(link rel="stylesheet" href|script src)="(.*?)" integrity="(.*?)"',
 	lambda m : '<' + m.group(1) + '="' + m.group(2) + '" integrity="' + make_integrity(m.group(2)) + '"',
 	content)
 with open(fn, 'w') as f:
 	f.write(content)
Author	SHA1	Message	Date
Joshua Tauberer	0c0a079354	v0.27	2018-06-14 07:49:20 -04:00
Joshua Tauberer	42e86610ba	changelog entry	2018-05-12 09:43:41 -04:00
yeah	7c62f4b8e9	Update Roundcube to 1.3.6 (#1376 )	2018-04-17 11:54:24 -04:00
Joshua Tauberer	1eba7b0616	send the mail_log.py report to the box admin every Monday	2018-02-25 11:55:06 -05:00
Joshua Tauberer	9c7820f422	mail_log.py: include sent mail in the logins report in a new smtp column	2018-02-24 09:24:15 -05:00
Joshua Tauberer	87ec4e9f82	mail_log.py: refactor the dovecot login collector	2018-02-24 09:24:14 -05:00
Joshua Tauberer	08becf7fa3	the hidden feature for proxying web requests now sets X-Forwarded-For	2018-02-24 09:24:14 -05:00
Joshua Tauberer	5eb4a53de1	remove old tools/update-subresource-integrity.py script which isn't used now that we download all admin page remote assets during setup	2018-02-24 09:24:14 -05:00
Joshua Tauberer	598ade3f7a	changelog entry	2018-02-24 09:24:09 -05:00
xetorixik	8f399df5bb	Update Roundcube to 1.3.4 and Z-push to 2.3.9 (#1354 )	2018-02-21 08:22:57 -05:00