Merge commit '40b21c466d11ab4fd25d7345693bcb5131d5c0cf'

* commit '40b21c466d11ab4fd25d7345693bcb5131d5c0cf':
  Fypo fix in users.html (#1748)
  Better return codes after errors in the setup scripts  (#1741)
  Allowing adding nginx aliases in www/custom.yaml (#1742)
  Delete unused function apt_add_repository_to_unattended_upgrades (#1721)
  Fix Feb 29 issue #1733
  mail_log.py reading forward #1593

management/mail_log.py

@@ -18,13 +18,13 @@ import utils
 
 
 LOG_FILES = (
-    '/var/log/mail.log',
-    '/var/log/mail.log.1',
-    '/var/log/mail.log.2.gz',
-    '/var/log/mail.log.3.gz',
-    '/var/log/mail.log.4.gz',
-    '/var/log/mail.log.5.gz',
     '/var/log/mail.log.6.gz',
+    '/var/log/mail.log.5.gz',
+    '/var/log/mail.log.4.gz',
+    '/var/log/mail.log.3.gz',
+    '/var/log/mail.log.2.gz',
+    '/var/log/mail.log.1',
+    '/var/log/mail.log',
 )
 
 TIME_DELTAS = OrderedDict([
@@ -80,7 +80,7 @@ def scan_files(collector):
             print("Processing file", fn, "...")
         fn = tmp_file.name if tmp_file else fn
 
-        for line in reverse_readline(fn):
+        for line in readline(fn):
             if scan_mail_log_line(line.strip(), collector) is False:
                 if stop_scan:
                     return
@@ -344,16 +344,22 @@ def scan_mail_log_line(line, collector):
 
     # Replaced the dateutil parser for a less clever way of parser that is roughly 4 times faster.
     # date = dateutil.parser.parse(date)
-    date = datetime.datetime.strptime(date, '%b %d %H:%M:%S')
-    date = date.replace(START_DATE.year)
+
+    # date = datetime.datetime.strptime(date, '%b %d %H:%M:%S')
+    # date = date.replace(START_DATE.year)
+
+    # strptime fails on Feb 29 if correct year is not provided. See https://bugs.python.org/issue26460
+    date = datetime.datetime.strptime(str(START_DATE.year) + ' ' + date, '%Y %b %d %H:%M:%S')
+    # print("date:", date)
 
     # Check if the found date is within the time span we are scanning
+    # END_DATE < START_DATE
     if date > START_DATE:
-        # Don't process, but continue
-        return True
-    elif date < END_DATE:
         # Don't process, and halt
         return False
+    elif date < END_DATE:
+        # Don't process, but continue
+        return True
 
     if service == "postfix/submission/smtpd":
         if SCAN_OUT:
@@ -453,9 +459,9 @@ def scan_postfix_smtpd_line(date, log, collector):
                     if m:
                         message = "domain blocked: " + m.group(2)
 
-                if data["latest"] is None:
-                    data["latest"] = date
-                data["earliest"] = date
+                if data["earliest"] is None:
+                    data["earliest"] = date
+                data["latest"] = date
                 data["blocked"].append((date, sender, message))
 
                 collector["rejected"][user] = data
@@ -487,9 +493,9 @@ def add_login(user, date, protocol_name, host, collector):
                 }
             )
 
-            if data["latest"] is None:
-                data["latest"] = date
-            data["earliest"] = date
+            if data["earliest"] is None:
+                data["earliest"] = date
+            data["latest"] = date
 
             data["totals_by_protocol"][protocol_name] += 1
             data["totals_by_protocol_and_host"][(protocol_name, host)] += 1
@@ -528,9 +534,9 @@ def scan_postfix_lmtp_line(date, log, collector):
             data["received_count"] += 1
             data["activity-by-hour"][date.hour] += 1
 
-            if data["latest"] is None:
-                data["latest"] = date
-            data["earliest"] = date
+            if data["earliest"] is None:
+                data["earliest"] = date
+            data["latest"] = date
 
             collector["received_mail"][user] = data
 
@@ -567,9 +573,9 @@ def scan_postfix_submission_line(date, log, collector):
             data["hosts"].add(client)
             data["activity-by-hour"][date.hour] += 1
 
-            if data["latest"] is None:
-                data["latest"] = date
-            data["earliest"] = date
+            if data["earliest"] is None:
+                data["earliest"] = date
+            data["latest"] = date
 
             collector["sent_mail"][user] = data
 
@@ -578,42 +584,15 @@ def scan_postfix_submission_line(date, log, collector):
 
 # Utility functions
 
-def reverse_readline(filename, buf_size=8192):
-    """ A generator that returns the lines of a file in reverse order
-
-    http://stackoverflow.com/a/23646049/801870
-
+def readline(filename):
+    """ A generator that returns the lines of a file
     """
-
-    with open(filename) as fh:
-        segment = None
-        offset = 0
-        fh.seek(0, os.SEEK_END)
-        file_size = remaining_size = fh.tell()
-        while remaining_size > 0:
-            offset = min(file_size, offset + buf_size)
-            fh.seek(file_size - offset)
-            buff = fh.read(min(remaining_size, buf_size))
-            remaining_size -= buf_size
-            lines = buff.split('\n')
-            # the first line of the buffer is probably not a complete line so
-            # we'll save it and append it to the last line of the next buffer
-            # we read
-            if segment is not None:
-                # if the previous chunk starts right from the beginning of line
-                # do not concat the segment to the last line of new chunk
-                # instead, yield the segment first
-                if buff[-1] is not '\n':
-                    lines[-1] += segment
-                else:
-                    yield segment
-            segment = lines[0]
-            for index in range(len(lines) - 1, 0, -1):
-                if len(lines[index]):
-                    yield lines[index]
-        # Don't yield None if the file was empty
-        if segment is not None:
-            yield segment
+    with open(filename) as file:
+        while True:
+          line = file.readline()
+          if not line:
+              break
+          yield line
 
 
 def user_match(user):

management/templates/users.html

@@ -99,7 +99,7 @@
 <thead><th>Verb</th> <th>Action</th><th></th></thead>
 <tr><td>GET</td><td><i>(none)</i></td> <td>Returns a list of existing mail users. Adding <code>?format=json</code> to the URL will give JSON-encoded results.</td></tr>
 <tr><td>POST</td><td>/add</td> <td>Adds a new mail user. Required POST-body parameters are <code>email</code> and <code>password</code>.</td></tr>
-<tr><td>POST</td><td>/remove</td> <td>Removes a mail user. Required POST-by parameter is <code>email</code>.</td></tr>
+<tr><td>POST</td><td>/remove</td> <td>Removes a mail user. Required POST-body parameter is <code>email</code>.</td></tr>
 <tr><td>POST</td><td>/privileges/add</td> <td>Used to make a mail user an admin. Required POST-body parameters are <code>email</code> and <code>privilege=admin</code>.</td></tr>
 <tr><td>POST</td><td>/privileges/remove</td> <td>Used to remove the admin privilege from a mail user. Required POST-body parameter is <code>email</code>.</td></tr>
 </table>

management/web_update.py

@@ -162,6 +162,10 @@ def make_domain_config(domain, templates, ssl_certificates, env):
 				nginx_conf_extra += "\n\t\tproxy_pass %s;" % url
 				nginx_conf_extra += "\n\t\tproxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;"
 				nginx_conf_extra += "\n\t}\n"
+			for path, alias in yaml.get("aliases", {}).items():
+				nginx_conf_extra += "\tlocation %s {" % path
+				nginx_conf_extra += "\n\t\talias %s;" % alias
+				nginx_conf_extra += "\n\t}\n"
 			for path, url in yaml.get("redirects", {}).items():
 				nginx_conf_extra += "\trewrite %s %s permanent;\n" % (path, url)
 

setup/bootstrap.sh

@@ -35,14 +35,14 @@ if [ -z "$TAG" ]; then
 
 	else
 		echo "This script must be run on a system running Ubuntu 18.04 or Ubuntu 14.04."
-		exit
+		exit 1
 	fi
 fi
 
 # Are we running as root?
 if [[ $EUID -ne 0 ]]; then
 	echo "This script must be run as root. Did you leave out sudo?"
-	exit
+	exit 1
 fi
 
 # Clone the Mail-in-a-Box repository if it doesn't exist.
@@ -73,7 +73,7 @@ if [ "$TAG" != `git describe` ]; then
 	git fetch --depth 1 --force --prune origin tag $TAG
 	if ! git checkout -q $TAG; then
 		echo "Update failed. Did you modify something in `pwd`?"
-		exit
+		exit 1
 	fi
 	echo
 fi

setup/functions.sh

@@ -57,15 +57,6 @@ function apt_install {
 	apt_get_quiet install $PACKAGES
 }
 
-function apt_add_repository_to_unattended_upgrades {
-	if [ -f /etc/apt/apt.conf.d/50unattended-upgrades ]; then
-		if ! grep -q "$1" /etc/apt/apt.conf.d/50unattended-upgrades; then
-			sed -i "/Allowed-Origins/a \
-	    \"$1\";" /etc/apt/apt.conf.d/50unattended-upgrades
-		fi
-	fi
-}
-
 function get_default_hostname {
 	# Guess the machine's hostname. It should be a fully qualified
 	# domain name suitable for DNS. None of these calls may provide

setup/preflight.sh

@@ -4,7 +4,7 @@ if [[ $EUID -ne 0 ]]; then
 	echo
 	echo "sudo $0"
 	echo
-	exit
+	exit 1
 fi
 
 # Check that we are running on Ubuntu 18.04 LTS (or 18.04.xx).
@@ -14,7 +14,7 @@ if [ "`lsb_release -d | sed 's/.*:\s*//' | sed 's/18\.04\.[0-9]/18.04/' `" != "U
 	lsb_release -d | sed 's/.*:\s*//'
 	echo
 	echo "We can't write scripts that run on every possible setup, sorry."
-	exit
+	exit 1
 fi
 
 # Check that we have enough memory.

tools/owncloud-restore.sh

@@ -22,7 +22,7 @@ fi
 
 if [ ! -f $1/config.php ]; then
 	echo "This isn't a valid backup location"
-	exit
+	exit 1
 fi
 
 echo "Restoring backup from $1"