From f908bc364ef96a20b25c0bf9e00bdf6a2c6741ae Mon Sep 17 00:00:00 2001 From: Jarek Jurasz Date: Tue, 3 Mar 2020 20:03:18 +0100 Subject: [PATCH 1/6] mail_log.py reading forward #1593 --- management/mail_log.py | 87 +++++++++++++++--------------------------- 1 file changed, 30 insertions(+), 57 deletions(-) diff --git a/management/mail_log.py b/management/mail_log.py index 79d6ea56..a0dd1c09 100755 --- a/management/mail_log.py +++ b/management/mail_log.py @@ -18,13 +18,13 @@ import utils LOG_FILES = ( - '/var/log/mail.log', - '/var/log/mail.log.1', - '/var/log/mail.log.2.gz', - '/var/log/mail.log.3.gz', - '/var/log/mail.log.4.gz', - '/var/log/mail.log.5.gz', '/var/log/mail.log.6.gz', + '/var/log/mail.log.5.gz', + '/var/log/mail.log.4.gz', + '/var/log/mail.log.3.gz', + '/var/log/mail.log.2.gz', + '/var/log/mail.log.1', + '/var/log/mail.log', ) TIME_DELTAS = OrderedDict([ @@ -80,7 +80,7 @@ def scan_files(collector): print("Processing file", fn, "...") fn = tmp_file.name if tmp_file else fn - for line in reverse_readline(fn): + for line in readline(fn): if scan_mail_log_line(line.strip(), collector) is False: if stop_scan: return @@ -349,11 +349,11 @@ def scan_mail_log_line(line, collector): # Check if the found date is within the time span we are scanning if date > START_DATE: - # Don't process, but continue - return True - elif date < END_DATE: # Don't process, and halt return False + elif date < END_DATE: + # Don't process, but continue + return True if service == "postfix/submission/smtpd": if SCAN_OUT: @@ -453,9 +453,9 @@ def scan_postfix_smtpd_line(date, log, collector): if m: message = "domain blocked: " + m.group(2) - if data["latest"] is None: - data["latest"] = date - data["earliest"] = date + if data["earliest"] is None: + data["earliest"] = date + data["latest"] = date data["blocked"].append((date, sender, message)) collector["rejected"][user] = data @@ -487,9 +487,9 @@ def add_login(user, date, protocol_name, host, collector): } ) - if data["latest"] is None: - data["latest"] = date - data["earliest"] = date + if data["earliest"] is None: + data["earliest"] = date + data["latest"] = date data["totals_by_protocol"][protocol_name] += 1 data["totals_by_protocol_and_host"][(protocol_name, host)] += 1 @@ -528,9 +528,9 @@ def scan_postfix_lmtp_line(date, log, collector): data["received_count"] += 1 data["activity-by-hour"][date.hour] += 1 - if data["latest"] is None: - data["latest"] = date - data["earliest"] = date + if data["earliest"] is None: + data["earliest"] = date + data["latest"] = date collector["received_mail"][user] = data @@ -567,9 +567,9 @@ def scan_postfix_submission_line(date, log, collector): data["hosts"].add(client) data["activity-by-hour"][date.hour] += 1 - if data["latest"] is None: - data["latest"] = date - data["earliest"] = date + if data["earliest"] is None: + data["earliest"] = date + data["latest"] = date collector["sent_mail"][user] = data @@ -578,42 +578,15 @@ def scan_postfix_submission_line(date, log, collector): # Utility functions -def reverse_readline(filename, buf_size=8192): - """ A generator that returns the lines of a file in reverse order - - http://stackoverflow.com/a/23646049/801870 - +def readline(filename): + """ A generator that returns the lines of a file """ - - with open(filename) as fh: - segment = None - offset = 0 - fh.seek(0, os.SEEK_END) - file_size = remaining_size = fh.tell() - while remaining_size > 0: - offset = min(file_size, offset + buf_size) - fh.seek(file_size - offset) - buff = fh.read(min(remaining_size, buf_size)) - remaining_size -= buf_size - lines = buff.split('\n') - # the first line of the buffer is probably not a complete line so - # we'll save it and append it to the last line of the next buffer - # we read - if segment is not None: - # if the previous chunk starts right from the beginning of line - # do not concat the segment to the last line of new chunk - # instead, yield the segment first - if buff[-1] is not '\n': - lines[-1] += segment - else: - yield segment - segment = lines[0] - for index in range(len(lines) - 1, 0, -1): - if len(lines[index]): - yield lines[index] - # Don't yield None if the file was empty - if segment is not None: - yield segment + with open(filename) as file: + while True: + line = file.readline() + if not line: + break + yield line def user_match(user): From db9637ce4f0515671bd192cfa61a385f7c4a93f5 Mon Sep 17 00:00:00 2001 From: Jarek Jurasz Date: Tue, 3 Mar 2020 20:59:28 +0100 Subject: [PATCH 2/6] Fix Feb 29 issue #1733 --- management/mail_log.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/management/mail_log.py b/management/mail_log.py index a0dd1c09..9e08df77 100755 --- a/management/mail_log.py +++ b/management/mail_log.py @@ -344,10 +344,16 @@ def scan_mail_log_line(line, collector): # Replaced the dateutil parser for a less clever way of parser that is roughly 4 times faster. # date = dateutil.parser.parse(date) - date = datetime.datetime.strptime(date, '%b %d %H:%M:%S') - date = date.replace(START_DATE.year) + + # date = datetime.datetime.strptime(date, '%b %d %H:%M:%S') + # date = date.replace(START_DATE.year) + + # strptime fails on Feb 29 if correct year is not provided. See https://bugs.python.org/issue26460 + date = datetime.datetime.strptime(str(START_DATE.year) + ' ' + date, '%Y %b %d %H:%M:%S') + # print("date:", date) # Check if the found date is within the time span we are scanning + # END_DATE < START_DATE if date > START_DATE: # Don't process, and halt return False From e224fc66569d63ff23cc96fa4e5aa04a6da141e4 Mon Sep 17 00:00:00 2001 From: Daniel Davis Date: Sun, 8 Mar 2020 09:49:39 -0400 Subject: [PATCH 3/6] Delete unused function apt_add_repository_to_unattended_upgrades (#1721) The function apt_add_repository_to_unattended_upgrades is defined but never called anywhere. It appears that automatic apt updates are handled in system.sh where the file /etc/apt/apt.conf.d/02periodic is created. The last call was removed in bbfa01f33ae14d1ecfb2bca24aed7b607d1d638f. Co-authored-by: ddavis32 --- setup/functions.sh | 9 --------- 1 file changed, 9 deletions(-) diff --git a/setup/functions.sh b/setup/functions.sh index 3bb96b7a..b36d14bc 100644 --- a/setup/functions.sh +++ b/setup/functions.sh @@ -57,15 +57,6 @@ function apt_install { apt_get_quiet install $PACKAGES } -function apt_add_repository_to_unattended_upgrades { - if [ -f /etc/apt/apt.conf.d/50unattended-upgrades ]; then - if ! grep -q "$1" /etc/apt/apt.conf.d/50unattended-upgrades; then - sed -i "/Allowed-Origins/a \ - \"$1\";" /etc/apt/apt.conf.d/50unattended-upgrades - fi - fi -} - function get_default_hostname { # Guess the machine's hostname. It should be a fully qualified # domain name suitable for DNS. None of these calls may provide From d67e09f33412ebcbcb9abcd399ceef93c4e70e12 Mon Sep 17 00:00:00 2001 From: Sumit Date: Sat, 11 Apr 2020 20:17:46 +0200 Subject: [PATCH 4/6] Allowing adding nginx aliases in www/custom.yaml (#1742) with this nginx will keep on proxying requests and serve static content instead of passing this responsibility to proxied server Without this the one needs to run an additional server to server static content on the proxied url --- management/web_update.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/management/web_update.py b/management/web_update.py index 72295c21..e2498e77 100644 --- a/management/web_update.py +++ b/management/web_update.py @@ -159,6 +159,10 @@ def make_domain_config(domain, templates, ssl_certificates, env): nginx_conf_extra += "\n\t\tproxy_pass %s;" % url nginx_conf_extra += "\n\t\tproxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;" nginx_conf_extra += "\n\t}\n" + for path, alias in yaml.get("aliases", {}).items(): + nginx_conf_extra += "\tlocation %s {" % path + nginx_conf_extra += "\n\t\talias %s;" % alias + nginx_conf_extra += "\n\t}\n" for path, url in yaml.get("redirects", {}).items(): nginx_conf_extra += "\trewrite %s %s permanent;\n" % (path, url) From f52749b403ec51936dafbf4ef56eca72acb95519 Mon Sep 17 00:00:00 2001 From: Stefan Date: Sat, 11 Apr 2020 20:18:44 +0200 Subject: [PATCH 5/6] Better return codes after errors in the setup scripts (#1741) --- setup/bootstrap.sh | 6 +++--- setup/preflight.sh | 4 ++-- tools/owncloud-restore.sh | 2 +- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/setup/bootstrap.sh b/setup/bootstrap.sh index 238109dc..db596798 100644 --- a/setup/bootstrap.sh +++ b/setup/bootstrap.sh @@ -35,14 +35,14 @@ if [ -z "$TAG" ]; then else echo "This script must be run on a system running Ubuntu 18.04 or Ubuntu 14.04." - exit + exit 1 fi fi # Are we running as root? if [[ $EUID -ne 0 ]]; then echo "This script must be run as root. Did you leave out sudo?" - exit + exit 1 fi # Clone the Mail-in-a-Box repository if it doesn't exist. @@ -73,7 +73,7 @@ if [ "$TAG" != `git describe` ]; then git fetch --depth 1 --force --prune origin tag $TAG if ! git checkout -q $TAG; then echo "Update failed. Did you modify something in `pwd`?" - exit + exit 1 fi echo fi diff --git a/setup/preflight.sh b/setup/preflight.sh index 2547c410..acaf80c9 100644 --- a/setup/preflight.sh +++ b/setup/preflight.sh @@ -4,7 +4,7 @@ if [[ $EUID -ne 0 ]]; then echo echo "sudo $0" echo - exit + exit 1 fi # Check that we are running on Ubuntu 18.04 LTS (or 18.04.xx). @@ -14,7 +14,7 @@ if [ "`lsb_release -d | sed 's/.*:\s*//' | sed 's/18\.04\.[0-9]/18.04/' `" != "U lsb_release -d | sed 's/.*:\s*//' echo echo "We can't write scripts that run on every possible setup, sorry." - exit + exit 1 fi # Check that we have enough memory. diff --git a/tools/owncloud-restore.sh b/tools/owncloud-restore.sh index c93a322c..4b0ba4de 100755 --- a/tools/owncloud-restore.sh +++ b/tools/owncloud-restore.sh @@ -22,7 +22,7 @@ fi if [ ! -f $1/config.php ]; then echo "This isn't a valid backup location" - exit + exit 1 fi echo "Restoring backup from $1" From 40b21c466d11ab4fd25d7345693bcb5131d5c0cf Mon Sep 17 00:00:00 2001 From: Michael Becker <7737034+elbakerino@users.noreply.github.com> Date: Tue, 14 Apr 2020 04:10:52 +0200 Subject: [PATCH 6/6] Fypo fix in users.html (#1748) --- management/templates/users.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/management/templates/users.html b/management/templates/users.html index dee79d42..78fef61a 100644 --- a/management/templates/users.html +++ b/management/templates/users.html @@ -99,7 +99,7 @@ Verb Action GET(none) Returns a list of existing mail users. Adding ?format=json to the URL will give JSON-encoded results. POST/add Adds a new mail user. Required POST-body parameters are email and password. -POST/remove Removes a mail user. Required POST-by parameter is email. +POST/remove Removes a mail user. Required POST-body parameter is email. POST/privileges/add Used to make a mail user an admin. Required POST-body parameters are email and privilege=admin. POST/privileges/remove Used to remove the admin privilege from a mail user. Required POST-body parameter is email.