From f908bc364ef96a20b25c0bf9e00bdf6a2c6741ae Mon Sep 17 00:00:00 2001 From: Jarek Jurasz Date: Tue, 3 Mar 2020 20:03:18 +0100 Subject: [PATCH 01/12] mail_log.py reading forward #1593 --- management/mail_log.py | 87 +++++++++++++++--------------------------- 1 file changed, 30 insertions(+), 57 deletions(-) diff --git a/management/mail_log.py b/management/mail_log.py index 79d6ea56..a0dd1c09 100755 --- a/management/mail_log.py +++ b/management/mail_log.py @@ -18,13 +18,13 @@ import utils LOG_FILES = ( - '/var/log/mail.log', - '/var/log/mail.log.1', - '/var/log/mail.log.2.gz', - '/var/log/mail.log.3.gz', - '/var/log/mail.log.4.gz', - '/var/log/mail.log.5.gz', '/var/log/mail.log.6.gz', + '/var/log/mail.log.5.gz', + '/var/log/mail.log.4.gz', + '/var/log/mail.log.3.gz', + '/var/log/mail.log.2.gz', + '/var/log/mail.log.1', + '/var/log/mail.log', ) TIME_DELTAS = OrderedDict([ @@ -80,7 +80,7 @@ def scan_files(collector): print("Processing file", fn, "...") fn = tmp_file.name if tmp_file else fn - for line in reverse_readline(fn): + for line in readline(fn): if scan_mail_log_line(line.strip(), collector) is False: if stop_scan: return @@ -349,11 +349,11 @@ def scan_mail_log_line(line, collector): # Check if the found date is within the time span we are scanning if date > START_DATE: - # Don't process, but continue - return True - elif date < END_DATE: # Don't process, and halt return False + elif date < END_DATE: + # Don't process, but continue + return True if service == "postfix/submission/smtpd": if SCAN_OUT: @@ -453,9 +453,9 @@ def scan_postfix_smtpd_line(date, log, collector): if m: message = "domain blocked: " + m.group(2) - if data["latest"] is None: - data["latest"] = date - data["earliest"] = date + if data["earliest"] is None: + data["earliest"] = date + data["latest"] = date data["blocked"].append((date, sender, message)) collector["rejected"][user] = data @@ -487,9 +487,9 @@ def add_login(user, date, protocol_name, host, collector): } ) - if data["latest"] is None: - data["latest"] = date - data["earliest"] = date + if data["earliest"] is None: + data["earliest"] = date + data["latest"] = date data["totals_by_protocol"][protocol_name] += 1 data["totals_by_protocol_and_host"][(protocol_name, host)] += 1 @@ -528,9 +528,9 @@ def scan_postfix_lmtp_line(date, log, collector): data["received_count"] += 1 data["activity-by-hour"][date.hour] += 1 - if data["latest"] is None: - data["latest"] = date - data["earliest"] = date + if data["earliest"] is None: + data["earliest"] = date + data["latest"] = date collector["received_mail"][user] = data @@ -567,9 +567,9 @@ def scan_postfix_submission_line(date, log, collector): data["hosts"].add(client) data["activity-by-hour"][date.hour] += 1 - if data["latest"] is None: - data["latest"] = date - data["earliest"] = date + if data["earliest"] is None: + data["earliest"] = date + data["latest"] = date collector["sent_mail"][user] = data @@ -578,42 +578,15 @@ def scan_postfix_submission_line(date, log, collector): # Utility functions -def reverse_readline(filename, buf_size=8192): - """ A generator that returns the lines of a file in reverse order - - http://stackoverflow.com/a/23646049/801870 - +def readline(filename): + """ A generator that returns the lines of a file """ - - with open(filename) as fh: - segment = None - offset = 0 - fh.seek(0, os.SEEK_END) - file_size = remaining_size = fh.tell() - while remaining_size > 0: - offset = min(file_size, offset + buf_size) - fh.seek(file_size - offset) - buff = fh.read(min(remaining_size, buf_size)) - remaining_size -= buf_size - lines = buff.split('\n') - # the first line of the buffer is probably not a complete line so - # we'll save it and append it to the last line of the next buffer - # we read - if segment is not None: - # if the previous chunk starts right from the beginning of line - # do not concat the segment to the last line of new chunk - # instead, yield the segment first - if buff[-1] is not '\n': - lines[-1] += segment - else: - yield segment - segment = lines[0] - for index in range(len(lines) - 1, 0, -1): - if len(lines[index]): - yield lines[index] - # Don't yield None if the file was empty - if segment is not None: - yield segment + with open(filename) as file: + while True: + line = file.readline() + if not line: + break + yield line def user_match(user): From db9637ce4f0515671bd192cfa61a385f7c4a93f5 Mon Sep 17 00:00:00 2001 From: Jarek Jurasz Date: Tue, 3 Mar 2020 20:59:28 +0100 Subject: [PATCH 02/12] Fix Feb 29 issue #1733 --- management/mail_log.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/management/mail_log.py b/management/mail_log.py index a0dd1c09..9e08df77 100755 --- a/management/mail_log.py +++ b/management/mail_log.py @@ -344,10 +344,16 @@ def scan_mail_log_line(line, collector): # Replaced the dateutil parser for a less clever way of parser that is roughly 4 times faster. # date = dateutil.parser.parse(date) - date = datetime.datetime.strptime(date, '%b %d %H:%M:%S') - date = date.replace(START_DATE.year) + + # date = datetime.datetime.strptime(date, '%b %d %H:%M:%S') + # date = date.replace(START_DATE.year) + + # strptime fails on Feb 29 if correct year is not provided. See https://bugs.python.org/issue26460 + date = datetime.datetime.strptime(str(START_DATE.year) + ' ' + date, '%Y %b %d %H:%M:%S') + # print("date:", date) # Check if the found date is within the time span we are scanning + # END_DATE < START_DATE if date > START_DATE: # Don't process, and halt return False From e224fc66569d63ff23cc96fa4e5aa04a6da141e4 Mon Sep 17 00:00:00 2001 From: Daniel Davis Date: Sun, 8 Mar 2020 09:49:39 -0400 Subject: [PATCH 03/12] Delete unused function apt_add_repository_to_unattended_upgrades (#1721) The function apt_add_repository_to_unattended_upgrades is defined but never called anywhere. It appears that automatic apt updates are handled in system.sh where the file /etc/apt/apt.conf.d/02periodic is created. The last call was removed in bbfa01f33ae14d1ecfb2bca24aed7b607d1d638f. Co-authored-by: ddavis32 --- setup/functions.sh | 9 --------- 1 file changed, 9 deletions(-) diff --git a/setup/functions.sh b/setup/functions.sh index 3bb96b7a..b36d14bc 100644 --- a/setup/functions.sh +++ b/setup/functions.sh @@ -57,15 +57,6 @@ function apt_install { apt_get_quiet install $PACKAGES } -function apt_add_repository_to_unattended_upgrades { - if [ -f /etc/apt/apt.conf.d/50unattended-upgrades ]; then - if ! grep -q "$1" /etc/apt/apt.conf.d/50unattended-upgrades; then - sed -i "/Allowed-Origins/a \ - \"$1\";" /etc/apt/apt.conf.d/50unattended-upgrades - fi - fi -} - function get_default_hostname { # Guess the machine's hostname. It should be a fully qualified # domain name suitable for DNS. None of these calls may provide From d67e09f33412ebcbcb9abcd399ceef93c4e70e12 Mon Sep 17 00:00:00 2001 From: Sumit Date: Sat, 11 Apr 2020 20:17:46 +0200 Subject: [PATCH 04/12] Allowing adding nginx aliases in www/custom.yaml (#1742) with this nginx will keep on proxying requests and serve static content instead of passing this responsibility to proxied server Without this the one needs to run an additional server to server static content on the proxied url --- management/web_update.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/management/web_update.py b/management/web_update.py index 72295c21..e2498e77 100644 --- a/management/web_update.py +++ b/management/web_update.py @@ -159,6 +159,10 @@ def make_domain_config(domain, templates, ssl_certificates, env): nginx_conf_extra += "\n\t\tproxy_pass %s;" % url nginx_conf_extra += "\n\t\tproxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;" nginx_conf_extra += "\n\t}\n" + for path, alias in yaml.get("aliases", {}).items(): + nginx_conf_extra += "\tlocation %s {" % path + nginx_conf_extra += "\n\t\talias %s;" % alias + nginx_conf_extra += "\n\t}\n" for path, url in yaml.get("redirects", {}).items(): nginx_conf_extra += "\trewrite %s %s permanent;\n" % (path, url) From f52749b403ec51936dafbf4ef56eca72acb95519 Mon Sep 17 00:00:00 2001 From: Stefan Date: Sat, 11 Apr 2020 20:18:44 +0200 Subject: [PATCH 05/12] Better return codes after errors in the setup scripts (#1741) --- setup/bootstrap.sh | 6 +++--- setup/preflight.sh | 4 ++-- tools/owncloud-restore.sh | 2 +- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/setup/bootstrap.sh b/setup/bootstrap.sh index 238109dc..db596798 100644 --- a/setup/bootstrap.sh +++ b/setup/bootstrap.sh @@ -35,14 +35,14 @@ if [ -z "$TAG" ]; then else echo "This script must be run on a system running Ubuntu 18.04 or Ubuntu 14.04." - exit + exit 1 fi fi # Are we running as root? if [[ $EUID -ne 0 ]]; then echo "This script must be run as root. Did you leave out sudo?" - exit + exit 1 fi # Clone the Mail-in-a-Box repository if it doesn't exist. @@ -73,7 +73,7 @@ if [ "$TAG" != `git describe` ]; then git fetch --depth 1 --force --prune origin tag $TAG if ! git checkout -q $TAG; then echo "Update failed. Did you modify something in `pwd`?" - exit + exit 1 fi echo fi diff --git a/setup/preflight.sh b/setup/preflight.sh index 2547c410..acaf80c9 100644 --- a/setup/preflight.sh +++ b/setup/preflight.sh @@ -4,7 +4,7 @@ if [[ $EUID -ne 0 ]]; then echo echo "sudo $0" echo - exit + exit 1 fi # Check that we are running on Ubuntu 18.04 LTS (or 18.04.xx). @@ -14,7 +14,7 @@ if [ "`lsb_release -d | sed 's/.*:\s*//' | sed 's/18\.04\.[0-9]/18.04/' `" != "U lsb_release -d | sed 's/.*:\s*//' echo echo "We can't write scripts that run on every possible setup, sorry." - exit + exit 1 fi # Check that we have enough memory. diff --git a/tools/owncloud-restore.sh b/tools/owncloud-restore.sh index c93a322c..4b0ba4de 100755 --- a/tools/owncloud-restore.sh +++ b/tools/owncloud-restore.sh @@ -22,7 +22,7 @@ fi if [ ! -f $1/config.php ]; then echo "This isn't a valid backup location" - exit + exit 1 fi echo "Restoring backup from $1" From 40b21c466d11ab4fd25d7345693bcb5131d5c0cf Mon Sep 17 00:00:00 2001 From: Michael Becker <7737034+elbakerino@users.noreply.github.com> Date: Tue, 14 Apr 2020 04:10:52 +0200 Subject: [PATCH 06/12] Fypo fix in users.html (#1748) --- management/templates/users.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/management/templates/users.html b/management/templates/users.html index dee79d42..78fef61a 100644 --- a/management/templates/users.html +++ b/management/templates/users.html @@ -99,7 +99,7 @@ Verb Action GET(none) Returns a list of existing mail users. Adding ?format=json to the URL will give JSON-encoded results. POST/add Adds a new mail user. Required POST-body parameters are email and password. -POST/remove Removes a mail user. Required POST-by parameter is email. +POST/remove Removes a mail user. Required POST-body parameter is email. POST/privileges/add Used to make a mail user an admin. Required POST-body parameters are email and privilege=admin. POST/privileges/remove Used to remove the admin privilege from a mail user. Required POST-body parameter is email. From c19f8c9ee64e9b5e33d7817c64180ae806ce3561 Mon Sep 17 00:00:00 2001 From: Joshua Tauberer Date: Sun, 10 May 2020 19:29:01 -0400 Subject: [PATCH 07/12] Change Mozilla autoconfig useGlobalPreferredServer property to false Fixes #1736. --- conf/mozilla-autoconfig.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conf/mozilla-autoconfig.xml b/conf/mozilla-autoconfig.xml index 03e2fef3..22834622 100644 --- a/conf/mozilla-autoconfig.xml +++ b/conf/mozilla-autoconfig.xml @@ -21,7 +21,7 @@ %EMAILADDRESS% password-cleartext true - true + false From 1353949e4282ea6b7636a0ee7255bbac9fd8e246 Mon Sep 17 00:00:00 2001 From: Joshua Tauberer Date: Sun, 10 May 2020 19:44:12 -0400 Subject: [PATCH 08/12] Upgrade Roundcube to 1.4.4, Nextcloud to 17.0.6, Z-Push to 2.5.2 --- CHANGELOG.md | 9 +++++++++ setup/nextcloud.sh | 8 ++++---- setup/webmail.sh | 4 ++-- setup/zpush.sh | 4 ++-- 4 files changed, 17 insertions(+), 8 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d4247085..579793af 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,15 @@ CHANGELOG ========= +In Development +-------------- + +Software updates: + +* Upgraded Roundcube from 1.4.2 to 1.4.4. +* Upgraded Nextcloud from 17.0.2 to 17.0.6 (with Contacts from 3.1.6 to 3.3.0 and Calendar from 1.7.1 to v2.0.3) +* Upgraded Z-Push to 2.5.2. + v0.44 (February 15, 2020) ------------------------- diff --git a/setup/nextcloud.sh b/setup/nextcloud.sh index 3ab21176..76c04800 100755 --- a/setup/nextcloud.sh +++ b/setup/nextcloud.sh @@ -40,11 +40,11 @@ InstallNextcloud() { # their github repositories. mkdir -p /usr/local/lib/owncloud/apps - wget_verify https://github.com/nextcloud/contacts/releases/download/v3.1.6/contacts.tar.gz d331dc6db2ecf7c8e6166926a055dfa3b59722c3 /tmp/contacts.tgz + wget_verify https://github.com/nextcloud/contacts/releases/download/v3.3.0/contacts.tar.gz e55d0357c6785d3b1f3b5f21780cb6d41d32443a /tmp/contacts.tgz tar xf /tmp/contacts.tgz -C /usr/local/lib/owncloud/apps/ rm /tmp/contacts.tgz - wget_verify https://github.com/nextcloud/calendar/releases/download/v1.7.1/calendar.tar.gz bd7c846bad06da6d6ba04280f6fbf37ef846c2ad /tmp/calendar.tgz + wget_verify https://github.com/nextcloud/calendar/releases/download/v2.0.3/calendar.tar.gz 9d9717b29337613b72c74e9914c69b74b346c466 /tmp/calendar.tgz tar xf /tmp/calendar.tgz -C /usr/local/lib/owncloud/apps/ rm /tmp/calendar.tgz @@ -91,8 +91,8 @@ InstallNextcloud() { } # Nextcloud Version to install. Checks are done down below to step through intermediate versions. -nextcloud_ver=17.0.2 -nextcloud_hash=8095fb46e9e0c536163708aee3d17fab8b498ad6 +nextcloud_ver=17.0.6 +nextcloud_hash=50b98d2c2f18510b9530e558ced9ab51eb4f11b0 # Current Nextcloud Version, #1623 # Checking /usr/local/lib/owncloud/version.php shows version of the Nextcloud application, not the DB diff --git a/setup/webmail.sh b/setup/webmail.sh index 89ed1722..20d43c57 100755 --- a/setup/webmail.sh +++ b/setup/webmail.sh @@ -28,8 +28,8 @@ apt_install \ # Install Roundcube from source if it is not already present or if it is out of date. # Combine the Roundcube version number with the commit hash of plugins to track # whether we have the latest version of everything. -VERSION=1.4.2 -HASH=d53fcd7f1109a63364d5d4a43f879c6f47d34a89 +VERSION=1.4.4 +HASH=4e425263f5bec27d39c07bde524f421bda205c07 PERSISTENT_LOGIN_VERSION=6b3fc450cae23ccb2f393d0ef67aa319e877e435 HTML5_NOTIFIER_VERSION=4b370e3cd60dabd2f428a26f45b677ad1b7118d5 CARDDAV_VERSION=3.0.3 diff --git a/setup/zpush.sh b/setup/zpush.sh index a1253d2d..0cedf967 100755 --- a/setup/zpush.sh +++ b/setup/zpush.sh @@ -22,8 +22,8 @@ apt_install \ phpenmod -v php imap # Copy Z-Push into place. -VERSION=2.5.1 -TARGETHASH=4fa55863a429b0033497ae477aca4c8699b8f332 +VERSION=2.5.2 +TARGETHASH=2dc3dbd791b96b0ba2638df0d3d1e03c7e1cbab2 needs_update=0 #NODOC if [ ! -f /usr/local/lib/z-push/version ]; then needs_update=1 #NODOC From c202a5cbc63a8972fa66595dc0944e3f657ac791 Mon Sep 17 00:00:00 2001 From: Joshua Tauberer Date: Sun, 10 May 2020 19:46:25 -0400 Subject: [PATCH 09/12] Changlog entries --- CHANGELOG.md | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 579793af..c723f3b5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,19 @@ Software updates: * Upgraded Nextcloud from 17.0.2 to 17.0.6 (with Contacts from 3.1.6 to 3.3.0 and Calendar from 1.7.1 to v2.0.3) * Upgraded Z-Push to 2.5.2. +System: + +* Fix for mail log statistics report on leap days. +* Fix Mozilla autoconfig useGlobalPreferredServer setting. + +Web: + +* Add a new hidden feature to set nginx alias in www/custom.yaml. + +Setup: + +* Improved error handling. + v0.44 (February 15, 2020) ------------------------- From 8fe33da85d3e15d379bbf61e91f5f934d6263f24 Mon Sep 17 00:00:00 2001 From: clonejo Date: Sun, 3 May 2020 16:34:42 +0200 Subject: [PATCH 10/12] Run nightly tasks on a random minute after 03:00 to avoid overload (#1754) - The MIAB version check regularly fails at 03:00, presumably because a large portion of installations is checking mailinabox.email at the same time. - At installation time, the time of the nightly clock is configured to run at a random minute after 03:00, but before 04:00. - Users might expect the nightly tasks to be over at a certain time and run their own custom tasks afterwards. This could thus interfere with custom backup routines. - This breaks reproducibility of the installation process. - Users might also be surprised by the nightly task time changing after updating MIAB. --- CHANGELOG.md | 1 + setup/management.sh | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c723f3b5..2f28474c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,7 @@ Software updates: System: +* Nightly backups now occur on a random minute in the 3am hour (in the system time zone). The minute is chosen during Mail-in-a-Box installation/upgrade and remains the same until the next upgrade. * Fix for mail log statistics report on leap days. * Fix Mozilla autoconfig useGlobalPreferredServer setting. diff --git a/setup/management.sh b/setup/management.sh index 3caf8eea..9d7c762c 100755 --- a/setup/management.sh +++ b/setup/management.sh @@ -101,10 +101,11 @@ hide_output systemctl enable mailinabox.service # Perform nightly tasks at 3am in system time: take a backup, run # status checks and email the administrator any changes. +minute=$((RANDOM % 60)) # avoid overloading mailinabox.email cat > /etc/cron.d/mailinabox-nightly << EOF; # Mail-in-a-Box --- Do not edit / will be overwritten on update. # Run nightly tasks: backup, status checks. -0 3 * * * root (cd `pwd` && management/daily_tasks.sh) +$minute 3 * * * root (cd `pwd` && management/daily_tasks.sh) EOF # Start the management server. From c87b62b8c2d4356bd7ee508237575e115df1d0a1 Mon Sep 17 00:00:00 2001 From: yeuna92 <64936210+yeuna92@users.noreply.github.com> Date: Mon, 11 May 2020 08:59:42 -0400 Subject: [PATCH 11/12] Fix path to Roundcube error log in fail2ban jails.conf (#1761) --- conf/fail2ban/jails.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conf/fail2ban/jails.conf b/conf/fail2ban/jails.conf index 952dc35a..5de4fd48 100644 --- a/conf/fail2ban/jails.conf +++ b/conf/fail2ban/jails.conf @@ -50,7 +50,7 @@ findtime = 30 enabled = true port = http,https filter = miab-roundcube -logpath = /var/log/roundcubemail/errors +logpath = /var/log/roundcubemail/errors.log maxretry = 20 findtime = 30 From 7de8fc9bc0983f3d47ffaba306b03392b0a68a9c Mon Sep 17 00:00:00 2001 From: Joshua Tauberer Date: Sat, 16 May 2020 06:45:23 -0400 Subject: [PATCH 12/12] v0.45 --- CHANGELOG.md | 8 ++++++-- README.md | 4 ++-- setup/bootstrap.sh | 2 +- 3 files changed, 9 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2f28474c..3cd9e724 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,8 +1,12 @@ CHANGELOG ========= -In Development --------------- +v0.45 (May 16, 2020) +-------------------- + +Security fixes: + +* Fix missing brute force login protection for Roundcube logins. Software updates: diff --git a/README.md b/README.md index 4eef7ed9..e787c8d8 100644 --- a/README.md +++ b/README.md @@ -58,7 +58,7 @@ by him: $ curl -s https://keybase.io/joshdata/key.asc | gpg --import gpg: key C10BDD81: public key "Joshua Tauberer " imported - $ git verify-tag v0.44 + $ git verify-tag v0.45 gpg: Signature made ..... using RSA key ID C10BDD81 gpg: Good signature from "Joshua Tauberer " gpg: WARNING: This key is not certified with a trusted signature! @@ -71,7 +71,7 @@ and on his [personal homepage](https://razor.occams.info/). (Of course, if this Checkout the tag corresponding to the most recent release: - $ git checkout v0.44 + $ git checkout v0.45 Begin the installation. diff --git a/setup/bootstrap.sh b/setup/bootstrap.sh index db596798..4fcb85cc 100644 --- a/setup/bootstrap.sh +++ b/setup/bootstrap.sh @@ -20,7 +20,7 @@ if [ -z "$TAG" ]; then # want to display in status checks. if [ "`lsb_release -d | sed 's/.*:\s*//' | sed 's/18\.04\.[0-9]/18.04/' `" == "Ubuntu 18.04 LTS" ]; then # This machine is running Ubuntu 18.04. - TAG=v0.44 + TAG=v0.45 elif [ "`lsb_release -d | sed 's/.*:\s*//' | sed 's/14\.04\.[0-9]/14.04/' `" == "Ubuntu 14.04 LTS" ]; then # This machine is running Ubuntu 14.04.