From 7ce41e3865da598d2af4102b82e888d2ffed586d Mon Sep 17 00:00:00 2001 From: gumida Date: Sun, 15 Nov 2020 12:54:34 +0000 Subject: [PATCH 01/28] Changed mta-sts.txt end of line from LF to CRLF per RFC 8461 (#1863) --- conf/mta-sts.txt | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/conf/mta-sts.txt b/conf/mta-sts.txt index 26acc015..e7bdc4c4 100644 --- a/conf/mta-sts.txt +++ b/conf/mta-sts.txt @@ -1,4 +1,4 @@ -version: STSv1 -mode: MODE -mx: PRIMARY_HOSTNAME -max_age: 604800 +version: STSv1 +mode: MODE +mx: PRIMARY_HOSTNAME +max_age: 604800 From 7fd35bbd11a78f4f655e6f05a944a1b627b2e04e Mon Sep 17 00:00:00 2001 From: Joshua Tauberer Date: Sun, 15 Nov 2020 17:17:36 -0500 Subject: [PATCH 02/28] Disable default Nextcloud apps that we don't support Contacts and calendar are the only supported apps in Mail-in-a-Box. Files can't be disabled. Fixes #1864 --- setup/nextcloud.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/setup/nextcloud.sh b/setup/nextcloud.sh index c4b96f85..72b615aa 100755 --- a/setup/nextcloud.sh +++ b/setup/nextcloud.sh @@ -311,6 +311,9 @@ hide_output sudo -u www-data php /usr/local/lib/owncloud/console.php app:enable sudo -u www-data php /usr/local/lib/owncloud/occ upgrade if [ \( $? -ne 0 \) -a \( $? -ne 3 \) ]; then exit 1; fi +# Disable default apps that we don't support +sudo -u www-data php /usr/local/lib/owncloud/occ app:disable photos dashboard activity + # Set PHP FPM values to support large file uploads # (semicolon is the comment character in this file, hashes produce deprecation warnings) tools/editconf.py /etc/php/7.2/fpm/php.ini -c ';' \ From b85b86e6de8c6b0d134a24122ae1b1a6f8e1d3a7 Mon Sep 17 00:00:00 2001 From: Victor Date: Mon, 16 Nov 2020 12:03:41 +0100 Subject: [PATCH 03/28] Add download zonefile button to external DNS page (#1853) Co-authored-by: Joshua Tauberer --- api/mailinabox.yml | 27 +++++++++++++++++ management/daemon.py | 8 ++++++ management/dns_update.py | 11 +++++++ management/templates/custom-dns.html | 2 +- management/templates/external-dns.html | 40 ++++++++++++++++++++++++++ 5 files changed, 87 insertions(+), 1 deletion(-) diff --git a/api/mailinabox.yml b/api/mailinabox.yml index a9a2c124..6358afb4 100644 --- a/api/mailinabox.yml +++ b/api/mailinabox.yml @@ -743,6 +743,31 @@ paths: text/html: schema: type: string + /dns/zonefile/{zone}: + get: + tags: + - DNS + summary: Get DNS zonefile + description: Returns an array of all managed top-level domains. + operationId: getDnsZonefile + x-codeSamples: + - lang: curl + source: | + curl -X GET "https://{host}/admin/dns/zonefile/" \ + -u ":" + responses: + 200: + description: Successful operation + content: + application/json: + schema: + $ref: '#/components/schemas/DNSZonefileResponse' + 403: + description: Forbidden + content: + text/html: + schema: + type: string /dns/update: post: tags: @@ -2050,6 +2075,8 @@ components: items: $ref: '#/components/schemas/Hostname' description: DNS zones response. + DNSZonefileResponse: + type: string DNSSecondaryNameserverResponse: type: object required: diff --git a/management/daemon.py b/management/daemon.py index ffc6d5d5..3c19367b 100755 --- a/management/daemon.py +++ b/management/daemon.py @@ -1,3 +1,5 @@ +#!/usr/local/lib/mailinabox/env/bin/python3 + import os, os.path, re, json, time import multiprocessing.pool, subprocess @@ -338,6 +340,12 @@ def dns_get_dump(): from dns_update import build_recommended_dns return json_response(build_recommended_dns(env)) +@app.route('/dns/zonefile/') +@authorized_personnel_only +def dns_get_zonefile(zone): + from dns_update import get_dns_zonefile + return Response(get_dns_zonefile(zone, env), status=200, mimetype='text/plain') + # SSL @app.route('/ssl/status') diff --git a/management/dns_update.py b/management/dns_update.py index 748f87f1..ccca69cd 100755 --- a/management/dns_update.py +++ b/management/dns_update.py @@ -564,6 +564,17 @@ $TTL 1800 ; default time to live return True # file is updated +def get_dns_zonefile(zone, env): + for domain, fn in get_dns_zones(env): + if zone == domain: + break + else: + raise ValueError("%s is not a domain name that corresponds to a zone." % zone) + + nsd_zonefile = "/etc/nsd/zones/" + fn + with open(nsd_zonefile, "r") as f: + return f.read() + ######################################################################## def write_nsd_conf(zonefiles, additional_records, env): diff --git a/management/templates/custom-dns.html b/management/templates/custom-dns.html index 6984b081..b1b98b9b 100644 --- a/management/templates/custom-dns.html +++ b/management/templates/custom-dns.html @@ -89,7 +89,7 @@

- Multiple secondary servers can be separated with commas or spaces (i.e., ns2.hostingcompany.com ns3.hostingcompany.com). + Multiple secondary servers can be separated with commas or spaces (i.e., ns2.hostingcompany.com ns3.hostingcompany.com). To enable zone transfers to additional servers without listing them as secondary nameservers, add an IP address or subnet using xfr:10.20.30.40 or xfr:10.0.0.0/8.

+

Download zonefile

+

You can download your zonefiles here or use the table of records below.

+
+
+
+ + +
+ +
+
+ +

Records

@@ -57,6 +70,18 @@ From f66e609d3fb00f2e3c0ef8185f16975dd181b665 Mon Sep 17 00:00:00 2001 From: Richard Willis Date: Thu, 26 Nov 2020 11:56:04 +0000 Subject: [PATCH 04/28] Api spec cleanup (#1869) * Fix indentation * Add parameter definition and remove unused model * Update version * Quote example string --- api/mailinabox.yml | 66 +++++++++++++++++++++++----------------------- 1 file changed, 33 insertions(+), 33 deletions(-) diff --git a/api/mailinabox.yml b/api/mailinabox.yml index 6358afb4..14cf54de 100644 --- a/api/mailinabox.yml +++ b/api/mailinabox.yml @@ -15,7 +15,7 @@ info: license: name: CC0 1.0 Universal url: https://creativecommons.org/publicdomain/zero/1.0/legalcode - version: 0.47.0 + version: 0.51.0 x-logo: url: https://mailinabox.email/static/logo.png altText: Mail-in-a-Box logo @@ -744,30 +744,37 @@ paths: schema: type: string /dns/zonefile/{zone}: - get: - tags: - - DNS - summary: Get DNS zonefile - description: Returns an array of all managed top-level domains. - operationId: getDnsZonefile - x-codeSamples: - - lang: curl - source: | - curl -X GET "https://{host}/admin/dns/zonefile/" \ - -u ":" - responses: - 200: - description: Successful operation - content: - application/json: - schema: - $ref: '#/components/schemas/DNSZonefileResponse' - 403: - description: Forbidden - content: - text/html: - schema: - type: string + parameters: + - in: path + name: zone + schema: + $ref: '#/components/schemas/Hostname' + required: true + description: Hostname + get: + tags: + - DNS + summary: Get DNS zonefile + description: Returns a DNS zone file for a hostname. + operationId: getDnsZonefile + x-codeSamples: + - lang: curl + source: | + curl -X GET "https://{host}/admin/dns/zonefile/" \ + -u ":" + responses: + 200: + description: Successful operation + content: + application/json: + schema: + $ref: '#/components/schemas/DNSZonefileResponse' + 403: + description: Forbidden + content: + text/html: + schema: + type: string /dns/update: post: tags: @@ -1806,7 +1813,7 @@ components: text/plain: schema: type: string - example: 1.2.3.4 + example: '1.2.3.4' description: The value of the DNS record. example: '1.2.3.4' schemas: @@ -2690,13 +2697,6 @@ components: type: string MfaEnableSuccessResponse: type: string - MfaEnableBadRequestResponse: - type: object - required: - - error - properties: - error: - type: string MfaDisableRequest: type: object properties: From 82229ce04baef6aeabd74a2c88e414b51236884d Mon Sep 17 00:00:00 2001 From: Joshua Tauberer Date: Sat, 21 Nov 2020 08:09:37 -0500 Subject: [PATCH 05/28] Document how to start the control panel from the command line and in debugging use a stable API key --- management/daemon.py | 24 +++++++++++++++++++++++- 1 file changed, 23 insertions(+), 1 deletion(-) diff --git a/management/daemon.py b/management/daemon.py index 3c19367b..a0cfefa6 100755 --- a/management/daemon.py +++ b/management/daemon.py @@ -1,4 +1,11 @@ #!/usr/local/lib/mailinabox/env/bin/python3 +# +# During development, you can start the Mail-in-a-Box control panel +# by running this script, e.g.: +# +# service mailinabox stop # stop the system process +# DEBUG=1 management/daemon.py +# service mailinabox start # when done debugging, start it up again import os, os.path, re, json, time import multiprocessing.pool, subprocess @@ -680,7 +687,22 @@ def log_failed_login(request): # APP if __name__ == '__main__': - if "DEBUG" in os.environ: app.debug = True + if "DEBUG" in os.environ: + # Turn on Flask debugging. + app.debug = True + + # Use a stable-ish master API key so that login sessions don't restart on each run. + # Use /etc/machine-id to seed the key with a stable secret, but add something + # and hash it to prevent possibly exposing the machine id, using the time so that + # the key is not valid indefinitely. + import hashlib + with open("/etc/machine-id") as f: + api_key = f.read() + api_key += "|" + str(int(time.time() / (60*60*2))) + hasher = hashlib.sha1() + hasher.update(api_key.encode("ascii")) + auth_service.key = hasher.hexdigest() + if "APIKEY" in os.environ: auth_service.key = os.environ["APIKEY"] if not app.debug: From 8664afa99798c9dbd7b52cf67da7e90b3280bbf0 Mon Sep 17 00:00:00 2001 From: Hilko Date: Thu, 26 Nov 2020 13:13:31 +0100 Subject: [PATCH 06/28] Implement Backblaze for Backup (#1812) * Installing b2sdk for b2 support * Added Duplicity PPA so the most recent version is used * Implemented list_target_files for b2 * Implemented b2 in frontend * removed python2 boto package --- management/backup.py | 17 +++++++++ management/templates/system-backup.html | 46 +++++++++++++++++++++++-- setup/management.sh | 16 ++++----- setup/system.sh | 3 ++ 4 files changed, 71 insertions(+), 11 deletions(-) diff --git a/management/backup.py b/management/backup.py index e1651552..0a8a021e 100755 --- a/management/backup.py +++ b/management/backup.py @@ -456,6 +456,23 @@ def list_target_files(config): raise ValueError(e.reason) return [(key.name[len(path):], key.size) for key in bucket.list(prefix=path)] + elif target.scheme == 'b2': + from b2sdk.v1 import InMemoryAccountInfo, B2Api + from b2sdk.v1.exception import NonExistentBucket + info = InMemoryAccountInfo() + b2_api = B2Api(info) + + # Extract information from target + b2_application_keyid = target.netloc[:target.netloc.index(':')] + b2_application_key = target.netloc[target.netloc.index(':')+1:target.netloc.index('@')] + b2_bucket = target.netloc[target.netloc.index('@')+1:] + + try: + b2_api.authorize_account("production", b2_application_keyid, b2_application_key) + bucket = b2_api.get_bucket_by_name(b2_bucket) + except NonExistentBucket as e: + raise ValueError("B2 Bucket does not exist. Please double check your information!") + return [(key.file_name, key.size) for key, _ in bucket.ls()] else: raise ValueError(config["target"]) diff --git a/management/templates/system-backup.html b/management/templates/system-backup.html index 6afe62c8..7cdc3803 100644 --- a/management/templates/system-backup.html +++ b/management/templates/system-backup.html @@ -18,6 +18,7 @@ + @@ -111,6 +112,31 @@ + +
+
+

Backups are stored in a Backblaze B2 bucket. You must have a Backblaze account already.

+

You MUST manually copy the encryption password from to a safe and secure location. You will need this file to decrypt backup files. It is NOT stored in your Backblaze B2 bucket.

+
+
+
+ +
+ +
+
+
+ +
+ +
+
+
+ +
+ +
+
@@ -144,7 +170,7 @@ function toggle_form() { var target_type = $("#backup-target-type").val(); - $(".backup-target-local, .backup-target-rsync, .backup-target-s3").hide(); + $(".backup-target-local, .backup-target-rsync, .backup-target-s3, .backup-target-b2").hide(); $(".backup-target-" + target_type).show(); init_inputs(target_type); @@ -215,7 +241,7 @@ function show_system_backup() { } function show_custom_backup() { - $(".backup-target-local, .backup-target-rsync, .backup-target-s3").hide(); + $(".backup-target-local, .backup-target-rsync, .backup-target-s3, .backup-target-b2").hide(); api( "/system/backup/config", "GET", @@ -245,6 +271,15 @@ function show_custom_backup() { var host = hostpath.shift(); $("#backup-target-s3-host").val(host); $("#backup-target-s3-path").val(hostpath.join('/')); + } else if (r.target.substring(0, 5) == "b2://") { + $("#backup-target-type").val("b2"); + var targetPath = r.target.substring(5); + var b2_application_keyid = targetPath.split(':')[0]; + var b2_applicationkey = targetPath.split(':')[1].split('@')[0]; + var b2_bucket = targetPath.split('@')[1]; + $("#backup-target-b2-user").val(b2_application_keyid); + $("#backup-target-b2-pass").val(b2_applicationkey); + $("#backup-target-b2-bucket").val(b2_bucket); } toggle_form() }) @@ -264,6 +299,11 @@ function set_custom_backup() { target = "rsync://" + $("#backup-target-rsync-user").val() + "@" + $("#backup-target-rsync-host").val() + "/" + $("#backup-target-rsync-path").val(); target_user = ''; + } else if (target_type == "b2") { + target = 'b2://' + $('#backup-target-b2-user').val() + ':' + $('#backup-target-b2-pass').val() + + '@' + $('#backup-target-b2-bucket').val() + target_user = ''; + target_pass = ''; } @@ -303,4 +343,4 @@ function init_inputs(target_type) { set_host($('#backup-target-s3-host-select').val()); } } - + \ No newline at end of file diff --git a/setup/management.sh b/setup/management.sh index ae942985..f5685d4f 100755 --- a/setup/management.sh +++ b/setup/management.sh @@ -18,11 +18,7 @@ while [ -d /usr/local/lib/python3.4/dist-packages/acme ]; do pip3 uninstall -y acme; done -# duplicity is used to make backups of user data. It uses boto -# (via Python 2) to do backups to AWS S3. boto from the Ubuntu -# package manager is too out-of-date -- it doesn't support the newer -# S3 api used in some regions, which breaks backups to those regions. -# See #627, #653. +# duplicity is used to make backups of user data. # # virtualenv is used to isolate the Python 3 packages we # install via pip from the system-installed packages. @@ -30,7 +26,11 @@ done # certbot installs EFF's certbot which we use to # provision free TLS certificates. apt_install duplicity python-pip virtualenv certbot -hide_output pip2 install --upgrade boto + +# b2sdk is used for backblaze backups. +# boto is used for amazon aws backups. +# Both are installed outside the pipenv, so they can be used by duplicity +hide_output pip3 install --upgrade b2sdk boto # Create a virtualenv for the installation of Python 3 packages # used by the management daemon. @@ -50,8 +50,8 @@ hide_output $venv/bin/pip install --upgrade pip hide_output $venv/bin/pip install --upgrade \ rtyaml "email_validator>=1.0.0" "exclusiveprocess" \ flask dnspython python-dateutil \ - qrcode[pil] pyotp \ - "idna>=2.0.0" "cryptography==2.2.2" boto psutil postfix-mta-sts-resolver + qrcode[pil] pyotp \ + "idna>=2.0.0" "cryptography==2.2.2" boto psutil postfix-mta-sts-resolver b2sdk # CONFIGURATION diff --git a/setup/system.sh b/setup/system.sh index 4d33deb6..07f4aa1b 100755 --- a/setup/system.sh +++ b/setup/system.sh @@ -93,6 +93,9 @@ hide_output add-apt-repository -y universe # Install the certbot PPA. hide_output add-apt-repository -y ppa:certbot/certbot +# Install the duplicity PPA. +hide_output add-apt-repository -y ppa:duplicity-team/duplicity-release-git + # ### Update Packages # Update system packages to make sure we have the latest upstream versions From 3422cc61cede26d819d774a0a7cdabb0a85e3586 Mon Sep 17 00:00:00 2001 From: Hilko Date: Sun, 20 Dec 2020 01:11:58 +0100 Subject: [PATCH 07/28] Include en_US.UTF-8 locale in daemon startup (#1883) Fixes #1881. --- setup/management.sh | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/setup/management.sh b/setup/management.sh index f5685d4f..dcef0891 100755 --- a/setup/management.sh +++ b/setup/management.sh @@ -90,6 +90,12 @@ rm -f /tmp/bootstrap.zip # running after a reboot. cat > $inst_dir/start < Date: Fri, 25 Dec 2020 23:19:16 +0100 Subject: [PATCH 08/28] Adjust max-recursion-queries to fix alternating rdns status (#1876) --- setup/system.sh | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/setup/system.sh b/setup/system.sh index 07f4aa1b..208a35df 100755 --- a/setup/system.sh +++ b/setup/system.sh @@ -320,6 +320,9 @@ fi #NODOC # name server, on IPV6. # * The listen-on directive in named.conf.options restricts `bind9` to # binding to the loopback interface instead of all interfaces. +# * The max-recursion-queries directive increases the maximum number of iterative queries. +# If more queries than specified are sent, bind9 returns SERVFAIL. After flushing the cache during system checks, +# we ran into the limit thus we are increasing it from 75 (default value) to 100. apt_install bind9 tools/editconf.py /etc/default/bind9 \ "OPTIONS=\"-u bind -4\"" @@ -327,6 +330,10 @@ if ! grep -q "listen-on " /etc/bind/named.conf.options; then # Add a listen-on directive if it doesn't exist inside the options block. sed -i "s/^}/\n\tlisten-on { 127.0.0.1; };\n}/" /etc/bind/named.conf.options fi +if ! grep -q "max-recursion-queries " /etc/bind/named.conf.options; then + # Add a max-recursion-queries directive if it doesn't exist inside the options block. + sed -i "s/^}/\n\tmax-recursion-queries 100;\n}/" /etc/bind/named.conf.options +fi # First we'll disable systemd-resolved's management of resolv.conf and its stub server. # Breaking the symlink to /run/systemd/resolve/stub-resolv.conf means From c7280055a83085b3d3efd5a9296a1bea4923315c Mon Sep 17 00:00:00 2001 From: jvolkenant Date: Fri, 25 Dec 2020 14:22:24 -0800 Subject: [PATCH 09/28] Implement SPF/DMARC checks, add spam weight to those mails (#1836) --- setup/dkim.sh | 26 ++++++++++++++++++++++ setup/spamassassin.sh | 50 +++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 76 insertions(+) diff --git a/setup/dkim.sh b/setup/dkim.sh index 5bd32370..05221b27 100755 --- a/setup/dkim.sh +++ b/setup/dkim.sh @@ -64,6 +64,32 @@ tools/editconf.py /etc/opendmarc.conf -s \ "Syslog=true" \ "Socket=inet:8893@[127.0.0.1]" +# SPFIgnoreResults causes the filter to ignore any SPF results in the header +# of the message. This is useful if you want the filter to perfrom SPF checks +# itself, or because you don't trust the arriving header. This added header is +# used by spamassassin to evaluate the mail for spamminess. + +tools/editconf.py /etc/opendmarc.conf -s \ + "SPFIgnoreResults=true" + +# SPFSelfValidate causes the filter to perform a fallback SPF check itself +# when it can find no SPF results in the message header. If SPFIgnoreResults +# is also set, it never looks for SPF results in headers and always performs +# the SPF check itself when this is set. This added header is used by +# spamassassin to evaluate the mail for spamminess. + +tools/editconf.py /etc/opendmarc.conf -s \ + "SPFSelfValidate=true" + +# AlwaysAddARHeader Adds an "Authentication-Results:" header field even to +# unsigned messages from domains with no "signs all" policy. The reported DKIM +# result will be "none" in such cases. Normally unsigned mail from non-strict +# domains does not cause the results header field to be added. This added header +# is used by spamassassin to evaluate the mail for spamminess. + +tools/editconf.py /etc/opendkim.conf -s \ + "AlwaysAddARHeader=true" + # Add OpenDKIM and OpenDMARC as milters to postfix, which is how OpenDKIM # intercepts outgoing mail to perform the signing (by adding a mail header) # and how they both intercept incoming mail to add Authentication-Results diff --git a/setup/spamassassin.sh b/setup/spamassassin.sh index d6c8b83b..989bbff4 100755 --- a/setup/spamassassin.sh +++ b/setup/spamassassin.sh @@ -67,6 +67,56 @@ tools/editconf.py /etc/spamassassin/local.cf -s \ "add_header all Report"=_REPORT_ \ "add_header all Score"=_SCORE_ + +# Authentication-Results SPF/Dmarc checks +# --------------------------------------- +# OpenDKIM and OpenDMARC are configured to validate and add "Authentication-Results: ..." +# headers by checking the sender's SPF & DMARC policies. Instead of blocking mail that fails +# these checks, we can use these headers to evaluate the mail as spam. +# +# Our custom rules are added to their own file so that an update to the deb package config +# does not remove our changes. +# +# We need to escape period's in $PRIMARY_HOSTNAME since spamassassin config uses regex. + +escapedprimaryhostname="${PRIMARY_HOSTNAME//./\\.}" + +cat > /etc/spamassassin/miab_spf_dmarc.cf << EOF +# Evaluate DMARC Authentication-Results +header DMARC_PASS Authentication-Results =~ /$escapedprimaryhostname; dmarc=pass/ +describe DMARC_PASS DMARC check passed +score DMARC_PASS -0.1 + +header DMARC_NONE Authentication-Results =~ /$escapedprimaryhostname; dmarc=none/ +describe DMARC_NONE DMARC record not found +score DMARC_NONE 0.1 + +header DMARC_FAIL_NONE Authentication-Results =~ /$escapedprimaryhostname; dmarc=fail \(p=none/ +describe DMARC_FAIL_NONE DMARC check failed (p=none) +score DMARC_FAIL_NONE 2.0 + +header DMARC_FAIL_QUARANTINE Authentication-Results =~ /$escapedprimaryhostname; dmarc=fail \(p=quarantine/ +describe DMARC_FAIL_QUARANTINE DMARC check failed (p=quarantine) +score DMARC_FAIL_QUARANTINE 5.0 + +header DMARC_FAIL_REJECT Authentication-Results =~ /$escapedprimaryhostname; dmarc=fail \(p=reject/ +describe DMARC_FAIL_REJECT DMARC check failed (p=reject) +score DMARC_FAIL_REJECT 10.0 + +# Evaluate SPF Authentication-Results +header SPF_PASS Authentication-Results =~ /$escapedprimaryhostname; spf=pass/ +describe SPF_PASS SPF check passed +score SPF_PASS -0.1 + +header SPF_NONE Authentication-Results =~ /$escapedprimaryhostname; spf=none/ +describe SPF_NONE SPF record not found +score SPF_NONE 2.0 + +header SPF_FAIL Authentication-Results =~ /$escapedprimaryhostname; spf=fail/ +describe SPF_FAIL SPF check failed +score SPF_FAIL 5.0 +EOF + # Bayesean learning # ----------------- # From e26cf4512c3e2c82a46824ce17bfe99da292d58a Mon Sep 17 00:00:00 2001 From: Joshua Tauberer Date: Fri, 25 Dec 2020 17:28:34 -0500 Subject: [PATCH 10/28] Update CHANGELOG --- CHANGELOG.md | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index bd1745a8..fb146e80 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,17 @@ CHANGELOG ========= +In Development +-------------- + +* Incoming emails with SPF/DKIM/DMARC failures now have a higher spam score, and these messages are more likely to appear in the junk folder, since they are often spam/phishing. +* A new Download button in the control panel's External DNS page can be used to download the required DNS records in zonefile format. +* Blackblaze is now a supported backup protocol. +* Fixed the problem when the control panel would report DNS entries as Not Set by increasing a bind query limit. +* Fixed a control panel startup bug on some systems. +* Fixed the MTA-STS policy file's line endings. +* Nextcloud's photos, dashboard, and activity apps are disabled since we only support contacts and calendar. + v0.51 (November 14, 2020) ------------------------- From e2f9cd845a362cff0c4e0dfc8b387978f5b17dd3 Mon Sep 17 00:00:00 2001 From: jcm-shove-it Date: Mon, 28 Dec 2020 14:11:33 +0100 Subject: [PATCH 11/28] Update roundcube to 1.4.10 (#1891) --- setup/webmail.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/setup/webmail.sh b/setup/webmail.sh index 3e725027..de4ca528 100755 --- a/setup/webmail.sh +++ b/setup/webmail.sh @@ -28,8 +28,8 @@ apt_install \ # Install Roundcube from source if it is not already present or if it is out of date. # Combine the Roundcube version number with the commit hash of plugins to track # whether we have the latest version of everything. -VERSION=1.4.9 -HASH=df650f4d3eae9eaae2d5a5f06d68665691daf57d +VERSION=1.4.10 +HASH=36b2351030e1ebddb8e39190d7b0ba82b1bbec1b PERSISTENT_LOGIN_VERSION=6b3fc450cae23ccb2f393d0ef67aa319e877e435 HTML5_NOTIFIER_VERSION=4b370e3cd60dabd2f428a26f45b677ad1b7118d5 CARDDAV_VERSION=3.0.3 From 7a5d729a537221e043d5d24779ccb2a5f1b8cd05 Mon Sep 17 00:00:00 2001 From: Josh Brown Date: Sun, 3 Jan 2021 16:54:31 -0600 Subject: [PATCH 12/28] Fix misspelling (#1893) Change Blackblaze to Backblaze. Include B2 as the integration name. --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index fb146e80..579ab9f0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,7 +6,7 @@ In Development * Incoming emails with SPF/DKIM/DMARC failures now have a higher spam score, and these messages are more likely to appear in the junk folder, since they are often spam/phishing. * A new Download button in the control panel's External DNS page can be used to download the required DNS records in zonefile format. -* Blackblaze is now a supported backup protocol. +* Backblaze B2 is now a supported backup protocol. * Fixed the problem when the control panel would report DNS entries as Not Set by increasing a bind query limit. * Fixed a control panel startup bug on some systems. * Fixed the MTA-STS policy file's line endings. From 8025c41ee40707b2ce954a762b9d076bf48cc012 Mon Sep 17 00:00:00 2001 From: Nicolas North Date: Sun, 3 Jan 2021 23:57:54 +0100 Subject: [PATCH 13/28] Bump TTL for NS records to 1800 (30 min) to 86400 (1 day) as some registries require this (#1892) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Nicolas North [norðurljósahviða] --- management/dns_update.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/management/dns_update.py b/management/dns_update.py index ccca69cd..781fb1dc 100755 --- a/management/dns_update.py +++ b/management/dns_update.py @@ -470,14 +470,14 @@ def write_nsd_zone(domain, zonefile, records, env, force): zone = """ $ORIGIN {domain}. -$TTL 1800 ; default time to live +$TTL 86400 ; default time to live @ IN SOA ns1.{primary_domain}. hostmaster.{primary_domain}. ( __SERIAL__ ; serial number 7200 ; Refresh (secondary nameserver update interval) - 1800 ; Retry (when refresh fails, how often to try again) + 86400 ; Retry (when refresh fails, how often to try again) 1209600 ; Expire (when refresh fails, how long secondary nameserver will keep records around anyway) - 1800 ; Negative TTL (how long negative responses are cached) + 86400 ; Negative TTL (how long negative responses are cached) ) """ From 879467d358d375c83225e57056569f305a1d969d Mon Sep 17 00:00:00 2001 From: Josh Brown Date: Tue, 5 Jan 2021 20:12:01 -0600 Subject: [PATCH 14/28] Fix typo in users.html (#1895) lettters -> letters fixes #1888 --- management/templates/users.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/management/templates/users.html b/management/templates/users.html index 78fef61a..4b349875 100644 --- a/management/templates/users.html +++ b/management/templates/users.html @@ -31,7 +31,7 @@
    -
  • Passwords must be at least eight characters consisting of English lettters and numbers only. For best results, generate a random password.
    • +
  • Passwords must be at least eight characters consisting of English letters and numbers only. For best results, generate a random password.
  • Use aliases to create email addresses that forward to existing accounts.
  • Administrators get access to this control panel.
  • User accounts cannot contain any international (non-ASCII) characters, but aliases can.
    • From 50d50ba6538f03bcb35e6e2b580b413ae1275b45 Mon Sep 17 00:00:00 2001 From: jvolkenant Date: Thu, 28 Jan 2021 15:20:19 -0800 Subject: [PATCH 15/28] Update zpush to 2.6.1 (#1908) --- setup/zpush.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/setup/zpush.sh b/setup/zpush.sh index 0cedf967..b7e0aa43 100755 --- a/setup/zpush.sh +++ b/setup/zpush.sh @@ -22,8 +22,8 @@ apt_install \ phpenmod -v php imap # Copy Z-Push into place. -VERSION=2.5.2 -TARGETHASH=2dc3dbd791b96b0ba2638df0d3d1e03c7e1cbab2 +VERSION=2.6.1 +TARGETHASH=a4415f0dc0ed884acc8ad5c506944fc7e6d68eeb needs_update=0 #NODOC if [ ! -f /usr/local/lib/z-push/version ]; then needs_update=1 #NODOC From e3d98b781ea44218a9b063c01d3f9b603ac26427 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Felix=20Sp=C3=B6ttel?= <1682504+fspoettel@users.noreply.github.com> Date: Thu, 28 Jan 2021 23:22:43 +0000 Subject: [PATCH 16/28] Warn when connection to Spamhaus times out (#1817) --- management/status_checks.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/management/status_checks.py b/management/status_checks.py index 36da034a..631a82a2 100755 --- a/management/status_checks.py +++ b/management/status_checks.py @@ -293,6 +293,8 @@ def run_network_checks(env, output): zen = query_dns(rev_ip4+'.zen.spamhaus.org', 'A', nxdomain=None) if zen is None: output.print_ok("IP address is not blacklisted by zen.spamhaus.org.") + elif zen == "[timeout]": + output.print_warning("Connection to zen.spamhaus.org timed out. We could not determine whether your server's IP address is blacklisted. Please try again later.") else: output.print_error("""The IP address of this machine %s is listed in the Spamhaus Block List (code %s), which may prevent recipients from receiving your email. See http://www.spamhaus.org/query/ip/%s.""" @@ -678,6 +680,8 @@ def check_mail_domain(domain, env, output): dbl = query_dns(domain+'.dbl.spamhaus.org', "A", nxdomain=None) if dbl is None: output.print_ok("Domain is not blacklisted by dbl.spamhaus.org.") + elif dbl == "[timeout]": + output.print_warning("Connection to dbl.spamhaus.org timed out. We could not determine whether the domain {} is blacklisted. Please try again later.".format(domain)) else: output.print_error("""This domain is listed in the Spamhaus Domain Block List (code %s), which may prevent recipients from receiving your mail. From b1d703a5e717c086aea0e37e9b14f78150181880 Mon Sep 17 00:00:00 2001 From: Joshua Tauberer Date: Sun, 31 Jan 2021 08:33:20 -0500 Subject: [PATCH 17/28] Disable Backblaze B2 backups until #1899 is resolved --- CHANGELOG.md | 1 - management/templates/system-backup.html | 3 +-- setup/management.sh | 5 ++--- 3 files changed, 3 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 579ab9f0..ff8db287 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,7 +6,6 @@ In Development * Incoming emails with SPF/DKIM/DMARC failures now have a higher spam score, and these messages are more likely to appear in the junk folder, since they are often spam/phishing. * A new Download button in the control panel's External DNS page can be used to download the required DNS records in zonefile format. -* Backblaze B2 is now a supported backup protocol. * Fixed the problem when the control panel would report DNS entries as Not Set by increasing a bind query limit. * Fixed a control panel startup bug on some systems. * Fixed the MTA-STS policy file's line endings. diff --git a/management/templates/system-backup.html b/management/templates/system-backup.html index 7cdc3803..7c4fef13 100644 --- a/management/templates/system-backup.html +++ b/management/templates/system-backup.html @@ -18,7 +18,6 @@ -
@@ -343,4 +342,4 @@ function init_inputs(target_type) { set_host($('#backup-target-s3-host-select').val()); } } - \ No newline at end of file + diff --git a/setup/management.sh b/setup/management.sh index dcef0891..1a5ab53e 100755 --- a/setup/management.sh +++ b/setup/management.sh @@ -27,10 +27,9 @@ done # provision free TLS certificates. apt_install duplicity python-pip virtualenv certbot -# b2sdk is used for backblaze backups. # boto is used for amazon aws backups. # Both are installed outside the pipenv, so they can be used by duplicity -hide_output pip3 install --upgrade b2sdk boto +hide_output pip3 install --upgrade boto # Create a virtualenv for the installation of Python 3 packages # used by the management daemon. @@ -51,7 +50,7 @@ hide_output $venv/bin/pip install --upgrade \ rtyaml "email_validator>=1.0.0" "exclusiveprocess" \ flask dnspython python-dateutil \ qrcode[pil] pyotp \ - "idna>=2.0.0" "cryptography==2.2.2" boto psutil postfix-mta-sts-resolver b2sdk + "idna>=2.0.0" "cryptography==2.2.2" boto psutil postfix-mta-sts-resolver # CONFIGURATION From e81963e585f51b520461c951fd61f510d1b1679d Mon Sep 17 00:00:00 2001 From: Joshua Tauberer Date: Sun, 31 Jan 2021 08:47:33 -0500 Subject: [PATCH 18/28] Remove the instructions for checking that release tags are signed by me since I am not going to do that anymore --- README.md | 27 +++++---------------------- 1 file changed, 5 insertions(+), 22 deletions(-) diff --git a/README.md b/README.md index fcda83f9..02445a20 100644 --- a/README.md +++ b/README.md @@ -54,36 +54,18 @@ See the [setup guide](https://mailinabox.email/guide.html) for detailed, user-fr For experts, start with a completely fresh (really, I mean it) Ubuntu 18.04 LTS 64-bit machine. On the machine... -Clone this repository: +Clone this repository and checkout the tag corresponding to the most recent release: $ git clone https://github.com/mail-in-a-box/mailinabox $ cd mailinabox - -_Optional:_ Download Josh's PGP key and then verify that the sources were signed -by him: - - $ curl -s https://keybase.io/joshdata/key.asc | gpg --import - gpg: key C10BDD81: public key "Joshua Tauberer " imported - - $ git verify-tag v0.51 - gpg: Signature made ..... using RSA key ID C10BDD81 - gpg: Good signature from "Joshua Tauberer " - gpg: WARNING: This key is not certified with a trusted signature! - gpg: There is no indication that the signature belongs to the owner. - Primary key fingerprint: 5F4C 0E73 13CC D744 693B 2AEA B920 41F4 C10B DD81 - -You'll get a lot of warnings, but that's OK. Check that the primary key fingerprint matches the -fingerprint in the key details at [https://keybase.io/joshdata](https://keybase.io/joshdata) -and on his [personal homepage](https://razor.occams.info/). (Of course, if this repository has been compromised you can't trust these instructions.) - -Checkout the tag corresponding to the most recent release: - - $ git checkout v0.51 + $ git checkout v0.52 Begin the installation. $ sudo setup/start.sh +The installation will install, uninstall, and configure packages to turn the machine into a working, good mail server. + For help, DO NOT contact Josh directly --- I don't do tech support by email or tweet (no exceptions). Post your question on the [discussion forum](https://discourse.mailinabox.email/) instead, where maintainers and Mail-in-a-Box users may be able to help you. @@ -91,6 +73,7 @@ Post your question on the [discussion forum](https://discourse.mailinabox.email/ Note that while we want everything to "just work," we can't control the rest of the Internet. Other mail services might block or spam-filter email sent from your Mail-in-a-Box. This is a challenge faced by everyone who runs their own mail server, with or without Mail-in-a-Box. See our discussion forum for tips about that. + Contributing and Development ---------------------------- From 90d63fd208ebeca9378a31d97f844363296d6f51 Mon Sep 17 00:00:00 2001 From: Joshua Tauberer Date: Sun, 31 Jan 2021 08:39:46 -0500 Subject: [PATCH 19/28] v0.52 --- CHANGELOG.md | 31 +++++++++++++++++++++++++------ setup/bootstrap.sh | 2 +- 2 files changed, 26 insertions(+), 7 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ff8db287..a7d548ff 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,14 +1,33 @@ CHANGELOG ========= -In Development --------------- +v0.52 (January 31, 2021) +------------------------ + +Software updates: + +* Upgraded Roundcube to version 1.4.10. +* Upgraded zpush to 2.6.1. + +Mail: + +* Incoming emails with SPF/DKIM/DMARC failures now get a higher spam score, and these messages are more likely to appear in the junk folder, since they are often spam/phishing. +* Fixed the MTA-STS policy file's line endings. + +Control panel: -* Incoming emails with SPF/DKIM/DMARC failures now have a higher spam score, and these messages are more likely to appear in the junk folder, since they are often spam/phishing. * A new Download button in the control panel's External DNS page can be used to download the required DNS records in zonefile format. * Fixed the problem when the control panel would report DNS entries as Not Set by increasing a bind query limit. * Fixed a control panel startup bug on some systems. -* Fixed the MTA-STS policy file's line endings. +* Improved an error message on a DNS lookup timeout. +* A typo was fixed. + +DNS: + +* The TTL for NS records has been increased to 1 day to comply with some registrar requirements. + +System: + * Nextcloud's photos, dashboard, and activity apps are disabled since we only support contacts and calendar. v0.51 (November 14, 2020) @@ -23,7 +42,7 @@ Mail: * The MTA-STA max_age value was increased to the normal one week. -Control Panel: +Control panel: * Two-factor authentication can now be enabled for logins to the control panel. However, keep in mind that many online services (including domain name registrars, cloud server providers, and TLS certificate providers) may allow an attacker to take over your account or issue a fraudulent TLS certificate with only access to your email address, and this new two-factor authentication does not protect access to your inbox. It therefore remains very important that user accounts with administrative email addresses have strong passwords. * TLS certificate expiry dates are now shown in ISO8601 format for clarity. @@ -49,7 +68,7 @@ TLS: * TLS certificates are now provisioned in groups by parent domain to limit easy domain enumeration and make provisioning more resilient to errors for particular domains. -Control Panel: +Control panel: * The control panel API is now fully documented at https://mailinabox.email/api-docs.html. * User passwords can now have spaces. diff --git a/setup/bootstrap.sh b/setup/bootstrap.sh index b6110fa8..79c7d389 100644 --- a/setup/bootstrap.sh +++ b/setup/bootstrap.sh @@ -20,7 +20,7 @@ if [ -z "$TAG" ]; then # want to display in status checks. if [ "`lsb_release -d | sed 's/.*:\s*//' | sed 's/18\.04\.[0-9]/18.04/' `" == "Ubuntu 18.04 LTS" ]; then # This machine is running Ubuntu 18.04. - TAG=v0.51 + TAG=v0.52 elif [ "`lsb_release -d | sed 's/.*:\s*//' | sed 's/14\.04\.[0-9]/14.04/' `" == "Ubuntu 14.04 LTS" ]; then # This machine is running Ubuntu 14.04. From af62e7a99bbe2852632daf274a4d8ad29a0cd523 Mon Sep 17 00:00:00 2001 From: jvolkenant Date: Sat, 6 Feb 2021 13:49:43 -0800 Subject: [PATCH 20/28] Fixes unbound variable when upgrading from Nextcloud 13 (#1913) --- setup/nextcloud.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/setup/nextcloud.sh b/setup/nextcloud.sh index 72b615aa..200eba9e 100755 --- a/setup/nextcloud.sh +++ b/setup/nextcloud.sh @@ -24,8 +24,8 @@ InstallNextcloud() { hash_contacts=$4 version_calendar=$5 hash_calendar=$6 - version_user_external=$7 - hash_user_external=$8 + version_user_external=${7:-} + hash_user_external=${8:-} echo echo "Upgrading to Nextcloud version $version" From 82ca54df962196466a0f8c02d28a8a605e777716 Mon Sep 17 00:00:00 2001 From: jeremitu Date: Sun, 28 Feb 2021 13:59:26 +0100 Subject: [PATCH 21/28] Fixed #1894 log date over year change, START_DATE < END_DATE now. (#1905) * Fixed #1894 log date over year change, START_DATE < END_DATE now. * Corrected mail_log.py argument help and message. Co-authored-by: Jarek --- management/mail_log.py | 53 +++++++++++++++++++++--------------------- 1 file changed, 27 insertions(+), 26 deletions(-) diff --git a/management/mail_log.py b/management/mail_log.py index 9e08df77..1626f820 100755 --- a/management/mail_log.py +++ b/management/mail_log.py @@ -44,9 +44,8 @@ TIME_DELTAS = OrderedDict([ ('today', datetime.datetime.now() - datetime.datetime.now().replace(hour=0, minute=0, second=0)) ]) -# Start date > end date! -START_DATE = datetime.datetime.now() -END_DATE = None +END_DATE = NOW = datetime.datetime.now() +START_DATE = None VERBOSE = False @@ -121,7 +120,7 @@ def scan_mail_log(env): pass print("Scanning logs from {:%Y-%m-%d %H:%M:%S} to {:%Y-%m-%d %H:%M:%S}".format( - END_DATE, START_DATE) + START_DATE, END_DATE) ) # Scan the lines in the log files until the date goes out of range @@ -253,7 +252,7 @@ def scan_mail_log(env): if collector["postgrey"]: msg = "Greylisted Email {:%Y-%m-%d %H:%M:%S} and {:%Y-%m-%d %H:%M:%S}" - print_header(msg.format(END_DATE, START_DATE)) + print_header(msg.format(START_DATE, END_DATE)) print(textwrap.fill( "The following mail was greylisted, meaning the emails were temporarily rejected. " @@ -291,7 +290,7 @@ def scan_mail_log(env): if collector["rejected"]: msg = "Blocked Email {:%Y-%m-%d %H:%M:%S} and {:%Y-%m-%d %H:%M:%S}" - print_header(msg.format(END_DATE, START_DATE)) + print_header(msg.format(START_DATE, END_DATE)) data = OrderedDict(sorted(collector["rejected"].items(), key=email_sort)) @@ -344,20 +343,20 @@ def scan_mail_log_line(line, collector): # Replaced the dateutil parser for a less clever way of parser that is roughly 4 times faster. # date = dateutil.parser.parse(date) - - # date = datetime.datetime.strptime(date, '%b %d %H:%M:%S') - # date = date.replace(START_DATE.year) - - # strptime fails on Feb 29 if correct year is not provided. See https://bugs.python.org/issue26460 - date = datetime.datetime.strptime(str(START_DATE.year) + ' ' + date, '%Y %b %d %H:%M:%S') - # print("date:", date) + + # strptime fails on Feb 29 with ValueError: day is out of range for month if correct year is not provided. + # See https://bugs.python.org/issue26460 + date = datetime.datetime.strptime(str(NOW.year) + ' ' + date, '%Y %b %d %H:%M:%S') + # if log date in future, step back a year + if date > NOW: + date = date.replace(year = NOW.year - 1) + #print("date:", date) # Check if the found date is within the time span we are scanning - # END_DATE < START_DATE - if date > START_DATE: + if date > END_DATE: # Don't process, and halt return False - elif date < END_DATE: + elif date < START_DATE: # Don't process, but continue return True @@ -606,7 +605,7 @@ def email_sort(email): def valid_date(string): - """ Validate the given date string fetched from the --startdate argument """ + """ Validate the given date string fetched from the --enddate argument """ try: date = dateutil.parser.parse(string) except ValueError: @@ -820,12 +819,14 @@ if __name__ == "__main__": parser.add_argument("-t", "--timespan", choices=TIME_DELTAS.keys(), default='today', metavar='"); - hdr.find('h4').text(r[i].domain); + var hdr = $(""); + hdr.find('th').text(r[i].domain); $('#alias_table tbody').append(hdr); for (var k = 0; k < r[i].aliases.length; k++) { diff --git a/management/templates/custom-dns.html b/management/templates/custom-dns.html index b1b98b9b..c59624eb 100644 --- a/management/templates/custom-dns.html +++ b/management/templates/custom-dns.html @@ -57,7 +57,13 @@ -

+
+ sort by: + domain name + | + created +
+ @@ -192,36 +198,38 @@ function show_current_custom_dns() { $('#custom-dns-current').fadeIn(); else $('#custom-dns-current').fadeOut(); - - var reverse_fqdn = function(el) { - el.qname = el.qname.split('.').reverse().join('.'); - return el; - } - var sort = function(a, b) { - if(a.qname === b.qname) { - if(a.rtype === b.rtype) { - return a.value > b.value ? 1 : -1; - } - return a.rtype > b.rtype ? 1 : -1; - } - return a.qname > b.qname ? 1 : -1; - } + window.miab_custom_dns_data = data; + show_current_custom_dns_update_after_sort(); + }); +} - data = data.map(reverse_fqdn).sort(sort).map(reverse_fqdn); +function show_current_custom_dns_update_after_sort() { + var data = window.miab_custom_dns_data; + var sort_key = window.miab_custom_dns_data_sort_order || "qname"; - $('#custom-dns-current').find("tbody").text(''); + data.sort(function(a, b) { return a["sort-order"][sort_key] - b["sort-order"][sort_key] }); + + var tbody = $('#custom-dns-current').find("tbody"); + tbody.text(''); + var last_zone = null; for (var i = 0; i < data.length; i++) { + if (sort_key == "qname" && data[i].zone != last_zone) { + var r = $(""); + r.find("th").text(data[i].zone); + tbody.append(r); + last_zone = data[i].zone; + } + var tr = $(""); - $('#custom-dns-current').find("tbody").append(tr); + tbody.append(tr); tr.attr('data-qname', data[i].qname); tr.attr('data-rtype', data[i].rtype); tr.attr('data-value', data[i].value); tr.append($('')); } - }); } function delete_custom_dns_record(elem) { diff --git a/management/templates/users.html b/management/templates/users.html index 4b349875..24adf4a1 100644 --- a/management/templates/users.html +++ b/management/templates/users.html @@ -1,7 +1,6 @@

Users