diff --git a/CHANGELOG.md b/CHANGELOG.md index bd1745a8..a75a9a43 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,57 @@ CHANGELOG ========= +v0.53 (April 12, 2021) +---------------------- + +Software updates: + +* Upgraded Roundcube to version 1.4.11 addressing a security issue, and its desktop notifications plugin. +* Upgraded Z-Push (for Exchange/ActiveSync) to version 2.6.2. + +Control panel: + +* Backblaze B2 is now a supported backup protocol. +* Fixed an issue in the daily mail reports. +* Sort the Custom DNS by zone and qname, and add an option to go back to the old sort order (creation order). + +Mail: + +* Enable sending DMARC failure reports to senders that request them. + +Setup: + +* Fixed error when upgrading from Nextcloud 13. + +v0.52 (January 31, 2021) +------------------------ + +Software updates: + +* Upgraded Roundcube to version 1.4.10. +* Upgraded Z-Push to 2.6.1. + +Mail: + +* Incoming emails with SPF/DKIM/DMARC failures now get a higher spam score, and these messages are more likely to appear in the junk folder, since they are often spam/phishing. +* Fixed the MTA-STS policy file's line endings. + +Control panel: + +* A new Download button in the control panel's External DNS page can be used to download the required DNS records in zonefile format. +* Fixed the problem when the control panel would report DNS entries as Not Set by increasing a bind query limit. +* Fixed a control panel startup bug on some systems. +* Improved an error message on a DNS lookup timeout. +* A typo was fixed. + +DNS: + +* The TTL for NS records has been increased to 1 day to comply with some registrar requirements. + +System: + +* Nextcloud's photos, dashboard, and activity apps are disabled since we only support contacts and calendar. + v0.51 (November 14, 2020) ------------------------- @@ -13,7 +64,7 @@ Mail: * The MTA-STA max_age value was increased to the normal one week. -Control Panel: +Control panel: * Two-factor authentication can now be enabled for logins to the control panel. However, keep in mind that many online services (including domain name registrars, cloud server providers, and TLS certificate providers) may allow an attacker to take over your account or issue a fraudulent TLS certificate with only access to your email address, and this new two-factor authentication does not protect access to your inbox. It therefore remains very important that user accounts with administrative email addresses have strong passwords. * TLS certificate expiry dates are now shown in ISO8601 format for clarity. @@ -39,7 +90,7 @@ TLS: * TLS certificates are now provisioned in groups by parent domain to limit easy domain enumeration and make provisioning more resilient to errors for particular domains. -Control Panel: +Control panel: * The control panel API is now fully documented at https://mailinabox.email/api-docs.html. * User passwords can now have spaces. diff --git a/README.md b/README.md index 2bbebdd9..53d9fa03 100644 --- a/README.md +++ b/README.md @@ -71,6 +71,14 @@ Issues Changes ------- +### v0.53-quota-0.22-beta + +* Update to v0.53 of Mail-in-a-Box + +### v0.52-quota-0.22-beta + +* Update to v0.52 of Mail-in-a-Box + ### v0.51-quota-0.22-beta * Update to v0.51 of Mail-in-a-Box @@ -244,36 +252,18 @@ See the [setup guide](https://mailinabox.email/guide.html) for detailed, user-fr For experts, start with a completely fresh (really, I mean it) Ubuntu 18.04 LTS 64-bit machine. On the machine... -Clone this repository: +Clone this repository and checkout the tag corresponding to the most recent release: $ git clone https://github.com/mail-in-a-box/mailinabox $ cd mailinabox - -_Optional:_ Download Josh's PGP key and then verify that the sources were signed -by him: - - $ curl -s https://keybase.io/joshdata/key.asc | gpg --import - gpg: key C10BDD81: public key "Joshua Tauberer " imported - - $ git verify-tag v0.51 - gpg: Signature made ..... using RSA key ID C10BDD81 - gpg: Good signature from "Joshua Tauberer " - gpg: WARNING: This key is not certified with a trusted signature! - gpg: There is no indication that the signature belongs to the owner. - Primary key fingerprint: 5F4C 0E73 13CC D744 693B 2AEA B920 41F4 C10B DD81 - -You'll get a lot of warnings, but that's OK. Check that the primary key fingerprint matches the -fingerprint in the key details at [https://keybase.io/joshdata](https://keybase.io/joshdata) -and on his [personal homepage](https://razor.occams.info/). (Of course, if this repository has been compromised you can't trust these instructions.) - -Checkout the tag corresponding to the most recent release: - - $ git checkout v0.51 + $ git checkout v0.53 Begin the installation. $ sudo setup/start.sh +The installation will install, uninstall, and configure packages to turn the machine into a working, good mail server. + For help, DO NOT contact Josh directly --- I don't do tech support by email or tweet (no exceptions). Post your question on the [discussion forum](https://discourse.mailinabox.email/) instead, where maintainers and Mail-in-a-Box users may be able to help you. @@ -281,6 +271,7 @@ Post your question on the [discussion forum](https://discourse.mailinabox.email/ Note that while we want everything to "just work," we can't control the rest of the Internet. Other mail services might block or spam-filter email sent from your Mail-in-a-Box. This is a challenge faced by everyone who runs their own mail server, with or without Mail-in-a-Box. See our discussion forum for tips about that. + Contributing and Development ---------------------------- diff --git a/api/mailinabox.yml b/api/mailinabox.yml index a9a2c124..14cf54de 100644 --- a/api/mailinabox.yml +++ b/api/mailinabox.yml @@ -15,7 +15,7 @@ info: license: name: CC0 1.0 Universal url: https://creativecommons.org/publicdomain/zero/1.0/legalcode - version: 0.47.0 + version: 0.51.0 x-logo: url: https://mailinabox.email/static/logo.png altText: Mail-in-a-Box logo @@ -743,6 +743,38 @@ paths: text/html: schema: type: string + /dns/zonefile/{zone}: + parameters: + - in: path + name: zone + schema: + $ref: '#/components/schemas/Hostname' + required: true + description: Hostname + get: + tags: + - DNS + summary: Get DNS zonefile + description: Returns a DNS zone file for a hostname. + operationId: getDnsZonefile + x-codeSamples: + - lang: curl + source: | + curl -X GET "https://{host}/admin/dns/zonefile/" \ + -u ":" + responses: + 200: + description: Successful operation + content: + application/json: + schema: + $ref: '#/components/schemas/DNSZonefileResponse' + 403: + description: Forbidden + content: + text/html: + schema: + type: string /dns/update: post: tags: @@ -1781,7 +1813,7 @@ components: text/plain: schema: type: string - example: 1.2.3.4 + example: '1.2.3.4' description: The value of the DNS record. example: '1.2.3.4' schemas: @@ -2050,6 +2082,8 @@ components: items: $ref: '#/components/schemas/Hostname' description: DNS zones response. + DNSZonefileResponse: + type: string DNSSecondaryNameserverResponse: type: object required: @@ -2663,13 +2697,6 @@ components: type: string MfaEnableSuccessResponse: type: string - MfaEnableBadRequestResponse: - type: object - required: - - error - properties: - error: - type: string MfaDisableRequest: type: object properties: diff --git a/conf/mta-sts.txt b/conf/mta-sts.txt index 26acc015..e7bdc4c4 100644 --- a/conf/mta-sts.txt +++ b/conf/mta-sts.txt @@ -1,4 +1,4 @@ -version: STSv1 -mode: MODE -mx: PRIMARY_HOSTNAME -max_age: 604800 +version: STSv1 +mode: MODE +mx: PRIMARY_HOSTNAME +max_age: 604800 diff --git a/management/backup.py b/management/backup.py index e1651552..0a8a021e 100755 --- a/management/backup.py +++ b/management/backup.py @@ -456,6 +456,23 @@ def list_target_files(config): raise ValueError(e.reason) return [(key.name[len(path):], key.size) for key in bucket.list(prefix=path)] + elif target.scheme == 'b2': + from b2sdk.v1 import InMemoryAccountInfo, B2Api + from b2sdk.v1.exception import NonExistentBucket + info = InMemoryAccountInfo() + b2_api = B2Api(info) + + # Extract information from target + b2_application_keyid = target.netloc[:target.netloc.index(':')] + b2_application_key = target.netloc[target.netloc.index(':')+1:target.netloc.index('@')] + b2_bucket = target.netloc[target.netloc.index('@')+1:] + + try: + b2_api.authorize_account("production", b2_application_keyid, b2_application_key) + bucket = b2_api.get_bucket_by_name(b2_bucket) + except NonExistentBucket as e: + raise ValueError("B2 Bucket does not exist. Please double check your information!") + return [(key.file_name, key.size) for key, _ in bucket.ls()] else: raise ValueError(config["target"]) diff --git a/management/daemon.py b/management/daemon.py index a1c07af7..b4dbd214 100755 --- a/management/daemon.py +++ b/management/daemon.py @@ -1,3 +1,12 @@ +#!/usr/local/lib/mailinabox/env/bin/python3 +# +# During development, you can start the Mail-in-a-Box control panel +# by running this script, e.g.: +# +# service mailinabox stop # stop the system process +# DEBUG=1 management/daemon.py +# service mailinabox start # when done debugging, start it up again + import os, os.path, re, json, time import multiprocessing.pool, subprocess @@ -292,17 +301,50 @@ def dns_set_secondary_nameserver(): @app.route('/dns/custom') @authorized_personnel_only def dns_get_records(qname=None, rtype=None): - from dns_update import get_custom_dns_config - return json_response([ - { - "qname": r[0], - "rtype": r[1], - "value": r[2], - } - for r in get_custom_dns_config(env) - if r[0] != "_secondary_nameserver" - and (not qname or r[0] == qname) - and (not rtype or r[1] == rtype) ]) + # Get the current set of custom DNS records. + from dns_update import get_custom_dns_config, get_dns_zones + records = get_custom_dns_config(env, only_real_records=True) + + # Filter per the arguments for the more complex GET routes below. + records = [r for r in records + if (not qname or r[0] == qname) + and (not rtype or r[1] == rtype) ] + + # Make a better data structure. + records = [ + { + "qname": r[0], + "rtype": r[1], + "value": r[2], + "sort-order": { }, + } for r in records ] + + # To help with grouping by zone in qname sorting, label each record with which zone it is in. + # There's an inconsistency in how we handle zones in get_dns_zones and in sort_domains, so + # do this first before sorting the domains within the zones. + zones = utils.sort_domains([z[0] for z in get_dns_zones(env)], env) + for r in records: + for z in zones: + if r["qname"] == z or r["qname"].endswith("." + z): + r["zone"] = z + break + + # Add sorting information. The 'created' order follows the order in the YAML file on disk, + # which tracs the order entries were added in the control panel since we append to the end. + # The 'qname' sort order sorts by our standard domain name sort (by zone then by qname), + # then by rtype, and last by the original order in the YAML file (since sorting by value + # may not make sense, unless we parse IP addresses, for example). + for i, r in enumerate(records): + r["sort-order"]["created"] = i + domain_sort_order = utils.sort_domains([r["qname"] for r in records], env) + for i, r in enumerate(sorted(records, key = lambda r : ( + zones.index(r["zone"]), + domain_sort_order.index(r["qname"]), + r["rtype"]))): + r["sort-order"]["qname"] = i + + # Return. + return json_response(records) @app.route('/dns/custom/', methods=['GET', 'POST', 'PUT', 'DELETE']) @app.route('/dns/custom//', methods=['GET', 'POST', 'PUT', 'DELETE']) @@ -362,6 +404,12 @@ def dns_get_dump(): from dns_update import build_recommended_dns return json_response(build_recommended_dns(env)) +@app.route('/dns/zonefile/') +@authorized_personnel_only +def dns_get_zonefile(zone): + from dns_update import get_dns_zonefile + return Response(get_dns_zonefile(zone, env), status=200, mimetype='text/plain') + # SSL @app.route('/ssl/status') @@ -719,7 +767,22 @@ def log_failed_login(request): # APP if __name__ == '__main__': - if "DEBUG" in os.environ: app.debug = True + if "DEBUG" in os.environ: + # Turn on Flask debugging. + app.debug = True + + # Use a stable-ish master API key so that login sessions don't restart on each run. + # Use /etc/machine-id to seed the key with a stable secret, but add something + # and hash it to prevent possibly exposing the machine id, using the time so that + # the key is not valid indefinitely. + import hashlib + with open("/etc/machine-id") as f: + api_key = f.read() + api_key += "|" + str(int(time.time() / (60*60*2))) + hasher = hashlib.sha1() + hasher.update(api_key.encode("ascii")) + auth_service.key = hasher.hexdigest() + if "APIKEY" in os.environ: auth_service.key = os.environ["APIKEY"] if not app.debug: diff --git a/management/dns_update.py b/management/dns_update.py index 748f87f1..b2901bc8 100755 --- a/management/dns_update.py +++ b/management/dns_update.py @@ -470,14 +470,14 @@ def write_nsd_zone(domain, zonefile, records, env, force): zone = """ $ORIGIN {domain}. -$TTL 1800 ; default time to live +$TTL 86400 ; default time to live @ IN SOA ns1.{primary_domain}. hostmaster.{primary_domain}. ( __SERIAL__ ; serial number 7200 ; Refresh (secondary nameserver update interval) - 1800 ; Retry (when refresh fails, how often to try again) + 86400 ; Retry (when refresh fails, how often to try again) 1209600 ; Expire (when refresh fails, how long secondary nameserver will keep records around anyway) - 1800 ; Negative TTL (how long negative responses are cached) + 86400 ; Negative TTL (how long negative responses are cached) ) """ @@ -564,6 +564,17 @@ $TTL 1800 ; default time to live return True # file is updated +def get_dns_zonefile(zone, env): + for domain, fn in get_dns_zones(env): + if zone == domain: + break + else: + raise ValueError("%s is not a domain name that corresponds to a zone." % zone) + + nsd_zonefile = "/etc/nsd/zones/" + fn + with open(nsd_zonefile, "r") as f: + return f.read() + ######################################################################## def write_nsd_conf(zonefiles, additional_records, env): @@ -742,7 +753,7 @@ def write_opendkim_tables(domains, env): ######################################################################## -def get_custom_dns_config(env): +def get_custom_dns_config(env, only_real_records=False): try: custom_dns = rtyaml.load(open(os.path.join(env['STORAGE_ROOT'], 'dns/custom.yaml'))) if not isinstance(custom_dns, dict): raise ValueError() # caught below @@ -750,6 +761,8 @@ def get_custom_dns_config(env): return [ ] for qname, value in custom_dns.items(): + if qname == "_secondary_nameserver" and only_real_records: continue # skip fake record + # Short form. Mapping a domain name to a string is short-hand # for creating A records. if isinstance(value, str): diff --git a/management/mail_log.py b/management/mail_log.py index 9e08df77..1626f820 100755 --- a/management/mail_log.py +++ b/management/mail_log.py @@ -44,9 +44,8 @@ TIME_DELTAS = OrderedDict([ ('today', datetime.datetime.now() - datetime.datetime.now().replace(hour=0, minute=0, second=0)) ]) -# Start date > end date! -START_DATE = datetime.datetime.now() -END_DATE = None +END_DATE = NOW = datetime.datetime.now() +START_DATE = None VERBOSE = False @@ -121,7 +120,7 @@ def scan_mail_log(env): pass print("Scanning logs from {:%Y-%m-%d %H:%M:%S} to {:%Y-%m-%d %H:%M:%S}".format( - END_DATE, START_DATE) + START_DATE, END_DATE) ) # Scan the lines in the log files until the date goes out of range @@ -253,7 +252,7 @@ def scan_mail_log(env): if collector["postgrey"]: msg = "Greylisted Email {:%Y-%m-%d %H:%M:%S} and {:%Y-%m-%d %H:%M:%S}" - print_header(msg.format(END_DATE, START_DATE)) + print_header(msg.format(START_DATE, END_DATE)) print(textwrap.fill( "The following mail was greylisted, meaning the emails were temporarily rejected. " @@ -291,7 +290,7 @@ def scan_mail_log(env): if collector["rejected"]: msg = "Blocked Email {:%Y-%m-%d %H:%M:%S} and {:%Y-%m-%d %H:%M:%S}" - print_header(msg.format(END_DATE, START_DATE)) + print_header(msg.format(START_DATE, END_DATE)) data = OrderedDict(sorted(collector["rejected"].items(), key=email_sort)) @@ -344,20 +343,20 @@ def scan_mail_log_line(line, collector): # Replaced the dateutil parser for a less clever way of parser that is roughly 4 times faster. # date = dateutil.parser.parse(date) - - # date = datetime.datetime.strptime(date, '%b %d %H:%M:%S') - # date = date.replace(START_DATE.year) - - # strptime fails on Feb 29 if correct year is not provided. See https://bugs.python.org/issue26460 - date = datetime.datetime.strptime(str(START_DATE.year) + ' ' + date, '%Y %b %d %H:%M:%S') - # print("date:", date) + + # strptime fails on Feb 29 with ValueError: day is out of range for month if correct year is not provided. + # See https://bugs.python.org/issue26460 + date = datetime.datetime.strptime(str(NOW.year) + ' ' + date, '%Y %b %d %H:%M:%S') + # if log date in future, step back a year + if date > NOW: + date = date.replace(year = NOW.year - 1) + #print("date:", date) # Check if the found date is within the time span we are scanning - # END_DATE < START_DATE - if date > START_DATE: + if date > END_DATE: # Don't process, and halt return False - elif date < END_DATE: + elif date < START_DATE: # Don't process, but continue return True @@ -606,7 +605,7 @@ def email_sort(email): def valid_date(string): - """ Validate the given date string fetched from the --startdate argument """ + """ Validate the given date string fetched from the --enddate argument """ try: date = dateutil.parser.parse(string) except ValueError: @@ -820,12 +819,14 @@ if __name__ == "__main__": parser.add_argument("-t", "--timespan", choices=TIME_DELTAS.keys(), default='today', metavar='