diff --git a/CHANGELOG.md b/CHANGELOG.md index a7d548ff..91d1d82d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,13 +1,35 @@ CHANGELOG ========= +In Development +-------------- + +Software updates: + +* Upgraded Roundcube to version 1.4.11 addressing a security issue, and its desktop notifications plugin. +* Upgraded Z-Push (for Exchange/ActiveSync) to version 2.6.2. + +Control panel: + +* Backblaze B2 is now a supported backup protocol. +* Fixed an issue in the daily mail reports. +* Sort the Custom DNS by zone and qname, and add an option to go back to the old sort order (creation order). + +Mail: + +* Enable sending DMARC failure reports to senders that request them. + +Setup: + +* Fixed error when upgrading from Nextcloud 13. + v0.52 (January 31, 2021) ------------------------ Software updates: * Upgraded Roundcube to version 1.4.10. -* Upgraded zpush to 2.6.1. +* Upgraded Z-Push to 2.6.1. Mail: diff --git a/management/daemon.py b/management/daemon.py index a0cfefa6..8490ee44 100755 --- a/management/daemon.py +++ b/management/daemon.py @@ -277,17 +277,50 @@ def dns_set_secondary_nameserver(): @app.route('/dns/custom') @authorized_personnel_only def dns_get_records(qname=None, rtype=None): - from dns_update import get_custom_dns_config - return json_response([ - { - "qname": r[0], - "rtype": r[1], - "value": r[2], - } - for r in get_custom_dns_config(env) - if r[0] != "_secondary_nameserver" - and (not qname or r[0] == qname) - and (not rtype or r[1] == rtype) ]) + # Get the current set of custom DNS records. + from dns_update import get_custom_dns_config, get_dns_zones + records = get_custom_dns_config(env, only_real_records=True) + + # Filter per the arguments for the more complex GET routes below. + records = [r for r in records + if (not qname or r[0] == qname) + and (not rtype or r[1] == rtype) ] + + # Make a better data structure. + records = [ + { + "qname": r[0], + "rtype": r[1], + "value": r[2], + "sort-order": { }, + } for r in records ] + + # To help with grouping by zone in qname sorting, label each record with which zone it is in. + # There's an inconsistency in how we handle zones in get_dns_zones and in sort_domains, so + # do this first before sorting the domains within the zones. + zones = utils.sort_domains([z[0] for z in get_dns_zones(env)], env) + for r in records: + for z in zones: + if r["qname"] == z or r["qname"].endswith("." + z): + r["zone"] = z + break + + # Add sorting information. The 'created' order follows the order in the YAML file on disk, + # which tracs the order entries were added in the control panel since we append to the end. + # The 'qname' sort order sorts by our standard domain name sort (by zone then by qname), + # then by rtype, and last by the original order in the YAML file (since sorting by value + # may not make sense, unless we parse IP addresses, for example). + for i, r in enumerate(records): + r["sort-order"]["created"] = i + domain_sort_order = utils.sort_domains([r["qname"] for r in records], env) + for i, r in enumerate(sorted(records, key = lambda r : ( + zones.index(r["zone"]), + domain_sort_order.index(r["qname"]), + r["rtype"]))): + r["sort-order"]["qname"] = i + + # Return. + return json_response(records) @app.route('/dns/custom/', methods=['GET', 'POST', 'PUT', 'DELETE']) @app.route('/dns/custom//', methods=['GET', 'POST', 'PUT', 'DELETE']) diff --git a/management/dns_update.py b/management/dns_update.py index 781fb1dc..b2901bc8 100755 --- a/management/dns_update.py +++ b/management/dns_update.py @@ -753,7 +753,7 @@ def write_opendkim_tables(domains, env): ######################################################################## -def get_custom_dns_config(env): +def get_custom_dns_config(env, only_real_records=False): try: custom_dns = rtyaml.load(open(os.path.join(env['STORAGE_ROOT'], 'dns/custom.yaml'))) if not isinstance(custom_dns, dict): raise ValueError() # caught below @@ -761,6 +761,8 @@ def get_custom_dns_config(env): return [ ] for qname, value in custom_dns.items(): + if qname == "_secondary_nameserver" and only_real_records: continue # skip fake record + # Short form. Mapping a domain name to a string is short-hand # for creating A records. if isinstance(value, str): diff --git a/management/mail_log.py b/management/mail_log.py index 9e08df77..1626f820 100755 --- a/management/mail_log.py +++ b/management/mail_log.py @@ -44,9 +44,8 @@ TIME_DELTAS = OrderedDict([ ('today', datetime.datetime.now() - datetime.datetime.now().replace(hour=0, minute=0, second=0)) ]) -# Start date > end date! -START_DATE = datetime.datetime.now() -END_DATE = None +END_DATE = NOW = datetime.datetime.now() +START_DATE = None VERBOSE = False @@ -121,7 +120,7 @@ def scan_mail_log(env): pass print("Scanning logs from {:%Y-%m-%d %H:%M:%S} to {:%Y-%m-%d %H:%M:%S}".format( - END_DATE, START_DATE) + START_DATE, END_DATE) ) # Scan the lines in the log files until the date goes out of range @@ -253,7 +252,7 @@ def scan_mail_log(env): if collector["postgrey"]: msg = "Greylisted Email {:%Y-%m-%d %H:%M:%S} and {:%Y-%m-%d %H:%M:%S}" - print_header(msg.format(END_DATE, START_DATE)) + print_header(msg.format(START_DATE, END_DATE)) print(textwrap.fill( "The following mail was greylisted, meaning the emails were temporarily rejected. " @@ -291,7 +290,7 @@ def scan_mail_log(env): if collector["rejected"]: msg = "Blocked Email {:%Y-%m-%d %H:%M:%S} and {:%Y-%m-%d %H:%M:%S}" - print_header(msg.format(END_DATE, START_DATE)) + print_header(msg.format(START_DATE, END_DATE)) data = OrderedDict(sorted(collector["rejected"].items(), key=email_sort)) @@ -344,20 +343,20 @@ def scan_mail_log_line(line, collector): # Replaced the dateutil parser for a less clever way of parser that is roughly 4 times faster. # date = dateutil.parser.parse(date) - - # date = datetime.datetime.strptime(date, '%b %d %H:%M:%S') - # date = date.replace(START_DATE.year) - - # strptime fails on Feb 29 if correct year is not provided. See https://bugs.python.org/issue26460 - date = datetime.datetime.strptime(str(START_DATE.year) + ' ' + date, '%Y %b %d %H:%M:%S') - # print("date:", date) + + # strptime fails on Feb 29 with ValueError: day is out of range for month if correct year is not provided. + # See https://bugs.python.org/issue26460 + date = datetime.datetime.strptime(str(NOW.year) + ' ' + date, '%Y %b %d %H:%M:%S') + # if log date in future, step back a year + if date > NOW: + date = date.replace(year = NOW.year - 1) + #print("date:", date) # Check if the found date is within the time span we are scanning - # END_DATE < START_DATE - if date > START_DATE: + if date > END_DATE: # Don't process, and halt return False - elif date < END_DATE: + elif date < START_DATE: # Don't process, but continue return True @@ -606,7 +605,7 @@ def email_sort(email): def valid_date(string): - """ Validate the given date string fetched from the --startdate argument """ + """ Validate the given date string fetched from the --enddate argument """ try: date = dateutil.parser.parse(string) except ValueError: @@ -820,12 +819,14 @@ if __name__ == "__main__": parser.add_argument("-t", "--timespan", choices=TIME_DELTAS.keys(), default='today', metavar='