diff --git a/CHANGELOG.md b/CHANGELOG.md index c4033df1..d5516d56 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,56 @@ CHANGELOG ========= +Version 62 (May 20, 2023) +------------------------- + +Package updates: + +* Nextcloud updated to 23.0.12 (and its apps also updated). +* Roundcube updated to 1.6.1. +* Z-Push to 2.7.0, which has compatibility for Ubuntu 22.04, so it works again. + +Mail: + +* Roundcube's password change page is now working again. + +Control panel: + +* Allow setting the backup location's S3 region name for non-AWS S3-compatible backup hosts. +* Control panel pages can be opened in a new tab/window and bookmarked and browser history navigation now works. +* Add a Copy button to put the rsync backup public key on clipboard. +* Allow secondary DNS xfr: items added in the control panel to be hostnames too. +* Fixed issue where sshkeygen fails when IPv6 is disabled. +* Fixed issue opening munin reports. +* Fixed report formatting in status emails sent to the administrator. + +Version 61.1 (January 28, 2023) +------------------------------- + +* Fixed rsync backups not working with the default port. +* Reverted "Improve error messages in the management tools when external command-line tools are run." because of the possibility of user secrets being included in error messages. +* Fix for TLS certificate SHA fingerprint not being displayed during setup. + +Version 61 (January 21, 2023) +----------------------------- + +System: + +* fail2ban didn't start after setup. + +Mail: + +* Disable Roundcube password plugin since it was corrupting the user database. + +Control panel: + +* Fix changing existing backup settings when the rsync type is used. +* Allow setting a custom port for rsync backups. +* Fixes to DNS lookups during status checks when there are timeouts, enforce timeouts better. +* A new check is added to ensure fail2ban is running. +* Fixed a color. +* Improve error messages in the management tools when external command-line tools are run. + Version 60.1 (October 30, 2022) ------------------------------- diff --git a/README.md b/README.md index 42792025..6c771656 100644 --- a/README.md +++ b/README.md @@ -60,7 +60,7 @@ Clone this repository and checkout the tag corresponding to the most recent rele $ git clone https://github.com/mail-in-a-box/mailinabox $ cd mailinabox - $ git checkout v60.1 + $ git checkout v62 Begin the installation. diff --git a/management/backup.py b/management/backup.py index 8a82c4ad..06285ba5 100755 --- a/management/backup.py +++ b/management/backup.py @@ -202,7 +202,9 @@ def get_duplicity_target_url(config): # the target URL must be the bucket name. The hostname is passed # via get_duplicity_additional_args. Move the first part of the # path (the bucket name) into the hostname URL component, and leave - # the rest for the path. + # the rest for the path. (The S3 region name is also stored in the + # hostname part of the URL, in the username portion, which we also + # have to drop here). target[1], target[2] = target[2].lstrip('/').split('/', 1) target = urlunsplit(target) @@ -213,16 +215,32 @@ def get_duplicity_additional_args(env): config = get_backup_config(env) if get_target_type(config) == 'rsync': + # Extract a port number for the ssh transport. Duplicity accepts the + # optional port number syntax in the target, but it doesn't appear to act + # on it, so we set the ssh port explicitly via the duplicity options. + from urllib.parse import urlsplit + try: + port = urlsplit(config["target"]).port + except ValueError: + port = 22 + if port is None: + port = 22 + return [ - "--ssh-options= -i /root/.ssh/id_rsa_miab", - "--rsync-options= -e \"/usr/bin/ssh -oStrictHostKeyChecking=no -oBatchMode=yes -p 22 -i /root/.ssh/id_rsa_miab\"", + f"--ssh-options= -i /root/.ssh/id_rsa_miab -p {port}", + f"--rsync-options= -e \"/usr/bin/ssh -oStrictHostKeyChecking=no -oBatchMode=yes -p {port} -i /root/.ssh/id_rsa_miab\"", ] elif get_target_type(config) == 's3': # See note about hostname in get_duplicity_target_url. + # The region name, which is required by some non-AWS endpoints, + # is saved inside the username portion of the URL. from urllib.parse import urlsplit, urlunsplit target = urlsplit(config["target"]) - endpoint_url = urlunsplit(("https", target.netloc, '', '', '')) - return ["--s3-endpoint-url", endpoint_url] + endpoint_url = urlunsplit(("https", target.hostname, '', '', '')) + args = ["--s3-endpoint-url", endpoint_url] + if target.username: # region name is stuffed here + args += ["--s3-region-name", target.username] + return args return [] @@ -408,6 +426,16 @@ def list_target_files(config): rsync_fn_size_re = re.compile(r'.* ([^ ]*) [^ ]* [^ ]* (.*)') rsync_target = '{host}:{path}' + # Strip off any trailing port specifier because it's not valid in rsync's + # DEST syntax. Explicitly set the port number for the ssh transport. + user_host, *_ = target.netloc.rsplit(':', 1) + try: + port = target.port + except ValueError: + port = 22 + if port is None: + port = 22 + target_path = target.path if not target_path.endswith('/'): target_path = target_path + '/' @@ -416,11 +444,11 @@ def list_target_files(config): rsync_command = [ 'rsync', '-e', - '/usr/bin/ssh -i /root/.ssh/id_rsa_miab -oStrictHostKeyChecking=no -oBatchMode=yes', + f'/usr/bin/ssh -i /root/.ssh/id_rsa_miab -oStrictHostKeyChecking=no -oBatchMode=yes -p {port}', '--list-only', '-r', rsync_target.format( - host=target.netloc, + host=user_host, path=target_path) ] @@ -531,7 +559,8 @@ def get_backup_config(env, for_save=False, for_ui=False): # Merge in anything written to custom.yaml. try: - custom_config = rtyaml.load(open(os.path.join(backup_root, 'custom.yaml'))) + with open(os.path.join(backup_root, 'custom.yaml'), 'r') as f: + custom_config = rtyaml.load(f) if not isinstance(custom_config, dict): raise ValueError() # caught below config.update(custom_config) except: @@ -556,7 +585,8 @@ def get_backup_config(env, for_save=False, for_ui=False): config["target"] = "file://" + config["file_target_directory"] ssh_pub_key = os.path.join('/root', '.ssh', 'id_rsa_miab.pub') if os.path.exists(ssh_pub_key): - config["ssh_pub_key"] = open(ssh_pub_key, 'r').read() + with open(ssh_pub_key, 'r') as f: + config["ssh_pub_key"] = f.read() return config diff --git a/management/cli.py b/management/cli.py index 1b91b003..b32089c6 100755 --- a/management/cli.py +++ b/management/cli.py @@ -47,7 +47,8 @@ def read_password(): return first def setup_key_auth(mgmt_uri): - key = open('/var/lib/mailinabox/api.key').read().strip() + with open('/var/lib/mailinabox/api.key', 'r') as f: + key = f.read().strip() auth_handler = urllib.request.HTTPBasicAuthHandler() auth_handler.add_password( diff --git a/management/daemon.py b/management/daemon.py index cbbfd6bf..47548531 100755 --- a/management/daemon.py +++ b/management/daemon.py @@ -709,7 +709,7 @@ def munin_cgi(filename): support infrastructure like spawn-fcgi. """ - COMMAND = 'su - munin --preserve-environment --shell=/bin/bash -c /usr/lib/munin/cgi/munin-cgi-graph' + COMMAND = 'su munin --preserve-environment --shell=/bin/bash -c /usr/lib/munin/cgi/munin-cgi-graph' # su changes user, we use the munin user here # --preserve-environment retains the environment, which is where Popen's `env` data is # --shell=/bin/bash ensures the shell used is bash diff --git a/management/dns_update.py b/management/dns_update.py index 29a6453f..ac123567 100755 --- a/management/dns_update.py +++ b/management/dns_update.py @@ -465,7 +465,7 @@ def build_sshfp_records(): pass break - keys = shell("check_output", ["ssh-keyscan", "-t", "rsa,dsa,ecdsa,ed25519", "-p", str(port), "localhost"]) + keys = shell("check_output", ["ssh-keyscan", "-4", "-t", "rsa,dsa,ecdsa,ed25519", "-p", str(port), "localhost"]) keys = sorted(keys.split("\n")) for key in keys: @@ -815,7 +815,8 @@ def write_opendkim_tables(domains, env): def get_custom_dns_config(env, only_real_records=False): try: - custom_dns = rtyaml.load(open(os.path.join(env['STORAGE_ROOT'], 'dns/custom.yaml'))) + with open(os.path.join(env['STORAGE_ROOT'], 'dns/custom.yaml'), 'r') as f: + custom_dns = rtyaml.load(f) if not isinstance(custom_dns, dict): raise ValueError() # caught below except: return [ ] @@ -992,6 +993,7 @@ def set_custom_dns_record(qname, rtype, value, action, env): def get_secondary_dns(custom_dns, mode=None): resolver = dns.resolver.get_default_resolver() resolver.timeout = 10 + resolver.lifetime = 10 values = [] for qname, rtype, value in custom_dns: @@ -1003,25 +1005,33 @@ def get_secondary_dns(custom_dns, mode=None): values.append(hostname) continue - # This is a hostname. Before including in zone xfr lines, - # resolve to an IP address. Otherwise just return the hostname. - # It may not resolve to IPv6, so don't throw an exception if it - # doesn't. - if not hostname.startswith("xfr:"): - if mode == "xfr": - response = dns.resolver.resolve(hostname+'.', "A", raise_on_no_answer=False) - values.extend(map(str, response)) - response = dns.resolver.resolve(hostname+'.', "AAAA", raise_on_no_answer=False) - values.extend(map(str, response)) - continue - values.append(hostname) + # If the entry starts with "xfr:" only include it in the zone transfer settings. + if hostname.startswith("xfr:"): + if mode != "xfr": continue + hostname = hostname[4:] - # This is a zone-xfer-only IP address. Do not return if - # we're querying for NS record hostnames. Only return if - # we're querying for zone xfer IP addresses - return the - # IP address. - elif mode == "xfr": - values.append(hostname[4:]) + # If is a hostname, before including in zone xfr lines, + # resolve to an IP address. + # It may not resolve to IPv6, so don't throw an exception if it + # doesn't. Skip the entry if there is a DNS error. + if mode == "xfr": + try: + ipaddress.ip_interface(hostname) # test if it's an IP address or CIDR notation + values.append(hostname) + except ValueError: + try: + response = dns.resolver.resolve(hostname+'.', "A", raise_on_no_answer=False) + values.extend(map(str, response)) + except dns.exception.DNSException: + pass + try: + response = dns.resolver.resolve(hostname+'.', "AAAA", raise_on_no_answer=False) + values.extend(map(str, response)) + except dns.exception.DNSException: + pass + + else: + values.append(hostname) return values @@ -1030,15 +1040,17 @@ def set_secondary_dns(hostnames, env): # Validate that all hostnames are valid and that all zone-xfer IP addresses are valid. resolver = dns.resolver.get_default_resolver() resolver.timeout = 5 + resolver.lifetime = 5 + for item in hostnames: if not item.startswith("xfr:"): # Resolve hostname. try: response = resolver.resolve(item, "A") - except (dns.resolver.NoNameservers, dns.resolver.NXDOMAIN, dns.resolver.NoAnswer): + except (dns.resolver.NoNameservers, dns.resolver.NXDOMAIN, dns.resolver.NoAnswer, dns.resolver.Timeout): try: response = resolver.resolve(item, "AAAA") - except (dns.resolver.NoNameservers, dns.resolver.NXDOMAIN, dns.resolver.NoAnswer): + except (dns.resolver.NoNameservers, dns.resolver.NXDOMAIN, dns.resolver.NoAnswer, dns.resolver.Timeout): raise ValueError("Could not resolve the IP address of %s." % item) else: # Validate IP address. @@ -1071,7 +1083,7 @@ def get_custom_dns_records(custom_dns, qname, rtype): def build_recommended_dns(env): ret = [] for (domain, zonefile, records) in build_zones(env): - # remove records that we don't dislay + # remove records that we don't display records = [r for r in records if r[3] is not False] # put Required at the top, then Recommended, then everythiing else diff --git a/management/email_administrator.py b/management/email_administrator.py index 8ed6e2a8..c87eda40 100755 --- a/management/email_administrator.py +++ b/management/email_administrator.py @@ -29,7 +29,7 @@ content = sys.stdin.read().strip() # If there's nothing coming in, just exit. if content == "": - sys.exit(0) + sys.exit(0) # create MIME message msg = MIMEMultipart('alternative') @@ -41,7 +41,7 @@ msg['From'] = "\"%s\" <%s>" % (env['PRIMARY_HOSTNAME'], admin_addr) msg['To'] = admin_addr msg['Subject'] = "[%s] %s" % (env['PRIMARY_HOSTNAME'], subject) -content_html = "
{}
".format(html.escape(content)) +content_html = '
{}
'.format(html.escape(content)) msg.attach(MIMEText(content, 'plain')) msg.attach(MIMEText(content_html, 'html')) diff --git a/management/mail_log.py b/management/mail_log.py index bdf757cc..1a963966 100755 --- a/management/mail_log.py +++ b/management/mail_log.py @@ -73,7 +73,8 @@ def scan_files(collector): continue elif fn[-3:] == '.gz': tmp_file = tempfile.NamedTemporaryFile() - shutil.copyfileobj(gzip.open(fn), tmp_file) + with gzip.open(fn, 'rb') as f: + shutil.copyfileobj(f, tmp_file) if VERBOSE: print("Processing file", fn, "...") diff --git a/management/ssl_certificates.py b/management/ssl_certificates.py index ab4f2dc8..19f0266a 100755 --- a/management/ssl_certificates.py +++ b/management/ssl_certificates.py @@ -535,7 +535,8 @@ def check_certificate(domain, ssl_certificate, ssl_private_key, warn_if_expiring # Second, check that the certificate matches the private key. if ssl_private_key is not None: try: - priv_key = load_pem(open(ssl_private_key, 'rb').read()) + with open(ssl_private_key, 'rb') as f: + priv_key = load_pem(f.read()) except ValueError as e: return ("The private key file %s is not a private key file: %s" % (ssl_private_key, str(e)), None) diff --git a/management/status_checks.py b/management/status_checks.py index ba7a682b..dd47c75e 100755 --- a/management/status_checks.py +++ b/management/status_checks.py @@ -94,6 +94,12 @@ def run_services_checks(env, output, pool): fatal = fatal or fatal2 output2.playback(output) + # Check fail2ban. + code, ret = shell('check_output', ["fail2ban-client", "status"], capture_stderr=True, trap=True) + if code != 0: + output.print_error("fail2ban is not running.") + all_running = False + if all_running: output.print_ok("All system services are running.") @@ -206,7 +212,8 @@ def check_ssh_password(env, output): # the configuration file. if not os.path.exists("/etc/ssh/sshd_config"): return - sshd = open("/etc/ssh/sshd_config").read() + with open("/etc/ssh/sshd_config", "r") as f: + sshd = f.read() if re.search("\nPasswordAuthentication\s+yes", sshd) \ or not re.search("\nPasswordAuthentication\s+no", sshd): output.print_error("""The SSH server on this machine permits password-based login. A more secure @@ -307,6 +314,8 @@ def run_network_checks(env, output): output.print_ok("IP address is not blacklisted by zen.spamhaus.org.") elif zen == "[timeout]": output.print_warning("Connection to zen.spamhaus.org timed out. We could not determine whether your server's IP address is blacklisted. Please try again later.") + elif zen == "[Not Set]": + output.print_warning("Could not connect to zen.spamhaus.org. We could not determine whether your server's IP address is blacklisted. Please try again later.") else: output.print_error("""The IP address of this machine %s is listed in the Spamhaus Block List (code %s), which may prevent recipients from receiving your email. See http://www.spamhaus.org/query/ip/%s.""" @@ -540,7 +549,7 @@ def check_dns_zone(domain, env, output, dns_zonefiles): for ns in custom_secondary_ns: # We must first resolve the nameserver to an IP address so we can query it. ns_ips = query_dns(ns, "A") - if not ns_ips: + if not ns_ips or ns_ips in {'[Not Set]', '[timeout]'}: output.print_error("Secondary nameserver %s is not valid (it doesn't resolve to an IP address)." % ns) continue # Choose the first IP if nameserver returns multiple @@ -591,7 +600,8 @@ def check_dnssec(domain, env, output, dns_zonefiles, is_checking_primary=False): # record that we suggest using is for the KSK (and that's how the DS records were generated). # We'll also give the nice name for the key algorithm. dnssec_keys = load_env_vars_from_file(os.path.join(env['STORAGE_ROOT'], 'dns/dnssec/%s.conf' % alg_name_map[ds_alg])) - dnsssec_pubkey = open(os.path.join(env['STORAGE_ROOT'], 'dns/dnssec/' + dnssec_keys['KSK'] + '.key')).read().split("\t")[3].split(" ")[3] + with open(os.path.join(env['STORAGE_ROOT'], 'dns/dnssec/' + dnssec_keys['KSK'] + '.key'), 'r') as f: + dnsssec_pubkey = f.read().split("\t")[3].split(" ")[3] expected_ds_records[ (ds_keytag, ds_alg, ds_digalg, ds_digest) ] = { "record": rr_ds, @@ -743,6 +753,8 @@ def check_mail_domain(domain, env, output): output.print_ok("Domain is not blacklisted by dbl.spamhaus.org.") elif dbl == "[timeout]": output.print_warning("Connection to dbl.spamhaus.org timed out. We could not determine whether the domain {} is blacklisted. Please try again later.".format(domain)) + elif dbl == "[Not Set]": + output.print_warning("Could not connect to dbl.spamhaus.org. We could not determine whether the domain {} is blacklisted. Please try again later.".format(domain)) else: output.print_error("""This domain is listed in the Spamhaus Domain Block List (code %s), which may prevent recipients from receiving your mail. @@ -787,12 +799,17 @@ def query_dns(qname, rtype, nxdomain='[Not Set]', at=None, as_list=False): # running unbound server), or if the 'at' argument is specified, use that host # as the nameserver. resolver = dns.resolver.get_default_resolver() - if at: + + # Make sure at is not a string that cannot be used as a nameserver + if at and at not in {'[Not set]', '[timeout]'}: resolver = dns.resolver.Resolver() resolver.nameservers = [at] # Set a timeout so that a non-responsive server doesn't hold us back. resolver.timeout = 5 + # The number of seconds to spend trying to get an answer to the question. If the + # lifetime expires a dns.exception.Timeout exception will be raised. + resolver.lifetime = 5 # Do the query. try: @@ -946,7 +963,8 @@ def run_and_output_changes(env, pool): # Load previously saved status checks. cache_fn = "/var/cache/mailinabox/status_checks.json" if os.path.exists(cache_fn): - prev = json.load(open(cache_fn)) + with open(cache_fn, 'r') as f: + prev = json.load(f) # Group the serial output into categories by the headings. def group_by_heading(lines): diff --git a/management/templates/aliases.html b/management/templates/aliases.html index c2a141f7..6c8b0376 100644 --- a/management/templates/aliases.html +++ b/management/templates/aliases.html @@ -7,7 +7,7 @@

Add a mail alias

-

Aliases are email forwarders. An alias can forward email to a mail user or to any email address.

+

Aliases are email forwarders. An alias can forward email to a mail user or to any email address.

To use an alias or any address besides your own login username in outbound mail, the sending user must be included as a permitted sender for the alias.

diff --git a/management/templates/custom-dns.html b/management/templates/custom-dns.html index c59624eb..7d3b6b37 100644 --- a/management/templates/custom-dns.html +++ b/management/templates/custom-dns.html @@ -77,7 +77,7 @@

Using a secondary nameserver

-

If your TLD requires you to have two separate nameservers, you can either set up external DNS and ignore the DNS server on this box entirely, or use the DNS server on this box but add a secondary (aka “slave”) nameserver.

+

If your TLD requires you to have two separate nameservers, you can either set up external DNS and ignore the DNS server on this box entirely, or use the DNS server on this box but add a secondary (aka “slave”) nameserver.

If you choose to use a secondary nameserver, you must find a secondary nameserver service provider. Your domain name registrar or virtual cloud provider may provide this service for you. Once you set up the secondary nameserver service, enter the hostname (not the IP address) of their secondary nameserver in the box below.

@@ -96,7 +96,7 @@

Multiple secondary servers can be separated with commas or spaces (i.e., ns2.hostingcompany.com ns3.hostingcompany.com). - To enable zone transfers to additional servers without listing them as secondary nameservers, add an IP address or subnet using xfr:10.20.30.40 or xfr:10.0.0.0/8. + To enable zone transfers to additional servers without listing them as secondary nameservers, prefix a hostname, IP address, or subnet with xfr:, e.g. xfr:10.20.30.40 or xfr:10.0.0.0/8.