From eeada2b9b56ffd74a6c09d5300e1115e19d4f8ec Mon Sep 17 00:00:00 2001 From: "github@kiekerjan.isdronken.nl" Date: Tue, 19 Oct 2021 23:07:02 +0200 Subject: [PATCH] merge changes from V55 upstream --- CHANGELOG.md | 26 ++++ README.md | 4 +- api/mailinabox.yml | 34 ++++- management/auth.py | 196 +++++++++++++------------ management/daemon.py | 99 ++++++++----- management/dns_update.py | 2 +- management/mailconfig.py | 78 +++++----- management/status_checks.py | 2 +- management/templates/aliases.html | 4 +- management/templates/external-dns.html | 2 +- management/templates/index.html | 109 ++++++++++---- management/templates/login.html | 34 ++++- management/templates/munin.html | 20 +++ management/templates/sync-guide.html | 6 +- management/templates/users.html | 6 +- management/templates/welcome.html | 16 ++ setup/bootstrap.sh | 4 +- setup/dns.sh | 8 +- setup/mail-users.sh | 11 +- setup/management.sh | 4 +- setup/migrate.py | 5 + setup/webmail.sh | 5 +- tests/fail2ban.py | 2 +- 23 files changed, 439 insertions(+), 238 deletions(-) create mode 100644 management/templates/munin.html create mode 100644 management/templates/welcome.html diff --git a/CHANGELOG.md b/CHANGELOG.md index 9338b052..67fcdd94 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,32 @@ CHANGELOG ========= +Version 55 (October 18, 2021) +----------------------------- + +Mail: + +* "SMTPUTF8" is now disabled in Postfix. Because Dovecot still does not support SMTPUTF8, incoming mail to internationalized addresses was bouncing. This fixes incoming mail to internationalized domains (which was probably working prior to v0.40), but it will prevent sending outbound mail to addresses with internationalized local-parts. +* Upgraded to Roundcube 1.5. + +Control panel: + +* The control panel menus are now hidden before login, but now non-admins can log in to access the mail and contacts/calendar instruction pages. +* The login form now disables browser autocomplete in the two-factor authentication code field. +* After logging in, the default page is now a fast-loading welcome page rather than the slow-loading system status checks page. +* The backup retention period option now displays for B2 backup targets. +* The DNSSEC DS record recommendations are cleaned up and now recommend changing records that use SHA1. +* The Munin monitoring pages no longer require a separate HTTP basic authentication login and can be used if two-factor authentication is turned on. +* Control panel logins are now tied to a session backend that allows true logouts (rather than an encrypted cookie). +* Failed logins no longer directly reveal whether the email address corresponds to a user account. +* Browser dark mode now inverts the color scheme. + +Other: + +* Fail2ban's IPv6 support is enabled. +* The mail log tool now doesn't crash if there are email addresess in log messages with invalid UTF-8 characters. +* Additional nsd.conf files can be placed in /etc/nsd.conf.d. + v0.54 (June 20, 2021) --------------------- diff --git a/README.md b/README.md index deebab71..bc0f7a27 100644 --- a/README.md +++ b/README.md @@ -98,6 +98,8 @@ It also includes system management tools: * A control panel for adding/removing mail users, aliases, custom DNS records, configuring backups, etc. * An API for all of the actions on the control panel +Internationalized domain names are supported and configured easily (but SMTPUTF8 is not supported, unfortunately). + It also supports static website hosting since the box is serving HTTPS anyway. (To serve a website for your domains elsewhere, just add a custom DNS "A" record in you Mail-in-a-Box's control panel to point domains to another server.) For more information on how Mail-in-a-Box handles your privacy, see the [security details page](security.md). @@ -114,7 +116,7 @@ Clone this repository and checkout the tag corresponding to the most recent rele $ git clone https://github.com/mail-in-a-box/mailinabox $ cd mailinabox - $ git checkout v0.54 + $ git checkout v55 Begin the installation. diff --git a/api/mailinabox.yml b/api/mailinabox.yml index 14cf54de..bd4b203b 100644 --- a/api/mailinabox.yml +++ b/api/mailinabox.yml @@ -54,24 +54,24 @@ tags: System operations, which include system status checks, new version checks and reboot status. paths: - /me: - get: + /login: + post: tags: - User - summary: Get user information + summary: Exchange a username and password for a session API key. description: | - Returns user information. Used for user authentication. + Returns user information and a session API key. Authenticate a user by supplying the auth token as a base64 encoded string in format `email:password` using basic authentication headers. If successful, a long-lived `api_key` is returned which can be used for subsequent - requests to the API. - operationId: getMe + requests to the API in place of the password. + operationId: login x-codeSamples: - lang: curl source: | - curl -X GET "https://{host}/admin/me" \ + curl -X GET "https://{host}/admin/login" \ -u ":" responses: 200: @@ -92,6 +92,24 @@ paths: privileges: - admin status: ok + /logout: + post: + tags: + - User + summary: Invalidates a session API key. + description: | + Invalidates a session API key so that it cannot be used after this API call. + operationId: logout + x-codeSamples: + - lang: curl + source: | + curl -X GET "https://{host}/admin/logout" \ + -u ":" + responses: + 200: + description: Successful operation + content: + application/json: /system/status: post: tags: @@ -1803,7 +1821,7 @@ components: The `access-token` is comprised of the Base64 encoding of `username:password`. The `username` is the mail user's email address, and `password` can either be the mail user's - password, or the `api_key` returned from the `getMe` operation. + password, or the `api_key` returned from the `login` operation. When using `curl`, you can supply user credentials using the `-u` or `--user` parameter. requestBodies: diff --git a/management/auth.py b/management/auth.py index fd143c76..0a88c457 100644 --- a/management/auth.py +++ b/management/auth.py @@ -1,6 +1,7 @@ -import base64, os, os.path, hmac, json +import base64, os, os.path, hmac, json, secrets +from datetime import timedelta -from flask import make_response +from expiringdict import ExpiringDict import utils from mailconfig import get_mail_password, get_mail_user_privileges @@ -9,25 +10,18 @@ from mfa import get_hash_mfa_state, validate_auth_mfa DEFAULT_KEY_PATH = '/var/lib/mailinabox/api.key' DEFAULT_AUTH_REALM = 'Mail-in-a-Box Management Server' -class KeyAuthService: - """Generate an API key for authenticating clients - - Clients must read the key from the key file and send the key with all HTTP - requests. The key is passed as the username field in the standard HTTP - Basic Auth header. - """ +class AuthService: def __init__(self): self.auth_realm = DEFAULT_AUTH_REALM - self.key = self._generate_key() self.key_path = DEFAULT_KEY_PATH + self.max_session_duration = timedelta(days=2) - def write_key(self): - """Write key to file so authorized clients can get the key + self.init_system_api_key() + self.sessions = ExpiringDict(max_len=64, max_age_seconds=self.max_session_duration.total_seconds()) + + def init_system_api_key(self): + """Write an API key to a local file so local processes can use the API""" - The key file is created with mode 0640 so that additional users can be - authorized to access the API by granting group/ACL read permissions on - the key file. - """ def create_file_with_mode(path, mode): # Based on answer by A-B-B: http://stackoverflow.com/a/15015748 old_umask = os.umask(0) @@ -36,123 +30,137 @@ class KeyAuthService: finally: os.umask(old_umask) + self.key = secrets.token_hex(32) + os.makedirs(os.path.dirname(self.key_path), exist_ok=True) with create_file_with_mode(self.key_path, 0o640) as key_file: key_file.write(self.key + '\n') - def authenticate(self, request, env): - """Test if the client key passed in HTTP Authorization header matches the service key - or if the or username/password passed in the header matches an administrator user. + def authenticate(self, request, env, login_only=False, logout=False): + """Test if the HTTP Authorization header's username matches the system key, a session key, + or if the username/password passed in the header matches a local user. Returns a tuple of the user's email address and list of user privileges (e.g. ('my@email', []) or ('my@email', ['admin']); raises a ValueError on login failure. - If the user used an API key, the user's email is returned as None.""" + If the user used the system API key, the user's email is returned as None since + this key is not associated with a user.""" - def decode(s): - return base64.b64decode(s.encode('ascii')).decode('ascii') - - def parse_basic_auth(header): + def parse_http_authorization_basic(header): + def decode(s): + return base64.b64decode(s.encode('ascii')).decode('ascii') if " " not in header: return None, None scheme, credentials = header.split(maxsplit=1) if scheme != 'Basic': return None, None - credentials = decode(credentials) if ":" not in credentials: return None, None username, password = credentials.split(':', maxsplit=1) return username, password - header = request.headers.get('Authorization') - if not header: - raise ValueError("No authorization header provided.") - - username, password = parse_basic_auth(header) - + username, password = parse_http_authorization_basic(request.headers.get('Authorization', '')) if username in (None, ""): raise ValueError("Authorization header invalid.") - elif username == self.key: - # The user passed the master API key which grants administrative privs. + + if username.strip() == "" and password.strip() == "": + raise ValueError("No email address, password, session key, or API key provided.") + + # If user passed the system API key, grant administrative privs. This key + # is not associated with a user. + if username == self.key and not login_only: return (None, ["admin"]) + + # If the password corresponds with a session token for the user, grant access for that user. + if self.get_session(username, password, "login", env) and not login_only: + sessionid = password + session = self.sessions[sessionid] + if logout: + # Clear the session. + del self.sessions[sessionid] + else: + # Re-up the session so that it does not expire. + self.sessions[sessionid] = session + + # If no password was given, but a username was given, we're missing some information. + elif password.strip() == "": + raise ValueError("Enter a password.") + else: - # The user is trying to log in with a username and either a password - # (and possibly a MFA token) or a user-specific API key. - return (username, self.check_user_auth(username, password, request, env)) + # The user is trying to log in with a username and a password + # (and possibly a MFA token). On failure, an exception is raised. + self.check_user_auth(username, password, request, env) + + # Get privileges for authorization. This call should never fail because by this + # point we know the email address is a valid user --- unless the user has been + # deleted after the session was granted. On error the call will return a tuple + # of an error message and an HTTP status code. + privs = get_mail_user_privileges(username, env) + if isinstance(privs, tuple): raise ValueError(privs[0]) + + # Return the authorization information. + return (username, privs) def check_user_auth(self, email, pw, request, env): # Validate a user's login email address and password. If MFA is enabled, # check the MFA token in the X-Auth-Token header. # - # On success returns a list of privileges (e.g. [] or ['admin']). On login - # failure, raises a ValueError with a login error message. + # On login failure, raises a ValueError with a login error message. On + # success, nothing is returned. - # Sanity check. - if email == "" or pw == "": - raise ValueError("Enter an email address and password.") - - # The password might be a user-specific API key. create_user_key raises - # a ValueError if the user does not exist. - if hmac.compare_digest(self.create_user_key(email, env), pw): - # OK. - pass - else: + # Authenticate. + try: # Get the hashed password of the user. Raise a ValueError if the - # email address does not correspond to a user. + # email address does not correspond to a user. But wrap it in the + # same exception as if a password fails so we don't easily reveal + # if an email address is valid. pw_hash = get_mail_password(email, env) - # Authenticate. - try: - # Use 'doveadm pw' to check credentials. doveadm will return - # a non-zero exit status if the credentials are no good, - # and check_call will raise an exception in that case. - utils.shell('check_call', [ - "/usr/bin/doveadm", "pw", - "-p", pw, - "-t", pw_hash, - ]) - except: - # Login failed. - raise ValueError("Invalid password.") + # Use 'doveadm pw' to check credentials. doveadm will return + # a non-zero exit status if the credentials are no good, + # and check_call will raise an exception in that case. + utils.shell('check_call', [ + "/usr/bin/doveadm", "pw", + "-p", pw, + "-t", pw_hash, + ]) + except: + # Login failed. + raise ValueError("Incorrect email address or password.") - # If MFA is enabled, check that MFA passes. - status, hints = validate_auth_mfa(email, request, env) - if not status: - # Login valid. Hints may have more info. - raise ValueError(",".join(hints)) + # If MFA is enabled, check that MFA passes. + status, hints = validate_auth_mfa(email, request, env) + if not status: + # Login valid. Hints may have more info. + raise ValueError(",".join(hints)) - # Get privileges for authorization. This call should never fail because by this - # point we know the email address is a valid user. But on error the call will - # return a tuple of an error message and an HTTP status code. - privs = get_mail_user_privileges(email, env) - if isinstance(privs, tuple): raise ValueError(privs[0]) - - # Return a list of privileges. - return privs - - def create_user_key(self, email, env): - # Create a user API key, which is a shared secret that we can re-generate from - # static information in our database. The shared secret contains the user's - # email address, current hashed password, and current MFA state, so that the - # key becomes invalid if any of that information changes. - # - # Use an HMAC to generate the API key using our master API key as a key, - # which also means that the API key becomes invalid when our master API key - # changes --- i.e. when this process is restarted. - # - # Raises ValueError via get_mail_password if the user doesn't exist. - - # Construct the HMAC message from the user's email address and current password. - msg = b"AUTH:" + email.encode("utf8") + b" " + get_mail_password(email, env).encode("utf8") + def create_user_password_state_token(self, email, env): + # Create a token that changes if the user's password or MFA options change + # so that sessions become invalid if any of that information changes. + msg = get_mail_password(email, env).encode("utf8") # Add to the message the current MFA state, which is a list of MFA information. # Turn it into a string stably. msg += b" " + json.dumps(get_hash_mfa_state(email, env), sort_keys=True).encode("utf8") - # Make the HMAC. + # Make a HMAC using the system API key as a hash key. hash_key = self.key.encode('ascii') return hmac.new(hash_key, msg, digestmod="sha256").hexdigest() - def _generate_key(self): - raw_key = os.urandom(32) - return base64.b64encode(raw_key).decode('ascii') + def create_session_key(self, username, env, type=None): + # Create a new session. + token = secrets.token_hex(32) + self.sessions[token] = { + "email": username, + "password_token": self.create_user_password_state_token(username, env), + "type": type, + } + return token + + def get_session(self, user_email, session_key, session_type, env): + if session_key not in self.sessions: return None + session = self.sessions[session_key] + if session_type == "login" and session["email"] != user_email: return None + if session["type"] != session_type: return None + if session["password_token"] != self.create_user_password_state_token(session["email"], env): return None + return session diff --git a/management/daemon.py b/management/daemon.py index 31aaf56c..e8e679e4 100755 --- a/management/daemon.py +++ b/management/daemon.py @@ -1,5 +1,8 @@ #!/usr/local/lib/mailinabox/env/bin/python3 # +# The API can be accessed on the command line, e.g. use `curl` like so: +# curl --user $(') @authorized_personnel_only -def munin(filename=""): - # Checks administrative access (@authorized_personnel_only) and then just proxies - # the request to static files. +def munin_start(): + # Munin pages, static images, and dynamically generated images are served + # outside of the AJAX API. We'll start with a 'start' API that sets a cookie + # that subsequent requests will read for authorization. (We don't use cookies + # for the API to avoid CSRF vulnerabilities.) + response = make_response("OK") + response.set_cookie("session", auth_service.create_session_key(request.user_email, env, type='cookie'), + max_age=60*30, secure=True, httponly=True, samesite="Strict") # 30 minute duration + return response + +def check_request_cookie_for_admin_access(): + session = auth_service.get_session(None, request.cookies.get("session", ""), "cookie", env) + if not session: return False + privs = get_mail_user_privileges(session["email"], env) + if not isinstance(privs, list): return False + if "admin" not in privs: return False + return True + +def authorized_personnel_only_via_cookie(f): + @wraps(f) + def g(*args, **kwargs): + if not check_request_cookie_for_admin_access(): + return Response("Unauthorized", status=403, mimetype='text/plain', headers={}) + return f(*args, **kwargs) + return g + +@app.route('/munin/') +@authorized_personnel_only_via_cookie +def munin_static_file(filename=""): + # Proxy the request to static files. if filename == "": filename = "index.html" return send_from_directory("/var/cache/munin/www", filename) @app.route('/munin/cgi-graph/') -@authorized_personnel_only +@authorized_personnel_only_via_cookie def munin_cgi(filename): """ Relay munin cgi dynazoom requests /usr/lib/munin/cgi/munin-cgi-graph is a perl cgi script in the munin package @@ -727,30 +766,10 @@ if __name__ == '__main__': # Turn on Flask debugging. app.debug = True - # Use a stable-ish master API key so that login sessions don't restart on each run. - # Use /etc/machine-id to seed the key with a stable secret, but add something - # and hash it to prevent possibly exposing the machine id, using the time so that - # the key is not valid indefinitely. - import hashlib - with open("/etc/machine-id") as f: - api_key = f.read() - api_key += "|" + str(int(time.time() / (60*60*2))) - hasher = hashlib.sha1() - hasher.update(api_key.encode("ascii")) - auth_service.key = hasher.hexdigest() - - if "APIKEY" in os.environ: auth_service.key = os.environ["APIKEY"] - if not app.debug: app.logger.addHandler(utils.create_syslog_handler()) - # For testing on the command line, you can use `curl` like so: - # curl --user $( %s)\n" % (address, administrator)) - - for address in required_aliases: - ensure_admin_alias_exists(address) - - # Remove auto-generated postmaster/admin on domains we no - # longer have any other email addresses for. - for address, forwards_to, *_ in existing_alias_records: + # Remove auto-generated postmaster/admin/abuse alises from the main aliases table. + # They are now stored in the auto_aliases table. + for address, forwards_to, permitted_senders, auto in get_mail_aliases(env): user, domain = address.split("@") if user in ("postmaster", "admin", "abuse") \ and address not in required_aliases \ - and forwards_to == get_system_administrator(env): + and forwards_to == get_system_administrator(env) \ + and not auto: remove_mail_alias(address, env, do_kick=False) results.append("removed alias %s (was to %s; domain no longer used for email)\n" % (address, forwards_to)) diff --git a/management/status_checks.py b/management/status_checks.py index 33641ed2..90b4d175 100755 --- a/management/status_checks.py +++ b/management/status_checks.py @@ -663,7 +663,7 @@ def check_dnssec(domain, env, output, dns_zonefiles, is_checking_primary=False): output.print_line("Option " + str(i+1) + ":") output.print_line("----------") output.print_line("Key Tag: " + ds_suggestion['keytag']) - output.print_line("Key Flags: KSK") + output.print_line("Key Flags: KSK (256)") output.print_line("Algorithm: %s / %s" % (ds_suggestion['alg'], ds_suggestion['alg_name'])) output.print_line("Digest Type: %s / %s" % (ds_suggestion['digalg'], ds_suggestion['digalg_name'])) output.print_line("Digest: " + ds_suggestion['digest']) diff --git a/management/templates/aliases.html b/management/templates/aliases.html index 3d37c4fd..c2a141f7 100644 --- a/management/templates/aliases.html +++ b/management/templates/aliases.html @@ -1,6 +1,6 @@

Aliases

@@ -163,7 +163,7 @@ function show_aliases() { var n = $("#alias-template").clone(); n.attr('id', ''); - if (alias.required) n.addClass('alias-required'); + if (alias.auto) n.addClass('alias-auto'); n.attr('data-address', alias.address_display); // this is decoded from IDNA, but will get re-coded to IDNA on the backend n.find('td.address').text(alias.address_display) for (var j = 0; j < alias.forwards_to.length; j++) diff --git a/management/templates/external-dns.html b/management/templates/external-dns.html index 0634ec82..4532c3f5 100644 --- a/management/templates/external-dns.html +++ b/management/templates/external-dns.html @@ -38,7 +38,7 @@ diff --git a/management/templates/index.html b/management/templates/index.html index 03e01687..f9c87f2c 100644 --- a/management/templates/index.html +++ b/management/templates/index.html @@ -62,6 +62,37 @@ ol li { margin-bottom: 1em; } + + .if-logged-in { display: none; } + .if-logged-in-admin { display: none; } + + /* The below only gets used if it is supported */ + @media (prefers-color-scheme: dark) { + /* Invert invert lightness but not hue */ + html { + filter: invert(100%) hue-rotate(180deg); + } + + /* Set explicit background color (necessary for Firefox) */ + html { + background-color: #111; + } + + /* Override Boostrap theme here to give more contrast. The black turns to white by the filter. */ + .form-control { + color: black !important; + } + + /* Revert the invert for the navbar */ + button, div.navbar { + filter: invert(100%) hue-rotate(180deg); + } + + /* Revert the revert for the dropdowns */ + ul.dropdown-menu { + filter: invert(100%) hue-rotate(180deg); + } + } @@ -83,7 +114,7 @@
-
+
+ {% include "welcome.html" %} +
+
{% include "system-status.html" %}
@@ -166,6 +202,10 @@ {% include "ssl.html" %}
+
+ {% include "munin.html" %} +
+