From 7cda439c8055d36f12d79efb167a12d8f594be0c Mon Sep 17 00:00:00 2001 From: Steve Hay Date: Sat, 17 Sep 2022 07:57:12 -0400 Subject: [PATCH 1/7] Port boto to boto3 and fix asyncio issue in the management daemon (#2156) Co-authored-by: Steve Hay --- management/backup.py | 41 ++++++++++++------------------------- management/daemon.py | 6 ++++-- management/status_checks.py | 2 +- 3 files changed, 18 insertions(+), 31 deletions(-) diff --git a/management/backup.py b/management/backup.py index 2e88c8d0..012ccccb 100755 --- a/management/backup.py +++ b/management/backup.py @@ -446,25 +446,13 @@ def list_target_files(config): raise ValueError("Connection to rsync host failed: {}".format(reason)) elif target.scheme == "s3": - # match to a Region - import boto.s3 - from boto.exception import BotoServerError - custom_region = False - for region in boto.s3.regions(): - if region.endpoint == target.hostname: - break - else: - # If region is not found this is a custom region - custom_region = True - + import boto3.s3 + from botocore.exceptions import ClientError + + # separate bucket from path in target bucket = target.path[1:].split('/')[0] path = '/'.join(target.path[1:].split('/')[1:]) + '/' - # Create a custom region with custom endpoint - if custom_region: - from boto.s3.connection import S3Connection - region = boto.s3.S3RegionInfo(name=bucket, endpoint=target.hostname, connection_cls=S3Connection) - # If no prefix is specified, set the path to '', otherwise boto won't list the files if path == '/': path = '' @@ -474,18 +462,15 @@ def list_target_files(config): # connect to the region & bucket try: - conn = region.connect(aws_access_key_id=config["target_user"], aws_secret_access_key=config["target_pass"]) - bucket = conn.get_bucket(bucket) - except BotoServerError as e: - if e.status == 403: - raise ValueError("Invalid S3 access key or secret access key.") - elif e.status == 404: - raise ValueError("Invalid S3 bucket name.") - elif e.status == 301: - raise ValueError("Incorrect region for this bucket.") - raise ValueError(e.reason) - - return [(key.name[len(path):], key.size) for key in bucket.list(prefix=path)] + s3 = boto3.client('s3', \ + endpoint_url=f'https://{target.hostname}', \ + aws_access_key_id=config['target_user'], \ + aws_secret_access_key=config['target_pass']) + bucket_objects = s3.list_objects_v2(Bucket=bucket, Prefix=path)['Contents'] + backup_list = [(key['Key'][len(path):], key['Size']) for key in bucket_objects] + except ClientError as e: + raise ValueError(e) + return backup_list elif target.scheme == 'b2': from b2sdk.v1 import InMemoryAccountInfo, B2Api from b2sdk.v1.exception import NonExistentBucket diff --git a/management/daemon.py b/management/daemon.py index 98c6689c..2be32504 100755 --- a/management/daemon.py +++ b/management/daemon.py @@ -121,8 +121,10 @@ def index(): no_users_exist = (len(get_mail_users(env)) == 0) no_admins_exist = (len(get_admins(env)) == 0) - import boto.s3 - backup_s3_hosts = [(r.name, r.endpoint) for r in boto.s3.regions()] + import boto3.s3 + from urllib.parse import urlparse + backup_s3_hosts = [(r, f"s3.{r}.amazonaws.com") for r in boto3.session.Session().get_available_regions('s3')] + return render_template('index.html', hostname=env['PRIMARY_HOSTNAME'], diff --git a/management/status_checks.py b/management/status_checks.py index 12b4440d..0d555441 100755 --- a/management/status_checks.py +++ b/management/status_checks.py @@ -715,7 +715,7 @@ def check_mail_domain(domain, env, output): output.print_ok(good_news) # Check MTA-STS policy. - loop = asyncio.get_event_loop() + loop = asyncio.new_event_loop() sts_resolver = postfix_mta_sts_resolver.resolver.STSResolver(loop=loop) valid, policy = loop.run_until_complete(sts_resolver.resolve(domain)) if valid == postfix_mta_sts_resolver.resolver.STSFetchResult.VALID: From 3fd2e3efa9078593ff19a209fcd72743d4d91490 Mon Sep 17 00:00:00 2001 From: Steve Hay Date: Sat, 17 Sep 2022 08:03:16 -0400 Subject: [PATCH 2/7] Replace Flask built-in WSGI server with gunicorn (#2158) --- conf/mailinabox.service | 1 + management/auth.py | 16 ++-------------- management/wsgi.py | 7 +++++++ setup/management.sh | 10 ++++++++-- 4 files changed, 18 insertions(+), 16 deletions(-) create mode 100644 management/wsgi.py diff --git a/conf/mailinabox.service b/conf/mailinabox.service index b4cfa6cf..c1d98a03 100644 --- a/conf/mailinabox.service +++ b/conf/mailinabox.service @@ -4,6 +4,7 @@ After=multi-user.target [Service] Type=idle +IgnoreSIGPIPE=False ExecStart=/usr/local/lib/mailinabox/start [Install] diff --git a/management/auth.py b/management/auth.py index 0a88c457..c576d01c 100644 --- a/management/auth.py +++ b/management/auth.py @@ -22,20 +22,8 @@ class AuthService: def init_system_api_key(self): """Write an API key to a local file so local processes can use the API""" - def create_file_with_mode(path, mode): - # Based on answer by A-B-B: http://stackoverflow.com/a/15015748 - old_umask = os.umask(0) - try: - return os.fdopen(os.open(path, os.O_WRONLY | os.O_CREAT, mode), 'w') - finally: - os.umask(old_umask) - - self.key = secrets.token_hex(32) - - os.makedirs(os.path.dirname(self.key_path), exist_ok=True) - - with create_file_with_mode(self.key_path, 0o640) as key_file: - key_file.write(self.key + '\n') + with open(self.key_path, 'r') as file: + self.key = file.read() def authenticate(self, request, env, login_only=False, logout=False): """Test if the HTTP Authorization header's username matches the system key, a session key, diff --git a/management/wsgi.py b/management/wsgi.py new file mode 100644 index 00000000..86cf3af4 --- /dev/null +++ b/management/wsgi.py @@ -0,0 +1,7 @@ +from daemon import app +import auth, utils + +app.logger.addHandler(utils.create_syslog_handler()) + +if __name__ == "__main__": + app.run(port=10222) \ No newline at end of file diff --git a/setup/management.sh b/setup/management.sh index cfac5db9..cebed8d5 100755 --- a/setup/management.sh +++ b/setup/management.sh @@ -50,7 +50,7 @@ hide_output $venv/bin/pip install --upgrade pip # NOTE: email_validator is repeated in setup/questions.sh, so please keep the versions synced. hide_output $venv/bin/pip install --upgrade \ rtyaml "email_validator>=1.0.0" "exclusiveprocess" \ - flask dnspython python-dateutil expiringdict \ + flask dnspython python-dateutil expiringdict gunicorn \ qrcode[pil] pyotp \ "idna>=2.0.0" "cryptography==37.0.2" psutil postfix-mta-sts-resolver \ b2sdk boto3 @@ -90,6 +90,7 @@ rm -f /tmp/bootstrap.zip # Create an init script to start the management daemon and keep it # running after a reboot. +# Note: Authentication currently breaks with more than 1 gunicorn worker. cat > $inst_dir/start < /var/lib/mailinabox/api.key +chmod 640 /var/lib/mailinabox/api.key + source $venv/bin/activate -exec python $(pwd)/management/daemon.py +export PYTHONPATH=$(pwd)/management +exec gunicorn -b localhost:10222 -w 1 wsgi:app EOF chmod +x $inst_dir/start cp --remove-destination conf/mailinabox.service /lib/systemd/system/mailinabox.service # target was previously a symlink so remove it first From 58ded74181b750ae654ef8ca19f10527471b5e82 Mon Sep 17 00:00:00 2001 From: Joshua Tauberer Date: Mon, 12 Sep 2022 18:13:31 -0400 Subject: [PATCH 3/7] Restore the backup S3 host select box if an S3 target has been set Also remove unnecessary import added in 7cda439c. Was a mistake from edits during PR review. --- management/daemon.py | 1 - management/templates/system-backup.html | 1 + 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/management/daemon.py b/management/daemon.py index 2be32504..dc59c19b 100755 --- a/management/daemon.py +++ b/management/daemon.py @@ -122,7 +122,6 @@ def index(): no_admins_exist = (len(get_admins(env)) == 0) import boto3.s3 - from urllib.parse import urlparse backup_s3_hosts = [(r, f"s3.{r}.amazonaws.com") for r in boto3.session.Session().get_available_regions('s3')] diff --git a/management/templates/system-backup.html b/management/templates/system-backup.html index 3075b912..5450b6e5 100644 --- a/management/templates/system-backup.html +++ b/management/templates/system-backup.html @@ -269,6 +269,7 @@ function show_custom_backup() { $("#backup-target-type").val("s3"); var hostpath = r.target.substring(5).split('/'); var host = hostpath.shift(); + $("#backup-target-s3-host-select").val(host); $("#backup-target-s3-host").val(host); $("#backup-target-s3-path").val(hostpath.join('/')); } else if (r.target.substring(0, 5) == "b2://") { From 84da4e600013e0ce7ee67a8676f442362973ef6a Mon Sep 17 00:00:00 2001 From: Steve Hay Date: Mon, 5 Sep 2022 19:25:20 -0400 Subject: [PATCH 4/7] Update dovecot to use same DH parameters file as the other services Originally from #2157. --- setup/mail-dovecot.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/setup/mail-dovecot.sh b/setup/mail-dovecot.sh index 394ede8b..a4bb563b 100755 --- a/setup/mail-dovecot.sh +++ b/setup/mail-dovecot.sh @@ -87,7 +87,8 @@ tools/editconf.py /etc/dovecot/conf.d/10-ssl.conf \ "ssl_min_protocol=TLSv1.2" \ "ssl_cipher_list=ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:DHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES256-GCM-SHA384" \ "ssl_prefer_server_ciphers=no" \ - "ssl_dh_parameters_length=2048" + "ssl_dh_parameters_length=2048" \ + "ssl_dh=<$STORAGE_ROOT/ssl/dh2048.pem" # Disable in-the-clear IMAP/POP because there is no reason for a user to transmit # login credentials outside of an encrypted connection. Only the over-TLS versions From 30631b0fc535d2d83944e2175379cd41d015e397 Mon Sep 17 00:00:00 2001 From: downtownallday Date: Sat, 25 Jun 2022 12:35:03 -0400 Subject: [PATCH 5/7] Fix undefined variable 'val' in tools/editconf.py (#2137) Merges #2137. --- tools/editconf.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tools/editconf.py b/tools/editconf.py index e80742e4..dc184966 100755 --- a/tools/editconf.py +++ b/tools/editconf.py @@ -136,9 +136,10 @@ while len(input_lines) > 0: # Put any settings we didn't see at the end of the file, # except settings being cleared. for i in range(len(settings)): - if (i not in found) and not (not val and erase_setting): + if i not in found: name, val = settings[i].split("=", 1) - buf += name + delimiter + val + "\n" + if not (not val and erase_setting): + buf += name + delimiter + val + "\n" if not testing: # Write out the new file. From 56074ae03592e9a4b590409b2e756aa12998ef86 Mon Sep 17 00:00:00 2001 From: downtownallday Date: Tue, 28 Jun 2022 07:46:24 -0400 Subject: [PATCH 6/7] Tighten roundcube session config (#2138) Merges #2138. --- CHANGELOG.md | 4 ++++ setup/webmail.sh | 4 ++++ 2 files changed, 8 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1234a898..4188f5cc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -20,6 +20,10 @@ No features of Mail-in-a-Box have changed in this release, but with the newer ve * fail2ban is upgraded to 0.11.2. * nginx is upgraded to 1.18. +Also: + +* Roundcube's login session cookie was tightened. Existing sessions may require a manual logout. + Version 57a (June 19, 2022) --------------------------- diff --git a/setup/webmail.sh b/setup/webmail.sh index e064a201..131f7aa5 100755 --- a/setup/webmail.sh +++ b/setup/webmail.sh @@ -141,6 +141,10 @@ cat > $RCM_CONFIG < EOF From d584a41e604e1a1393f9ceb0b118c4ce97c1cd33 Mon Sep 17 00:00:00 2001 From: kiekerjan Date: Sat, 17 Sep 2022 15:20:20 +0200 Subject: [PATCH 7/7] Update Roundcube to 1.6.0 (#2153) --- CHANGELOG.md | 4 +++- setup/webmail.sh | 19 +++++++++---------- 2 files changed, 12 insertions(+), 11 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4188f5cc..72a2608a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,10 +15,12 @@ LINK TBD No features of Mail-in-a-Box have changed in this release, but with the newer version of Ubuntu the following software packages we use are updated: * dovecot is upgraded to 2.3.16, postfix to 3.6.4, opendmark to 1.4 (which adds ARC-Authentication-Results headers), and spampd to 2.53 (alleviating a mail delivery rate limiting bug). -* Nextcloud is upgraded to 23.0.4 with PHP updated from 7.2 to 8.0. +* Nextcloud is upgraded to 23.0.4. +* Roundcube is upgraded to 1.6.0. * certbot is upgraded to 1.21 (via the Ubuntu repository instead of a PPA). * fail2ban is upgraded to 0.11.2. * nginx is upgraded to 1.18. +* PHP is upgraded from 7.2 to 8.0. Also: diff --git a/setup/webmail.sh b/setup/webmail.sh index 131f7aa5..791bda57 100755 --- a/setup/webmail.sh +++ b/setup/webmail.sh @@ -35,12 +35,12 @@ apt_install \ # https://github.com/mstilkerich/rcmcarddav/releases # The easiest way to get the package hashes is to run this script and get the hash from # the error message. -VERSION=1.5.2 -HASH=208ce4ca0be423cc0f7070ff59bd03588b4439bf -PERSISTENT_LOGIN_VERSION=59ca1b0d3a02cff5fa621c1ad581d15f9d642fe8 +VERSION=1.6.0 +HASH=fd84b4fac74419bb73e7a3bcae1978d5589c52de +PERSISTENT_LOGIN_VERSION=bde7b6840c7d91de627ea14e81cf4133cbb3c07a # version 5.2 HTML5_NOTIFIER_VERSION=68d9ca194212e15b3c7225eb6085dbcf02fd13d7 # version 0.6.4+ -CARDDAV_VERSION=4.3.0 -CARDDAV_HASH=4ad7df8843951062878b1375f77c614f68bc5c61 +CARDDAV_VERSION=4.4.3 +CARDDAV_HASH=74f8ba7aee33e78beb9de07f7f44b81f6071b644 UPDATE_KEY=$VERSION:$PERSISTENT_LOGIN_VERSION:$HTML5_NOTIFIER_VERSION:$CARDDAV_VERSION @@ -83,7 +83,7 @@ if [ $needs_update == 1 ]; then # download and verify the full release of the carddav plugin wget_verify \ - https://github.com/blind-coder/rcmcarddav/releases/download/v${CARDDAV_VERSION}/carddav-v${CARDDAV_VERSION}.tar.gz \ + https://github.com/mstilkerich/rcmcarddav/releases/download/v${CARDDAV_VERSION}/carddav-v${CARDDAV_VERSION}.tar.gz \ $CARDDAV_HASH \ /tmp/carddav.tar.gz @@ -115,8 +115,7 @@ cat > $RCM_CONFIG < array( 'verify_peer' => false, @@ -124,7 +123,7 @@ cat > $RCM_CONFIG < array( 'verify_peer' => false, @@ -158,7 +157,7 @@ cat > ${RCM_PLUGIN_DIR}/carddav/config.inc.php < 'ownCloud', 'username' => '%u', // login username 'password' => '%p', // login password - 'url' => 'https://${PRIMARY_HOSTNAME}/cloud/remote.php/carddav/addressbooks/%u/contacts', + 'url' => 'https://${PRIMARY_HOSTNAME}/cloud/remote.php/dav/addressbooks/users/%u/contacts/', 'active' => true, 'readonly' => false, 'refresh_time' => '02:00:00',