From 32d6728dc9bb9e0c585cab5b880f0cdb788ada65 Mon Sep 17 00:00:00 2001 From: Joshua Tauberer Date: Wed, 1 Feb 2017 09:14:01 -0500 Subject: [PATCH 01/11] fix pip breaking due to setuptools/pip/cryptography problem pip<6.1 + setuptools>=34 have a problem with packages that try to update setuptools during installation, like cryptography. See https://github.com/pypa/pip/issues/4253. The Ubuntu 14.04 package versions are pip 1.5.4 and setuptools 3.3. When we install cryptography under those versions, it tries to update setuptools to version 34, which became available about 10 days ago, and then pip gets permanently broken with errors like "ImportError: No module named 'packaging'". The easiest work-around on systems that aren't already broken is to upgrade pip and setuptools individually before we install any package that tries to update setuptools. Also try to detect a broken system and forcibly remove setuptools first before trying to install/upgrade pip. fixes #1080, fixes #1081, fixes #1086 see #1083 see https://discourse.mailinabox.email/t/error-with-pip-and-python/1880 see https://discourse.mailinabox.email/t/error-installing-mib/1875 --- setup/management.sh | 30 +++++++++++++++++++++++++++++- 1 file changed, 29 insertions(+), 1 deletion(-) diff --git a/setup/management.sh b/setup/management.sh index cfc6aad4..02909dfe 100755 --- a/setup/management.sh +++ b/setup/management.sh @@ -4,7 +4,10 @@ source setup/functions.sh echo "Installing Mail-in-a-Box system management daemon..." -# Install packages. +# DEPENDENCIES + +# Install Python packages that are available from the Ubuntu +# apt repository: # flask, yaml, dnspython, and dateutil are all for our Python 3 management daemon itself. # duplicity does backups. python-pip is so we can 'pip install boto' for Python 2, for duplicity, so it can do backups to AWS S3. apt_install python3-flask links duplicity libyaml-dev python3-dnspython python3-dateutil python-pip @@ -12,6 +15,29 @@ apt_install python3-flask links duplicity libyaml-dev python3-dnspython python3- # These are required to pip install cryptography. apt_install build-essential libssl-dev libffi-dev python3-dev +# pip<6.1 + setuptools>=34 have a problem with packages that +# try to update setuptools during installation, like cryptography. +# See https://github.com/pypa/pip/issues/4253. The Ubuntu 14.04 +# package versions are pip 1.5.4 and setuptools 3.3. When we +# install cryptography under those versions, it tries to update +# setuptools to version 34, which now creates the conflict, and +# then pip gets permanently broken with errors like +# "ImportError: No module named 'packaging'". +# +# Let's test for the error: +if ! python3 -c "from pkg_resources import load_entry_point" 2&> /dev/null; then + # This system seems to be broken already. + echo "Fixing broken pip and setuptools..." + rm -rf /usr/local/lib/python3.4/dist-packages/{pkg_resources,setuptools}* + apt-get install --reinstall python3-setuptools python3-pip python3-pkg-resources +fi +# +# The easiest work-around on systems that aren't already broken is +# to upgrade pip (to >=9.0.1) and setuptools (to >=34.1) individually +# before we install any package that tries to update setuptools. +hide_output pip3 install --upgrade pip +hide_output pip3 install --upgrade setuptools + # Install other Python 3 packages used by the management daemon. # The first line is the packages that Josh maintains himself! # NOTE: email_validator is repeated in setup/questions.sh, so please keep the versions synced. @@ -24,6 +50,8 @@ hide_output pip3 install --upgrade \ # S3 api used in some regions, which breaks backups to those regions. See #627, #653. hide_output pip install --upgrade boto +# CONFIGURATION + # Create a backup directory and a random key for encrypting backups. mkdir -p $STORAGE_ROOT/backup if [ ! -f $STORAGE_ROOT/backup/secret_key.txt ]; then From 2e00530944e3bb7e7dfddb68d523d28e5408a80a Mon Sep 17 00:00:00 2001 From: Joshua Tauberer Date: Wed, 1 Feb 2017 11:00:54 -0500 Subject: [PATCH 02/11] upgrade acme package --- setup/management.sh | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/setup/management.sh b/setup/management.sh index 02909dfe..902e64b6 100755 --- a/setup/management.sh +++ b/setup/management.sh @@ -41,9 +41,12 @@ hide_output pip3 install --upgrade setuptools # Install other Python 3 packages used by the management daemon. # The first line is the packages that Josh maintains himself! # NOTE: email_validator is repeated in setup/questions.sh, so please keep the versions synced. +# Force acme to be updated because it seems to need it after the +# pip/setuptools breakage (see above) and the ACME protocol may +# have changed (I got an error on one of my systems). hide_output pip3 install --upgrade \ rtyaml "email_validator>=1.0.0" "free_tls_certificates>=0.1.3" \ - "idna>=2.0.0" "cryptography>=1.0.2" boto psutil + "idna>=2.0.0" "cryptography>=1.0.2" acme boto psutil # duplicity uses python 2 so we need to get the python 2 package of boto to have backups to S3. # boto from the Ubuntu package manager is too out-of-date -- it doesn't support the newer From 3c05fc94ffd24f810e218fac1b522c3bdfc6d69d Mon Sep 17 00:00:00 2001 From: Joshua Tauberer Date: Wed, 1 Feb 2017 10:07:15 -0500 Subject: [PATCH 03/11] v0.21c --- CHANGELOG.md | 7 +++++++ README.md | 4 ++-- setup/bootstrap.sh | 2 +- 3 files changed, 10 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5901e3f0..7bb1c6c7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,13 @@ CHANGELOG ========= +v0.21c (February 1, 2017) +------------------------- + +Installations and upgrades started failing about 10 days ago with the error "ImportError: No module named 'packaging'" after an upstream package (Python's setuptools) was updated by its maintainers. The updated package conflicted with Ubuntu 14.04's version of another package (Python's pip). This update upgrades both packages to remove the conflict. + +If you already encountered the error during installation or upgrade of Mail-in-a-Box, this update may not correct the problem on your existing system. See https://discourse.mailinabox.email/t/v0-21c-release-fixes-python-package-installation-issue/1881 for help if the problem persists after upgrading to this version of Mail-in-a-Box. + v0.21b (December 4, 2016) ------------------------- diff --git a/README.md b/README.md index 173a159f..81d8952d 100644 --- a/README.md +++ b/README.md @@ -59,7 +59,7 @@ by me: $ curl -s https://keybase.io/joshdata/key.asc | gpg --import gpg: key C10BDD81: public key "Joshua Tauberer " imported - $ git verify-tag v0.21b + $ git verify-tag v0.21c gpg: Signature made ..... using RSA key ID C10BDD81 gpg: Good signature from "Joshua Tauberer " gpg: WARNING: This key is not certified with a trusted signature! @@ -72,7 +72,7 @@ and on my [personal homepage](https://razor.occams.info/). (Of course, if this r Checkout the tag corresponding to the most recent release: - $ git checkout v0.21b + $ git checkout v0.21c Begin the installation. diff --git a/setup/bootstrap.sh b/setup/bootstrap.sh index 51aa7af0..25d59f53 100644 --- a/setup/bootstrap.sh +++ b/setup/bootstrap.sh @@ -7,7 +7,7 @@ ######################################################### if [ -z "$TAG" ]; then - TAG=v0.21b + TAG=v0.21c fi # Are we running as root? From f2ff14100e851af3f4743b8c8663e19707dbfaf7 Mon Sep 17 00:00:00 2001 From: Norman S Date: Fri, 10 Feb 2017 15:43:11 +0100 Subject: [PATCH 04/11] Change password min-length to four characters (#1094) in order to correlate with the management interface. --- setup/webmail.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup/webmail.sh b/setup/webmail.sh index 56efc59b..b0b916f7 100755 --- a/setup/webmail.sh +++ b/setup/webmail.sh @@ -185,7 +185,7 @@ cp ${RCM_PLUGIN_DIR}/password/config.inc.php.dist \ ${RCM_PLUGIN_DIR}/password/config.inc.php tools/editconf.py ${RCM_PLUGIN_DIR}/password/config.inc.php \ - "\$config['password_minimum_length']=6;" \ + "\$config['password_minimum_length']=4;" \ "\$config['password_db_dsn']='sqlite:///$STORAGE_ROOT/mail/users.sqlite';" \ "\$config['password_query']='UPDATE users SET password=%D WHERE email=%u';" \ "\$config['password_dovecotpw']='/usr/bin/doveadm pw';" \ From f6b20a810f7afd2859b4355ae72531be89502daf Mon Sep 17 00:00:00 2001 From: Norman S Date: Fri, 10 Feb 2017 15:44:40 +0100 Subject: [PATCH 05/11] Enforce pip to use python 2.7 for boto (#1093) --- setup/management.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup/management.sh b/setup/management.sh index a521dd85..26c2c38e 100755 --- a/setup/management.sh +++ b/setup/management.sh @@ -51,7 +51,7 @@ hide_output pip3 install --upgrade \ # duplicity uses python 2 so we need to get the python 2 package of boto to have backups to S3. # boto from the Ubuntu package manager is too out-of-date -- it doesn't support the newer # S3 api used in some regions, which breaks backups to those regions. See #627, #653. -hide_output pip install --upgrade boto +hide_output pip2 install --upgrade boto # CONFIGURATION From 36bef2ee16362eec1a0051f2b96034b1fbc63963 Mon Sep 17 00:00:00 2001 From: Dominik Murzynowski Date: Tue, 14 Feb 2017 20:24:59 +0100 Subject: [PATCH 06/11] Change password min-length to 8 characters (#1098) --- management/mailconfig.py | 4 ++-- management/templates/users.html | 4 ++-- setup/webmail.sh | 2 +- tools/mail.py | 4 ++-- 4 files changed, 7 insertions(+), 7 deletions(-) diff --git a/management/mailconfig.py b/management/mailconfig.py index d044c1ab..4cb57027 100755 --- a/management/mailconfig.py +++ b/management/mailconfig.py @@ -599,8 +599,8 @@ def validate_password(pw): raise ValueError("No password provided.") if re.search(r"[\s]", pw): raise ValueError("Passwords cannot contain spaces.") - if len(pw) < 4: - raise ValueError("Passwords must be at least four characters.") + if len(pw) < 8: + raise ValueError("Passwords must be at least eight characters.") if __name__ == "__main__": diff --git a/management/templates/users.html b/management/templates/users.html index 2cda327c..e0545835 100644 --- a/management/templates/users.html +++ b/management/templates/users.html @@ -31,7 +31,7 @@
    -
  • Passwords must be at least four characters and may not contain spaces. For best results, generate a random password.
    • +
  • Passwords must be at least eight characters and may not contain spaces. For best results, generate a random password.
  • Use aliases to create email addresses that forward to existing accounts.
  • Administrators get access to this control panel.
  • User accounts cannot contain any international (non-ASCII) characters, but aliases can.
    • @@ -296,7 +296,7 @@ function mod_priv(elem, add_remove) { function generate_random_password() { var pw = ""; var charset = "ABCDEFGHJKLMNPQRSTUVWXYZabcdefghijkmnopqrstuvwxyz23456789"; // confusable characters skipped - for (var i = 0; i < 10; i++) + for (var i = 0; i < 12; i++) pw += charset.charAt(Math.floor(Math.random() * charset.length)); show_modal_error("Random Password", "

    Here, try this:

    " + pw + " Date: Thu, 16 Feb 2017 00:24:32 +0100 Subject: [PATCH 07/11] Fix date delta display for deltas greater than 1 year (#1099) --- management/backup.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/management/backup.py b/management/backup.py index 92f8a740..d6189cfe 100755 --- a/management/backup.py +++ b/management/backup.py @@ -39,6 +39,8 @@ def backup_status(env): def reldate(date, ref, clip): if ref < date: return clip rd = dateutil.relativedelta.relativedelta(ref, date) + if rd.years > 1: return "%d years, %d months" % (rd.years, rd.months) + if rd.years == 1: return "%d year, %d months" % (rd.years, rd.months) if rd.months > 1: return "%d months, %d days" % (rd.months, rd.days) if rd.months == 1: return "%d month, %d days" % (rd.months, rd.days) if rd.days >= 7: return "%d days" % rd.days From f88c907a29d575784886aa8d5cb8c6c7f10d20f8 Mon Sep 17 00:00:00 2001 From: NatCC Date: Tue, 21 Feb 2017 14:32:28 +0000 Subject: [PATCH 08/11] Update jails.conf - SSH fail2ban jail (#1105) SSH fail2ban jail is not enabled by default and so the jail does not load. --- conf/fail2ban/jails.conf | 1 + 1 file changed, 1 insertion(+) diff --git a/conf/fail2ban/jails.conf b/conf/fail2ban/jails.conf index 0146b64c..290a75bb 100644 --- a/conf/fail2ban/jails.conf +++ b/conf/fail2ban/jails.conf @@ -73,6 +73,7 @@ action = iptables-allports[name=recidive] enabled = true [ssh] +enabled = true maxretry = 7 bantime = 3600 From d4baac2363b576cb131ff0f23fd7bf22daba917f Mon Sep 17 00:00:00 2001 From: Manuel Date: Wed, 1 Mar 2017 13:57:03 +0100 Subject: [PATCH 09/11] at the end of setup show SHA256 tls cert hash instead of SHA1 hash (#1108) --- setup/start.sh | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/setup/start.sh b/setup/start.sh index 790afe18..04096474 100755 --- a/setup/start.sh +++ b/setup/start.sh @@ -147,17 +147,17 @@ if management/status_checks.py --check-primary-hostname; then echo https://$PRIMARY_HOSTNAME/admin echo echo "If you have a DNS problem put the box's IP address in the URL" - echo "(https://$PUBLIC_IP/admin) but then check the SSL fingerprint:" - openssl x509 -in $STORAGE_ROOT/ssl/ssl_certificate.pem -noout -fingerprint \ - | sed "s/SHA1 Fingerprint=//" + echo "(https://$PUBLIC_IP/admin) but then check the TLS fingerprint:" + openssl x509 -in $STORAGE_ROOT/ssl/ssl_certificate.pem -noout -fingerprint -sha256\ + | sed "s/SHA256 Fingerprint=//" else echo https://$PUBLIC_IP/admin echo echo You will be alerted that the website has an invalid certificate. Check that echo the certificate fingerprint matches: echo - openssl x509 -in $STORAGE_ROOT/ssl/ssl_certificate.pem -noout -fingerprint \ - | sed "s/SHA1 Fingerprint=//" + openssl x509 -in $STORAGE_ROOT/ssl/ssl_certificate.pem -noout -fingerprint -sha256\ + | sed "s/SHA256 Fingerprint=//" echo echo Then you can confirm the security exception and continue. echo From 3830facf78b57fc094c7154c9e1c40a509acea12 Mon Sep 17 00:00:00 2001 From: Jan Schulz-Hofen Date: Wed, 1 Mar 2017 19:59:48 +0700 Subject: [PATCH 10/11] set dovecot vsz_limit to 1/3 of available memory (#1096) The `default_vsz_limit` is the maximum amount of virtual memory that can be allocated. It should be set *reasonably high* to avoid allocation issues with larger mailboxes. We're setting it to 1/3 of the total available memory (physical mem + swap) to be sure. See here for discussion: - https://www.dovecot.org/list/dovecot/2012-August/137569.html - https://www.dovecot.org/list/dovecot/2011-December/132455.html --- setup/mail-dovecot.sh | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/setup/mail-dovecot.sh b/setup/mail-dovecot.sh index 2c5533ec..b92384f5 100755 --- a/setup/mail-dovecot.sh +++ b/setup/mail-dovecot.sh @@ -37,8 +37,16 @@ apt_install \ # of active IMAP connections (at, say, 5 open connections per user that # would be 20 users). Set it to 250 times the number of cores this # machine has, so on a two-core machine that's 500 processes/100 users). +# The `default_vsz_limit` is the maximum amount of virtual memory that +# can be allocated. It should be set *reasonably high* to avoid allocation +# issues with larger mailboxes. We're setting it to 1/3 of the total +# available memory (physical mem + swap) to be sure. +# See here for discussion: +# - https://www.dovecot.org/list/dovecot/2012-August/137569.html +# - https://www.dovecot.org/list/dovecot/2011-December/132455.html tools/editconf.py /etc/dovecot/conf.d/10-master.conf \ default_process_limit=$(echo "`nproc` * 250" | bc) \ + default_vsz_limit=$(echo "`free -tom | tail -1 | awk '{print $2}'` / 3" | bc)M \ log_path=/var/log/mail.log # The inotify `max_user_instances` default is 128, which constrains From 368b9c50d0f676495fae16feffd723ebd8f6365b Mon Sep 17 00:00:00 2001 From: Sean Watson Date: Wed, 1 Mar 2017 05:02:41 -0800 Subject: [PATCH 11/11] add DSA and ED25519 SSHFP records if those keys are present (#1078) --- CHANGELOG.md | 1 + management/dns_update.py | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8a33f476..28286d2b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,7 @@ In Development * Allow larger messages to be checked by SpamAssassin. * Made nightly re-provisioning of TLS certificates less noisy. * Fixed bugs in rsync backup method. +* Added support for DSA and ED25519 SSHFP records. v0.21c (February 1, 2017) ------------------------- diff --git a/management/dns_update.py b/management/dns_update.py index c231c8c4..165f52ef 100755 --- a/management/dns_update.py +++ b/management/dns_update.py @@ -342,6 +342,7 @@ def build_sshfp_records(): "ssh-rsa": 1, "ssh-dss": 2, "ecdsa-sha2-nistp256": 3, + "ssh-ed25519": 4, } # Get our local fingerprints by running ssh-keyscan. The output looks @@ -359,7 +360,7 @@ def build_sshfp_records(): ports = ports + [s[1]] # the keys are the same at each port, so we only need to get # them at the first port found (may not be port 22) - keys = shell("check_output", ["ssh-keyscan", "-p", ports[0], "localhost"]) + keys = shell("check_output", ["ssh-keyscan", "-t", "rsa,dsa,ecdsa,ed25519", "-p", ports[0], "localhost"]) for key in sorted(keys.split("\n")): if key.strip() == "" or key[0] == "#": continue try: