From a3d7e0dfaed596b62b1adef5a9f90427855a8d17 Mon Sep 17 00:00:00 2001 From: Jeff Volkenant Date: Wed, 16 Jan 2019 10:21:19 -0800 Subject: [PATCH 01/72] Adapted MIAB Solr install script from https://github.com/jkaberg/ for Ubuntu Bionic --- conf/cronjob/dovecot | 1 + conf/cronjob/solr | 2 ++ setup/solr.sh | 63 ++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 66 insertions(+) create mode 100644 conf/cronjob/dovecot create mode 100644 conf/cronjob/solr create mode 100644 setup/solr.sh diff --git a/conf/cronjob/dovecot b/conf/cronjob/dovecot new file mode 100644 index 00000000..8de53e63 --- /dev/null +++ b/conf/cronjob/dovecot @@ -0,0 +1 @@ +/usr/bin/doveadm fts rescan -A diff --git a/conf/cronjob/solr b/conf/cronjob/solr new file mode 100644 index 00000000..217f0191 --- /dev/null +++ b/conf/cronjob/solr @@ -0,0 +1,2 @@ +*/1 * * * * root /usr/bin/curl http://127.0.0.1:8080/solr/update?commit=true &>/dev/null +30 3 * * * root /usr/bin/curl http://127.0.0.1:8080/solr/update?optimize=true &>/dev/null diff --git a/setup/solr.sh b/setup/solr.sh new file mode 100644 index 00000000..1e9f597f --- /dev/null +++ b/setup/solr.sh @@ -0,0 +1,63 @@ +#!/bin/bash +# +# Inspired by the solr.sh from jkaberg (https://github.com/jkaberg/mailinabox-sogo) +# with some modifications +# +# IMAP search with lucene via solr +# -------------------------------- +# +# By default dovecot uses its own Squat search index that has awful performance +# on large mailboxes. Dovecot 2.1+ has support for using Lucene internally but +# this didn't make it into the Ubuntu packages, so we use Solr instead to run +# Lucene for us. +# +# Solr runs as a tomcat process. The dovecot solr plugin talks to solr via its +# HTTP interface, causing mail to be indexed when searches occur, and getting +# results back. + +source setup/functions.sh # load our functions +source /etc/mailinabox.conf # load global vars + +# Install packages and basic configuation +# --------------------------------------- + +echo "Installing Solr..." + +# Install packages +apt_install solr-tomcat dovecot-solr + +# Solr requires a schema to tell it how to index data, this is provided by dovecot +cp /usr/share/dovecot/solr-schema.xml /etc/solr/conf/schema.xml + +# Update the dovecot plugin configuration +# +# Break-imap-search makes search work the way users expect, rather than the way +# the IMAP specification expects +tools/editconf.py /etc/dovecot/conf.d/10-mail.conf \ + mail_plugins="fts fts_solr" + +cat > /etc/dovecot/conf.d/90-plugin-fts.conf << EOF; +plugin { + fts = solr + fts_autoindex = yes + fts_solr = break-imap-search url=http://127.0.0.1:8080/solr/ +} +EOF + +# Bump memory allocation for Solr. +# Not needed? I'll let it sit here for a while. +#echo 'export JAVA_OPTS=-Xms512M -Xmx1024M' > /usr/share/tomcat7/bin/setenv.sh + +# Install cronjobs to keep FTS up to date +hide_output install -m 755 conf/cronjob/dovecot /etc/cron.daily/ +hide_output install -m 644 conf/cronjob/solr /etc/cron.d/ + +# PERMISSIONS + +# Ensure configuration files are owned by dovecot and not world readable. +chown -R mail:dovecot /etc/dovecot +chmod -R o-rwx /etc/dovecot + +# Restart services to reload solr schema & dovecot plugins +restart_service tomcat8 +restart_service dovecot From 4a23a522e1ed2263645c5b73a004b037dc0b0a32 Mon Sep 17 00:00:00 2001 From: Jeff Volkenant Date: Wed, 16 Jan 2019 10:29:21 -0800 Subject: [PATCH 02/72] added solr.sh to start.sh --- setup/start.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/setup/start.sh b/setup/start.sh index 0b145022..14c5700c 100755 --- a/setup/start.sh +++ b/setup/start.sh @@ -102,6 +102,7 @@ source setup/dns.sh source setup/mail-postfix.sh source setup/mail-dovecot.sh source setup/mail-users.sh +source setup/solr.sh source setup/dkim.sh source setup/spamassassin.sh source setup/web.sh From 2303ac3394b99ef8a6695a27aaa3cd6c29c0b8b2 Mon Sep 17 00:00:00 2001 From: Jeff Volkenant Date: Wed, 16 Jan 2019 11:32:16 -0800 Subject: [PATCH 03/72] Force kickoff of Solr indexing at install time --- setup/solr.sh | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/setup/solr.sh b/setup/solr.sh index 1e9f597f..72f8af61 100644 --- a/setup/solr.sh +++ b/setup/solr.sh @@ -61,3 +61,18 @@ chmod -R o-rwx /etc/dovecot # Restart services to reload solr schema & dovecot plugins restart_service tomcat8 restart_service dovecot + + +# Kickoff building the index + +# Per doveadm-fts manpage: Scan what mails exist in the full text search index +# and compare those to what actually exist in mailboxes. +# This removes mails from the index that have already been expunged and makes +# sure that the next doveadm index will index all the missing mails (if any). +doveadm fts rescan -A + +# Adds unindexed files to the fts database +# * `-q`: Queues the indexing to be run by indexer process. (will background the indexing) +# * `-A`: All users +# * `'*'`: All folders +doveadm index -q -A '*' From 2fce29d775dfb041454c497dbf2a33adf13fddb6 Mon Sep 17 00:00:00 2001 From: Jeff Volkenant Date: Wed, 16 Jan 2019 20:46:52 -0800 Subject: [PATCH 04/72] Added Solr (Tomcat) to status_checks.py --- management/status_checks.py | 1 + 1 file changed, 1 insertion(+) diff --git a/management/status_checks.py b/management/status_checks.py index 6f9bb1ef..82310d16 100755 --- a/management/status_checks.py +++ b/management/status_checks.py @@ -38,6 +38,7 @@ def get_services(): { "name": "Mail Filters (Sieve/dovecot)", "port": 4190, "public": True, }, { "name": "HTTP Web (nginx)", "port": 80, "public": True, }, { "name": "HTTPS Web (nginx)", "port": 443, "public": True, }, + { "name": "Solr Full Text Search (tomcat)", "port": 8080, "public": False, }, ] def run_checks(rounded_values, env, output, pool): From c302606de44a8b4c4c313e5d98f42f8964f482d1 Mon Sep 17 00:00:00 2001 From: Jeff Volkenant Date: Tue, 29 Jan 2019 13:46:35 -0800 Subject: [PATCH 05/72] Extended timeout for php/roundcube for text searches that take a long time --- setup/web.sh | 7 +++++++ setup/webmail.sh | 2 +- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/setup/web.sh b/setup/web.sh index ed37e5e3..2e82fe6a 100755 --- a/setup/web.sh +++ b/setup/web.sh @@ -48,6 +48,13 @@ tools/editconf.py /etc/php/7.2/fpm/php.ini -c ';' \ tools/editconf.py /etc/php/7.2/fpm/php.ini -c ';' \ default_charset="UTF-8" +# Set higher timeout since searches with Roundcube and Solr may take longer +# than the default 60 seconds. We will also match Roundcube's timeout to the +# same value +tools/editconf.py /etc/php/7.2/fpm/php.ini -c ';' \ + default_socket_timeout=180 + + # Switch from the dynamic process manager to the ondemand manager see #1216 tools/editconf.py /etc/php/7.2/fpm/pool.d/www.conf -c ';' \ pm=ondemand diff --git a/setup/webmail.sh b/setup/webmail.sh index b0e11c9b..d082382f 100755 --- a/setup/webmail.sh +++ b/setup/webmail.sh @@ -108,7 +108,7 @@ cat > $RCM_CONFIG < false, ), ); -\$config['imap_timeout'] = 15; +\$config['imap_timeout'] = 180; \$config['smtp_server'] = 'tls://127.0.0.1'; \$config['smtp_port'] = 587; \$config['smtp_user'] = '%u'; From 1dee84949838d90be1aa2d8f85995cd17b602074 Mon Sep 17 00:00:00 2001 From: Jeff Volkenant Date: Wed, 30 Jan 2019 11:33:43 -0800 Subject: [PATCH 06/72] Removed extra blankline --- setup/web.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/setup/web.sh b/setup/web.sh index 2e82fe6a..4ba646e4 100755 --- a/setup/web.sh +++ b/setup/web.sh @@ -54,7 +54,6 @@ tools/editconf.py /etc/php/7.2/fpm/php.ini -c ';' \ tools/editconf.py /etc/php/7.2/fpm/php.ini -c ';' \ default_socket_timeout=180 - # Switch from the dynamic process manager to the ondemand manager see #1216 tools/editconf.py /etc/php/7.2/fpm/pool.d/www.conf -c ';' \ pm=ondemand From fc1f211af56a3d49a77a933c069d49cd5fbddeaf Mon Sep 17 00:00:00 2001 From: John Supplee Date: Sun, 10 Feb 2019 23:39:38 +0200 Subject: [PATCH 07/72] initial work on extended configuration --- conf/nginx.conf | 6 ++++-- management/auth.py | 2 +- management/daemon.py | 28 ++++++++++++++++++++++++++++ management/web_update.py | 18 ++++++++++++++++++ setup/mail-postfix.sh | 6 +++++- setup/start.sh | 16 ++++++++++++++++ setup/web.sh | 9 ++++++--- tools/dns-auth.sh | 2 ++ 8 files changed, 80 insertions(+), 7 deletions(-) create mode 100755 tools/dns-auth.sh diff --git a/conf/nginx.conf b/conf/nginx.conf index fafd3409..25910764 100644 --- a/conf/nginx.conf +++ b/conf/nginx.conf @@ -1,5 +1,6 @@ ## $HOSTNAME +#BEGIN_HTTP # Redirect all HTTP to HTTPS *except* the ACME challenges (Let's Encrypt TLS certificate # domain validation challenges) path, which must be served over HTTP per the ACME spec # (due to some Apache vulnerability). @@ -28,11 +29,12 @@ server { alias $STORAGE_ROOT/ssl/lets_encrypt/webroot/.well-known/acme-challenge/; } } +#END_HTTP # The secure HTTPS server. server { - listen 443 ssl http2; - listen [::]:443 ssl http2; + listen $HTTP_SSL_PORT ssl http2; + listen [::]:$HTTP_SSL_PORT ssl http2; server_name $HOSTNAME; diff --git a/management/auth.py b/management/auth.py index 55f59664..0b082580 100644 --- a/management/auth.py +++ b/management/auth.py @@ -59,7 +59,7 @@ class KeyAuthService: credentials = decode(credentials) if ":" not in credentials: - return None, None + return credentials, None username, password = credentials.split(':', maxsplit=1) return username, password diff --git a/management/daemon.py b/management/daemon.py index 572b6b4a..3869cbaa 100755 --- a/management/daemon.py +++ b/management/daemon.py @@ -324,6 +324,34 @@ def dns_get_dump(): from dns_update import build_recommended_dns return json_response(build_recommended_dns(env)) +@app.route('/letsencrypt/dns-auth//', methods=['GET']) +@authorized_personnel_only +def letsencrypt_dns_auth(domain, token): + from dns_update import do_dns_update, set_custom_dns_record + try: + qname = '_acme-challenge.' + domain + if set_custom_dns_record(qname, 'TXT', token, 'add', env): + if not do_dns_update(env): + return ("Error updating DNS", 400) + return "OK" + + except ValueError as e: + return (str(e), 400) + +@app.route('/letsencrypt/dns-cleanup/', methods=['GET']) +@authorized_personnel_only +def letsencrypt_dns_cleanup(domain): + from dns_update import do_dns_update, set_custom_dns_record + try: + qname = '_acme-challenge.' + domain + if set_custom_dns_record(qname, 'TXT', None, 'remove', env): + if not do_dns_update(env): + return ("Error updating DNS", 400) + return "OK" + + except ValueError as e: + return (str(e), 400) + # SSL @app.route('/ssl/status') diff --git a/management/web_update.py b/management/web_update.py index 61b38a7b..616698ba 100644 --- a/management/web_update.py +++ b/management/web_update.py @@ -94,6 +94,20 @@ def do_web_update(env): # Add default 'www.' redirect. nginx_conf += make_domain_config(domain, [template0, template3], ssl_certificates, env) + if str(env['HTTP_SSL_PORT']) != "443": + in_http = False + new_conf = '' + for line in nginx_conf.split('\n'): + if line.strip() == '#BEGIN_HTTP': + in_http = True + elif line.strip() == '#END_HTTP': + in_http = False + + if not in_http: + new_conf += line + '\n' + + nginx_conf = new_conf + # Did the file change? If not, don't bother writing & restarting nginx. nginx_conf_fn = "/etc/nginx/conf.d/local.conf" if os.path.exists(nginx_conf_fn): @@ -178,8 +192,12 @@ def make_domain_config(domain, templates, ssl_certificates, env): nginx_conf = re.sub("[ \t]*# ADDITIONAL DIRECTIVES HERE *\n", t, nginx_conf) # Replace substitution strings in the template & return. + if int(env['HTTP_SSL_PORT']) != 443: + # disable the regular HTTP server + nginx_conf = re.sub(r'#BEGIN_HTTP.*?#END_HTTP', repl='', string=nginx_conf, flags=re.MULTILINE) nginx_conf = nginx_conf.replace("$STORAGE_ROOT", env['STORAGE_ROOT']) nginx_conf = nginx_conf.replace("$HOSTNAME", domain) + nginx_conf = nginx_conf.replace("$HTTP_SSL_PORT", env['HTTP_SSL_PORT']) nginx_conf = nginx_conf.replace("$ROOT", root) nginx_conf = nginx_conf.replace("$SSL_KEY", tls_cert["private-key"]) nginx_conf = nginx_conf.replace("$SSL_CERTIFICATE", tls_cert["certificate"]) diff --git a/setup/mail-postfix.sh b/setup/mail-postfix.sh index 0c9bc97c..1ea64a50 100755 --- a/setup/mail-postfix.sh +++ b/setup/mail-postfix.sh @@ -193,9 +193,13 @@ tools/editconf.py /etc/postfix/main.cf virtual_transport=lmtp:[127.0.0.1]:10025 # so these IPs get mail delivered quickly. But when an IP is not listed in the permit_dnswl_client list (i.e. it is not #NODOC # whitelisted) then postfix does a DEFER_IF_REJECT, which results in all "unknown user" sorts of messages turning into #NODOC # "450 4.7.1 Client host rejected: Service unavailable". This is a retry code, so the mail doesn't properly bounce. #NODOC +RECIPIENT_RESTRICTIONS=permit_sasl_authenticated,permit_mynetworks,\"reject_rbl_client zen.spamhaus.org\",reject_unlisted_recipient +if [ $NO_GREYLISTING != "1" ]; then + RECIPIENT_RESTRICTIONS=${RECIPIENT_RESTRICTIONS},\"check_policy_service inet:127.0.0.1:10023\" +fi tools/editconf.py /etc/postfix/main.cf \ smtpd_sender_restrictions="reject_non_fqdn_sender,reject_unknown_sender_domain,reject_authenticated_sender_login_mismatch,reject_rhsbl_sender dbl.spamhaus.org" \ - smtpd_recipient_restrictions=permit_sasl_authenticated,permit_mynetworks,"reject_rbl_client zen.spamhaus.org",reject_unlisted_recipient,"check_policy_service inet:127.0.0.1:10023" + smtpd_recipient_restrictions=$RECIPIENT_RESTRICTIONS # Postfix connects to Postgrey on the 127.0.0.1 interface specifically. Ensure that # Postgrey listens on the same interface (and not IPv6, for instance). diff --git a/setup/start.sh b/setup/start.sh index 0b145022..60a0349e 100755 --- a/setup/start.sh +++ b/setup/start.sh @@ -42,6 +42,20 @@ else FIRST_TIME_SETUP=1 fi +if [ -z "${DEFAULT_HTTP_SSL_PORT:-}" ]; then + HTTP_SSL_PORT=443 +else + HTTP_SSL_PORT=$DEFAULT_HTTP_SSL_PORT +fi + +if [ -z "${DEFAULT_NO_GREYLISTING:-}" ]; then + NO_GREYLISTING=0 +elif (($DEFAULT_NO_GREYLISTING > 0)); then + NO_GREYLISTING=1 +else + NO_GREYLISTING=0 +fi + # Put a start script in a global location. We tell the user to run 'mailinabox' # in the first dialog prompt, so we should do this before that starts. cat > /usr/local/bin/mailinabox << EOF; @@ -93,6 +107,8 @@ PUBLIC_IP=$PUBLIC_IP PUBLIC_IPV6=$PUBLIC_IPV6 PRIVATE_IP=$PRIVATE_IP PRIVATE_IPV6=$PRIVATE_IPV6 +HTTP_SSL_PORT=$HTTP_SSL_PORT +NO_GREYLISTING=$NO_GREYLISTING EOF # Start service configuration. diff --git a/setup/web.sh b/setup/web.sh index ed37e5e3..ad7ad310 100755 --- a/setup/web.sh +++ b/setup/web.sh @@ -96,6 +96,9 @@ restart_service nginx restart_service php7.2-fpm # Open ports. -ufw_allow http -ufw_allow https - +if [ $HTTP_SSL_PORT == 443 ]; then + ufw_allow http + ufw_allow https +else + ufw_allow $HTTP_SSL_PORT +fi diff --git a/tools/dns-auth.sh b/tools/dns-auth.sh new file mode 100755 index 00000000..20d602bd --- /dev/null +++ b/tools/dns-auth.sh @@ -0,0 +1,2 @@ +#!/usr/bin/env bash + From 514619b44ab9927f3922e6bfac5096eadd304bce Mon Sep 17 00:00:00 2001 From: John Supplee Date: Mon, 11 Feb 2019 16:05:50 +0200 Subject: [PATCH 08/72] setup dns authentication for letsencrypt --- .gitignore | 1 + management/ssl_certificates.py | 6 +++++- tools/dns-auth.sh | 8 ++++++++ tools/dns-cleanup.sh | 8 ++++++++ 4 files changed, 22 insertions(+), 1 deletion(-) create mode 100755 tools/dns-cleanup.sh diff --git a/.gitignore b/.gitignore index f3cdb1bc..94072693 100644 --- a/.gitignore +++ b/.gitignore @@ -5,3 +5,4 @@ tools/__pycache__/ externals/ .env .vagrant +.idea/ diff --git a/management/ssl_certificates.py b/management/ssl_certificates.py index 76b0f8fa..6d433b54 100755 --- a/management/ssl_certificates.py +++ b/management/ssl_certificates.py @@ -313,6 +313,7 @@ def provision_certificates(env, limit_domains): webroot = os.path.join(account_path, 'webroot') os.makedirs(webroot, exist_ok=True) with tempfile.TemporaryDirectory() as d: + miab_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) cert_file = os.path.join(d, 'cert_and_chain.pem') print("Provisioning TLS certificates for " + ", ".join(domain_list) + ".") certbotret = subprocess.check_output([ @@ -328,7 +329,10 @@ def provision_certificates(env, limit_domains): "--chain-path", os.path.join(d, 'chain'), # we only use the full chain "--fullchain-path", cert_file, - "--webroot", "--webroot-path", webroot, + "--manual", + "--preferred-challenge", "dns", + "--manual-auth-hook", os.path.join(miab_dir, "/tools/dns-auth.sh"), + "--manual-cleanup-hook", os.path.join(miab_dir, "/tools/dns-cleanup.sh"), "--config-dir", account_path, #"--staging", diff --git a/tools/dns-auth.sh b/tools/dns-auth.sh index 20d602bd..e3ee77c8 100755 --- a/tools/dns-auth.sh +++ b/tools/dns-auth.sh @@ -1,2 +1,10 @@ #!/usr/bin/env bash +# TODO: Make work with port other than 443 + +API_KEY=`cat /var/lib/mailinabox/api.key` +HOSTNAME=`hostname` + +curl -s -X PUT -d "$CERTBOT_VALIDATION" --user "$API_KEY:" https://$HOSTNAME/admin/dns/custom/_acme-challenge.$CERTBOT_DOMAIN/TXT + +sleep 15 diff --git a/tools/dns-cleanup.sh b/tools/dns-cleanup.sh new file mode 100755 index 00000000..f9ac582d --- /dev/null +++ b/tools/dns-cleanup.sh @@ -0,0 +1,8 @@ +#!/usr/bin/env bash + +# TODO: Make work with port other than 443 + +API_KEY=`cat /var/lib/mailinabox/api.key` +HOSTNAME=`hostname` + +curl -s -X DELETE --user "$API_KEY:" https://$HOSTNAME/admin/dns/custom/_acme-challenge.$CERTBOT_DOMAIN/TXT \ No newline at end of file From aa234a504e008dff8d71a962ef01289862e89e9e Mon Sep 17 00:00:00 2001 From: John Supplee Date: Wed, 27 Feb 2019 12:48:34 +0200 Subject: [PATCH 09/72] changes for SPF on incoming email --- setup/mail-postfix.sh | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/setup/mail-postfix.sh b/setup/mail-postfix.sh index 0c9bc97c..4802adeb 100755 --- a/setup/mail-postfix.sh +++ b/setup/mail-postfix.sh @@ -42,7 +42,8 @@ source /etc/mailinabox.conf # load global vars # * `ca-certificates`: A trust store used to squelch postfix warnings about # untrusted opportunistically-encrypted connections. echo "Installing Postfix (SMTP server)..." -apt_install postfix postfix-sqlite postfix-pcre postgrey ca-certificates +apt_install postfix postfix-sqlite postfix-pcre postgrey ca-certificates \ + postfix-policyd-spf-python # ### Basic Settings @@ -97,7 +98,9 @@ tools/editconf.py /etc/postfix/master.cf -s -w \ -o cleanup_service_name=authclean" \ "authclean=unix n - - - 0 cleanup -o header_checks=pcre:/etc/postfix/outgoing_mail_header_filters - -o nested_header_checks=" + -o nested_header_checks=" \ + "policy-spf=unix - n n - - spawn + user=nobody argv=/usr/bin/policyd-spf" # Install the `outgoing_mail_header_filters` file required by the new 'authclean' service. cp conf/postfix_outgoing_mail_header_filters /etc/postfix/outgoing_mail_header_filters @@ -195,7 +198,7 @@ tools/editconf.py /etc/postfix/main.cf virtual_transport=lmtp:[127.0.0.1]:10025 # "450 4.7.1 Client host rejected: Service unavailable". This is a retry code, so the mail doesn't properly bounce. #NODOC tools/editconf.py /etc/postfix/main.cf \ smtpd_sender_restrictions="reject_non_fqdn_sender,reject_unknown_sender_domain,reject_authenticated_sender_login_mismatch,reject_rhsbl_sender dbl.spamhaus.org" \ - smtpd_recipient_restrictions=permit_sasl_authenticated,permit_mynetworks,"reject_rbl_client zen.spamhaus.org",reject_unlisted_recipient,"check_policy_service inet:127.0.0.1:10023" + smtpd_recipient_restrictions=permit_sasl_authenticated,permit_mynetworks,"reject_rbl_client zen.spamhaus.org",reject_unlisted_recipient,"check_policy_service inet:127.0.0.1:10023","check_policy_service unix:private/policy-spf" # Postfix connects to Postgrey on the 127.0.0.1 interface specifically. Ensure that # Postgrey listens on the same interface (and not IPv6, for instance). @@ -214,6 +217,7 @@ tools/editconf.py /etc/postfix/main.cf \ # Allow the two SMTP ports in the firewall. + ufw_allow smtp ufw_allow submission From 68f4d1c426ca239b40e1f077305709ed0c6d1feb Mon Sep 17 00:00:00 2001 From: John Supplee Date: Wed, 27 Feb 2019 17:30:59 +0200 Subject: [PATCH 10/72] add SRS support --- setup/mail-postfix.sh | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/setup/mail-postfix.sh b/setup/mail-postfix.sh index e59de474..e974fb91 100755 --- a/setup/mail-postfix.sh +++ b/setup/mail-postfix.sh @@ -43,7 +43,7 @@ source /etc/mailinabox.conf # load global vars # untrusted opportunistically-encrypted connections. echo "Installing Postfix (SMTP server)..." apt_install postfix postfix-sqlite postfix-pcre postgrey ca-certificates \ - postfix-policyd-spf-python + postfix-policyd-spf-python postsrsd # ### Basic Settings @@ -201,7 +201,7 @@ tools/editconf.py /etc/postfix/main.cf lmtp_destination_recipient_limit=1 # "450 4.7.1 Client host rejected: Service unavailable". This is a retry code, so the mail doesn't properly bounce. #NODOC tools/editconf.py /etc/postfix/main.cf \ smtpd_sender_restrictions="reject_non_fqdn_sender,reject_unknown_sender_domain,reject_authenticated_sender_login_mismatch,reject_rhsbl_sender dbl.spamhaus.org" \ - smtpd_recipient_restrictions=permit_sasl_authenticated,permit_mynetworks,"reject_rbl_client zen.spamhaus.org",reject_unlisted_recipient,"check_policy_service inet:127.0.0.1:10023","check_policy_service unix:private/policy-spf" + smtpd_recipient_restrictions=permit_sasl_authenticated,permit_mynetworks,"reject_rbl_client zen.spamhaus.org",reject_unlisted_recipient,"check_policy_service unix:private/policy-spf","check_policy_service inet:127.0.0.1:10023" # Postfix connects to Postgrey on the 127.0.0.1 interface specifically. Ensure that # Postgrey listens on the same interface (and not IPv6, for instance). @@ -218,6 +218,16 @@ tools/editconf.py /etc/default/postgrey \ tools/editconf.py /etc/postfix/main.cf \ message_size_limit=134217728 +# Setup SRS +postconf -e \ + sender_canonical_maps=tcp:localhost:10001 \ + sender_canonical_classes=envelope_sender \ + recipient_canonical_maps=tcp:localhost:10002 \ + recipient_canonical_classes=envelope_recipient,header_recipient + +hide_output systemctl enable postsrsd +hide_output systemctl restart postsrsd + # Allow the two SMTP ports in the firewall. From 887e2927480303014992427ec1bb1a4d023c0785 Mon Sep 17 00:00:00 2001 From: John Supplee Date: Sat, 2 Mar 2019 15:30:08 +0200 Subject: [PATCH 11/72] Add changes for mailgraph webserver --- conf/nginx-primaryonly.conf | 10 ++++++++++ setup/web.sh | 2 +- 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/conf/nginx-primaryonly.conf b/conf/nginx-primaryonly.conf index d8d912ca..be6588e2 100644 --- a/conf/nginx-primaryonly.conf +++ b/conf/nginx-primaryonly.conf @@ -64,4 +64,14 @@ rewrite ^/.well-known/carddav /cloud/remote.php/carddav/ redirect; rewrite ^/.well-known/caldav /cloud/remote.php/caldav/ redirect; + location ~ ^/mailgraph/.*\.cgi { + root /usr/share/mailgraph; + include /etc/nginx/fastcgi_params; + fastcgi_param SCRIPT_FILENAME /usr/share/mailgraph/mailgraph.$ + fastcgi_pass unix:/var/run/fcgiwrap.socket; + + auth_basic "box"; + auth_basic_user_file /etc/nginx/htpasswd; + } + # ADDITIONAL DIRECTIVES HERE diff --git a/setup/web.sh b/setup/web.sh index ed37e5e3..6cb1a30b 100755 --- a/setup/web.sh +++ b/setup/web.sh @@ -19,7 +19,7 @@ fi echo "Installing Nginx (web server)..." -apt_install nginx php-cli php-fpm +apt_install nginx php-cli php-fpm fcgiwrap mailgraph rm -f /etc/nginx/sites-enabled/default From 69a50ed9541712b39939a28d9a2d0336a6fb80ac Mon Sep 17 00:00:00 2001 From: John Supplee Date: Sat, 2 Mar 2019 15:39:19 +0200 Subject: [PATCH 12/72] fix error in nginx configuration --- conf/nginx-primaryonly.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conf/nginx-primaryonly.conf b/conf/nginx-primaryonly.conf index be6588e2..2eeacc6f 100644 --- a/conf/nginx-primaryonly.conf +++ b/conf/nginx-primaryonly.conf @@ -67,7 +67,7 @@ location ~ ^/mailgraph/.*\.cgi { root /usr/share/mailgraph; include /etc/nginx/fastcgi_params; - fastcgi_param SCRIPT_FILENAME /usr/share/mailgraph/mailgraph.$ + fastcgi_param SCRIPT_FILENAME /usr/share/mailgraph/mailgraph.cgi fastcgi_pass unix:/var/run/fcgiwrap.socket; auth_basic "box"; From 8180707cc63ef234f094fc240bbd6f8d2cbcf102 Mon Sep 17 00:00:00 2001 From: John Supplee Date: Sat, 2 Mar 2019 15:43:30 +0200 Subject: [PATCH 13/72] add missing semicolon to nginx conf --- conf/nginx-primaryonly.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conf/nginx-primaryonly.conf b/conf/nginx-primaryonly.conf index 2eeacc6f..552cf28a 100644 --- a/conf/nginx-primaryonly.conf +++ b/conf/nginx-primaryonly.conf @@ -67,7 +67,7 @@ location ~ ^/mailgraph/.*\.cgi { root /usr/share/mailgraph; include /etc/nginx/fastcgi_params; - fastcgi_param SCRIPT_FILENAME /usr/share/mailgraph/mailgraph.cgi + fastcgi_param SCRIPT_FILENAME /usr/share/mailgraph/mailgraph.cgi; fastcgi_pass unix:/var/run/fcgiwrap.socket; auth_basic "box"; From 5615031ef8a040a1fc0a90d65da562cfb6fca787 Mon Sep 17 00:00:00 2001 From: John Supplee Date: Sat, 2 Mar 2019 17:23:58 +0200 Subject: [PATCH 14/72] initial work to integrate with control panel --- management/templates/index.html | 1 + management/templates/mailgraph.html | 32 +++++++++++++++++++++++++++++ 2 files changed, 33 insertions(+) create mode 100644 management/templates/mailgraph.html diff --git a/management/templates/index.html b/management/templates/index.html index 2c0d5a9a..31fe5916 100644 --- a/management/templates/index.html +++ b/management/templates/index.html @@ -102,6 +102,7 @@
  • Instructions
  • Users
  • Aliases
    • +
  • Reports
  • Contacts/Calendar
    • diff --git a/management/templates/mailgraph.html b/management/templates/mailgraph.html new file mode 100644 index 00000000..9d495b0c --- /dev/null +++ b/management/templates/mailgraph.html @@ -0,0 +1,32 @@ +

    Mail statistics

    + + +

    Last Day

    +

    mailgraph

    +

    mailgraph

    +

    mailgraph

    + +

    Last Week

    +

    mailgraph

    +

    mailgraph

    +

    mailgraph

    + +

    Last Month

    +

    mailgraph

    +

    mailgraph

    +

    mailgraph

    + +

    Last Year

    +

    mailgraph

    +

    mailgraph

    +

    mailgraph

    + +
    + +

    Mailgraph 1.14 by David Schweikert +(built on Tobi Oetiker's RRDtool)

    From fefb5ebc3342cf8a07520091b4fbd7f0da4cdfb6 Mon Sep 17 00:00:00 2001 From: John Supplee Date: Sat, 2 Mar 2019 20:16:39 +0200 Subject: [PATCH 15/72] more work on control panel integration --- management/daemon.py | 14 ++++++++++++ management/templates/index.html | 2 +- management/templates/mailgraph.html | 34 ++++++++++++++--------------- 3 files changed, 32 insertions(+), 18 deletions(-) diff --git a/management/daemon.py b/management/daemon.py index 572b6b4a..918370ef 100755 --- a/management/daemon.py +++ b/management/daemon.py @@ -520,6 +520,20 @@ def privacy_status_set(): utils.write_settings(config, env) return "OK" +# Mailgraph + +@app.route('/mailgraph/image.cgi') +@authorized_personnel_only +def mailgraph(): + if request.query_string: + return utils.shell( + "check_output", + ["/usr/share/mailgraph/mailgraph.cgi"], + env={"QUERY_STRING": request.query_string} + ) + return '' + + # MUNIN @app.route('/munin/') diff --git a/management/templates/index.html b/management/templates/index.html index 31fe5916..9f8b8652 100644 --- a/management/templates/index.html +++ b/management/templates/index.html @@ -102,7 +102,7 @@
  • Instructions
  • Users
  • Aliases
    • -
  • Reports
    • +
  • Mailgraph
  • Contacts/Calendar
    • diff --git a/management/templates/mailgraph.html b/management/templates/mailgraph.html index 9d495b0c..8b1ad893 100644 --- a/management/templates/mailgraph.html +++ b/management/templates/mailgraph.html @@ -1,4 +1,4 @@ -

    Mail statistics

    +

    Mail statistics

    -

    Last Day

    -

    mailgraph

    -

    mailgraph

    -

    mailgraph

    +

    Last Day

    +

    mailgraph

    +

    mailgraph

    +

    mailgraph

    -

    Last Week

    -

    mailgraph

    -

    mailgraph

    -

    mailgraph

    +

    Last Week

    +

    mailgraph

    +

    mailgraph

    +

    mailgraph

    -

    Last Month

    -

    mailgraph

    -

    mailgraph

    -

    mailgraph

    +

    Last Month

    +

    mailgraph

    +

    mailgraph

    +

    mailgraph

    -

    Last Year

    -

    mailgraph

    -

    mailgraph

    -

    mailgraph

    +

    Last Year

    +

    mailgraph

    +

    mailgraph

    +

    mailgraph

    From fd239db7c1e2ae92bb9bcdfec5c8b65cec373723 Mon Sep 17 00:00:00 2001 From: John Supplee Date: Sat, 2 Mar 2019 20:22:08 +0200 Subject: [PATCH 16/72] integrate template --- management/templates/index.html | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/management/templates/index.html b/management/templates/index.html index 9f8b8652..26d25e4a 100644 --- a/management/templates/index.html +++ b/management/templates/index.html @@ -152,6 +152,10 @@ {% include "sync-guide.html" %} +
    + {% include "mailgraph.html" %} +
    +
    {% include "web.html" %}
    From d41cdb844cb6965aa2958b5627b8e88d149ff2de Mon Sep 17 00:00:00 2001 From: John Supplee Date: Sat, 2 Mar 2019 20:31:24 +0200 Subject: [PATCH 17/72] remove nginx config for Mailgraph --- conf/nginx-primaryonly.conf | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/conf/nginx-primaryonly.conf b/conf/nginx-primaryonly.conf index 552cf28a..d8d912ca 100644 --- a/conf/nginx-primaryonly.conf +++ b/conf/nginx-primaryonly.conf @@ -64,14 +64,4 @@ rewrite ^/.well-known/carddav /cloud/remote.php/carddav/ redirect; rewrite ^/.well-known/caldav /cloud/remote.php/caldav/ redirect; - location ~ ^/mailgraph/.*\.cgi { - root /usr/share/mailgraph; - include /etc/nginx/fastcgi_params; - fastcgi_param SCRIPT_FILENAME /usr/share/mailgraph/mailgraph.cgi; - fastcgi_pass unix:/var/run/fcgiwrap.socket; - - auth_basic "box"; - auth_basic_user_file /etc/nginx/htpasswd; - } - # ADDITIONAL DIRECTIVES HERE From 5ffa71999ac42001fcc825a717a53741f54c32ea Mon Sep 17 00:00:00 2001 From: John Supplee Date: Sat, 2 Mar 2019 20:47:54 +0200 Subject: [PATCH 18/72] work on the daemon for mailgraph --- management/daemon.py | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/management/daemon.py b/management/daemon.py index 918370ef..54b1a565 100755 --- a/management/daemon.py +++ b/management/daemon.py @@ -526,12 +526,21 @@ def privacy_status_set(): @authorized_personnel_only def mailgraph(): if request.query_string: - return utils.shell( + print("QUERY_STRING=%s" % request.query_string) + + code, bin_out = utils.shell( "check_output", ["/usr/share/mailgraph/mailgraph.cgi"], - env={"QUERY_STRING": request.query_string} + env={"QUERY_STRING": request.query_string}, + return_bytes=True ) - return '' + + if code != 0: + return ('Error generating mailgraph image: %s' % request.query_string, 500) + + return make_response(bin_out) + + return ('Mailgraph: no image requested', 500) # MUNIN From 89677584132d62c0d1152335459887202be599f5 Mon Sep 17 00:00:00 2001 From: John Supplee Date: Sat, 2 Mar 2019 23:18:44 +0200 Subject: [PATCH 19/72] Only load mailgraph images when the page is viewed --- management/templates/mailgraph.html | 32 ++++++++++++++++++----------- 1 file changed, 20 insertions(+), 12 deletions(-) diff --git a/management/templates/mailgraph.html b/management/templates/mailgraph.html index 8b1ad893..fc96f9b1 100644 --- a/management/templates/mailgraph.html +++ b/management/templates/mailgraph.html @@ -7,26 +7,34 @@

    Last Day

    -

    mailgraph

    -

    mailgraph

    -

    mailgraph

    +

    mailgraph

    +

    mailgraph

    +

    mailgraph

    Last Week

    -

    mailgraph

    -

    mailgraph

    -

    mailgraph

    +

    mailgraph

    +

    mailgraph

    +

    mailgraph

    Last Month

    -

    mailgraph

    -

    mailgraph

    -

    mailgraph

    +

    mailgraph

    +

    mailgraph

    +

    mailgraph

    Last Year

    -

    mailgraph

    -

    mailgraph

    -

    mailgraph

    +

    mailgraph

    +

    mailgraph

    +

    mailgraph

    Mailgraph 1.14 by David Schweikert (built on Tobi Oetiker's RRDtool)

    + + From 5b5087c9dd1bdf35c35b35c411be4bbf550dfd15 Mon Sep 17 00:00:00 2001 From: John Supplee Date: Sat, 2 Mar 2019 23:33:45 +0200 Subject: [PATCH 20/72] fix query params for mailgraph images --- management/templates/mailgraph.html | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/management/templates/mailgraph.html b/management/templates/mailgraph.html index fc96f9b1..086aa9cc 100644 --- a/management/templates/mailgraph.html +++ b/management/templates/mailgraph.html @@ -7,24 +7,24 @@

    Last Day

    -

    mailgraph

    -

    mailgraph

    -

    mailgraph

    +

    mailgraph

    +

    mailgraph

    +

    mailgraph

    Last Week

    -

    mailgraph

    -

    mailgraph

    -

    mailgraph

    +

    mailgraph

    +

    mailgraph

    +

    mailgraph

    Last Month

    -

    mailgraph

    -

    mailgraph

    -

    mailgraph

    +

    mailgraph

    +

    mailgraph

    +

    mailgraph

    Last Year

    -

    mailgraph

    -

    mailgraph

    -

    mailgraph

    +

    mailgraph

    +

    mailgraph

    +

    mailgraph

    From dce4058705714f30c4e8271d26315da75d6a14e3 Mon Sep 17 00:00:00 2001 From: John Supplee Date: Sun, 3 Mar 2019 00:34:41 +0200 Subject: [PATCH 21/72] process images returned from mailgraph --- management/daemon.py | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/management/daemon.py b/management/daemon.py index 54b1a565..b53c3c40 100755 --- a/management/daemon.py +++ b/management/daemon.py @@ -522,23 +522,29 @@ def privacy_status_set(): # Mailgraph -@app.route('/mailgraph/image.cgi') +@app.route('/mailgraph/image.cgi', methods=['GET']) @authorized_personnel_only def mailgraph(): if request.query_string: - print("QUERY_STRING=%s" % request.query_string) + app.logger.error("QUERY_STRING=%s" % request.query_string) code, bin_out = utils.shell( "check_output", ["/usr/share/mailgraph/mailgraph.cgi"], env={"QUERY_STRING": request.query_string}, - return_bytes=True + return_bytes=True, + trap=True ) if code != 0: return ('Error generating mailgraph image: %s' % request.query_string, 500) - return make_response(bin_out) + headers, image_bytes = bin_out.split(b'\n\n', 1) + response = make_response(image_bytes) + for line in headers.splitlines(): + name, value = line.decode("utf8").split(':', 1) + response.headers[name] = value + return response return ('Mailgraph: no image requested', 500) From 6883a60f5d024437a3a8b8a60688a5d73844a91e Mon Sep 17 00:00:00 2001 From: John Supplee Date: Sun, 3 Mar 2019 19:15:59 +0200 Subject: [PATCH 22/72] load images as base64 --- management/daemon.py | 9 +++----- management/templates/mailgraph.html | 33 +++++++++++++++++------------ 2 files changed, 23 insertions(+), 19 deletions(-) diff --git a/management/daemon.py b/management/daemon.py index b53c3c40..7e0d4936 100755 --- a/management/daemon.py +++ b/management/daemon.py @@ -521,7 +521,7 @@ def privacy_status_set(): return "OK" # Mailgraph - +import base64 @app.route('/mailgraph/image.cgi', methods=['GET']) @authorized_personnel_only def mailgraph(): @@ -540,11 +540,8 @@ def mailgraph(): return ('Error generating mailgraph image: %s' % request.query_string, 500) headers, image_bytes = bin_out.split(b'\n\n', 1) - response = make_response(image_bytes) - for line in headers.splitlines(): - name, value = line.decode("utf8").split(':', 1) - response.headers[name] = value - return response + + return base64.b64encode(image_bytes) return ('Mailgraph: no image requested', 500) diff --git a/management/templates/mailgraph.html b/management/templates/mailgraph.html index 086aa9cc..25428509 100644 --- a/management/templates/mailgraph.html +++ b/management/templates/mailgraph.html @@ -7,24 +7,24 @@

    Last Day

    -

    mailgraph

    -

    mailgraph

    -

    mailgraph

    +

    mailgraph

    +

    mailgraph

    +

    mailgraph

    Last Week

    -

    mailgraph

    -

    mailgraph

    -

    mailgraph

    +

    mailgraph

    +

    mailgraph

    +

    mailgraph

    Last Month

    -

    mailgraph

    -

    mailgraph

    -

    mailgraph

    +

    mailgraph

    +

    mailgraph

    +

    mailgraph

    Last Year

    -

    mailgraph

    -

    mailgraph

    -

    mailgraph

    +

    mailgraph

    +

    mailgraph

    +

    mailgraph

    @@ -34,7 +34,14 @@ From e29e3a5cba43ac3094dd0b776dc1cfd78b5c26ae Mon Sep 17 00:00:00 2001 From: John Supplee Date: Sun, 3 Mar 2019 19:23:58 +0200 Subject: [PATCH 23/72] fix QUERY_STRING to only have the image request --- management/daemon.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/management/daemon.py b/management/daemon.py index 7e0d4936..b3f5689c 100755 --- a/management/daemon.py +++ b/management/daemon.py @@ -526,12 +526,16 @@ import base64 @authorized_personnel_only def mailgraph(): if request.query_string: - app.logger.error("QUERY_STRING=%s" % request.query_string) + query = request.query_string + if '&' in query: + query = query.split('&')[0] + + app.logger.error("QUERY_STRING=%s" % query) code, bin_out = utils.shell( "check_output", ["/usr/share/mailgraph/mailgraph.cgi"], - env={"QUERY_STRING": request.query_string}, + env={"QUERY_STRING": query}, return_bytes=True, trap=True ) From 7600e727c118e92c0338bb32c2908addc2e06445 Mon Sep 17 00:00:00 2001 From: John Supplee Date: Sun, 3 Mar 2019 21:15:16 +0200 Subject: [PATCH 24/72] debugging of image display --- management/daemon.py | 10 ++++++---- management/templates/mailgraph.html | 3 ++- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/management/daemon.py b/management/daemon.py index b3f5689c..9925245d 100755 --- a/management/daemon.py +++ b/management/daemon.py @@ -1,5 +1,7 @@ import os, os.path, re, json, time import subprocess +import base64 +import sys from functools import wraps @@ -521,16 +523,16 @@ def privacy_status_set(): return "OK" # Mailgraph -import base64 + @app.route('/mailgraph/image.cgi', methods=['GET']) @authorized_personnel_only def mailgraph(): if request.query_string: - query = request.query_string + query = request.query_string.decode('utf-8', 'ignore') if '&' in query: query = query.split('&')[0] - app.logger.error("QUERY_STRING=%s" % query) + print("QUERY_STRING=%s" % query, file=sys.stderr) code, bin_out = utils.shell( "check_output", @@ -541,7 +543,7 @@ def mailgraph(): ) if code != 0: - return ('Error generating mailgraph image: %s' % request.query_string, 500) + return ('Error generating mailgraph image: %s' % query, 500) headers, image_bytes = bin_out.split(b'\n\n', 1) diff --git a/management/templates/mailgraph.html b/management/templates/mailgraph.html index 25428509..22d76088 100644 --- a/management/templates/mailgraph.html +++ b/management/templates/mailgraph.html @@ -34,8 +34,9 @@ From 0d4c6937927a498bf7faf8e468cba4d37eafe745 Mon Sep 17 00:00:00 2001 From: Michael Heuberger Date: Mon, 13 May 2019 00:10:34 +1200 Subject: [PATCH 44/72] Add missing login form method to keep LastPass happy (#1565) --- management/templates/login.html | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/management/templates/login.html b/management/templates/login.html index 04c27279..b6e74df6 100644 --- a/management/templates/login.html +++ b/management/templates/login.html @@ -17,13 +17,13 @@ sudo tools/mail.py user make-admin me@{{hostname}} {% endif %}
    - + {% endif %}

    Log in here for your Mail-in-a-Box control panel.

    -
    +
    @@ -76,7 +76,7 @@ function do_login() { "/me", "GET", { }, - function(response){ + function(response){ // This API call always succeeds. It returns a JSON object indicating // whether the request was authenticated or not. if (response.status != "ok") { From 4232a1205c300254be8052c1e9f105b519e44d66 Mon Sep 17 00:00:00 2001 From: jvolkenant Date: Wed, 15 May 2019 11:46:52 -0700 Subject: [PATCH 45/72] fix dovecot message about SSLv2 not supported by OpenSSL (#1580) --- setup/mail-dovecot.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/setup/mail-dovecot.sh b/setup/mail-dovecot.sh index 4bcc53aa..6098e295 100755 --- a/setup/mail-dovecot.sh +++ b/setup/mail-dovecot.sh @@ -80,11 +80,12 @@ tools/editconf.py /etc/dovecot/conf.d/10-auth.conf \ # Enable SSL, specify the location of the SSL certificate and private key files. # Disable obsolete SSL protocols and allow only good ciphers per http://baldric.net/2013/12/07/tls-ciphers-in-postfix-and-dovecot/. # Enable strong ssl dh parameters + tools/editconf.py /etc/dovecot/conf.d/10-ssl.conf \ ssl=required \ "ssl_cert=<$STORAGE_ROOT/ssl/ssl_certificate.pem" \ "ssl_key=<$STORAGE_ROOT/ssl/ssl_private_key.pem" \ - "ssl_protocols=!SSLv3 !SSLv2" \ + "ssl_protocols=!SSLv3" \ "ssl_cipher_list=ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:DHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-AES128-SHA256:ECDHE-RSA-AES128-SHA256:ECDHE-ECDSA-AES128-SHA:ECDHE-RSA-AES256-SHA384:ECDHE-RSA-AES128-SHA:ECDHE-ECDSA-AES256-SHA384:ECDHE-ECDSA-AES256-SHA:ECDHE-RSA-AES256-SHA:DHE-RSA-AES128-SHA256:DHE-RSA-AES128-SHA:DHE-RSA-AES256-SHA256:DHE-RSA-AES256-SHA:ECDHE-ECDSA-DES-CBC3-SHA:ECDHE-RSA-DES-CBC3-SHA:EDH-RSA-DES-CBC3-SHA:AES128-GCM-SHA256:AES256-GCM-SHA384:AES128-SHA256:AES256-SHA256:AES128-SHA:AES256-SHA:DES-CBC3-SHA:!DSS" \ "ssl_prefer_server_ciphers = yes" \ "ssl_dh_parameters_length = 2048" From 85e59245fdfd2a2e633a890927a89adba79e1bf5 Mon Sep 17 00:00:00 2001 From: cmharper <1422608+cmharper@users.noreply.github.com> Date: Wed, 15 May 2019 18:57:06 +0000 Subject: [PATCH 46/72] hide 'RTNETLINK answers: Network is unreachable' error message during setup if IPv6 is not available (#1576) --- setup/functions.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/setup/functions.sh b/setup/functions.sh index 1a74edfd..3bb96b7a 100644 --- a/setup/functions.sh +++ b/setup/functions.sh @@ -1,7 +1,7 @@ # Turn on "strict mode." See http://redsymbol.net/articles/unofficial-bash-strict-mode/. # -e: exit if any command unexpectedly fails. # -u: exit if we have a variable typo. -# -o pipefail: don't ignore errors in the non-last command in a pipeline +# -o pipefail: don't ignore errors in the non-last command in a pipeline set -euo pipefail function hide_output { @@ -127,7 +127,7 @@ function get_default_privateip { if [ "$1" == "6" ]; then target=2001:4860:4860::8888; fi # Get the route information. - route=$(ip -$1 -o route get $target | grep -v unreachable) + route=$(ip -$1 -o route get $target 2>/dev/null | grep -v unreachable) # Parse the address out of the route information. address=$(echo $route | sed "s/.* src \([^ ]*\).*/\1/") From c6fa0d23df986f9065108b58f984026204cd4e20 Mon Sep 17 00:00:00 2001 From: jvolkenant Date: Wed, 15 May 2019 11:58:40 -0700 Subject: [PATCH 47/72] check that munin-cron is not running (via cron) when it is run in setup, fixes #660 (#1579) --- setup/munin.sh | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/setup/munin.sh b/setup/munin.sh index 3cb1cd9d..df7af601 100755 --- a/setup/munin.sh +++ b/setup/munin.sh @@ -76,4 +76,8 @@ restart_service munin-node # generate initial statistics so the directory isn't empty # (We get "Pango-WARNING **: error opening config file '/root/.config/pango/pangorc': Permission denied" # if we don't explicitly set the HOME directory when sudo'ing.) -sudo -H -u munin munin-cron +# We check to see if munin-cron is already running, if it is, there is no need to run it simultaneously +# generating an error. +if [ ! -f /var/run/munin/munin-update.lock ]; then + sudo -H -u munin munin-cron +fi From 6e5ceab0f8d6496ae9dda5ae4cb134f16fa11626 Mon Sep 17 00:00:00 2001 From: jvolkenant Date: Wed, 15 May 2019 11:59:32 -0700 Subject: [PATCH 48/72] hide virtualenv output (#1578) --- setup/management.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup/management.sh b/setup/management.sh index f7621a8b..3a6e187b 100755 --- a/setup/management.sh +++ b/setup/management.sh @@ -38,7 +38,7 @@ inst_dir=/usr/local/lib/mailinabox mkdir -p $inst_dir venv=$inst_dir/env if [ ! -d $venv ]; then - virtualenv -ppython3 $venv + hide_output virtualenv -ppython3 $venv fi # Upgrade pip because the Ubuntu-packaged version is out of date. From 79759ea5a39c9398460d33453d986c90db9d7d54 Mon Sep 17 00:00:00 2001 From: jvolkenant Date: Sun, 16 Jun 2019 08:07:45 -0700 Subject: [PATCH 49/72] Upgrade Z-Push to 2.5.0 (#1581) --- CHANGELOG.md | 3 +++ setup/zpush.sh | 4 ++-- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 32f04763..14afe39d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,9 @@ In Development -------------- * Update to Roundcube 1.3.9. +Z-Push: + * Upgraded Z-Push from 2.4.4 to 2.5.0. + v0.41 (February 26, 2019) ------------------------- diff --git a/setup/zpush.sh b/setup/zpush.sh index 32fc4992..0cbd30ad 100755 --- a/setup/zpush.sh +++ b/setup/zpush.sh @@ -22,8 +22,8 @@ apt_install \ phpenmod -v php imap # Copy Z-Push into place. -VERSION=2.4.4 -TARGETHASH=104d44426852429dac8ec2783a4e9ad7752d4682 +VERSION=2.5.0 +TARGETHASH=30ce5c1af3f10939036361b6032d1187651b621e needs_update=0 #NODOC if [ ! -f /usr/local/lib/z-push/version ]; then needs_update=1 #NODOC From 193763f8f0a6e113053e2c8b5ae2bb12bbd5eb03 Mon Sep 17 00:00:00 2001 From: jvolkenant Date: Sun, 16 Jun 2019 08:10:53 -0700 Subject: [PATCH 50/72] Update to Nextcloud 15.0.8, Contacts to 3.1.1, and Calendar to 1.6.5 (#1577) * Update to Nextcloud 15.0.7, Contacts to 3.1.1, and Calendar to 1.6.5 * Enabled localhost-only insecure IMAP login for localhost Nextcloud auth * Add package php-imagick and BigInt conversion * added support for /cloud/oc[sm]-provider/ endpoint --- CHANGELOG.md | 11 ++++++++++- conf/nginx-primaryonly.conf | 9 +++++++++ setup/mail-dovecot.sh | 8 ++++++++ setup/nextcloud.sh | 38 +++++++++++++++++++++++++++---------- 4 files changed, 55 insertions(+), 11 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 14afe39d..faa4a918 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,7 +3,16 @@ CHANGELOG In Development -------------- - * Update to Roundcube 1.3.9. + +Mail: + +* Update to Roundcube 1.3.9. + +Contacts/Calendar: + +* Upgraded Nextcloud from 14.0.6 to 15.0.8. +* Upgraded Contacts from 2.1.8 to 3.1.1. +* Upgraded Calendar from 1.6.4 to 1.6.5. Z-Push: * Upgraded Z-Push from 2.4.4 to 2.5.0. diff --git a/conf/nginx-primaryonly.conf b/conf/nginx-primaryonly.conf index d8d912ca..288fce40 100644 --- a/conf/nginx-primaryonly.conf +++ b/conf/nginx-primaryonly.conf @@ -19,6 +19,7 @@ rewrite ^/cloud/$ /cloud/index.php; rewrite ^/cloud/(contacts|calendar|files)$ /cloud/index.php/apps/$1/ redirect; rewrite ^(/cloud/core/doc/[^\/]+/)$ $1/index.html; + rewrite ^(/cloud/oc[sm]-provider)/$ $1/index.php redirect; location /cloud/ { alias /usr/local/lib/owncloud/; location ~ ^/cloud/(build|tests|config|lib|3rdparty|templates|data|README)/ { @@ -27,6 +28,14 @@ location ~ ^/cloud/(?:\.|autotest|occ|issue|indie|db_|console) { deny all; } + # Enable paths for service and cloud federation discovery + # Resolves warning in Nextcloud Settings panel + location ~ ^/cloud/(oc[sm]-provider)?/([^/]+\.php)$ { + index index.php; + include fastcgi_params; + fastcgi_param SCRIPT_FILENAME /usr/local/lib/owncloud/$1/$2; + fastcgi_pass php-fpm; + } } location ~ ^(/cloud)((?:/ocs)?/[^/]+\.php)(/.*)?$ { # note: ~ has precendence over a regular location block diff --git a/setup/mail-dovecot.sh b/setup/mail-dovecot.sh index 6098e295..c3d2ee57 100755 --- a/setup/mail-dovecot.sh +++ b/setup/mail-dovecot.sh @@ -136,6 +136,14 @@ service lmtp { } } +# Enable imap-login on localhost to allow the user_external plugin +# for Nextcloud to do imap authentication. (See #1577) +service imap-login { + inet_listener imap { + address = 127.0.0.1 + port = 143 + } +} protocol imap { mail_max_userip_connections = 20 } diff --git a/setup/nextcloud.sh b/setup/nextcloud.sh index 9476bbf8..8dc8d35a 100755 --- a/setup/nextcloud.sh +++ b/setup/nextcloud.sh @@ -13,7 +13,8 @@ apt-get purge -qq -y owncloud* # we used to use the package manager apt_install php php-fpm \ php-cli php-sqlite3 php-gd php-imap php-curl php-pear curl \ - php-dev php-gd php-xml php-mbstring php-zip php-apcu php-json php-intl + php-dev php-gd php-xml php-mbstring php-zip php-apcu php-json \ + php-intl php-imagick InstallNextcloud() { @@ -39,14 +40,20 @@ InstallNextcloud() { # their github repositories. mkdir -p /usr/local/lib/owncloud/apps - wget_verify https://github.com/nextcloud/contacts/releases/download/v2.1.8/contacts.tar.gz b5d5bbee33f0c32b124b46cb6aaab90c695ac170 /tmp/contacts.tgz + wget_verify https://github.com/nextcloud/contacts/releases/download/v3.1.1/contacts.tar.gz a06bd967197dcb03c94ec1dbd698c037018669e5 /tmp/contacts.tgz tar xf /tmp/contacts.tgz -C /usr/local/lib/owncloud/apps/ rm /tmp/contacts.tgz - wget_verify https://github.com/nextcloud/calendar/releases/download/v1.6.4/calendar.tar.gz d8a7950dba14803472b6c19625a8ceb23d6fd4ef /tmp/calendar.tgz + wget_verify https://github.com/nextcloud/calendar/releases/download/v1.6.5/calendar.tar.gz 79941255521a5172f7e4ce42dc7773838b5ede2f /tmp/calendar.tgz tar xf /tmp/calendar.tgz -C /usr/local/lib/owncloud/apps/ rm /tmp/calendar.tgz + # Starting with Nextcloud 15, the app user_external is no longer included in Nextcloud core, + # we will install from their github repository. + wget_verify https://github.com/nextcloud/user_external/releases/download/v0.6.3/user_external-0.6.3.tar.gz 0f756d35fef6b64a177d6a16020486b76ea5799c /tmp/user_external.tgz + tar -xf /tmp/user_external.tgz -C /usr/local/lib/owncloud/apps/ + rm /tmp/user_external.tgz + # Fix weird permissions. chmod 750 /usr/local/lib/owncloud/{apps,config} @@ -75,12 +82,14 @@ InstallNextcloud() { # Add missing indices. NextCloud didn't include this in the normal upgrade because it might take some time. sudo -u www-data php /usr/local/lib/owncloud/occ db:add-missing-indices + + # Run conversion to BigInt identifiers, this process may take some time on large tables. + sudo -u www-data php /usr/local/lib/owncloud/occ db:convert-filecache-bigint --no-interaction fi } -nextcloud_ver=14.0.6 -nextcloud_hash=4e43a57340f04c2da306c8eea98e30040399ae5a - +nextcloud_ver=15.0.8 +nextcloud_hash=4129d8d4021c435f2e86876225fb7f15adf764a3 # Check if Nextcloud dir exist, and check if version matches nextcloud_ver (if either doesn't - install/upgrade) if [ ! -d /usr/local/lib/owncloud/ ] \ || ! grep -q $nextcloud_ver /usr/local/lib/owncloud/version.php; then @@ -115,6 +124,11 @@ if [ ! -d /usr/local/lib/owncloud/ ] \ echo "Upgrades from Mail-in-a-Box prior to v0.28 (dated July 30, 2018) with Nextcloud < 13.0.6 (you have ownCloud 10, 11 or 12) are not supported. Upgrade to Mail-in-a-Box version v0.30 first. Setup aborting." exit 1 fi + # During the upgrade from Nextcloud 14 to 15, user_external may cause the upgrade to fail. + # We will disable it here before the upgrade and install it again after the upgrade. + if grep -q "OC_VersionString = '14\." /usr/local/lib/owncloud/version.php; then + hide_output sudo -u www-data php /usr/local/lib/owncloud/console.php app:disable user_external + fi fi InstallNextcloud $nextcloud_ver $nextcloud_hash @@ -142,10 +156,12 @@ if [ ! -f $STORAGE_ROOT/owncloud/owncloud.db ]; then 'overwritewebroot' => '/cloud', 'overwrite.cli.url' => '/cloud', 'user_backends' => array( - array( - 'class'=>'OC_User_IMAP', - 'arguments'=>array('{127.0.0.1:993/imap/ssl/novalidate-cert}') - ) + array( + 'class' => 'OC_User_IMAP', + 'arguments' => array( + '127.0.0.1', 143, null + ), + ), ), 'memcache.local' => '\OC\Memcache\APCu', 'mail_smtpmode' => 'sendmail', @@ -217,6 +233,8 @@ include("$STORAGE_ROOT/owncloud/config.php"); \$CONFIG['mail_domain'] = '$PRIMARY_HOSTNAME'; +\$CONFIG['user_backends'] = array(array('class' => 'OC_User_IMAP','arguments' => array('127.0.0.1', 143, null),),); + echo " Date: Sun, 16 Jun 2019 11:40:40 -0400 Subject: [PATCH 51/72] changelog updates --- CHANGELOG.md | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index faa4a918..0f232670 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,18 +4,20 @@ CHANGELOG In Development -------------- -Mail: +Changes: -* Update to Roundcube 1.3.9. +* Decreased the minimum supported RAM to 502 Mb. +* Improved mail client autoconfiguration. +* Added support for S3-compatible backup services besides Amazon S3. +* Fixed the control panel login page to let LastPass save passwords. +* Fixed an error in the user privileges API. +* Silenced some spurrious messages. -Contacts/Calendar: +Software updates: -* Upgraded Nextcloud from 14.0.6 to 15.0.8. -* Upgraded Contacts from 2.1.8 to 3.1.1. -* Upgraded Calendar from 1.6.4 to 1.6.5. - -Z-Push: - * Upgraded Z-Push from 2.4.4 to 2.5.0. +* Upgraded Roundcube from 1.3.8 to 1.3.9. +* Upgraded Nextcloud from 14.0.6 to 15.0.8 (with Contacts from 2.1.8 to 3.1.1 and Calendar from 1.6.4 to 1.6.5). +* Upgraded Z-Push from 2.4.4 to 2.5.0. v0.41 (February 26, 2019) ------------------------- From 39fd4ce16c6b665f1df886083dcc0a6afda6c08e Mon Sep 17 00:00:00 2001 From: Joshua Tauberer Date: Thu, 4 Jul 2019 21:34:55 -0400 Subject: [PATCH 52/72] v0.42 --- CHANGELOG.md | 4 ++-- README.md | 4 ++-- setup/bootstrap.sh | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0f232670..15311c84 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,8 +1,8 @@ CHANGELOG ========= -In Development --------------- +v0.42 (July 4, 2019) +-------------------- Changes: diff --git a/README.md b/README.md index 01997fd4..d93a0847 100644 --- a/README.md +++ b/README.md @@ -58,7 +58,7 @@ by him: $ curl -s https://keybase.io/joshdata/key.asc | gpg --import gpg: key C10BDD81: public key "Joshua Tauberer " imported - $ git verify-tag v0.41 + $ git verify-tag v0.42 gpg: Signature made ..... using RSA key ID C10BDD81 gpg: Good signature from "Joshua Tauberer " gpg: WARNING: This key is not certified with a trusted signature! @@ -71,7 +71,7 @@ and on his [personal homepage](https://razor.occams.info/). (Of course, if this Checkout the tag corresponding to the most recent release: - $ git checkout v0.41 + $ git checkout v0.42 Begin the installation. diff --git a/setup/bootstrap.sh b/setup/bootstrap.sh index 74bf5e16..b8dfcc64 100644 --- a/setup/bootstrap.sh +++ b/setup/bootstrap.sh @@ -20,7 +20,7 @@ if [ -z "$TAG" ]; then # want to display in status checks. if [ "`lsb_release -d | sed 's/.*:\s*//' | sed 's/18\.04\.[0-9]/18.04/' `" == "Ubuntu 18.04 LTS" ]; then # This machine is running Ubuntu 18.04. - TAG=v0.41 + TAG=v0.42 elif [ "`lsb_release -d | sed 's/.*:\s*//' | sed 's/14\.04\.[0-9]/14.04/' `" == "Ubuntu 14.04 LTS" ]; then # This machine is running Ubuntu 14.04. From 5fc1944f04dba9e111f3c8787a5969d4c296468f Mon Sep 17 00:00:00 2001 From: Joshua Tauberer Date: Fri, 5 Jul 2019 11:56:54 -0400 Subject: [PATCH 53/72] pull v0.42, go back to v0.41 --- setup/bootstrap.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup/bootstrap.sh b/setup/bootstrap.sh index b8dfcc64..74bf5e16 100644 --- a/setup/bootstrap.sh +++ b/setup/bootstrap.sh @@ -20,7 +20,7 @@ if [ -z "$TAG" ]; then # want to display in status checks. if [ "`lsb_release -d | sed 's/.*:\s*//' | sed 's/18\.04\.[0-9]/18.04/' `" == "Ubuntu 18.04 LTS" ]; then # This machine is running Ubuntu 18.04. - TAG=v0.42 + TAG=v0.41 elif [ "`lsb_release -d | sed 's/.*:\s*//' | sed 's/14\.04\.[0-9]/14.04/' `" == "Ubuntu 14.04 LTS" ]; then # This machine is running Ubuntu 14.04. From fd5b11823ce01cd7e9bf68a247758adb46f4f146 Mon Sep 17 00:00:00 2001 From: jvolkenant Date: Wed, 10 Jul 2019 03:28:37 -0700 Subject: [PATCH 54/72] Add AAAA records for autodiscover & autoconfig (#1606) --- management/dns_update.py | 22 ++++++++++++++-------- 1 file changed, 14 insertions(+), 8 deletions(-) diff --git a/management/dns_update.py b/management/dns_update.py index 006a00c2..7e006d0b 100755 --- a/management/dns_update.py +++ b/management/dns_update.py @@ -288,14 +288,20 @@ def build_zone(domain, all_domains, additional_records, www_redirect_domains, en if not has_rec(qname, "SRV"): records.append((qname, "SRV", "0 0 443 " + env["PRIMARY_HOSTNAME"] + ".", "Recommended. Specifies the hostname of the server that handles CardDAV/CalDAV services for email addresses on this domain.")) - # Adds autoconfiguration A records for all domains. - # This allows the following clients to automatically configure email addresses in the respective applications. - # autodiscover.* - Z-Push ActiveSync Autodiscover - # autoconfig.* - Thunderbird Autoconfig - if not has_rec("autodiscover", "A"): - records.append(("autodiscover", "A", env["PUBLIC_IP"], "Provides email configuration autodiscovery support for Z-Push ActiveSync Autodiscover.")) - if not has_rec("autoconfig", "A"): - records.append(("autoconfig", "A", env["PUBLIC_IP"], "Provides email configuration autodiscovery support for Thunderbird Autoconfig.")) + # Adds autoconfiguration A records for all domains. + # This allows the following clients to automatically configure email addresses in the respective applications. + # autodiscover.* - Z-Push ActiveSync Autodiscover + # autoconfig.* - Thunderbird Autoconfig + autodiscover_records = [ + ("autodiscover", "A", env["PUBLIC_IP"], "Provides email configuration autodiscovery support for Z-Push ActiveSync Autodiscover."), + ("autodiscover", "AAAA", env["PUBLIC_IPV6"], "Provides email configuration autodiscovery support for Z-Push ActiveSync Autodiscover."), + ("autoconfig", "A", env["PUBLIC_IP"], "Provides email configuration autodiscovery support for Thunderbird Autoconfig."), + ("autoconfig", "AAAA", env["PUBLIC_IPV6"], "Provides email configuration autodiscovery support for Thunderbird Autoconfig.") + ] + for qname, rtype, value, explanation in autodiscover_records: + if value is None or value.strip() == "": continue # skip IPV6 if not set + if not has_rec(qname, rtype): + records.append((qname, rtype, value, explanation)) # Sort the records. The None records *must* go first in the nsd zone file. Otherwise it doesn't matter. records.sort(key = lambda rec : list(reversed(rec[0].split(".")) if rec[0] is not None else "")) From bea5eb0dda083478ce0a27d0b8e6740bae26e2ab Mon Sep 17 00:00:00 2001 From: jvolkenant Date: Fri, 12 Jul 2019 03:41:16 -0700 Subject: [PATCH 55/72] Add interm upgrade step from Nextcloud 13 -> 14 (#1605) --- setup/nextcloud.sh | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/setup/nextcloud.sh b/setup/nextcloud.sh index 8dc8d35a..9f36ee56 100755 --- a/setup/nextcloud.sh +++ b/setup/nextcloud.sh @@ -50,9 +50,11 @@ InstallNextcloud() { # Starting with Nextcloud 15, the app user_external is no longer included in Nextcloud core, # we will install from their github repository. - wget_verify https://github.com/nextcloud/user_external/releases/download/v0.6.3/user_external-0.6.3.tar.gz 0f756d35fef6b64a177d6a16020486b76ea5799c /tmp/user_external.tgz - tar -xf /tmp/user_external.tgz -C /usr/local/lib/owncloud/apps/ - rm /tmp/user_external.tgz + if [[ $version =~ ^15 ]]; then + wget_verify https://github.com/nextcloud/user_external/releases/download/v0.6.3/user_external-0.6.3.tar.gz 0f756d35fef6b64a177d6a16020486b76ea5799c /tmp/user_external.tgz + tar -xf /tmp/user_external.tgz -C /usr/local/lib/owncloud/apps/ + rm /tmp/user_external.tgz + fi # Fix weird permissions. chmod 750 /usr/local/lib/owncloud/{apps,config} @@ -124,6 +126,11 @@ if [ ! -d /usr/local/lib/owncloud/ ] \ echo "Upgrades from Mail-in-a-Box prior to v0.28 (dated July 30, 2018) with Nextcloud < 13.0.6 (you have ownCloud 10, 11 or 12) are not supported. Upgrade to Mail-in-a-Box version v0.30 first. Setup aborting." exit 1 fi + # If we are running Nextcloud 13, upgrade to Nextcloud 14 + if grep -q "OC_VersionString = '13\." /usr/local/lib/owncloud/version.php; then + InstallNextcloud 14.0.6 4e43a57340f04c2da306c8eea98e30040399ae5a + + fi # During the upgrade from Nextcloud 14 to 15, user_external may cause the upgrade to fail. # We will disable it here before the upgrade and install it again after the upgrade. if grep -q "OC_VersionString = '14\." /usr/local/lib/owncloud/version.php; then From e37768ca8680106b037822975445de88e3e0da14 Mon Sep 17 00:00:00 2001 From: Joshua Tauberer Date: Sat, 3 Aug 2019 11:49:32 -0400 Subject: [PATCH 56/72] v0.42b --- CHANGELOG.md | 6 ++++-- README.md | 4 ++-- setup/bootstrap.sh | 2 +- 3 files changed, 7 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 15311c84..a07304b6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,8 +1,8 @@ CHANGELOG ========= -v0.42 (July 4, 2019) --------------------- +v0.42b (August 3, 2019) +----------------------- Changes: @@ -19,6 +19,8 @@ Software updates: * Upgraded Nextcloud from 14.0.6 to 15.0.8 (with Contacts from 2.1.8 to 3.1.1 and Calendar from 1.6.4 to 1.6.5). * Upgraded Z-Push from 2.4.4 to 2.5.0. +Note that v0.42 (July 4, 2019) was pulled shortly after it was released to fix a Nextcloud upgrade issue. + v0.41 (February 26, 2019) ------------------------- diff --git a/README.md b/README.md index d93a0847..925e62aa 100644 --- a/README.md +++ b/README.md @@ -58,7 +58,7 @@ by him: $ curl -s https://keybase.io/joshdata/key.asc | gpg --import gpg: key C10BDD81: public key "Joshua Tauberer " imported - $ git verify-tag v0.42 + $ git verify-tag v0.42b gpg: Signature made ..... using RSA key ID C10BDD81 gpg: Good signature from "Joshua Tauberer " gpg: WARNING: This key is not certified with a trusted signature! @@ -71,7 +71,7 @@ and on his [personal homepage](https://razor.occams.info/). (Of course, if this Checkout the tag corresponding to the most recent release: - $ git checkout v0.42 + $ git checkout v0.42b Begin the installation. diff --git a/setup/bootstrap.sh b/setup/bootstrap.sh index 74bf5e16..1135107b 100644 --- a/setup/bootstrap.sh +++ b/setup/bootstrap.sh @@ -20,7 +20,7 @@ if [ -z "$TAG" ]; then # want to display in status checks. if [ "`lsb_release -d | sed 's/.*:\s*//' | sed 's/18\.04\.[0-9]/18.04/' `" == "Ubuntu 18.04 LTS" ]; then # This machine is running Ubuntu 18.04. - TAG=v0.41 + TAG=v0.42b elif [ "`lsb_release -d | sed 's/.*:\s*//' | sed 's/14\.04\.[0-9]/14.04/' `" == "Ubuntu 14.04 LTS" ]; then # This machine is running Ubuntu 14.04. From 0657f9e875895adbc46cb36b72c6aa3f55a88e45 Mon Sep 17 00:00:00 2001 From: captainwasabi Date: Tue, 13 Aug 2019 05:47:11 -0400 Subject: [PATCH 57/72] add proper check for DNS error in list_target_files (#1625) The elif needed to check to see if the string was in the listing of results of the shell command. As it was the conditional was just the string which always evaluates to true and was therefore giving a misleading error message. --- management/backup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/management/backup.py b/management/backup.py index 93136bf5..cd1ee6fc 100755 --- a/management/backup.py +++ b/management/backup.py @@ -406,7 +406,7 @@ def list_target_files(config): reason = "Provided path {} is invalid.".format(target_path) elif 'Network is unreachable' in listing: reason = "The IP address {} is unreachable.".format(target.hostname) - elif 'Could not resolve hostname': + elif 'Could not resolve hostname' in listing: reason = "The hostname {} cannot be resolved.".format(target.hostname) else: reason = "Unknown error." \ From c4cb828f6562ad03497e64acd2ebb0af7294d36c Mon Sep 17 00:00:00 2001 From: captainwasabi Date: Tue, 13 Aug 2019 05:57:05 -0400 Subject: [PATCH 58/72] Fix rsync backup options string: extraneous single quotes causing problems (#1629) The resulting command had nested single quotes which doesn't work I think this fixes all/most of the issues in #1627. I am getting a full backup, then the next time it's run I get an incremental. running from the CLI with --status looks good, --verify looks good, and --list looks good. --- management/backup.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/management/backup.py b/management/backup.py index cd1ee6fc..11cdbb8a 100755 --- a/management/backup.py +++ b/management/backup.py @@ -15,8 +15,8 @@ from exclusiveprocess import Lock from utils import load_environment, shell, wait_for_service, fix_boto rsync_ssh_options = [ - "--ssh-options='-i /root/.ssh/id_rsa_miab'", - "--rsync-options=-e \"/usr/bin/ssh -oStrictHostKeyChecking=no -oBatchMode=yes -p 22 -i /root/.ssh/id_rsa_miab\"", + "--ssh-options= -i /root/.ssh/id_rsa_miab", + "--rsync-options= -e \"/usr/bin/ssh -oStrictHostKeyChecking=no -oBatchMode=yes -p 22 -i /root/.ssh/id_rsa_miab\"", ] def backup_status(env): From 7636a7b40b00ea8e38512767388feef797b14fe5 Mon Sep 17 00:00:00 2001 From: "John R. Supplee" Date: Wed, 28 Aug 2019 17:50:38 -0400 Subject: [PATCH 59/72] Update README for v0.42b --- README.md | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 88b3eef8..79e57083 100644 --- a/README.md +++ b/README.md @@ -20,7 +20,7 @@ Mailbox size recalculation by Dovecot can be forced using the command: Please report any bugs on github. -Installing v0.41-quota +Installing v0.42b-quota ---------------------- To install the latest version, log into your box and execute the following commands: @@ -34,7 +34,7 @@ Follow the standard directions for setting up an MiaB installation. There are n The default quota is set to `0` which means unlimited. If you want to set a different default quota, follow the directions above. -Upgrading v0.41 to v.0.41-quota +Upgrading v0.4x to v.0.42b-quota ------------------------------- This is experimental software. You have been warned. @@ -69,6 +69,10 @@ Issues Changes ------- +### v0.42b-quota-0.18-beta + +* Update to v0.42b of Mail-in-a-Box + ### v0.41-quota-0.18-beta * Bump version to add a new annotated tag. The last version had a plain tag which is not seen when checking for the latest version. From 295d481603025a603e3f39cb8da8629ba569f98f Mon Sep 17 00:00:00 2001 From: cmharper <1422608+cmharper@users.noreply.github.com> Date: Sat, 31 Aug 2019 11:55:38 +0000 Subject: [PATCH 60/72] Upgraded roundcube to 1.3.10 (#1634) --- setup/webmail.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/setup/webmail.sh b/setup/webmail.sh index cbe6bfca..6cbe55f9 100755 --- a/setup/webmail.sh +++ b/setup/webmail.sh @@ -28,8 +28,8 @@ apt_install \ # Install Roundcube from source if it is not already present or if it is out of date. # Combine the Roundcube version number with the commit hash of plugins to track # whether we have the latest version of everything. -VERSION=1.3.9 -HASH=02850972b416bbfa1c13580f16d06fd7ae2774aa +VERSION=1.3.10 +HASH=431625fc737e301f9b7e502cccc61e50a24786b8 PERSISTENT_LOGIN_VERSION=dc5ca3d3f4415cc41edb2fde533c8a8628a94c76 HTML5_NOTIFIER_VERSION=4b370e3cd60dabd2f428a26f45b677ad1b7118d5 CARDDAV_VERSION=3.0.3 From 08021ea19f1a5395d09e2bc557cbf8b5cdef18d4 Mon Sep 17 00:00:00 2001 From: Snacho Date: Sat, 31 Aug 2019 14:58:12 +0300 Subject: [PATCH 61/72] Fix an issue when Secondary NS has multiple A records (#1633) If a custom secondary NS server has multiple A records status_checks.py will fail with a timeout and Web UI won't load. --- management/status_checks.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/management/status_checks.py b/management/status_checks.py index 6f9bb1ef..a9d0595c 100755 --- a/management/status_checks.py +++ b/management/status_checks.py @@ -486,10 +486,12 @@ def check_dns_zone(domain, env, output, dns_zonefiles): if custom_secondary_ns and not probably_external_dns: for ns in custom_secondary_ns: # We must first resolve the nameserver to an IP address so we can query it. - ns_ip = query_dns(ns, "A") - if not ns_ip: + ns_ips = query_dns(ns, "A") + if not ns_ips: output.print_error("Secondary nameserver %s is not valid (it doesn't resolve to an IP address)." % ns) continue + # Choose the first IP if nameserver returns multiple + ns_ip = ns_ips.split('; ')[0] # Now query it to see what it says about this domain. ip = query_dns(domain, "A", at=ns_ip, nxdomain=None) From c7377e602d06a792b97a5a1047bbdeb32bdd377a Mon Sep 17 00:00:00 2001 From: Kim Schulz Date: Sat, 31 Aug 2019 14:00:18 +0200 Subject: [PATCH 62/72] make it possible to use subnet addresses for axfr (#1616) it is sometimes needed to be able to set axfr to more than just one ip address. This can be done with multiple xfr: in the secondary dns input but if you need to add an entire subnet segment (xxx.xxx.xxx.0/yy) then it will not work. With this patch it is now possible to use a subnet as input for xfr the same way as if it was an ip address. --- management/dns_update.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/management/dns_update.py b/management/dns_update.py index 7e006d0b..7eed50b5 100755 --- a/management/dns_update.py +++ b/management/dns_update.py @@ -903,8 +903,12 @@ def set_secondary_dns(hostnames, env): else: # Validate IP address. try: - v = ipaddress.ip_address(item[4:]) # raises a ValueError if there's a problem - if not isinstance(v, ipaddress.IPv4Address): raise ValueError("That's an IPv6 address.") + if "/" in item[4:]: + v = ipaddress.ip_network(item[4:] # raises a ValueError if there's a problem + if not isinstance(v, ipaddress.IPv4Network): raise ValueError("That's an IPv6 subnet.") + else: + v = ipaddress.ip_address(item[4:]) # raises a ValueError if there's a problem + if not isinstance(v, ipaddress.IPv4Address): raise ValueError("That's an IPv6 address.") except ValueError: raise ValueError("'%s' is not an IPv4 address." % item[4:]) From 1d6793d12434a407d47efa7dc276f63227ad29e5 Mon Sep 17 00:00:00 2001 From: Michael Kroes Date: Sat, 31 Aug 2019 14:38:41 +0200 Subject: [PATCH 63/72] Update the Postgrey whitelist to a newer version monthly (#1611) Automatically update the Postgrey whitelist to a newer version once a month. --- CHANGELOG.md | 5 +++++ setup/mail-postfix.sh | 29 ++++++++++++++++++++++++++++- 2 files changed, 33 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a07304b6..7c1849fe 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,11 @@ CHANGELOG ========= +In Development +-------------- + +* Fetch an updated whitelist for Postgrey on a monthly basis. + v0.42b (August 3, 2019) ----------------------- diff --git a/setup/mail-postfix.sh b/setup/mail-postfix.sh index 4d66cd58..283d08af 100755 --- a/setup/mail-postfix.sh +++ b/setup/mail-postfix.sh @@ -208,7 +208,34 @@ tools/editconf.py /etc/postfix/main.cf \ # e-mails really latter, delay of greylisting has been set to # 180 seconds (default is 300 seconds). tools/editconf.py /etc/default/postgrey \ - POSTGREY_OPTS=\"'--inet=127.0.0.1:10023 --delay=180'\" + POSTGREY_OPTS=\"'--inet=127.0.0.1:10023 --delay=180 --whitelist-recipients=/etc/postgrey/whitelist_clients'\" + + +# We are going to setup a newer whitelist for postgrey, the version included in the distribution is old +cat > /etc/cron.daily/mailinabox-postgrey-whitelist << EOF; +#!/bin/bash + +# Mail-in-a-Box + +# check we have a postgrey_whitelist_clients file and that it is not older than 28 days +if [ ! -f /etc/postgrey/whitelist_clients ] || find /etc/postgrey/whitelist_clients -mtime +28 > /dev/null ; then + # ok we need to update the file, so lets try to fetch it + if curl https://postgrey.schweikert.ch/pub/postgrey_whitelist_clients --output /tmp/postgrey_whitelist_clients -sS --fail > /dev/null 2>&1 ; then + # if fetching hasn't failed yet then check it is a plain text file + # curl manual states that --fail sometimes still produces output + # this final check will at least check the output is not html + # before moving it into place + if [ "\$(file -b --mime-type /tmp/postgrey_whitelist_clients)" == "text/plain" ]; then + mv /tmp/postgrey_whitelist_clients /etc/postgrey/whitelist_clients + service postgrey restart + else + rm /tmp/postgrey_whitelist_clients + fi + fi +fi +EOF +chmod +x /etc/cron.daily/mailinabox-postgrey-whitelist +/etc/cron.daily/mailinabox-postgrey-whitelist # Increase the message size limit from 10MB to 128MB. # The same limit is specified in nginx.conf for mail submitted via webmail and Z-Push. From d6becddbe5f120818fca111187a7313d4944fb71 Mon Sep 17 00:00:00 2001 From: jvolkenant Date: Sat, 31 Aug 2019 05:50:36 -0700 Subject: [PATCH 64/72] Change Nextcloud upgrade logic to look at STORAGE_ROOT's config.php version vs /usr/local's version.php version (#1632) * Download and verify Nextcloud download before deleting old install directory * Changed install logic to look at config.php and not version.php for database version number. When restoring from a backup, config.php in STORAGE_ROOT will hold the Nextcloud version that corresponds to the user's database and version.php in /usr/local won't even exist, so we were missing Nextcloud migration steps. In other cases they should be the same. --- setup/nextcloud.sh | 47 ++++++++++++++++++++++++++++------------------ 1 file changed, 29 insertions(+), 18 deletions(-) diff --git a/setup/nextcloud.sh b/setup/nextcloud.sh index 9f36ee56..d5a8e7c5 100755 --- a/setup/nextcloud.sh +++ b/setup/nextcloud.sh @@ -25,12 +25,12 @@ InstallNextcloud() { echo "Upgrading to Nextcloud version $version" echo + # Download and verify + wget_verify https://download.nextcloud.com/server/releases/nextcloud-$version.zip $hash /tmp/nextcloud.zip + # Remove the current owncloud/Nextcloud rm -rf /usr/local/lib/owncloud - # Download and verify - wget_verify https://download.nextcloud.com/server/releases/nextcloud-$version.zip $hash /tmp/nextcloud.zip - # Extract ownCloud/Nextcloud unzip -q /tmp/nextcloud.zip -d /usr/local/lib mv /usr/local/lib/nextcloud /usr/local/lib/owncloud @@ -90,11 +90,26 @@ InstallNextcloud() { fi } +# Nextcloud Version to install. Checks are done down below to step through intermediate versions. nextcloud_ver=15.0.8 nextcloud_hash=4129d8d4021c435f2e86876225fb7f15adf764a3 -# Check if Nextcloud dir exist, and check if version matches nextcloud_ver (if either doesn't - install/upgrade) -if [ ! -d /usr/local/lib/owncloud/ ] \ - || ! grep -q $nextcloud_ver /usr/local/lib/owncloud/version.php; then + +# Current Nextcloud Version, #1623 +# Checking /usr/local/lib/owncloud/version.php shows version of the Nextcloud application, not the DB +# $STORAGE_ROOT/owncloud is kept together even during a backup. It is better to rely on config.php than +# version.php since the restore procedure can leave the system in a state where you have a newer Nextcloud +# application version than the database. + +# If config.php exists, get version number, otherwise CURRENT_NEXTCLOUD_VER is empty. +if [ -f "$STORAGE_ROOT/owncloud/config.php" ]; then + CURRENT_NEXTCLOUD_VER=$(php -r "include(\"$STORAGE_ROOT/owncloud/config.php\"); echo(\$CONFIG['version']);") +else + CURRENT_NEXTCLOUD_VER="" +fi + +# If the Nextcloud directory is missing (never been installed before, or the nextcloud version to be installed is different +# from the version currently installed, do the install/upgrade +if [ ! -d /usr/local/lib/owncloud/ ] || [[ ! ${CURRENT_NEXTCLOUD_VER} =~ ^$nextcloud_ver ]]; then # Stop php-fpm if running. If theyre not running (which happens on a previously failed install), dont bail. service php7.2-fpm stop &> /dev/null || /bin/true @@ -115,25 +130,21 @@ if [ ! -d /usr/local/lib/owncloud/ ] \ fi # If ownCloud or Nextcloud was previously installed.... - if [ -e /usr/local/lib/owncloud/version.php ]; then + if [ ! -z ${CURRENT_NEXTCLOUD_VER} ]; then # Database migrations from ownCloud are no longer possible because ownCloud cannot be run under # PHP 7. - if grep -q "OC_VersionString = '[89]\." /usr/local/lib/owncloud/version.php; then + if [[ ${CURRENT_NEXTCLOUD_VER} =~ ^[89] ]]; then echo "Upgrades from Mail-in-a-Box prior to v0.28 (dated July 30, 2018) with Nextcloud < 13.0.6 (you have ownCloud 8 or 9) are not supported. Upgrade to Mail-in-a-Box version v0.30 first. Setup aborting." exit 1 - fi - if grep -q "OC_VersionString = '1[012]\." /usr/local/lib/owncloud/version.php; then + elif [[ ${CURRENT_NEXTCLOUD_VER} =~ ^1[012] ]]; then echo "Upgrades from Mail-in-a-Box prior to v0.28 (dated July 30, 2018) with Nextcloud < 13.0.6 (you have ownCloud 10, 11 or 12) are not supported. Upgrade to Mail-in-a-Box version v0.30 first. Setup aborting." exit 1 - fi - # If we are running Nextcloud 13, upgrade to Nextcloud 14 - if grep -q "OC_VersionString = '13\." /usr/local/lib/owncloud/version.php; then + elif [[ ${CURRENT_NEXTCLOUD_VER} =~ ^13 ]]; then + # If we are running Nextcloud 13, upgrade to Nextcloud 14 InstallNextcloud 14.0.6 4e43a57340f04c2da306c8eea98e30040399ae5a - - fi - # During the upgrade from Nextcloud 14 to 15, user_external may cause the upgrade to fail. - # We will disable it here before the upgrade and install it again after the upgrade. - if grep -q "OC_VersionString = '14\." /usr/local/lib/owncloud/version.php; then + elif [[ ${CURRENT_NEXTCLOUD_VER} =~ ^14 ]]; then + # During the upgrade from Nextcloud 14 to 15, user_external may cause the upgrade to fail. + # We will disable it here before the upgrade and install it again after the upgrade. hide_output sudo -u www-data php /usr/local/lib/owncloud/console.php app:disable user_external fi fi From 3ff9817325d6d113c03f2d8f3b1eef9623b07e87 Mon Sep 17 00:00:00 2001 From: Joshua Tauberer Date: Sat, 31 Aug 2019 08:15:38 -0400 Subject: [PATCH 65/72] document the xfr: CIDR notation, fix spaces vs tabs and syntax error, broken by c7377e602d06a792b97a5a1047bbdeb32bdd377a, #1616 --- management/dns_update.py | 14 +++++++------- management/templates/custom-dns.html | 2 +- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/management/dns_update.py b/management/dns_update.py index 7eed50b5..dba6dbc1 100755 --- a/management/dns_update.py +++ b/management/dns_update.py @@ -903,14 +903,14 @@ def set_secondary_dns(hostnames, env): else: # Validate IP address. try: - if "/" in item[4:]: - v = ipaddress.ip_network(item[4:] # raises a ValueError if there's a problem - if not isinstance(v, ipaddress.IPv4Network): raise ValueError("That's an IPv6 subnet.") - else: - v = ipaddress.ip_address(item[4:]) # raises a ValueError if there's a problem - if not isinstance(v, ipaddress.IPv4Address): raise ValueError("That's an IPv6 address.") + if "/" in item[4:]: + v = ipaddress.ip_network(item[4:]) # raises a ValueError if there's a problem + if not isinstance(v, ipaddress.IPv4Network): raise ValueError("That's an IPv6 subnet.") + else: + v = ipaddress.ip_address(item[4:]) # raises a ValueError if there's a problem + if not isinstance(v, ipaddress.IPv4Address): raise ValueError("That's an IPv6 address.") except ValueError: - raise ValueError("'%s' is not an IPv4 address." % item[4:]) + raise ValueError("'%s' is not an IPv4 address or subnet." % item[4:]) # Set. set_custom_dns_record("_secondary_nameserver", "A", " ".join(hostnames), "set", env) diff --git a/management/templates/custom-dns.html b/management/templates/custom-dns.html index c838d32a..56facdc9 100644 --- a/management/templates/custom-dns.html +++ b/management/templates/custom-dns.html @@ -90,7 +90,7 @@

    Multiple secondary servers can be separated with commas or spaces (i.e., ns2.hostingcompany.com ns3.hostingcompany.com). - To enable zone transfers to additional servers without listing them as secondary nameservers, add xfr:IPADDRESS. + To enable zone transfers to additional servers without listing them as secondary nameservers, add an IP address or subnet using xfr:10.20.30.40 or xfr:10.20.30.40/24.

    Examples:

    From a05a33051e3a8361fae04558ac910b629945c63c Mon Sep 17 00:00:00 2001 From: John Supplee Date: Fri, 4 Oct 2019 17:46:10 +0200 Subject: [PATCH 72/72] Increment quota version to 0.19-beta --- README.md | 10 +++++++--- setup/bootstrap.sh | 2 +- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index e8de1fd7..d74c5071 100644 --- a/README.md +++ b/README.md @@ -21,7 +21,7 @@ Please report any bugs on github. Installing v0.42b-quota ----------------------- +----------------------- To install the latest version, log into your box and execute the following commands: @@ -35,7 +35,7 @@ The default quota is set to `0` which means unlimited. If you want to set a dif Upgrading v0.4x to v.0.42b-quota -------------------------------- +-------------------------------- This is experimental software. You have been warned. @@ -51,7 +51,7 @@ This is experimental software. You have been warned. Upgrading MiaB with quotas to a New Version ---------------------------------------- +------------------------------------------- * `cd` into the `mailinabox` directory. @@ -69,6 +69,10 @@ Issues Changes ------- +### v0.43-quota-0.19-beta + +* Add user quota API documentation to the mail users page + ### v0.43-quota-0.18-beta * Update to v0.43 of Mail-in-a-Box diff --git a/setup/bootstrap.sh b/setup/bootstrap.sh index 3d020225..3190aa86 100644 --- a/setup/bootstrap.sh +++ b/setup/bootstrap.sh @@ -20,7 +20,7 @@ if [ -z "$TAG" ]; then # want to display in status checks. if [ "`lsb_release -d | sed 's/.*:\s*//' | sed 's/18\.04\.[0-9]/18.04/' `" == "Ubuntu 18.04 LTS" ]; then # This machine is running Ubuntu 18.04. - TAG=v0.43-quota-0.18-beta + TAG=v0.43-quota-0.19-beta elif [ "`lsb_release -d | sed 's/.*:\s*//' | sed 's/14\.04\.[0-9]/14.04/' `" == "Ubuntu 14.04 LTS" ]; then # This machine is running Ubuntu 14.04.