From fc1f211af56a3d49a77a933c069d49cd5fbddeaf Mon Sep 17 00:00:00 2001 From: John Supplee Date: Sun, 10 Feb 2019 23:39:38 +0200 Subject: [PATCH 1/6] initial work on extended configuration --- conf/nginx.conf | 6 ++++-- management/auth.py | 2 +- management/daemon.py | 28 ++++++++++++++++++++++++++++ management/web_update.py | 18 ++++++++++++++++++ setup/mail-postfix.sh | 6 +++++- setup/start.sh | 16 ++++++++++++++++ setup/web.sh | 9 ++++++--- tools/dns-auth.sh | 2 ++ 8 files changed, 80 insertions(+), 7 deletions(-) create mode 100755 tools/dns-auth.sh diff --git a/conf/nginx.conf b/conf/nginx.conf index fafd3409..25910764 100644 --- a/conf/nginx.conf +++ b/conf/nginx.conf @@ -1,5 +1,6 @@ ## $HOSTNAME +#BEGIN_HTTP # Redirect all HTTP to HTTPS *except* the ACME challenges (Let's Encrypt TLS certificate # domain validation challenges) path, which must be served over HTTP per the ACME spec # (due to some Apache vulnerability). @@ -28,11 +29,12 @@ server { alias $STORAGE_ROOT/ssl/lets_encrypt/webroot/.well-known/acme-challenge/; } } +#END_HTTP # The secure HTTPS server. server { - listen 443 ssl http2; - listen [::]:443 ssl http2; + listen $HTTP_SSL_PORT ssl http2; + listen [::]:$HTTP_SSL_PORT ssl http2; server_name $HOSTNAME; diff --git a/management/auth.py b/management/auth.py index 55f59664..0b082580 100644 --- a/management/auth.py +++ b/management/auth.py @@ -59,7 +59,7 @@ class KeyAuthService: credentials = decode(credentials) if ":" not in credentials: - return None, None + return credentials, None username, password = credentials.split(':', maxsplit=1) return username, password diff --git a/management/daemon.py b/management/daemon.py index 572b6b4a..3869cbaa 100755 --- a/management/daemon.py +++ b/management/daemon.py @@ -324,6 +324,34 @@ def dns_get_dump(): from dns_update import build_recommended_dns return json_response(build_recommended_dns(env)) +@app.route('/letsencrypt/dns-auth//', methods=['GET']) +@authorized_personnel_only +def letsencrypt_dns_auth(domain, token): + from dns_update import do_dns_update, set_custom_dns_record + try: + qname = '_acme-challenge.' + domain + if set_custom_dns_record(qname, 'TXT', token, 'add', env): + if not do_dns_update(env): + return ("Error updating DNS", 400) + return "OK" + + except ValueError as e: + return (str(e), 400) + +@app.route('/letsencrypt/dns-cleanup/', methods=['GET']) +@authorized_personnel_only +def letsencrypt_dns_cleanup(domain): + from dns_update import do_dns_update, set_custom_dns_record + try: + qname = '_acme-challenge.' + domain + if set_custom_dns_record(qname, 'TXT', None, 'remove', env): + if not do_dns_update(env): + return ("Error updating DNS", 400) + return "OK" + + except ValueError as e: + return (str(e), 400) + # SSL @app.route('/ssl/status') diff --git a/management/web_update.py b/management/web_update.py index 61b38a7b..616698ba 100644 --- a/management/web_update.py +++ b/management/web_update.py @@ -94,6 +94,20 @@ def do_web_update(env): # Add default 'www.' redirect. nginx_conf += make_domain_config(domain, [template0, template3], ssl_certificates, env) + if str(env['HTTP_SSL_PORT']) != "443": + in_http = False + new_conf = '' + for line in nginx_conf.split('\n'): + if line.strip() == '#BEGIN_HTTP': + in_http = True + elif line.strip() == '#END_HTTP': + in_http = False + + if not in_http: + new_conf += line + '\n' + + nginx_conf = new_conf + # Did the file change? If not, don't bother writing & restarting nginx. nginx_conf_fn = "/etc/nginx/conf.d/local.conf" if os.path.exists(nginx_conf_fn): @@ -178,8 +192,12 @@ def make_domain_config(domain, templates, ssl_certificates, env): nginx_conf = re.sub("[ \t]*# ADDITIONAL DIRECTIVES HERE *\n", t, nginx_conf) # Replace substitution strings in the template & return. + if int(env['HTTP_SSL_PORT']) != 443: + # disable the regular HTTP server + nginx_conf = re.sub(r'#BEGIN_HTTP.*?#END_HTTP', repl='', string=nginx_conf, flags=re.MULTILINE) nginx_conf = nginx_conf.replace("$STORAGE_ROOT", env['STORAGE_ROOT']) nginx_conf = nginx_conf.replace("$HOSTNAME", domain) + nginx_conf = nginx_conf.replace("$HTTP_SSL_PORT", env['HTTP_SSL_PORT']) nginx_conf = nginx_conf.replace("$ROOT", root) nginx_conf = nginx_conf.replace("$SSL_KEY", tls_cert["private-key"]) nginx_conf = nginx_conf.replace("$SSL_CERTIFICATE", tls_cert["certificate"]) diff --git a/setup/mail-postfix.sh b/setup/mail-postfix.sh index 0c9bc97c..1ea64a50 100755 --- a/setup/mail-postfix.sh +++ b/setup/mail-postfix.sh @@ -193,9 +193,13 @@ tools/editconf.py /etc/postfix/main.cf virtual_transport=lmtp:[127.0.0.1]:10025 # so these IPs get mail delivered quickly. But when an IP is not listed in the permit_dnswl_client list (i.e. it is not #NODOC # whitelisted) then postfix does a DEFER_IF_REJECT, which results in all "unknown user" sorts of messages turning into #NODOC # "450 4.7.1 Client host rejected: Service unavailable". This is a retry code, so the mail doesn't properly bounce. #NODOC +RECIPIENT_RESTRICTIONS=permit_sasl_authenticated,permit_mynetworks,\"reject_rbl_client zen.spamhaus.org\",reject_unlisted_recipient +if [ $NO_GREYLISTING != "1" ]; then + RECIPIENT_RESTRICTIONS=${RECIPIENT_RESTRICTIONS},\"check_policy_service inet:127.0.0.1:10023\" +fi tools/editconf.py /etc/postfix/main.cf \ smtpd_sender_restrictions="reject_non_fqdn_sender,reject_unknown_sender_domain,reject_authenticated_sender_login_mismatch,reject_rhsbl_sender dbl.spamhaus.org" \ - smtpd_recipient_restrictions=permit_sasl_authenticated,permit_mynetworks,"reject_rbl_client zen.spamhaus.org",reject_unlisted_recipient,"check_policy_service inet:127.0.0.1:10023" + smtpd_recipient_restrictions=$RECIPIENT_RESTRICTIONS # Postfix connects to Postgrey on the 127.0.0.1 interface specifically. Ensure that # Postgrey listens on the same interface (and not IPv6, for instance). diff --git a/setup/start.sh b/setup/start.sh index 0b145022..60a0349e 100755 --- a/setup/start.sh +++ b/setup/start.sh @@ -42,6 +42,20 @@ else FIRST_TIME_SETUP=1 fi +if [ -z "${DEFAULT_HTTP_SSL_PORT:-}" ]; then + HTTP_SSL_PORT=443 +else + HTTP_SSL_PORT=$DEFAULT_HTTP_SSL_PORT +fi + +if [ -z "${DEFAULT_NO_GREYLISTING:-}" ]; then + NO_GREYLISTING=0 +elif (($DEFAULT_NO_GREYLISTING > 0)); then + NO_GREYLISTING=1 +else + NO_GREYLISTING=0 +fi + # Put a start script in a global location. We tell the user to run 'mailinabox' # in the first dialog prompt, so we should do this before that starts. cat > /usr/local/bin/mailinabox << EOF; @@ -93,6 +107,8 @@ PUBLIC_IP=$PUBLIC_IP PUBLIC_IPV6=$PUBLIC_IPV6 PRIVATE_IP=$PRIVATE_IP PRIVATE_IPV6=$PRIVATE_IPV6 +HTTP_SSL_PORT=$HTTP_SSL_PORT +NO_GREYLISTING=$NO_GREYLISTING EOF # Start service configuration. diff --git a/setup/web.sh b/setup/web.sh index ed37e5e3..ad7ad310 100755 --- a/setup/web.sh +++ b/setup/web.sh @@ -96,6 +96,9 @@ restart_service nginx restart_service php7.2-fpm # Open ports. -ufw_allow http -ufw_allow https - +if [ $HTTP_SSL_PORT == 443 ]; then + ufw_allow http + ufw_allow https +else + ufw_allow $HTTP_SSL_PORT +fi diff --git a/tools/dns-auth.sh b/tools/dns-auth.sh new file mode 100755 index 00000000..20d602bd --- /dev/null +++ b/tools/dns-auth.sh @@ -0,0 +1,2 @@ +#!/usr/bin/env bash + From 514619b44ab9927f3922e6bfac5096eadd304bce Mon Sep 17 00:00:00 2001 From: John Supplee Date: Mon, 11 Feb 2019 16:05:50 +0200 Subject: [PATCH 2/6] setup dns authentication for letsencrypt --- .gitignore | 1 + management/ssl_certificates.py | 6 +++++- tools/dns-auth.sh | 8 ++++++++ tools/dns-cleanup.sh | 8 ++++++++ 4 files changed, 22 insertions(+), 1 deletion(-) create mode 100755 tools/dns-cleanup.sh diff --git a/.gitignore b/.gitignore index f3cdb1bc..94072693 100644 --- a/.gitignore +++ b/.gitignore @@ -5,3 +5,4 @@ tools/__pycache__/ externals/ .env .vagrant +.idea/ diff --git a/management/ssl_certificates.py b/management/ssl_certificates.py index 76b0f8fa..6d433b54 100755 --- a/management/ssl_certificates.py +++ b/management/ssl_certificates.py @@ -313,6 +313,7 @@ def provision_certificates(env, limit_domains): webroot = os.path.join(account_path, 'webroot') os.makedirs(webroot, exist_ok=True) with tempfile.TemporaryDirectory() as d: + miab_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) cert_file = os.path.join(d, 'cert_and_chain.pem') print("Provisioning TLS certificates for " + ", ".join(domain_list) + ".") certbotret = subprocess.check_output([ @@ -328,7 +329,10 @@ def provision_certificates(env, limit_domains): "--chain-path", os.path.join(d, 'chain'), # we only use the full chain "--fullchain-path", cert_file, - "--webroot", "--webroot-path", webroot, + "--manual", + "--preferred-challenge", "dns", + "--manual-auth-hook", os.path.join(miab_dir, "/tools/dns-auth.sh"), + "--manual-cleanup-hook", os.path.join(miab_dir, "/tools/dns-cleanup.sh"), "--config-dir", account_path, #"--staging", diff --git a/tools/dns-auth.sh b/tools/dns-auth.sh index 20d602bd..e3ee77c8 100755 --- a/tools/dns-auth.sh +++ b/tools/dns-auth.sh @@ -1,2 +1,10 @@ #!/usr/bin/env bash +# TODO: Make work with port other than 443 + +API_KEY=`cat /var/lib/mailinabox/api.key` +HOSTNAME=`hostname` + +curl -s -X PUT -d "$CERTBOT_VALIDATION" --user "$API_KEY:" https://$HOSTNAME/admin/dns/custom/_acme-challenge.$CERTBOT_DOMAIN/TXT + +sleep 15 diff --git a/tools/dns-cleanup.sh b/tools/dns-cleanup.sh new file mode 100755 index 00000000..f9ac582d --- /dev/null +++ b/tools/dns-cleanup.sh @@ -0,0 +1,8 @@ +#!/usr/bin/env bash + +# TODO: Make work with port other than 443 + +API_KEY=`cat /var/lib/mailinabox/api.key` +HOSTNAME=`hostname` + +curl -s -X DELETE --user "$API_KEY:" https://$HOSTNAME/admin/dns/custom/_acme-challenge.$CERTBOT_DOMAIN/TXT \ No newline at end of file From aa234a504e008dff8d71a962ef01289862e89e9e Mon Sep 17 00:00:00 2001 From: John Supplee Date: Wed, 27 Feb 2019 12:48:34 +0200 Subject: [PATCH 3/6] changes for SPF on incoming email --- setup/mail-postfix.sh | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/setup/mail-postfix.sh b/setup/mail-postfix.sh index 0c9bc97c..4802adeb 100755 --- a/setup/mail-postfix.sh +++ b/setup/mail-postfix.sh @@ -42,7 +42,8 @@ source /etc/mailinabox.conf # load global vars # * `ca-certificates`: A trust store used to squelch postfix warnings about # untrusted opportunistically-encrypted connections. echo "Installing Postfix (SMTP server)..." -apt_install postfix postfix-sqlite postfix-pcre postgrey ca-certificates +apt_install postfix postfix-sqlite postfix-pcre postgrey ca-certificates \ + postfix-policyd-spf-python # ### Basic Settings @@ -97,7 +98,9 @@ tools/editconf.py /etc/postfix/master.cf -s -w \ -o cleanup_service_name=authclean" \ "authclean=unix n - - - 0 cleanup -o header_checks=pcre:/etc/postfix/outgoing_mail_header_filters - -o nested_header_checks=" + -o nested_header_checks=" \ + "policy-spf=unix - n n - - spawn + user=nobody argv=/usr/bin/policyd-spf" # Install the `outgoing_mail_header_filters` file required by the new 'authclean' service. cp conf/postfix_outgoing_mail_header_filters /etc/postfix/outgoing_mail_header_filters @@ -195,7 +198,7 @@ tools/editconf.py /etc/postfix/main.cf virtual_transport=lmtp:[127.0.0.1]:10025 # "450 4.7.1 Client host rejected: Service unavailable". This is a retry code, so the mail doesn't properly bounce. #NODOC tools/editconf.py /etc/postfix/main.cf \ smtpd_sender_restrictions="reject_non_fqdn_sender,reject_unknown_sender_domain,reject_authenticated_sender_login_mismatch,reject_rhsbl_sender dbl.spamhaus.org" \ - smtpd_recipient_restrictions=permit_sasl_authenticated,permit_mynetworks,"reject_rbl_client zen.spamhaus.org",reject_unlisted_recipient,"check_policy_service inet:127.0.0.1:10023" + smtpd_recipient_restrictions=permit_sasl_authenticated,permit_mynetworks,"reject_rbl_client zen.spamhaus.org",reject_unlisted_recipient,"check_policy_service inet:127.0.0.1:10023","check_policy_service unix:private/policy-spf" # Postfix connects to Postgrey on the 127.0.0.1 interface specifically. Ensure that # Postgrey listens on the same interface (and not IPv6, for instance). @@ -214,6 +217,7 @@ tools/editconf.py /etc/postfix/main.cf \ # Allow the two SMTP ports in the firewall. + ufw_allow smtp ufw_allow submission From 68f4d1c426ca239b40e1f077305709ed0c6d1feb Mon Sep 17 00:00:00 2001 From: John Supplee Date: Wed, 27 Feb 2019 17:30:59 +0200 Subject: [PATCH 4/6] add SRS support --- setup/mail-postfix.sh | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/setup/mail-postfix.sh b/setup/mail-postfix.sh index e59de474..e974fb91 100755 --- a/setup/mail-postfix.sh +++ b/setup/mail-postfix.sh @@ -43,7 +43,7 @@ source /etc/mailinabox.conf # load global vars # untrusted opportunistically-encrypted connections. echo "Installing Postfix (SMTP server)..." apt_install postfix postfix-sqlite postfix-pcre postgrey ca-certificates \ - postfix-policyd-spf-python + postfix-policyd-spf-python postsrsd # ### Basic Settings @@ -201,7 +201,7 @@ tools/editconf.py /etc/postfix/main.cf lmtp_destination_recipient_limit=1 # "450 4.7.1 Client host rejected: Service unavailable". This is a retry code, so the mail doesn't properly bounce. #NODOC tools/editconf.py /etc/postfix/main.cf \ smtpd_sender_restrictions="reject_non_fqdn_sender,reject_unknown_sender_domain,reject_authenticated_sender_login_mismatch,reject_rhsbl_sender dbl.spamhaus.org" \ - smtpd_recipient_restrictions=permit_sasl_authenticated,permit_mynetworks,"reject_rbl_client zen.spamhaus.org",reject_unlisted_recipient,"check_policy_service inet:127.0.0.1:10023","check_policy_service unix:private/policy-spf" + smtpd_recipient_restrictions=permit_sasl_authenticated,permit_mynetworks,"reject_rbl_client zen.spamhaus.org",reject_unlisted_recipient,"check_policy_service unix:private/policy-spf","check_policy_service inet:127.0.0.1:10023" # Postfix connects to Postgrey on the 127.0.0.1 interface specifically. Ensure that # Postgrey listens on the same interface (and not IPv6, for instance). @@ -218,6 +218,16 @@ tools/editconf.py /etc/default/postgrey \ tools/editconf.py /etc/postfix/main.cf \ message_size_limit=134217728 +# Setup SRS +postconf -e \ + sender_canonical_maps=tcp:localhost:10001 \ + sender_canonical_classes=envelope_sender \ + recipient_canonical_maps=tcp:localhost:10002 \ + recipient_canonical_classes=envelope_recipient,header_recipient + +hide_output systemctl enable postsrsd +hide_output systemctl restart postsrsd + # Allow the two SMTP ports in the firewall. From 5167b3a62336e713a312ad5780d38db65485291d Mon Sep 17 00:00:00 2001 From: John Supplee Date: Fri, 8 Mar 2019 17:18:24 +0200 Subject: [PATCH 5/6] Change variable name for Postgrey setup and add others --- setup/mail-postfix.sh | 2 +- setup/start.sh | 26 +++++++++++++++----------- 2 files changed, 16 insertions(+), 12 deletions(-) diff --git a/setup/mail-postfix.sh b/setup/mail-postfix.sh index 1ea64a50..39045046 100755 --- a/setup/mail-postfix.sh +++ b/setup/mail-postfix.sh @@ -194,7 +194,7 @@ tools/editconf.py /etc/postfix/main.cf virtual_transport=lmtp:[127.0.0.1]:10025 # whitelisted) then postfix does a DEFER_IF_REJECT, which results in all "unknown user" sorts of messages turning into #NODOC # "450 4.7.1 Client host rejected: Service unavailable". This is a retry code, so the mail doesn't properly bounce. #NODOC RECIPIENT_RESTRICTIONS=permit_sasl_authenticated,permit_mynetworks,\"reject_rbl_client zen.spamhaus.org\",reject_unlisted_recipient -if [ $NO_GREYLISTING != "1" ]; then +if [ $GREYLISTING != "1" ]; then RECIPIENT_RESTRICTIONS=${RECIPIENT_RESTRICTIONS},\"check_policy_service inet:127.0.0.1:10023\" fi tools/editconf.py /etc/postfix/main.cf \ diff --git a/setup/start.sh b/setup/start.sh index 60a0349e..b84d720c 100755 --- a/setup/start.sh +++ b/setup/start.sh @@ -42,18 +42,20 @@ else FIRST_TIME_SETUP=1 fi -if [ -z "${DEFAULT_HTTP_SSL_PORT:-}" ]; then - HTTP_SSL_PORT=443 -else - HTTP_SSL_PORT=$DEFAULT_HTTP_SSL_PORT +if [ -z "${HTTP_SSL_PORT:-}" ]; then + HTTP_SSL_PORT=$([[ -z "${DEFAULT_HTTP_SSL_PORT:-}" ]] && echo "443" || echo "$DEFAULT_HTTP_SSL_PORT") fi -if [ -z "${DEFAULT_NO_GREYLISTING:-}" ]; then - NO_GREYLISTING=0 -elif (($DEFAULT_NO_GREYLISTING > 0)); then - NO_GREYLISTING=1 -else - NO_GREYLISTING=0 +if [ -z "${GREYLISTING:-}" ]; then + GREYLISTING=$([[ -z "${DEFAULT_GREYLISTING:-}" ]] && echo "1" || echo "$DEFAULT_GREYLISTING") +fi + +if [ -z "${POSTSRSD:-}" ]; then + POSTSRSD=$([[ -z "${DEFAULT_POSTSRSD:-}" ]] && echo "0" || echo "$DEFAULT_POSTSRSD") +fi + +if [ -z "${POLICY_SPF:-}" ]; then + POLICY_SPF=$([[ -z "${DEFAULT_POLICY_SPF:-}" ]] && echo "0" || echo "$DEFAULT_POLICY_SPF") fi # Put a start script in a global location. We tell the user to run 'mailinabox' @@ -108,7 +110,9 @@ PUBLIC_IPV6=$PUBLIC_IPV6 PRIVATE_IP=$PRIVATE_IP PRIVATE_IPV6=$PRIVATE_IPV6 HTTP_SSL_PORT=$HTTP_SSL_PORT -NO_GREYLISTING=$NO_GREYLISTING +GREYLISTING=$GREYLISTING +POSTSRSD=$POSTSRSD +POLICY_SPF=$POLICY_SPF EOF # Start service configuration. From a7f62f281bdfee328c979087524da2e547aa119a Mon Sep 17 00:00:00 2001 From: John Supplee Date: Sat, 9 Mar 2019 10:05:46 +0200 Subject: [PATCH 6/6] Initial work on dialog screen for options --- setup/options-dialog.py | 60 +++++++++++++++++++++++++++++++++++++++++ setup/questions.sh | 3 +++ 2 files changed, 63 insertions(+) create mode 100644 setup/options-dialog.py diff --git a/setup/options-dialog.py b/setup/options-dialog.py new file mode 100644 index 00000000..9c9bfc60 --- /dev/null +++ b/setup/options-dialog.py @@ -0,0 +1,60 @@ +#!/usr/bin/env python +# encoding: utf-8 + +import npyscreen +import sys +import os + + +class TestApp(npyscreen.NPSApp): + def main(self): + # These lines create the form and populate it with widgets. + # A fairly complex screen in only 8 or so lines of code - a line for each control. + npyscreen.setTheme(npyscreen.Themes.BlackOnWhiteTheme) + + form = npyscreen.Form(name = "Mail-in-a-Box Options",) + postgrey_text = form.add( + npyscreen.TitleFixedText, + name="POSTGREY", + value="Should Postgrey be used to greylist messages?", + editable=False + ) + form.add(npyscreen.FixedText) + + postgrey_text = form.add( + npyscreen.MultiLineEditable, + name="POSTSRSD", + value="The PostSRSd daemon performs return path rewriting using the SRS protocol.\n" + "Not that all messages, including locally delivered mail will have their return\n" + "paths rewritten", + max_height=4, + editable=False + ) + form.add(npyscreen.FixedText) + + options = form.add( + npyscreen.TitleMultiSelect, + max_height=-2, + value = [ + int(os.getenv('POSTGREY', 1)), + int(os.getenv('POSTSRSD', 0)), + int(os.getenv('POLICY_SPF', 0)) + ], + name="Options", + values= ["POSTGREY","POSTSRSD","POLICY_SPF"], + scroll_exit=True + ) + + # This lets the user interact with the Form. + form.edit() + + with open('_options.sh', 'w') as output: + print('POSTGREY=%i' % (1 if 0 in options.value else 0), file=output) + print('POSTSRSD=%i' % (1 if 1 in options.value else 0), file=output) + print('POLICY_SPF=%i' % (1 if 2 in options.value else 0), file=output) + # print(npyscreen.ThemeManager.default_colors, file=output) + + +if __name__ == "__main__": + App = TestApp() + App.run() diff --git a/setup/questions.sh b/setup/questions.sh index bf382f49..18b64f02 100644 --- a/setup/questions.sh +++ b/setup/questions.sh @@ -16,6 +16,7 @@ if [ -z "${NONINTERACTIVE:-}" ]; then # we install it inside a virtualenv. In this script, we don't have the virtualenv yet # so we install the python package globally. hide_output pip3 install "email_validator>=1.0.0" || exit 1 + hide_output pip3 install npyscreen || exit 1 message_box "Mail-in-a-Box Installation" \ "Hello and thanks for deploying a Mail-in-a-Box! @@ -193,6 +194,8 @@ if [ -z "${STORAGE_ROOT:-}" ]; then STORAGE_ROOT=$([[ -z "${DEFAULT_STORAGE_ROOT:-}" ]] && echo "/home/$STORAGE_USER" || echo "$DEFAULT_STORAGE_ROOT") fi +python3 setup/options-dialog.py + # Show the configuration, since the user may have not entered it manually. echo echo "Primary Hostname: $PRIMARY_HOSTNAME"