Merge branch 'miab-config' into extended

.gitignore

@@ -5,3 +5,4 @@ tools/__pycache__/
 externals/
 .env
 .vagrant
+.idea/

conf/nginx.conf

@@ -1,5 +1,6 @@
 ## $HOSTNAME
 
+#BEGIN_HTTP
 # Redirect all HTTP to HTTPS *except* the ACME challenges (Let's Encrypt TLS certificate
 # domain validation challenges) path, which must be served over HTTP per the ACME spec
 # (due to some Apache vulnerability).
@@ -28,11 +29,12 @@ server {
 		alias $STORAGE_ROOT/ssl/lets_encrypt/webroot/.well-known/acme-challenge/;
 	}
 }
+#END_HTTP
 
 # The secure HTTPS server.
 server {
-	listen 443 ssl http2;
+	listen $HTTP_SSL_PORT ssl http2;
-	listen [::]:443 ssl http2;
+	listen [::]:$HTTP_SSL_PORT ssl http2;
 
 	server_name $HOSTNAME;
 

management/auth.py

@@ -59,7 +59,7 @@ class KeyAuthService:
 
 			credentials = decode(credentials)
 			if ":" not in credentials:
-				return None, None
+				return credentials, None
 			username, password = credentials.split(':', maxsplit=1)
 			return username, password
 

management/daemon.py

@@ -347,6 +347,34 @@ def dns_get_dump():
 	from dns_update import build_recommended_dns
 	return json_response(build_recommended_dns(env))
 
+@app.route('/letsencrypt/dns-auth/<domain>/<token>', methods=['GET'])
+@authorized_personnel_only
+def letsencrypt_dns_auth(domain, token):
+	from dns_update import do_dns_update, set_custom_dns_record
+	try:
+		qname = '_acme-challenge.' + domain
+		if set_custom_dns_record(qname, 'TXT', token, 'add', env):
+			if not do_dns_update(env):
+				return ("Error updating DNS", 400)
+		return "OK"
+
+	except ValueError as e:
+		return (str(e), 400)
+
+@app.route('/letsencrypt/dns-cleanup/<domain>', methods=['GET'])
+@authorized_personnel_only
+def letsencrypt_dns_cleanup(domain):
+	from dns_update import do_dns_update, set_custom_dns_record
+	try:
+		qname = '_acme-challenge.' + domain
+		if set_custom_dns_record(qname, 'TXT', None, 'remove', env):
+			if not do_dns_update(env):
+				return ("Error updating DNS", 400)
+		return "OK"
+
+	except ValueError as e:
+		return (str(e), 400)
+
 # SSL
 
 @app.route('/ssl/status')

management/ssl_certificates.py

@@ -313,6 +313,7 @@ def provision_certificates(env, limit_domains):
 				webroot = os.path.join(account_path, 'webroot')
 				os.makedirs(webroot, exist_ok=True)
 				with tempfile.TemporaryDirectory() as d:
+					miab_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
 					cert_file = os.path.join(d, 'cert_and_chain.pem')
 					print("Provisioning TLS certificates for " + ", ".join(domain_list) + ".")
 					certbotret = subprocess.check_output([
@@ -328,7 +329,10 @@ def provision_certificates(env, limit_domains):
 						"--chain-path", os.path.join(d, 'chain'), # we only use the full chain
 						"--fullchain-path", cert_file,
 
-						"--webroot", "--webroot-path", webroot,
+						"--manual",
+						"--preferred-challenge", "dns",
+						"--manual-auth-hook", os.path.join(miab_dir, "/tools/dns-auth.sh"),
+						"--manual-cleanup-hook", os.path.join(miab_dir, "/tools/dns-cleanup.sh"),
 
 						"--config-dir", account_path,
 						#"--staging",

management/web_update.py

@@ -94,6 +94,20 @@ def do_web_update(env):
 			# Add default 'www.' redirect.
 			nginx_conf += make_domain_config(domain, [template0, template3], ssl_certificates, env)
 
+	if str(env['HTTP_SSL_PORT']) != "443":
+		in_http = False
+		new_conf = ''
+		for line in nginx_conf.split('\n'):
+			if line.strip() == '#BEGIN_HTTP':
+				in_http = True
+			elif line.strip() == '#END_HTTP':
+				in_http = False
+
+			if not in_http:
+				new_conf += line + '\n'
+
+		nginx_conf = new_conf
+
 	# Did the file change? If not, don't bother writing & restarting nginx.
 	nginx_conf_fn = "/etc/nginx/conf.d/local.conf"
 	if os.path.exists(nginx_conf_fn):
@@ -178,8 +192,12 @@ def make_domain_config(domain, templates, ssl_certificates, env):
 		nginx_conf = re.sub("[ \t]*# ADDITIONAL DIRECTIVES HERE *\n", t, nginx_conf)
 
 	# Replace substitution strings in the template & return.
+	if int(env['HTTP_SSL_PORT']) != 443:
+		# disable the regular HTTP server
+		nginx_conf = re.sub(r'#BEGIN_HTTP.*?#END_HTTP', repl='', string=nginx_conf, flags=re.MULTILINE)
 	nginx_conf = nginx_conf.replace("$STORAGE_ROOT", env['STORAGE_ROOT'])
 	nginx_conf = nginx_conf.replace("$HOSTNAME", domain)
+	nginx_conf = nginx_conf.replace("$HTTP_SSL_PORT", env['HTTP_SSL_PORT'])
 	nginx_conf = nginx_conf.replace("$ROOT", root)
 	nginx_conf = nginx_conf.replace("$SSL_KEY", tls_cert["private-key"])
 	nginx_conf = nginx_conf.replace("$SSL_CERTIFICATE", tls_cert["certificate"])

setup/mail-postfix.sh

@@ -42,7 +42,8 @@ source /etc/mailinabox.conf # load global vars
 # * `ca-certificates`: A trust store used to squelch postfix warnings about
 #   untrusted opportunistically-encrypted connections.
 echo "Installing Postfix (SMTP server)..."
-apt_install postfix postfix-sqlite postfix-pcre postgrey ca-certificates
+apt_install postfix postfix-sqlite postfix-pcre postgrey ca-certificates \
+	postfix-policyd-spf-python postsrsd
 
 # ### Basic Settings
 
@@ -97,7 +98,9 @@ tools/editconf.py /etc/postfix/master.cf -s -w \
 	  -o cleanup_service_name=authclean" \
 	"authclean=unix  n       -       -       -       0       cleanup
 	  -o header_checks=pcre:/etc/postfix/outgoing_mail_header_filters
-	  -o nested_header_checks="
+	  -o nested_header_checks=" \
+	"policy-spf=unix  -       n       n       -       -       spawn
+	  user=nobody argv=/usr/bin/policyd-spf"
 
 # Install the `outgoing_mail_header_filters` file required by the new 'authclean' service.
 cp conf/postfix_outgoing_mail_header_filters /etc/postfix/outgoing_mail_header_filters
@@ -196,9 +199,23 @@ tools/editconf.py /etc/postfix/main.cf lmtp_destination_recipient_limit=1
 # so these IPs get mail delivered quickly. But when an IP is not listed in the permit_dnswl_client list (i.e. it is not #NODOC
 # whitelisted) then postfix does a DEFER_IF_REJECT, which results in all "unknown user" sorts of messages turning into #NODOC
 # "450 4.7.1 Client host rejected: Service unavailable". This is a retry code, so the mail doesn't properly bounce. #NODOC
-tools/editconf.py /etc/postfix/main.cf \
+
-	smtpd_sender_restrictions="reject_non_fqdn_sender,reject_unknown_sender_domain,reject_authenticated_sender_login_mismatch,reject_rhsbl_sender dbl.spamhaus.org" \
+postconf -e smtpd_sender_restrictions="reject_non_fqdn_sender,reject_unknown_sender_domain,reject_authenticated_sender_login_mismatch,reject_rhsbl_sender dbl.spamhaus.org"
-	smtpd_recipient_restrictions=permit_sasl_authenticated,permit_mynetworks,"reject_rbl_client zen.spamhaus.org",reject_unlisted_recipient,"check_policy_service inet:127.0.0.1:10023","check_policy_service inet:127.0.0.1:12340"
+
+RECIPIENT_RESTRICTIONS="permit_sasl_authenticated,permit_mynetworks,reject_rbl_client zen.spamhaus.org,reject_unlisted_recipient"
+
+if [ $GREYLISTING != 1 ]; then
+    RECIPIENT_RESTRICTIONS="${RECIPIENT_RESTRICTIONS},check_policy_service inet:127.0.0.1:10023"
+fi
+
+if [ $POLICY_SPF == 1 ]; then
+    RECIPIENT_RESTRICTIONS="${RECIPIENT_RESTRICTIONS},check_policy_service unix:private/policy-spf"
+fi
+
+# Add quota check
+RECIPIENT_RESTRICTIONS="${RECIPIENT_RESTRICTIONS},check_policy_service inet:127.0.0.1:12340"
+
+postconf -e smtpd_recipient_restrictions="$RECIPIENT_RESTRICTIONS"
 
 # Postfix connects to Postgrey on the 127.0.0.1 interface specifically. Ensure that
 # Postgrey listens on the same interface (and not IPv6, for instance).
@@ -215,6 +232,29 @@ tools/editconf.py /etc/default/postgrey \
 tools/editconf.py /etc/postfix/main.cf \
 	message_size_limit=134217728
 
+if [ $POSTSRSD == "1" ]; then
+    # Setup SRS
+    postconf -e \
+        sender_canonical_maps=tcp:localhost:10001 \
+        sender_canonical_classes=envelope_sender \
+        recipient_canonical_maps=tcp:localhost:10002 \
+        recipient_canonical_classes=envelope_recipient,header_recipient
+
+    hide_output systemctl enable postsrsd
+    hide_output systemctl restart postsrsd
+
+else
+    postconf -e \
+        sender_canonical_maps= \
+        sender_canonical_classes= \
+        recipient_canonical_maps= \
+        recipient_canonical_classes=
+
+    hide_output systemctl disable postsrsd
+    hide_output systemctl stop postsrsd
+fi
+
+
 # Allow the two SMTP ports in the firewall.
 
 ufw_allow smtp
@@ -223,4 +263,12 @@ ufw_allow submission
 # Restart services
 
 restart_service postfix
-restart_service postgrey
+
+if [ $GREYLISTING == 1 ]; then
+    hide_output systemctl enable postgrey
+    restart_service postgrey
+
+else
+    hide_output systemctl disable postgrey
+    hide_output systemctl stop postgrey
+fi

setup/options-dialog.py

@@ -0,0 +1,60 @@
+#!/usr/bin/env python
+# encoding: utf-8
+
+import npyscreen
+import sys
+import os
+
+
+class TestApp(npyscreen.NPSApp):
+    def main(self):
+        # These lines create the form and populate it with widgets.
+        # A fairly complex screen in only 8 or so lines of code - a line for each control.
+        npyscreen.setTheme(npyscreen.Themes.BlackOnWhiteTheme)
+
+        form  = npyscreen.Form(name = "Mail-in-a-Box Options",)
+        postgrey_text = form.add(
+            npyscreen.TitleFixedText,
+            name="POSTGREY",
+            value="Should Postgrey be used to greylist messages?",
+            editable=False
+        )
+        form.add(npyscreen.FixedText)
+
+        postgrey_text = form.add(
+            npyscreen.MultiLineEditable,
+            name="POSTSRSD",
+            value="The PostSRSd daemon performs return path rewriting using the SRS protocol.\n"
+                  "Not that all messages, including locally delivered mail will have their return\n"
+                  "paths rewritten",
+            max_height=4,
+            editable=False
+        )
+        form.add(npyscreen.FixedText)
+
+        options = form.add(
+            npyscreen.TitleMultiSelect,
+            max_height=-2,
+            value = [
+                int(os.getenv('POSTGREY', 1)),
+                int(os.getenv('POSTSRSD', 0)),
+                int(os.getenv('POLICY_SPF', 0))
+            ],
+            name="Options",
+            values= ["POSTGREY","POSTSRSD","POLICY_SPF"],
+            scroll_exit=True
+        )
+
+        # This lets the user interact with the Form.
+        form.edit()
+
+        with open('_options.sh', 'w') as output:
+            print('POSTGREY=%i' % (1 if 0 in options.value else 0), file=output)
+            print('POSTSRSD=%i' % (1 if 1 in options.value else 0), file=output)
+            print('POLICY_SPF=%i' % (1 if 2 in options.value else 0), file=output)
+            # print(npyscreen.ThemeManager.default_colors, file=output)
+
+
+if __name__ == "__main__":
+    App = TestApp()
+    App.run()

setup/questions.sh

@@ -16,6 +16,7 @@ if [ -z "${NONINTERACTIVE:-}" ]; then
 	# we install it inside a virtualenv. In this script, we don't have the virtualenv yet
 	# so we install the python package globally.
 	hide_output pip3 install "email_validator>=1.0.0" || exit 1
+	hide_output pip3 install npyscreen || exit 1
 
 	message_box "Mail-in-a-Box Installation" \
 		"Hello and thanks for deploying a Mail-in-a-Box!
@@ -193,6 +194,8 @@ if [ -z "${STORAGE_ROOT:-}" ]; then
 	STORAGE_ROOT=$([[ -z "${DEFAULT_STORAGE_ROOT:-}" ]] && echo "/home/$STORAGE_USER" || echo "$DEFAULT_STORAGE_ROOT")
 fi
 
+python3 setup/options-dialog.py
+
 # Show the configuration, since the user may have not entered it manually.
 echo
 echo "Primary Hostname: $PRIMARY_HOSTNAME"

setup/start.sh

@@ -42,6 +42,22 @@ else
 	FIRST_TIME_SETUP=1
 fi
 
+if [ -z "${HTTP_SSL_PORT:-}" ]; then
+	HTTP_SSL_PORT=$([[ -z "${DEFAULT_HTTP_SSL_PORT:-}" ]] && echo "443" || echo "$DEFAULT_HTTP_SSL_PORT")
+fi
+
+if [ -z "${GREYLISTING:-}" ]; then
+	GREYLISTING=$([[ -z "${DEFAULT_GREYLISTING:-}" ]] && echo "1" || echo "$DEFAULT_GREYLISTING")
+fi
+
+if [ -z "${POSTSRSD:-}" ]; then
+	POSTSRSD=$([[ -z "${DEFAULT_POSTSRSD:-}" ]] && echo "0" || echo "$DEFAULT_POSTSRSD")
+fi
+
+if [ -z "${POLICY_SPF:-}" ]; then
+	POLICY_SPF=$([[ -z "${DEFAULT_POLICY_SPF:-}" ]] && echo "0" || echo "$DEFAULT_POLICY_SPF")
+fi
+
 # Put a start script in a global location. We tell the user to run 'mailinabox'
 # in the first dialog prompt, so we should do this before that starts.
 cat > /usr/local/bin/mailinabox << EOF;
@@ -93,6 +109,10 @@ PUBLIC_IP=$PUBLIC_IP
 PUBLIC_IPV6=$PUBLIC_IPV6
 PRIVATE_IP=$PRIVATE_IP
 PRIVATE_IPV6=$PRIVATE_IPV6
+HTTP_SSL_PORT=$HTTP_SSL_PORT
+GREYLISTING=$GREYLISTING
+POSTSRSD=$POSTSRSD
+POLICY_SPF=$POLICY_SPF
 EOF
 
 # Start service configuration.

setup/web.sh

@@ -96,6 +96,9 @@ restart_service nginx
 restart_service php7.2-fpm
 
 # Open ports.
-ufw_allow http
+if [ $HTTP_SSL_PORT == 443 ]; then
-ufw_allow https
+    ufw_allow http
-
+    ufw_allow https
+else
+    ufw_allow $HTTP_SSL_PORT
+fi

tools/dns-auth.sh

@@ -0,0 +1,10 @@
+#!/usr/bin/env bash
+
+# TODO: Make work with port other than 443
+
+API_KEY=`cat /var/lib/mailinabox/api.key`
+HOSTNAME=`hostname`
+
+curl -s -X PUT -d "$CERTBOT_VALIDATION" --user "$API_KEY:" https://$HOSTNAME/admin/dns/custom/_acme-challenge.$CERTBOT_DOMAIN/TXT
+
+sleep 15

tools/dns-cleanup.sh

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+
+# TODO: Make work with port other than 443
+
+API_KEY=`cat /var/lib/mailinabox/api.key`
+HOSTNAME=`hostname`
+
+curl -s -X DELETE --user "$API_KEY:" https://$HOSTNAME/admin/dns/custom/_acme-challenge.$CERTBOT_DOMAIN/TXT