Fix url redirection when a remote nextcloud is used so that .well-known/caldav and carddav work properly, as well as the redirecting /cloud to the remote Nextcloud. Since the nginx config is replaced by the management daemon whenever a new domain is added, this change adds a hooking mechanism for setup mods.Fix url redirection when a remote nextcloud is in use. This corrects redirection for /.well-known/caldav, /.well-known/carddav and /cloud to send the client to the remote nextcloud. This requires an nginx configuration change, and since the nginx config is replaced by the management daemon whenever a new domain is added, this change adds a hooking mechanism for setup mods allowing them to intercept and modify the resultant nginx config.

management/daemon.py

@@ -549,6 +549,13 @@ def web_update():
 	from web_update import do_web_update
 	return do_web_update(env)
 
+@app.route('/hooks/update', methods=['POST'])
+@authorized_personnel_only
+def hooks_update():
+	from hooks import update_hook_handlers
+	update_hook_handlers()
+	return "OK"
+
 # System
 
 @app.route('/system/version', methods=["GET"])
@@ -820,6 +827,8 @@ add_ui_common(app)
 from daemon_reports import add_reports
 add_reports(app, env, authorized_personnel_only)
 
+from hooks import update_hook_handlers
+update_hook_handlers()
 
 if __name__ == '__main__':
 	if "DEBUG" in os.environ:

management/hooks.py

@@ -0,0 +1,88 @@
+# -*- indent-tabs-mode: t; tab-width: 4; python-indent-offset: 4; -*-
+#####
+##### This file is part of Mail-in-a-Box-LDAP which is released under the
+##### terms of the GNU Affero General Public License as published by the
+##### Free Software Foundation, either version 3 of the License, or (at
+##### your option) any later version. See file LICENSE or go to
+##### https://github.com/downtownallday/mailinabox-ldap for full license
+##### details.
+#####
+
+import sys, os, stat, importlib
+from threading import Lock
+from utils import load_environment, load_env_vars_from_file
+import logging
+
+log = logging.getLogger(__name__)
+
+#
+# keep a list of hook handlers as a list of dictionaries. see
+# update_hook_handlers() for the format
+#
+mutex = Lock()
+handlers = [] 
+mods_env = {}  # dict derived from /etc/mailinabox_mods.conf
+
+def update_hook_handlers():
+	global handlers, mods_env
+	new_handlers= []
+	for dir in sys.path:
+		hooks_dir = os.path.join(dir, "management_hooks_d")
+		if not os.path.isdir(hooks_dir):
+			continue
+
+		# gather a list of applicable hook handlers
+		for item in os.listdir(hooks_dir):
+			item_path = os.path.join(hooks_dir, item)
+			mode = os.lstat(item_path).st_mode
+			if item.endswith('.py') and stat.S_ISREG(mode):
+				new_handlers.append({
+					'sort_id': item,
+					'path': "management_hooks_d.%s" % (item[0:-3]),
+					'type': "py"
+				})
+				log.info('hook handler: %s', item_path)
+	
+	# handlers are sorted alphabetically by file name
+	new_handlers = sorted(new_handlers, key=lambda path: path['sort_id'])
+	log.info('%s hook handlers', len(new_handlers))
+
+	# load /etc/mailinabox_mods.conf
+	new_mods_env = load_environment()
+	if os.path.isfile('/etc/mailinabox_mods.conf'):    
+		load_env_vars_from_file(
+			'/etc/mailinabox_mods.conf',
+			strip_quotes=True,
+			merge_env=new_mods_env
+		)
+
+	# update globals
+	mutex.acquire()
+	handlers = new_handlers
+	mods_env = new_mods_env
+	mutex.release()
+
+
+def exec_hooks(hook_name, data):
+	# `data` is a dictionary containing data from the hook caller, the
+	# contents of which are specific to the type of hook. Handlers may
+	# modify the dictionary to return updates to the caller.
+
+	mutex.acquire()
+	cur_handlers = handlers
+	cur_mods_env = mods_env
+	mutex.release()
+	
+	for handler in cur_handlers:
+		if handler['type'] == 'py':
+			# load the python code and run the `do_hook` function
+			log.debug('calling %s hook handler: %s' % (hook_name, handler['path']))
+			module = importlib.import_module(handler['path'])
+			do_hook = getattr(module, "do_hook")
+			do_hook(hook_name, data, cur_mods_env)
+
+		else:
+			log.error('Unknown hook handler type in %s: %s', handler['path'], handler['type'])
+
+	return len(cur_handlers)
+

management/web_update.py

@@ -18,6 +18,7 @@ from mailconfig import get_mail_domains
 from dns_update import get_custom_dns_config, get_dns_zones
 from ssl_certificates import get_ssl_certificates, get_domain_ssl_files, check_certificate
 from utils import shell, safe_domain_name, sort_domains
+import hooks
 
 def get_web_domains(env, include_www_redirects=True, include_auto=True, exclude_dns_elsewhere=True, categories=['mail', 'ssl']):
 	# What domains should we serve HTTP(S) for?
@@ -114,6 +115,11 @@ def do_web_update(env):
 			# Add default 'www.' redirect.
 			nginx_conf += make_domain_config(domain, [template0, template3], ssl_certificates, env)
 
+	# execute hooks
+	hook_data = {'nginx_conf': nginx_conf}
+	hooks.exec_hooks('web_update', hook_data)
+	nginx_conf = hook_data['nginx_conf']
+
 	# Did the file change? If not, don't bother writing & restarting nginx.
 	nginx_conf_fn = "/etc/nginx/conf.d/local.conf"
 	if os.path.exists(nginx_conf_fn):

setup/functions.sh

@@ -316,3 +316,14 @@ say_verbose() {
 say() {
 	echo "$@"
 }
+
+install_hook_handler() {
+	# this is used by local setup mods to install a hook handler for
+	# the management daemon
+	local handler_file="$1"
+	local dst="${LOCAL_MODS_DIR:-local}/management_hooks_d"
+	mkdir -p "$dst"	
+	cp "$handler_file" "$dst"
+	# let the daemon know there's a new hook handler
+	tools/hooks_update >/dev/null
+}

setup/management.sh

@@ -113,8 +113,8 @@ tr -cd '[:xdigit:]' < /dev/urandom | head -c 32 > /var/lib/mailinabox/api.key
 chmod 640 /var/lib/mailinabox/api.key
 
 source $venv/bin/activate
-export PYTHONPATH=$(pwd)/management
-exec gunicorn -b localhost:10222 -w 1 wsgi:app
+export PYTHONPATH=$(pwd)/management:${LOCAL_MODS_DIR:-$(pwd)/local}
+exec gunicorn --log-level ${MGMT_LOG_LEVEL:-info} -b localhost:10222 -w 1 wsgi:app
 EOF
 chmod +x $inst_dir/start
 cp --remove-destination conf/mailinabox.service /lib/systemd/system/mailinabox.service # target was previously a symlink so remove it first

setup/mods.available/hooks/remote-nextcloud-mgmt-hooks.py

@@ -0,0 +1,63 @@
+
+#
+# This is a web_update management hook for the remote-nextcloud setup
+# mod.
+#
+# When management/web_update.py creates a new nginx configuration file
+# "local.conf", this mod will ensure that .well-known/caldav and
+# .well-known/carddav urls are redirected to the remote nextcloud.
+#
+# The hook is enabled by placing the file in directory
+# LOCAL_MODS_DIR/managment_hooks_d.
+#
+
+import os
+import logging
+
+log = logging.getLogger(__name__)
+
+
+def do_hook(hook_name, hook_data, mods_env):
+    if hook_name != 'web_update':
+        # we only care about hooking web_update
+        log.debug('hook - ignoring %s' % hook_name)
+        return False
+
+    if 'NC_HOST' not in mods_env or mods_env['NC_HOST'].strip() == '':
+        # not configured for a remote nextcloud
+        log.debug('hook - not configured for a remote nextcloud')
+        return False
+    
+    # get the remote nextcloud url and ensure no tailing /
+    nc_url = "%s://%s:%s%s" % (
+        mods_env['NC_PROTO'],
+        mods_env['NC_HOST'],
+        mods_env['NC_PORT'],
+        mods_env['NC_PREFIX'][0:-1] if mods_env['NC_PREFIX'].endswith('/') else mods_env['NC_PREFIX']
+    )
+
+    #
+    # modify nginx_conf
+    #
+    def do_replace(find_str, replace_with):
+        if hook_data['nginx_conf'].find(find_str) == -1:
+            log.warning('remote-nextcloud hook: string "%s" not found in proposed nginx_conf' % (find_str))
+            return False
+        hook_data['nginx_conf'] = hook_data['nginx_conf'].replace(
+            find_str,
+            replace_with
+        )
+        return True
+            
+    # 1. change the .well-known/(caldav|carddav) redirects
+    do_replace(
+        '/cloud/remote.php/dav/',
+        '%s/remote.php/dav/' % nc_url
+    )
+    
+    # 2. redirect /cloud to the remote nextcloud
+    do_replace(
+        'rewrite ^/cloud/$ /cloud/index.php;',
+        'rewrite ^/cloud/(.*)$ %s/$1 redirect;' % nc_url
+    )
+

setup/mods.available/remote-nextcloud.sh

@@ -89,6 +89,12 @@ EOF
 }
 
 
+update_mobileconfig() {
+    local url="$1"
+    sed -i "s|<string>/cloud/remote.php|<string>${url%/}/remote.php|g" /var/lib/mailinabox/mobileconfig.xml
+}
+
+
 
 remote_nextcloud_handler() {
     echo ""
@@ -184,6 +190,10 @@ remote_nextcloud_handler() {
         
         # configure zpush (which links to contacts & calendar)
         configure_zpush
+
+        # update ios mobileconfig.xml
+        update_mobileconfig "$new_url"
+
         
         # prevent nginx from serving any miab-installed nextcloud
         # files and remove owncloud cron job
@@ -225,6 +235,16 @@ remote_nextcloud_handler() {
                       "NC_PORT=$NC_PORT" \
                       "NC_PREFIX=$NC_PREFIX" \
                       "NC_HOST_SRC_IP='${NC_HOST_SRC_IP:-}'"
+
+    # Hook the management daemon, even if no remote nextcloud
+    # (NC_HOST==''). Must be done after writing mailinabox_mods.conf
+
+    # 1. install hooking code
+    install_hook_handler "setup/mods.available/hooks/remote-nextcloud-mgmt-hooks.py"
+    # 2. trigger hooking code for a web_update event, which updates
+    # the systems nginx configuration
+    tools/web_update
 }
 
 remote_nextcloud_handler
+

tests/bin/m-debug.sh

@@ -17,5 +17,6 @@ export FLASK_DEBUG=1
 if ! systemctl is-active --quiet miabldap-capture; then
     export CAPTURE_STORAGE_ROOT=/mailinabox/management/reporting/capture/tests
 fi
+export PYTHONPATH=${LOCAL_MODS_DIR:-/local}
 python3 --version
 python3 ./daemon.py

tests/lib/rest.sh

@@ -57,6 +57,10 @@ rest_urlencoded() {
     
 	local data=()
 	local item output onlydata="false"
+
+	if [ ! -z "$auth_user" ]; then
+		data+=("--user" "${auth_user}:${auth_pass}")
+	fi
 	
 	for item; do
         case "$item" in
@@ -86,9 +90,9 @@ rest_urlencoded() {
         esac
     done
 
-	echo "spawn: curl -w \"%{http_code}\" -X $verb --user \"${auth_user}:xxx\" ${data[@]} $url" 1>&2
+	echo "spawn: curl -w \"%{http_code}\" -X $verb ${data[@]} $url" 1>&2
 	# pipe through 'tr' to avoid bash "warning: command substitution: ignored null byte in input" where curl places a \0 between output and http_code
-	output=$(curl -s -S -w "%{http_code}" -X $verb --user "${auth_user}:${auth_pass}" "${data[@]}" $url | tr -d '\0')
+	output=$(curl -s -S -w "%{http_code}" -X $verb "${data[@]}" $url | tr -d '\0')
 	local code=$?
 
 	# http status is last 3 characters of output, extract it

tests/suites/remote-nextcloud.sh

@@ -140,10 +140,66 @@ test_nextcloud_contacts() {
     test_end
 }
 
+test_web_config() {
+    test_start "web-config"
+
+	if ! assert_is_configured; then
+		test_end
+		return
+	fi
+
+	local code
+	
+	# nginx should be configured to redirect .well-known/caldav and
+	# .well-known/carddav to the remote nextcloud
+	if grep '\.well-known/carddav[\t ]*/cloud/' /etc/nginx/conf.d/local.conf >/dev/null; then
+		test_failure "/.well-known/carddav redirects to the local nextcloud, but should redirect to $NC_HOST:$NC_PORT"
+	else
+		# ensure the url works
+		record "[test /.well-known/carddav url]"
+		rest_urlencoded GET "/.well-known/carddav" "" "" --location 2>>$TEST_OF
+		code=$?
+		record "code=$code"
+		record "status=$REST_HTTP_CODE"
+		record "output=$REST_OUTPUT"
+		if [ $code -eq 0 ]; then
+			test_failure "carddav url works, but expecting 401/NotAuthenticated from server"
+		elif [ $code -eq 1 -o $REST_HTTP_CODE -ne 401 ] || ! grep "NotAuthenticated" <<<"$REST_OUTPUT" >/dev/null; then
+			test_failure "carddav url doesn't work: $REST_ERROR"
+		fi
+	fi
+	
+	if grep '\.well-known/caldav[\t ]*/cloud/' /etc/nginx/conf.d/local.conf >/dev/null; then
+		test_failure "/.well-known/caldav redirects to the local nextcloud, but should redirect to $NC_HOST:$NC_PORT"
+	else
+		# ensure the url works
+		record "[test /.well-known/caldav url]"
+		rest_urlencoded GET "/.well-known/caldav" "" "" --location 2>>$TEST_OF
+		code=$?
+		record "code=$code"
+		record "status=$REST_HTTP_CODE"
+		record "output=$REST_OUTPUT"
+		if [ $code -eq 0 ]; then
+			test_failure "caldav url works, but expecting 401/NotAuthenticated from server"
+		elif [ $code -eq 1 -o $REST_HTTP_CODE -ne 401 ] || ! grep "NotAuthenticated" <<<"$REST_OUTPUT" >/dev/null; then
+			test_failure "caldav url doesn't work: $REST_ERROR"
+		fi
+	fi
+
+	# ios/osx mobileconfig should be configured to redirect carddav to the
+	# remote nectcloud
+	if grep -A 1 CardDAVPrincipalURL /var/lib/mailinabox/mobileconfig.xml | tail -1 | grep -F "<string>/cloud/remote.php" >/dev/null; then
+		test_failure "ios mobileconfig redirects to the local nextcloud, but should redirect to $NC_HOST:$NC_PORT"
+	fi
+	
+    test_end    
+}
+
 
 suite_start "remote-nextcloud" mgmt_start
 
 #test_mail_from_nextcloud
+test_web_config
 test_nextcloud_contacts
 
 suite_end mgmt_end

tests/system-setup/setup-defaults.sh

@@ -33,6 +33,7 @@ else
 fi
 export DOWNLOAD_CACHE_DIR="${DOWNLOAD_CACHE_DIR:-$(pwd)/downloads}"
 export DOWNLOAD_NEXTCLOUD_FROM_GITHUB="${DOWNLOAD_NEXTCLOUD_FROM_GITHUB:-false}"
+export MGMT_LOG_LEVEL=${MGMT_LOG_LEVEL:-debug}
 
 
 # Used by ehdd/start-encrypted.sh

tests/vagrant/Vagrantfile

@@ -25,7 +25,7 @@ export FEATURE_MUNIN=false
 export EHDD_KEYFILE=$HOME/keyfile
 echo -n "boo" >$EHDD_KEYFILE
 tests/system-setup/remote-nextcloud-docker.sh || exit 1
-tests/runner.sh ehdd remote-nextcloud default || exit 2
+tests/runner.sh remote-nextcloud ehdd default || exit 2
 SH
     end
   end

tools/hooks_update

@@ -0,0 +1,18 @@
+#!/bin/bash
+#####
+##### This file is part of Mail-in-a-Box-LDAP which is released under the
+##### terms of the GNU Affero General Public License as published by the
+##### Free Software Foundation, either version 3 of the License, or (at
+##### your option) any later version. See file LICENSE or go to
+##### https://github.com/downtownallday/mailinabox-ldap for full license
+##### details.
+#####
+
+# use this when a hook handler is added or removed from managment to
+# enable the hook without having to restart the management daemon.
+#
+# this only works for an addition or removal, if a hook handler file
+# was replaced, the daemon must be restarted
+#
+
+curl -s -d POSTDATA --user $(</var/lib/mailinabox/api.key): http://127.0.0.1:10222/hooks/update