Test upgrade to LDAP from upstream Mail-in-a-Box/sqlite

.travis.yml

@@ -1,37 +1,46 @@
 # travisci config
 env:
   global:
-  - PRIMARY_HOSTNAME=box.abc.com
+  - MIAB_LDAP=true
 
 language: shell
 os: linux
 dist: bionic
-  
-before_install:
-  - echo "==== ENVIRONMENT ===="
-  - env | sort
-  - echo "UMASK=$(umask)"
-  #
-  - echo "==== AppArmor Status ===="
-  - (sudo aa-status; true)
-  #
-  - echo "==== NETWORK INFO ===="
-  - hostname -I
-  - hostname -i
-  - hostname
-  - hostname --fqdn
-  - ip add
-  - sysctl -a 2>/dev/null | grep -i ipv6 | grep disable
-  
-install:
-  - sudo tests/system-setup/remote-nextcloud-docker.sh
-  - hostname || true
-  - hostname --fqdn || true
-  - getent hosts box.abc.com || true
 
-script:
+jobs:
-  #
+  fast_finish: true
-  # launch automated tests, but skip tests that require remote
+  include:
-  # smtp support because Travis-CI blocks outgoing port 25
+  # JOB: MiaB-LDAP connected to a remote Nextcloud
-  - sudo touch /etc/dovecot/sieve-spam.svbin
+  - env: PRIMARY_HOSTNAME=box1.abc.com
-  - sudo tests/runner.sh -dumpoutput -no-smtp-remote default remote-nextcloud
+    name: remote-nextcloud-docker
+    before_install:
+      - echo "==== ENVIRONMENT ===="
+      - env | sort
+      - echo "UMASK=$(umask)"
+      - echo "==== AppArmor Status ===="
+      - (sudo aa-status; true)
+      - echo "==== NETWORK INFO ===="
+      - hostname -I
+      - hostname -i
+      - hostname
+      - hostname --fqdn
+      - ip add
+      - sysctl -a 2>/dev/null | grep -i ipv6 | grep disable
+    install:
+      - sudo tests/system-setup/remote-nextcloud-docker.sh
+    script:
+      # launch automated tests, but skip tests that require remote
+      # smtp support because Travis-CI blocks outgoing port 25
+      - sudo touch /etc/dovecot/sieve-spam.svbin
+      - sudo tests/runner.sh -dumpoutput -no-smtp-remote default remote-nextcloud
+
+  # JOB: Upgrade from upstream install
+  - env: PRIMARY_HOSTNAME=box2.abc.com
+    name: upgrade-from-upstream
+    install:
+      - sudo tests/system-setup/upgrade-from-upstream.sh
+    script:
+      # launch automated tests, but skip tests that require remote
+      # smtp support because Travis-CI blocks outgoing port 25
+      - sudo touch /etc/dovecot/sieve-spam.svbin
+      - sudo tests/runner.sh -dumpoutput -no-smtp-remote

management/mailconfig.py

@@ -321,7 +321,7 @@ def get_mail_aliases(env, as_map=False):
 	
 	# make a dict of permitted senders, key=mail(lowercase) value=members
 	permitted_senders = { rec["mail"][0].lower(): rec["member"] for rec in pager }
-	
+
 	# get all alias groups
 	pager = c.paged_search(env.LDAP_ALIASES_BASE, "(objectClass=mailGroup)", attributes=['mail','member','rfc822MailMember'])
 	
@@ -362,7 +362,7 @@ def get_mail_aliases(env, as_map=False):
 			alias = aliases[address]
 			xft = ",".join(alias["forward_tos"])
 			xas = ",".join(alias["permitted_senders"])
-			list.append( (address, xft, xas) )
+			list.append( (address, xft, None if xas is "" else xas) )
 		return list
 	
 	else:
@@ -432,7 +432,7 @@ def get_domain(emailaddr, as_unicode=True):
 			pass
 	return ret
 
-def get_mail_domains(env, as_map=False, filter_aliases=None, category=None, users_only=False):
+def get_mail_domains(env, as_map=False, filter_aliases=lambda alias: True, category=None, users_only=False):
 	# Retrieves all domains, IDNA-encoded, we accept mail for.
 	#
 	# If as_map is False, the function returns the lowercase domain
@@ -457,17 +457,18 @@ def get_mail_domains(env, as_map=False, filter_aliases=None, category=None, user
 	# make it easy for dns_update to get ssl domains]
 	#
 	# If users_only is True, only return domains with email addresses
-	# that correspond to user accounts. [TODO: This currently has no
+	# that correspond to user accounts.
-	# effect - this function only returns user mail domains]
 	#
 	conn = open_database(env)
 	filter = "(&(objectClass=domain)(businessCategory=mail))"
 	if category:
 		filter = "(&(objectClass=domain)(businessCategory=%s))" % category
+
+	domains=None
+
+	# user mail domains
 	id = conn.search(env.LDAP_DOMAINS_BASE, filter, attributes="dc")
 	response = conn.wait(id)
-	filter_candidates=[]
-	domains=None
 	if as_map:
 		domains = {}
 		for rec in response:
@@ -478,36 +479,23 @@ def get_mail_domains(env, as_map=False, filter_aliases=None, category=None, user
 			if filter_aliases: filter_candidates.append(rec['dc'][0].lower())
 	else:
 		domains = set([ rec["dc"][0].lower() for rec in response ])
-		if filter_aliases: filter_candidates += domains
-	
-	for candidate in filter_candidates:
-		# with the filter, there has to be at least one user or
-		# filtered (included) alias in the domain for the domain to be
-		# part of the returned set
 
-		# any users ?
-		response = conn.wait( conn.search(env.LDAP_USERS_BASE, "(&(objectClass=mailUser)(mail=*@%s))" % candidate, size_limit=1) )
-		if response.next():
-			# yes, that domain needs to be in the returned set
-			continue
 
-		# any filtered aliases ?
+	# alias domains
-		pager = conn.paged_search(
+	if not users_only:
-			env.LDAP_ALIASES_BASE,
+		pager = conn.paged_search(env.LDAP_ALIASES_BASE, "(objectClass=mailGroup)", attributes="mail")
-			"(&(objectClass=mailGroup)(mail=*@%s))" % candidate,
+		if as_map:
-			attributes=['mail'])
+			for rec in pager:
+				if filter_aliases(rec["mail"][0].lower()):
+					domain = get_domain(rec["mail"][0].lower(),as_unicode=False)
+					domains[domain] = {
+						"dn": None,
+						"domain": domain
+					}
 
-		remove = True
+		else:
-		for rec in pager:
+			domains = domains.union(set([ get_domain(rec["mail"][0].lower(), as_unicode=False) for rec in pager if filter_aliases(rec["mail"][0].lower()) ]))
-			if filter_aliases(rec['mail'][0]):
+					
-				remove = False
-				pager.abandon()
-				break
-
-		if remove:
-			if as_map: del domains[candidate]
-			else: domains.remove(candidate)
-			
 	return domains
 
 

setup/ldap.sh

@@ -857,7 +857,13 @@ cat > /etc/logrotate.d/slapd <<EOF;
 EOF
 
 # Modify olc server config like TLS
-modify_global_config
+# Skip this step if no ca_certificate.pem exists - this indicates
+# that the system hasn't yet been migrated from sqlite
+if [ -e "$STORAGE_ROOT/ssl/ca_certificate.pem" ]; then
+	modify_global_config
+else
+	say_debug "Not enabling TLS at this time - ca_certificate hasn't been generated yet"
+fi
 
 # Add overlays and ensure mail-related attributes are indexed
 add_overlays

setup/mods.available/remote-nextcloud-use-miab.sh

@@ -6,8 +6,9 @@
 #
 # The script will:
 #   1. enable the "LDAP user and group backend" in Nextcloud
-#   2. configure Nextcloud to access MiaB-LDAP for users and groups
+#   2. install calendar and contacts
-#   3. optionally install and configure ssmtp so system mail is
+#   3. configure Nextcloud to access MiaB-LDAP for users and groups
+#   4. optionally install and configure ssmtp so system mail is
 #      sent to MiaB-LDAP
 #
 VERBOSE=0

setup/ssl.sh

@@ -96,11 +96,6 @@ if [ ! -s $STORAGE_ROOT/ssl/ssl_private_key.pem ]; then
 	# Set the umask so the key file is never world-readable.
 	(umask 037; hide_output \
 		openssl genrsa -out $STORAGE_ROOT/ssl/ssl_private_key.pem 2048)
-
-	# Give the group 'ssl-cert' read access so slapd can read it
-	groupadd -fr ssl-cert
-	chgrp ssl-cert $STORAGE_ROOT/ssl/ssl_private_key.pem
-	chmod g+r $STORAGE_ROOT/ssl/ssl_private_key.pem
 	
 	# Remove the ssl_certificate.pem symbolic link to force a
 	# regeneration of the server certificate. It needs to be
@@ -110,6 +105,11 @@ if [ ! -s $STORAGE_ROOT/ssl/ssl_private_key.pem ]; then
 	fi
 fi
 
+# Give the group 'ssl-cert' read access so slapd can read it
+groupadd -fr ssl-cert
+chgrp ssl-cert $STORAGE_ROOT/ssl/ssl_private_key.pem
+chmod g+r $STORAGE_ROOT/ssl/ssl_private_key.pem
+
 #
 # Generate a root CA certificate
 #

tests/lib/all.sh

@@ -0,0 +1,16 @@
+#
+# source all lib scripts
+#
+# from your script, supply the path to this directory as the first argument
+#
+#    eg source "tests/lib/all.sh" "tests/lib"
+#
+# failure to load any script is fatal!
+
+. "$1/color-output.sh" || exit 1
+. "$1/locations.sh"    || exit 2
+. "$1/misc.sh"         || exit 3
+. "$1/rest.sh"         || exit 4
+. "$1/system.sh"       || exit 5
+
+

tests/lib/color-output.sh

@@ -0,0 +1,32 @@
+# ansi escapes for hilighting text
+F_DANGER=$(echo -e "\033[31m")
+F_WARN=$(echo -e "\033[93m")
+F_RESET=$(echo -e "\033[39m")
+
+
+danger() {
+    local echoarg
+    case "$1" in
+        -n )
+            echoarg="$1"
+            shift
+            ;;
+        * )
+            echoarg=""
+    esac
+    echo $echoarg "${F_DANGER}$1${F_RESET}"
+}
+
+warn() {
+    local echoarg
+    case "$1" in
+        -n )
+            echoarg="$1"
+            shift
+            ;;
+        * )
+            echoarg=""
+    esac
+    echo "${F_WARN}$1${F_RESET}"
+}
+

tests/lib/misc.sh

@@ -0,0 +1,65 @@
+#
+# misc helpful functions
+#
+# requirements:
+#   system packages: [ python3 ]
+
+
+array_contains() {
+	local searchfor="$1"
+	shift
+	local item
+	for item; do
+		[ "$item" == "$searchfor" ] && return 0
+	done
+	return 1
+}
+
+is_true() {
+    # empty string is not true
+    if [ "$1" == "true" \
+              -o "$1" == "TRUE" \
+              -o "$1" == "True" \
+              -o "$1" == "yes" \
+              -o "$1" == "YES" \
+              -o "$1" == "Yes" \
+              -o "$1" == "1" ]
+    then
+        return 0
+    else
+        return 1
+    fi
+}
+
+is_false() {
+    if is_true $@; then return 1; fi
+    return 0
+}
+
+email_localpart() {
+    local addr="$1"
+    awk -F@ '{print $1}' <<<"$addr"
+}
+
+email_domainpart() {
+    local addr="$1"
+    awk -F@ '{print $2}' <<<"$addr"
+}
+
+
+generate_uuid() {
+	local uuid
+	uuid=$(python3 -c "import uuid; print(uuid.uuid4())")
+	[ $? -ne 0 ] && die "Unable to generate a uuid"
+	echo "$uuid"
+}
+
+generate_qa_password() {
+    echo "Test1234."
+}
+
+sha1() {
+	local txt="$1"
+	python3 -c "import hashlib; m=hashlib.sha1(); m.update(bytearray(r'''$txt''','utf-8')); print(m.hexdigest());" || die "Unable to generate sha1 hash"
+}
+

tests/lib/rest.sh

@@ -0,0 +1,106 @@
+#
+# REST helper functions
+#
+# requirements:
+#   system packages: [ curl ]
+#   lib scripts: [ color-output.sh ]
+#
+
+rest_urlencoded() {
+	# Issue a REST call having data urlencoded
+    #
+    # eg: rest_urlencoded POST /admin/mail/users/add "email=alice@abc.com" "password=secret"
+    #
+    # When providing a URI (/mail/users/add) and not a URL
+    # (https://host/mail/users/add), PRIMARY_HOSTNAME must be set!
+    #
+    # The function will set the following global variables regardless
+    # of exit c ode:
+    #     REST_HTTP_CODE
+    #     REST_OUTPUT
+    #     REST_ERROR
+    # 
+    # Return values:
+    #   0 indicates success (curl returned 0 or a code deemed to be
+    #     successful and HTTP status is >=200  but <300)
+    #   1 curl returned with non-zero code that indicates and error
+    #   2 the response status was <200 or >= 300
+    #
+    # Messages, errors, and output are all sent to stderr, there is no
+    # stdout output
+    #   
+	local verb="$1" # eg "POST"
+	local uri="$2"  # eg "/mail/users/add"
+	local auth_user="$3"
+	local auth_pass="$4"
+	shift; shift; shift; shift  # remaining arguments are data or curl args
+
+	local url
+    case "$uri" in
+        http:* | https:* )
+            url="$uri"
+            ;;
+        * )
+            url="https://$PRIMARY_HOSTNAME${uri}"
+            ;;
+    esac
+    
+	local data=()
+	local item output onlydata="false"
+	
+	for item; do
+        case "$item" in
+            -- )
+                onlydata="true"
+                ;;
+            --* )
+                # curl argument
+                if $onlydata; then
+                    data+=("--data-urlencode" "$item");
+                else
+                    data+=("$item")
+                fi
+                ;;
+            * )
+                onlydata="true"
+                data+=("--data-urlencode" "$item");
+                ;;
+        esac
+    done
+
+	echo "spawn: curl -w \"%{http_code}\" -X $verb --user \"${auth_user}:xxx\" ${data[@]} $url" 1>&2
+	output=$(curl -s -S -w "%{http_code}" -X $verb --user "${auth_user}:${auth_pass}" "${data[@]}" $url)
+	local code=$?
+
+	# http status is last 3 characters of output, extract it
+	REST_HTTP_CODE=$(awk '{S=substr($0,length($0)-2)} END {print S}' <<<"$output")
+	REST_OUTPUT=$(awk 'BEGIN{L=""}{ if(L!="") print L; L=$0 } END { print substr(L,1,length(L)-3) }' <<<"$output")
+	REST_ERROR=""
+	[ -z "$REST_HTTP_CODE" ] && REST_HTTP_CODE="000"
+
+	if [ $code -ne 0 ]; then
+		if [ $code -eq 56 -a $REST_HTTP_CODE -eq 200 ]; then
+			# this is okay, I guess. happens sometimes during
+			# POST /admin/mail/aliases/remove
+			# 56=Unexpected EOF
+			echo "Ignoring curl return code 56 due to 200 status" 1>&2
+			
+		elif [ $code -ne 16 -o $REST_HTTP_CODE -ne 200 ]; then
+			# any error code will fail the rest call except for a 16
+			# with a 200 HTTP status.
+			# 16="a problem was detected in the HTTP2 framing layer. This is somewhat generic and can be one out of several problems"
+			REST_ERROR="CURL failed with code $code"
+			echo "${F_DANGER}$REST_ERROR${F_RESET}" 1>&2
+			echo "$output" 1>&2
+			return 1
+		fi
+	fi
+	if [ $REST_HTTP_CODE -lt 200 -o $REST_HTTP_CODE -ge 300 ]; then
+		REST_ERROR="REST status $REST_HTTP_CODE: $REST_OUTPUT"
+		echo "${F_DANGER}$REST_ERROR${F_RESET}" 1>&2
+		return 2
+	fi
+	echo "CURL succeded, HTTP status $REST_HTTP_CODE" 1>&2
+	echo "$output" 1>&2
+	return 0	
+}

tests/lib/system.sh

@@ -0,0 +1,102 @@
+
+wait_for_apt() {
+    # check to see if other package managers have a lock on new
+    # installs, and wait for them to finish
+    #
+    # returns non-zero if waiting times out (currently ~600 seconds)
+    local count=0
+    while fuser /var/lib/dpkg/lock >/dev/null 2>&1 || fuser /var/lib/dpkg/lock-frontend >/dev/null 2>&1; do
+        sleep 6
+        let count+=1
+        if [ $count -eq 1 ]; then
+            echo -n "Waiting for other package manager to finish..."
+        elif [ $count -gt 100 ]; then
+            echo -n "FAILED"
+            return 1
+        else
+            echo -n "${count}.."
+        fi
+    done
+    [ $count -ge 1 ] && echo ""
+}
+
+dump_file() {
+    local log_file="$1"
+    local lines="$2"
+    local title="DUMP OF $log_file"
+    echo ""
+    echo "--------"
+    echo -n "-------- $log_file"
+    if [ ! -z "$lines" ]; then
+        echo " (last $line lines)"
+    else
+        echo ""
+    fi
+    echo "--------"
+
+    if [ !-e "$log_file" ]; then
+        echo "DOES NOT EXIST"
+    elif [ ! -z "$lines" ]; then
+        tail -$lines "$log_file"
+    else
+        cat "$log_file"
+    fi
+}
+
+dump_file_if_exists() {
+    [ ! -e "$1" ] && return
+    dump_file "$@"
+}
+
+update_system_time() {
+    if [ ! -x /usr/sbin/ntpdate ]; then
+        wait_for_apt
+        apt-get install -y -qq ntpdate || return 1
+    fi
+    ntpdate -s ntp.ubuntu.com && echo "System time updated"
+}
+
+set_system_hostname() {
+    # set the system hostname to the FQDN specified or
+    # PRIMARY_HOSTNAME if no FQDN was given
+    local fqdn="${1:-$PRIMARY_HOSTNAME}"
+    local host="$(awk -F. '{print $1}' <<< "$fqdn")"
+    sed -i 's/^127\.0\.1\.1[ \t].*/127.0.1.1 '"$fqdn $host ip4-loopback/" /etc/hosts || return 1
+    #hostname "$host" || return 1
+    #echo "$host" > /etc/hostname
+    return 0
+}
+
+install_docker() {
+    if [ -x /usr/bin/docker ]; then
+        echo "Docker already installed"
+        return 0
+    fi
+    
+    wait_for_apt
+    apt-get install -y -qq \
+            apt-transport-https \
+            ca-certificates \
+            curl \
+            gnupg-agent \
+            software-properties-common \
+        || return 1
+       
+    wait_for_apt
+    curl -fsSL https://download.docker.com/linux/ubuntu/gpg | apt-key add - \
+        || return 2
+    
+    wait_for_apt
+    apt-key fingerprint 0EBFCD88 || return 3
+    
+    wait_for_apt
+    add-apt-repository -y --update "deb [arch=amd64] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" || return 4
+    
+    wait_for_apt
+    apt-get install -y -qq \
+            docker-ce \
+            docker-ce-cli \
+            containerd.io \
+        || return 5
+}
+

tests/suites/_init.sh

@@ -6,7 +6,7 @@
 set +eu
 
 # load test suite helper functions
-. suites/_locations.sh      || exit 1
+. lib/all.sh "lib"      || exit 1
 . suites/_ldap-functions.sh || exit 1
 . suites/_mail-functions.sh || exit 1
 . suites/_mgmt-functions.sh || exit 1
@@ -20,10 +20,6 @@ declare -i OVERALL_SKIPPED=0
 declare -i OVERALL_COUNT=0
 declare -i OVERALL_COUNT_SUITES=0
 
-# ansi escapes for hilighting text
-F_DANGER=$(echo -e "\033[31m")
-F_WARN=$(echo -e "\033[93m")
-F_RESET=$(echo -e "\033[39m")
 
 # options
 FAILURE_IS_FATAL=no
@@ -157,7 +153,12 @@ test_skip() {
 }
 
 skip_test() {
-	# return 0 if we should skip the current test
+	# call from within a test to check whether the test will be
+	# skipped
+	#
+	# returns 0 if the current test was skipped in which case your test
+	# function must immediately call 'test_end' and return
+	#
 	if [ "$SKIP_REMOTE_SMTP_TESTS" == "yes" ] &&
 		   array_contains "remote-smtp" "$@";
 	then
@@ -191,16 +192,6 @@ die() {
 	exit 1
 }
 
-array_contains() {
-	local searchfor="$1"
-	shift
-	local item
-	for item; do
-		[ "$item" == "$searchfor" ] && return 0
-	done
-	return 1
-}
-
 python_error() {
 	# finds tracebacks and outputs just the final error message of
 	# each

tests/suites/_ldap-functions.sh

@@ -1,16 +1,10 @@
 # -*- indent-tabs-mode: t; tab-width: 4; -*-
 
-generate_uuid() {
+# requirements:
-	local uuid
+#   system packages: [ ldap-utils ]
-	uuid=$(python3 -c "import uuid; print(uuid.uuid4())")
+#   setup scripts:   [ functions-ldap.sh ]
-	[ $? -ne 0 ] && die "Unable to generate a uuid"
+#   setup artifacts: [ miab_ldap.conf ]
-	echo "$uuid"
-}
 
-sha1() {
-	local txt="$1"
-	python3 -c "import hashlib; m=hashlib.sha1(); m.update(bytearray(r'''$txt''','utf-8')); print(m.hexdigest());" || die "Unable to generate sha1 hash"
-}
 
 delete_user() {
 	local email="$1"

tests/suites/_mgmt-functions.sh

@@ -44,49 +44,9 @@ mgmt_rest() {
 	local uri="$2"  # eg "/mail/users/add"
 	shift; shift;   # remaining arguments are data
 
-	local auth_user="${MGMT_ADMIN_EMAIL}"
+	# call function from lib/rest.sh
-	local auth_pass="${MGMT_ADMIN_PW}"
+	rest_urlencoded "$verb" "$uri" "${MGMT_ADMIN_EMAIL}" "${MGMT_ADMIN_PW}" "$@" 2>>$TEST_OF
-	local url="https://$PRIMARY_HOSTNAME${uri}"
+	return $?
-	local data=()
-	local item output
-	
-	for item; do data+=("--data-urlencode" "$item"); done
-
-	record "spawn: curl -w \"%{http_code}\" -X $verb --user \"${auth_user}:xxx\" ${data[@]} $url"
-	output=$(curl -s -S -w "%{http_code}" -X $verb --user "${auth_user}:${auth_pass}" "${data[@]}" $url 2>>$TEST_OF)
-	local code=$?
-
-	# http status is last 3 characters of output, extract it
-	REST_HTTP_CODE=$(awk '{S=substr($0,length($0)-2)} END {print S}' <<<"$output")
-	REST_OUTPUT=$(awk 'BEGIN{L=""}{ if(L!="") print L; L=$0 } END { print substr(L,1,length(L)-3) }' <<<"$output")
-	REST_ERROR=""
-	[ -z "$REST_HTTP_CODE" ] && REST_HTTP_CODE="000"
-
-	if [ $code -ne 0 ]; then
-		if [ $code -eq 56 -a $REST_HTTP_CODE -eq 200 ]; then
-			# this is okay, I guess. happens sometimes during
-			# POST /admin/mail/aliases/remove
-			# 56=Unexpected EOF
-			record "Ignoring curl return code 56 due to 200 status"
-			
-		elif [ $code -ne 16 -o $REST_HTTP_CODE -ne 200 ]; then
-			# any error code will fail the rest call except for a 16
-			# with a 200 HTTP status.
-			# 16="a problem was detected in the HTTP2 framing layer. This is somewhat generic and can be one out of several problems"
-			REST_ERROR="CURL failed with code $code"
-			record "${F_DANGER}$REST_ERROR${F_RESET}"
-			record "$output"
-			return 1
-		fi
-	fi
-	if [ $REST_HTTP_CODE -lt 200 -o $REST_HTTP_CODE -ge 300 ]; then
-		REST_ERROR="REST status $REST_HTTP_CODE: $REST_OUTPUT"
-		record "${F_DANGER}$REST_ERROR${F_RESET}"
-		return 2
-	fi
-	record "CURL succeded, HTTP status $REST_HTTP_CODE"
-	record "$output"
-	return 0	
 }
 
 

tests/system-setup/remote-nextcloud-docker.sh

@@ -27,9 +27,9 @@
 
 
 usage() {
-    echo "Usage: $(basename "$0") [\"before-miab-install\"|\"miab-install\"|\"after-miab-install\"]"
+    echo "Usage: $(basename "$0")"
-    echo "Install MiaB-LDAP and a remote Nextcloud running under docker exposed as localhost:8000"
+    echo "Install MiaB-LDAP and a remote Nextcloud running under docker"
-    echo "With no arguments, all three stages are run."
+    echo "Nextcloud is exposed as http://localhost:8000"
     exit 1
 }
 
@@ -40,10 +40,9 @@ if [ ! -d "tests/system-setup" ]; then
 fi
 
 # load helper scripts
-. "tests/system-setup/setup-defaults.sh" \
+. "tests/lib/all.sh" "tests/lib" || die "Could not load lib scripts"
-    || die "Could not load setup-defaults"
+. "tests/system-setup/setup-defaults.sh" || die "Could not load setup-defaults"
-. "tests/system-setup/setup-funcs.sh" \
+. "tests/system-setup/setup-funcs.sh" || die "Could not load setup-funcs"
-    || die "Could not load setup-funcs"
 
 # ensure running as root
 if [ "$EUID" != "0" ]; then
@@ -54,42 +53,15 @@ fi
 
 before_miab_install() {
     H1 "BEFORE MIAB-LDAP INSTALL"
-
+    system_init
-    H2 "Update /etc/hosts"
+    miab_testing_init || die "Initialization failed"
-    set_system_hostname || die "Could not set hostname"
-
-    # update system time
-    H2 "Set system time"
-    update_system_time || echo "Ignoring error..."
     
-    # update package lists before installing anything
-    H2 "apt-get update"
-    wait_for_apt
-    apt-get update -qq || die "apt-get update failed!"
-
-    # upgrade packages - if we don't do this and something like bind
-    # is upgraded through automatic upgrades (because maybe MiaB was
-    # previously installed), it may cause problems with the rest of
-    # the setup, such as with name resolution failures
-    if is_false "$TRAVIS"; then
-        H2 "apt-get upgrade"
-        wait_for_apt
-        apt-get upgrade -qq || die "apt-get upgrade failed!"
-    fi
-    
-    # install prerequisites
-    H2 "QA pre-setup prerequisites"
-    install_pre_setup_qa_prerequisites \
-        || die "Error installing QA prerequisites"
-
     # enable the remote Nextcloud setup mod, which tells MiaB-LDAP to use
     # the remote Nextcloud for calendar and contacts instead of the
     # MiaB-installed one
-    H2 "Create local/remote-nextcloud.sh symbolic link"
+    H2 "Enable local mod remote-nextcloud"
-    if [ ! -e "local/remote-nextcloud.sh" ]; then
+    enable_miab_mod "remote-nextcloud" \
-        mkdir -p local
+        || die "Could not enable remote-nextcloud mod"
-        ln -s "../setup/mods.available/remote-nextcloud.sh" "local/remote-nextcloud.sh" || die "Could not create remote-nextcloud.sh symlink"
-    fi
     
     # install Docker
     H2 "Install Docker"
@@ -101,7 +73,7 @@ miab_install() {
     H1 "MIAB-LDAP INSTALL"
     if ! setup/start.sh; then
         H1 "OUTPUT OF SELECT FILES"
-        dump_log "/var/log/syslog" 100
+        dump_file "/var/log/syslog" 100
         dump_conf_files "$TRAVIS"
         H2; H2 "End"; H2
         die "setup/start.sh failed!"
@@ -119,22 +91,28 @@ after_miab_install() {
     
     # run Nextcloud docker image
     H2 "Start Nextcloud docker container"
-    docker run -d --name NC -p 8000:80 \
+    local container_started="true"
-           --env SQLITE_DATABASE=nextclouddb.sqlite \
+    if [ -z "$(docker ps -f NAME=NC -q)" ]; then
-           --env NEXTCLOUD_ADMIN_USER="$NC_ADMIN_USER" \
+        docker run -d --name NC -p 8000:80 \
-           --env NEXTCLOUD_ADMIN_PASSWORD="$NC_ADMIN_PASSWORD" \
+               --env SQLITE_DATABASE=nextclouddb.sqlite \
-           --env NEXTCLOUD_TRUSTED_DOMAINS="127.0.0.1 ::1" \
+               --env NEXTCLOUD_ADMIN_USER="$NC_ADMIN_USER" \
-           --env NEXTCLOUD_UPDATE=1 \
+               --env NEXTCLOUD_ADMIN_PASSWORD="$NC_ADMIN_PASSWORD" \
-           --env SMTP_HOST="$PRIMARY_HOSTNAME" \
+               --env NEXTCLOUD_TRUSTED_DOMAINS="127.0.0.1 ::1" \
-           --env SMTP_SECURE="tls" \
+               --env NEXTCLOUD_UPDATE=1 \
-           --env SMTP_PORT=587 \
+               --env SMTP_HOST="$PRIMARY_HOSTNAME" \
-           --env SMTP_AUTHTYPE="LOGIN" \
+               --env SMTP_SECURE="tls" \
-           --env SMTP_NAME="$EMAIL_ADDR" \
+               --env SMTP_PORT=587 \
-           --env SMTP_PASSWORD="$EMAIL_PW" \
+               --env SMTP_AUTHTYPE="LOGIN" \
-           --env SMTP_FROM_ADDRESS="$(awk -F@ '{print $1}' <<< "$EMAIL_ADDR")" \
+               --env SMTP_NAME="$EMAIL_ADDR" \
-           --env MAIL_DOMAIN="$(awk -F@ '{print $2}' <<< "$EMAIL_ADDR")" \
+               --env SMTP_PASSWORD="$EMAIL_PW" \
-           nextcloud:latest \
+               --env SMTP_FROM_ADDRESS="$(email_localpart "$EMAIL_ADDR")" \
-        || die "Docker run failed!"
+               --env MAIL_DOMAIN="$(email_domainpart "$EMAIL_ADDR")" \
+               nextcloud:latest \
+            || die "Docker run failed!"
+    else
+        echo "Container already running"
+        container_started="false"
+    fi
 
     H2 "docker: Update /etc/hosts so it can find MiaB-LDAP by name"
     echo "$PRIVATE_IP $PRIMARY_HOSTNAME" | \
@@ -160,32 +138,20 @@ after_miab_install() {
 
     # wait for Nextcloud installation to complete
     H2 "Wait for Nextcloud installation to complete"
-    echo -n "Waiting ..."
+    wait_for_docker_nextcloud NC installed || die "Giving up"
-    local count=0
-    while true; do
-        if [ $count -ge 10 ]; then
-            echo "FAILED"
-            die "Giving up"
-        fi
-        sleep 6
-        let count+=1
-        if [ $(docker exec NC php -n -r "include 'config/config.php'; print \$CONFIG['installed']?'true':'false';") == "true" ]; then
-            echo "ok"
-            break
-        fi
-        echo -n "${count}..."
-    done
     
-    # install and enable Nextcloud and apps
+    # install and enable Nextcloud apps
     H2 "docker: install Nextcloud calendar app"
     docker exec -u www-data NC ./occ app:install calendar \
-        || die "docker: installing calendar app failed"
+        || $container_started \
+            && die "docker: installing calendar app failed ($?)"
     H2 "docker: install Nextcloud contacts app"
     docker exec -u www-data NC ./occ app:install contacts \
-        || die "docker: installing contacts app failed"
+        || $container_started \
+            && die "docker: installing contacts app failed ($?)"
     H2 "docker: enable user_ldap"
     docker exec -u www-data NC ./occ app:enable user_ldap \
-        || die "docker: enabling user_ldap failed"
+        || die "docker: enabling user_ldap failed ($?)"
 
     # integrate Nextcloud with MiaB-LDAP
     H2 "docker: integrate Nextcloud with MiaB-LDAP"
@@ -201,28 +167,26 @@ after_miab_install() {
 }
 
 
-
 #
-# process command line
+# Main
 #
-
+case "${1:-all}" in
-case "$1" in
+    before-install )
-    before-miab-install )
         before_miab_install
         ;;
-    after-miab-install )
+    install )
-        after_miab_install
-        ;;
-    miab-install )
         miab_install
         ;;
-    "" )
+    after-install )
+        after_miab_install
+        ;;
+    all )
         before_miab_install
         miab_install
         after_miab_install
         ;;
     * )
-        
         usage
         ;;
 esac
+

tests/system-setup/setup-defaults.sh

@@ -1,6 +1,6 @@
 #!/bin/bash
 
-# Used by MiaB-LDAP setup/start.sh
+# Used by setup/start.sh
 export NONINTERACTIVE=${NONINTERACTIVE:-1}
 export SKIP_NETWORK_CHECKS=${SKIP_NETWORK_CHECKS:-1}
 export STORAGE_USER="${STORAGE_USER:-user-data}"
@@ -15,7 +15,6 @@ elif [ -z "$PRIMARY_HOSTNAME" ]; then
     export PRIMARY_HOSTNAME=${PRIMARY_HOSTNAME:-$(hostname --fqdn || hostname)}
 fi
 
-
 # Placing this var in STORAGE_ROOT/ldap/miab_ldap.conf before running
 # setup/start.sh will avoid a random password from being used for the
 # Nextcloud LDAP service account
@@ -28,6 +27,9 @@ export NC_HOST=${NC_HOST:-127.0.0.1}
 export NC_PORT=${NC_PORT:-8000}
 export NC_PREFIX=${NC_PREFIX:-/}
 
-# For setup scripts that are installing a remote Nextcloud
+# For setup scripts that may be installing a remote Nextcloud
 export NC_ADMIN_USER="${NC_ADMIN_USER:-admin}"
 export NC_ADMIN_PASSWORD="${NC_ADMIN_PASSWORD:-Test_1234}"
+
+# For setup scripts that install upstream versions
+export MIAB_UPSTREAM_GIT="https://github.com/mail-in-a-box/mailinabox.git"

tests/system-setup/setup-funcs.sh

@@ -1,4 +1,9 @@
 
+#
+# requires:
+#
+#   test scripts: [ lib/misc.sh, lib/system.sh ]
+#
 
 die() {
     local msg="$1"
@@ -25,64 +30,28 @@ H2() {
     fi
 }
 
-dump_log() {
-    local log_file="$1"
-    local lines="$2"
-    local title="DUMP OF $log_file"
-    echo ""
-    echo "--------"
-    echo -n "-------- $log_file"
-    if [ ! -z "$lines" ]; then
-        echo " (last $line lines)"
-    else
-        echo ""
-    fi
-    echo "--------"
-      
-    if [ ! -z "$lines" ]; then
-        tail -$lines "$log_file"
-    else
-        cat "$log_file"
-    fi
-}
 
-is_true() {
+wait_for_docker_nextcloud() {
-    # empty string is not true
+    local container="$1"
-    if [ "$1" == "true" \
+    local config_key="$2"
-              -o "$1" == "TRUE" \
+    echo -n "Waiting ..."
-              -o "$1" == "True" \
+    local count=0
-              -o "$1" == "yes" \
+    while true; do
-              -o "$1" == "YES" \
+        if [ $count -ge 10 ]; then
-              -o "$1" == "Yes" \
+            echo "FAILED"
-              -o "$1" == "1" ]
+            return 1
-    then
+        fi
-        return 0
+        sleep 6
-    else
+        let count+=1
-        return 1
+        if [ $(docker exec "$container" php -n -r "include 'config/config.php'; print \$CONFIG['$config_key']?'true':'false';") == "true" ]; then
-    fi
+            echo "ok"
-}
+            break
-
+        fi
-is_false() {
+        echo -n "${count}..."
-    if is_true $@; then return 1; fi
+    done
     return 0
 }
 
-wait_for_apt() {
-    local count=0
-    while fuser /var/lib/dpkg/lock >/dev/null 2>&1 || fuser /var/lib/dpkg/lock-frontend >/dev/null 2>&1; do
-        sleep 6
-        let count+=1
-        if [ $count -eq 1 ]; then
-            echo -n "Waiting for other package manager to finish..."
-        elif [ $count -gt 100 ]; then
-            echo -n "FAILED"
-            return 1
-        else
-            echo -n "${count}.."
-        fi
-    done
-    [ $count -ge 1 ] && echo ""
-}
 
 dump_conf_files() {
     local skip
@@ -98,99 +67,58 @@ dump_conf_files() {
         done
     fi
     if [ "$skip" == "false" ]; then
-        dump_log "/etc/mailinabox.conf"
+        dump_file "/etc/mailinabox.conf"
-        dump_log "/etc/hosts"
+        dump_file_if_exists "/etc/mailinabox_mods.conf"
-        dump_log "/etc/nsswitch.conf"
+        dump_file "/etc/hosts"
-        dump_log "/etc/resolv.conf"
+        dump_file "/etc/nsswitch.conf"
-        dump_log "/etc/nsd/nsd.conf"
+        dump_file "/etc/resolv.conf"
-        dump_log "/etc/postfix/main.cf"
+        dump_file "/etc/nsd/nsd.conf"
+        #dump_file "/etc/postfix/main.cf"
     fi
 }
 
 
-update_system_time() {
+
-    if [ ! -x /usr/sbin/ntpdate ]; then
+#
+# Initialize the test system
+#   hostname, time, apt update/upgrade, etc
+#
+system_init() {
+    H2 "Update /etc/hosts"
+    set_system_hostname || die "Could not set hostname"
+
+    # update system time
+    H2 "Set system time"
+    update_system_time || echo "Ignoring error..."
+    
+    # update package lists before installing anything
+    H2 "apt-get update"
+    wait_for_apt
+    apt-get update -qq || die "apt-get update failed!"
+
+    # upgrade packages - if we don't do this and something like bind
+    # is upgraded through automatic upgrades (because maybe MiaB was
+    # previously installed), it may cause problems with the rest of
+    # the setup, such as with name resolution failures
+    if is_false "$TRAVIS"; then
+        H2 "apt-get upgrade"
         wait_for_apt
-        apt-get install -y -qq ntpdate || return 1
+        apt-get upgrade -qq || die "apt-get upgrade failed!"
     fi
-    ntpdate -s ntp.ubuntu.com && echo "System time updated"
-}
-
-update_hosts() {
-    local host="$1"
-    shift
-    local ip
-    for ip; do
-        if [ ! -z "$ip" ]; then
-            local line="$ip $host"
-            if ! grep -F "$line" /etc/hosts 1>/dev/null; then
-                echo "$line" >>/etc/hosts
-            fi
-        fi
-    done
-}
-
-update_hosts_for_private_ip() {
-    # create /etc/hosts entry for PRIVATE_IP and PRIVATE_IPV6
-    # PRIMARY_HOSTNAME must already be set
-    local ip4=$(source setup/functions.sh; get_default_privateip 4)
-    local ip6=$(source setup/functions.sh; get_default_privateip 6)
-    [ -z "$ip4" -a -z "$ip6" ] && return 1
-    [ -z "$ip6" ] && ip6="::1"
-    update_hosts "$PRIMARY_HOSTNAME" "$ip4" "$ip6" || return 1
-}
-
-set_system_hostname() {
-    # set the system hostname to the FQDN specified or
-    # PRIMARY_HOSTNAME if no FQDN was given
-    local fqdn="${1:-$PRIMARY_HOSTNAME}"
-    local host="$(awk -F. '{print $1}' <<< "$fqdn")"
-    sed -i 's/^127\.0\.1\.1[ \t].*/127.0.1.1 '"$fqdn $host ip4-loopback/" /etc/hosts || return 1
-    #hostname "$host" || return 1
-    #echo "$host" > /etc/hostname
-    return 0
 }
 
 
-install_docker() {
+#
-    if [ -x /usr/bin/docker ]; then
+# Initialize the test system with QA prerequisites
-        echo "Docker already installed"
+# Anything needed to use the test runner, speed up the installation,
-        return 0
+# etc
-    fi
+#
-    
+miab_testing_init() {
-    wait_for_apt
-    apt-get install -y -qq \
-            apt-transport-https \
-            ca-certificates \
-            curl \
-            gnupg-agent \
-            software-properties-common \
-        || return 1
-       
-    wait_for_apt
-    curl -fsSL https://download.docker.com/linux/ubuntu/gpg | apt-key add - \
-        || return 2
-    
-    wait_for_apt
-    apt-key fingerprint 0EBFCD88 || return 3
-    
-    wait_for_apt
-    add-apt-repository -y --update "deb [arch=amd64] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" || return 4
-    
-    wait_for_apt
-    apt-get install -y -qq \
-            docker-ce \
-            docker-ce-cli \
-            containerd.io \
-        || return 5
-}
-
-
-install_pre_setup_qa_prerequisites() {
     [ -z "$STORAGE_ROOT" ] \
         && echo "Error: STORAGE_ROOT not set" 1>&2 \
         && return 1
 
+    H2 "QA prerequisites"
     local rc=0
     
     # python3-dnspython: is used by the python scripts in 'tests' and is
@@ -221,22 +149,20 @@ install_pre_setup_qa_prerequisites() {
 }
 
 
-travis_fix_nsd() {
+enable_miab_mod() {
-    if [ "$TRAVIS" != "true" ]; then
+    local name="${1}.sh"
-        return 0
+    if [ ! -e "local/$name" ]; then
+        mkdir -p local
+        ln -s "../setup/mods.available/$name" "local/$name"
     fi
-    
-    # nsd won't start on Travis-CI without the changes below: ip6 off and
-    # control-enable set to no. Even though the nsd docs say the
-    # default value for control-enable is no, running "nsd-checkconf -o
-    # control-enable /etc/nsd/nsd.conf" returns "yes", so we explicitly
-    # set it here.
-    #
-    # we're assuming that the "ip-address" line is the last line in the
-    # "server" section of nsd.conf. if this generated file output
-    # changes, the sed command below may need to be adjusted.
-    sed -i 's/ip-address\(.\)\(.*\)/ip-address\1\2\n  do-ip4\1 yes\n  do-ip6\1 no\n  verbosity\1 3\nremote-control\1\n  control-enable\1 no/' /etc/nsd/nsd.conf || return 1
-    cat /etc/nsd/nsd.conf
-    systemctl reset-failed nsd.service || return 2
-    systemctl restart nsd.service || return 3
 }
+
+tag_from_readme() {
+    # extract the recommended TAG from README.md
+    # sets a global "TAG"
+    local readme="${1:-README.md}"
+    TAG="$(grep -F 'git checkout' "$readme" | sed 's/.*\(v[0123456789]*\.[0123456789]*\).*/\1/')"
+    [ $? -ne 0 -o -z "$TAG" ] && return 1
+    return 0
+}
+

tests/system-setup/upgrade-from-upstream.sh

@@ -0,0 +1,289 @@
+#!/bin/bash
+
+# setup MiaB-LDAP by:
+#    1. installing upstream MiaB
+#    2. adding some data (users/aliases/etc)
+#    3. upgrading to MiaB-LDAP
+#
+# See setup-defaults.sh for usernames and passwords.
+#
+
+
+usage() {
+    echo "Usage: $(basename "$0")"
+    echo "Install MiaB-LDAP after installing upstream MiaB"
+    exit 1
+}
+
+# ensure working directory
+if [ ! -d "tests/system-setup" ]; then
+    echo "This script must be run from the MiaB root directory"
+    exit 1
+fi
+
+# load helper scripts
+. "tests/lib/all.sh" "tests/lib" || die "Could not load lib scripts"
+. "tests/system-setup/setup-defaults.sh" || die "Could not load setup-defaults"
+. "tests/system-setup/setup-funcs.sh" || die "Could not load setup-funcs"
+
+# ensure running as root
+if [ "$EUID" != "0" ]; then
+    die "This script must be run as root (sudo)"
+fi
+
+
+before_install() {
+    H1 "INIT"
+    system_init
+    miab_testing_init || die "Initialization failed"
+}
+
+upstream_install() {
+    local upstream_dir="$HOME/mailinabox-upstream"
+    H1 "INSTALL UPSTREAM"
+    [ ! -x /usr/bin/git ] && apt-get install -y -qq git
+    
+    if [ ! -d "$upstream_dir" ] || [ -z "$(ls -A "$upstream_dir")" ] ; then
+        H2 "Cloning $MIAB_UPSTREAM_GIT"
+        rm -rf "$upstream_dir"
+        git clone "$MIAB_UPSTREAM_GIT" "$upstream_dir"
+        if [ $? -ne 0 ]; then
+            rm -rf "$upstream_dir"
+            die "git clone upstream failed!"
+        fi
+        if [ -z "$TAG" ]; then
+            tag_from_readme "$upstream_dir/README.md"
+            if [ $? -ne 0 ]; then
+                rm -rf "$upstream_dir"
+                die "Failed to extract TAG from $upstream_dir/README.md"
+            fi
+        fi
+    fi
+
+    pushd "$upstream_dir" >/dev/null
+    if [ ! -z "$TAG" ]; then
+        H2 "Checkout $TAG"
+        git checkout "$TAG" || die "git checkout $TAG failed"
+    fi
+    
+    H2 "Run upstream setup"
+    setup/start.sh || die "Upstream setup failed!"
+    popd >/dev/null
+    
+    H2 "Upstream info"
+    echo "Code version: $(git describe)"
+    echo "Migration version: $(cat "$STORAGE_ROOT/mailinabox.version")"
+}
+
+
+add_data() {
+    H1 "Add some Mail-in-a-Box data"
+    local users=()
+    users+="betsy@$(email_domainpart "$EMAIL_ADDR")"
+
+    local alises=()
+    aliases+="goalias@testdom.com > ${users[0]}"
+    aliases+="nested@testdom.com > goalias@testdom.com"
+
+    local pw="$(generate_qa_password)"
+
+
+    #
+    # get the existing users and aliases
+    #
+    local current_users=() current_aliases=()
+    local user alias
+    if ! rest_urlencoded GET /admin/mail/users "$EMAIL_ADDR" "$EMAIL_PW" --insecure 2>/dev/null; then
+        die "Unable to enumerate users: rc=$? err=$REST_ERROR"
+    fi
+    for user in $REST_OUTPUT; do
+        current_users+=("$user")
+    done
+
+    if ! rest_urlencoded GET /admin/mail/aliases "$EMAIL_ADDR" "$EMAIL_PW" --insecure 2>/dev/null; then
+        die "Unable to enumerate aliases: rc=$? err=$REST_ERROR"
+    fi
+    for alias in $REST_OUTPUT; do
+        current_aliases+=("$alias")
+    done
+
+    
+    #
+    # add users
+    #
+    for user in "${users[@]}"; do
+        if array_contains "$user" "${current_users[@]}"; then
+            echo "Not adding user $user: already exists"
+
+        elif ! rest_urlencoded POST /admin/mail/users/add "$EMAIL_ADDR" "$EMAIL_PW" --insecure -- "email=$user" "password=$pw" 2>/dev/null
+        then
+            die "Unable to add user $user: rc=$? err=$REST_ERROR"
+        fi
+    done
+
+    #
+    # add aliases
+    #
+    local aliasdef
+    for aliasdef in "${aliases[@]}"; do
+        alias="$(awk -F'[> ]' '{print $1}' <<<"$aliasdef")"
+        local forwards_to="$(sed 's/.*> *\(.*\)/\1/' <<<"$aliasdef")"
+        if array_contains "$alias" "${current_aliases[@]}"; then
+            echo "Not adding alias $alias: already exists"
+            
+        elif ! rest_urlencoded POST /admin/mail/aliases/add "$EMAIL_ADDR" "$EMAIL_PW" --insecure -- "address=$alias" "forwards_to=$forwards_to" 2>/dev/null
+        then 
+            die "Unable to add alias $alias: rc=$? err=$REST_ERROR"
+        fi
+    done
+}
+
+capture_state() {
+    # users and aliases lists
+    # dns zone files
+    # tls certificates: expected CN's
+
+    local state_dir="$1"
+    local infojson="$state_dir/info.json"
+
+    H1 "Capture server state to $state_dir"
+    
+    # nuke saved state, if any
+    rm -rf "$state_dir"
+    mkdir -p "$state_dir"
+
+    # create info.json
+    H2 "create info.json"
+    echo "VERSION='$(git describe --abbrev=0)'" >"$infojson"
+    echo "MIGRATION_VERSION=$(cat "$STORAGE_ROOT/mailinabox.version")" >>"$infojson"
+
+    # record users
+    H2 "record users"
+    rest_urlencoded GET "/admin/mail/users?format=json" "$EMAIL_ADDR" "$EMAIL_PW" --insecure 2>/dev/null \
+        || die "Unable to get users: rc=$? err=$REST_ERROR"
+    echo "$REST_OUTPUT" > "$state_dir/users.json"
+
+    # record aliases
+    H2 "record aliases"
+    rest_urlencoded GET "/admin/mail/aliases?format=json" "$EMAIL_ADDR" "$EMAIL_PW" --insecure 2>/dev/null \
+        || die "Unable to get aliases: rc=$? err=$REST_ERROR"
+    echo "$REST_OUTPUT" > "$state_dir/aliases.json"
+
+    # record dns config
+    H2 "record dns details"
+    local file
+    mkdir -p "$state_dir/zones"
+    for file in ls /etc/nsd/zones/*.signed; do
+        cp "$file" "$state_dir/zones"
+    done
+    
+}
+
+miab_ldap_install() {
+    # ensure we're in a MiaB-LDAP working directory
+    if [ ! -e setup/ldap.sh ]; then
+        die "The working directory is not MiaB-LDAP!"
+    fi
+    setup/start.sh -v || die "Upgrade to MiaB-LDAP failed !!!!!!"
+}
+
+compare_state() {
+    local s1="$1"
+    local s2="$2"
+    
+    local output
+    local changed="false"
+
+    H1 "COMPARE STATES $(basename "$s1") TO $(basename "$2")"
+    H2 "Users"
+    # users
+    output="$(diff "$s1/users.json" "$s2/users.json" 2>&1)"
+    if [ $? -ne 0 ]; then
+        changed="true"
+        echo "USERS ARE DIFFERENT!"
+        echo "$output"
+    else
+        echo "OK"
+    fi
+
+    H2 "Aliases"
+    output="$(diff "$s1/aliases.json" "$s2/aliases.json" 2>&1)"
+    if [ $? -ne 0 ]; then
+        change="true"
+        echo "ALIASES ARE DIFFERENT!"
+        echo "$output"
+    else
+        echo "OK"
+    fi
+
+    H2 "DNS - zones missing"
+    local zone
+    for zone in $(cd "$s1/zones"; ls *.signed); do
+        if [ ! -e "$s2/zones/$zone" ]; then
+            echo "MISSING zone: $zone"
+            changed="true"
+        fi
+    done
+
+    H2 "DNS - zones added"
+    for zone in $(cd "$s2/zones"; ls *.signed); do
+        if [ ! -e "$s2/zones/$zone" ]; then
+            echo "ADDED zone: $zone"
+            changed="true"
+        fi
+    done
+
+    H2 "DNS - zones changed"
+    for zone in $(cd "$s1/zones"; ls *.signed); do
+        if [ -e "$s2/zones/$zone" ]; then
+            output="$(diff "$s1/zones/$zone" "$s2/zones/$zone" 2>&1)"
+            if [ $? -ne 0 ]; then
+                echo "CHANGED zone: $zone"
+                echo "$output"
+                changed="true"
+            fi
+        fi
+    done
+
+    if $changed; then
+        return 1
+    else
+        return 0
+    fi
+}
+
+
+if [ "$1" == "c" ]; then
+    capture_state "tests/system-setup/state/miab-ldap"
+    exit $?
+fi
+
+
+# install basic stuff, set the hostname, time, etc
+before_install
+
+# if MiaB-LDAP is already migrated, do not run upstream setup
+if [ -e "$STORAGE_ROOT/mailinabox.version" ] &&
+       [ $(cat "$STORAGE_ROOT/mailinabox.version") -ge 13 ]
+then
+    echo "Warning: MiaB-LDAP is already installed! Skipping installation of upstream"
+else
+    # install upstream
+    upstream_install
+    add_data
+    capture_state "tests/system-setup/state/upstream"
+fi
+
+# install miab-ldap
+miab_ldap_install
+capture_state "tests/system-setup/state/miab-ldap"
+
+# compare states
+if ! compare_state "tests/system-setup/state/upstream" "tests/system-setup/state/miab-ldap"; then
+    die "Upstream and upgraded states are different !"
+fi
+
+#
+# actual verification that mail sends/receives properly is done via
+# the test runner ...
+#