From 1bd7b2c4c77df2168b3fc53bb74528b623e6304c Mon Sep 17 00:00:00 2001
From: downtownallday <downtownallday@gmail.com>
Date: Fri, 19 Jun 2020 12:12:49 -0400
Subject: [PATCH] 1. Better code organization & simplify 2. Add "populate" data
 for upgrades - enabled in both system-setup scripts 3. Add "upgrade" test
 runner suite

---
 .travis.yml                                   |  11 +-
 tests/lib/all.sh                              |   3 +
 tests/lib/carddav.sh                          | 280 ++++++++++++++++++
 tests/lib/color-output.sh                     |  20 +-
 tests/lib/installed-state.sh                  | 141 +++++++++
 tests/lib/misc.sh                             |   6 +-
 tests/lib/populate.sh                         |  99 +++++++
 tests/lib/rest.sh                             |   4 +-
 tests/runner.sh                               |  43 +--
 tests/suites/_init.sh                         |   2 +
 tests/suites/remote-nextcloud.sh              | 229 ++------------
 tests/suites/upgrade.sh                       |  48 +++
 tests/system-setup/populate/README.txt        |  31 ++
 tests/system-setup/populate/basic-data.sh     |  10 +
 tests/system-setup/populate/basic-populate.sh |  48 +++
 tests/system-setup/populate/basic-verify.sh   |  53 ++++
 tests/system-setup/remote-nextcloud-docker.sh | 155 ++++++----
 tests/system-setup/setup-defaults.sh          |   2 +-
 tests/system-setup/setup-funcs.sh             |  65 ++--
 tests/system-setup/upgrade-from-upstream.sh   | 246 +++------------
 20 files changed, 987 insertions(+), 509 deletions(-)
 create mode 100644 tests/lib/carddav.sh
 create mode 100644 tests/lib/installed-state.sh
 create mode 100644 tests/lib/populate.sh
 create mode 100644 tests/suites/upgrade.sh
 create mode 100644 tests/system-setup/populate/README.txt
 create mode 100755 tests/system-setup/populate/basic-data.sh
 create mode 100755 tests/system-setup/populate/basic-populate.sh
 create mode 100755 tests/system-setup/populate/basic-verify.sh

diff --git a/.travis.yml b/.travis.yml
index 3b514ecc..71f87913 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -29,12 +29,15 @@ jobs:
       - ip add
       - sysctl -a 2>/dev/null | grep -i ipv6 | grep disable
     install:
-      - sudo tests/system-setup/remote-nextcloud-docker.sh
+      # setup with 'basic' data before setting up again using
+      # a remote nextcloud to verify ownCloud contacts are still
+      # available
+      - sudo tests/system-setup/remote-nextcloud-docker.sh basic
     script:
       # launch automated tests, but skip tests that require remote
       # smtp support because Travis-CI blocks outgoing port 25
       - sudo touch /etc/dovecot/sieve-spam.svbin
-      - sudo tests/runner.sh -dumpoutput -no-smtp-remote default remote-nextcloud
+      - sudo tests/runner.sh -dumpoutput -no-smtp-remote default remote-nextcloud upgrade-basic
 
   # JOB: Upgrade from upstream install
   - env:
@@ -42,9 +45,9 @@ jobs:
       - UPSTREAM_TAG=master
     name: upgrade-from-upstream
     install:
-      - sudo tests/system-setup/upgrade-from-upstream.sh
+      - sudo tests/system-setup/upgrade-from-upstream.sh basic
     script:
       # launch automated tests, but skip tests that require remote
       # smtp support because Travis-CI blocks outgoing port 25
       - sudo touch /etc/dovecot/sieve-spam.svbin
-      - sudo tests/runner.sh -dumpoutput -no-smtp-remote
+      - sudo tests/runner.sh -dumpoutput -no-smtp-remote default upgrade-basic
diff --git a/tests/lib/all.sh b/tests/lib/all.sh
index 20a0c6de..0fa22f6c 100644
--- a/tests/lib/all.sh
+++ b/tests/lib/all.sh
@@ -12,5 +12,8 @@
 . "$1/misc.sh"         || exit 3
 . "$1/rest.sh"         || exit 4
 . "$1/system.sh"       || exit 5
+. "$1/carddav.sh"      || exit 6
 
+. "$1/populate.sh"     || exit 7
+. "$1/installed-state.sh" || exit 8
 
diff --git a/tests/lib/carddav.sh b/tests/lib/carddav.sh
new file mode 100644
index 00000000..6ff58011
--- /dev/null
+++ b/tests/lib/carddav.sh
@@ -0,0 +1,280 @@
+#
+# requires:
+#    system packages: [ curl, python3, sqlite3 ]
+#    scripts: [ color-output.sh, misc.sh, locations.sh ]
+#
+# ASSETS_DIR: where the assets directory is located (defaults to
+# tests/assets)
+#
+
+nextcloud_url() {
+    # eg: http://localhost/cloud/
+    carddav_url | sed 's|\(.*\)/remote.php/.*|\1/|'
+}
+
+carddav_url() {
+    # get the carddav url as configured in z-push for the user specified
+    # eg: http://localhost/cloud/remote.php/dav/addressbooks/users/admin/contacts/
+    local user="${1:-%u}"
+    local path="${2:-CARDDAV_DEFAULT_PATH}"
+    local php="include \"$ZPUSH_DIR/backend/carddav/config.php\"; print CARDDAV_PROTOCOL . \"://\" . CARDDAV_SERVER . \":\" . CARDDAV_PORT . "
+    php="$php$path;"
+    local url
+    url="$(php -n -r "$php")"
+    [ $? -ne 0 ] && die "Unable to run php to extract carddav url from z-push"
+    sed "s/%u/$user/" <<< "$url"
+}
+
+carddav_rest() {
+    # issue a CardDAV rest call to Nextcloud
+    # SEE: https://tools.ietf.org/html/rfc6352
+    #
+    # The function will set the following global variables regardless
+    # of exit code:
+    #     REST_HTTP_CODE
+    #     REST_OUTPUT
+    #     REST_ERROR
+    #     REST_ERROR_BRIEF
+    # 
+    # Return values:
+    #   0 indicates success (curl returned 0 or a code deemed to be
+    #     successful and HTTP status is >=200  but <300)
+    #   1 curl returned with non-zero code that indicates and error
+    #   2 the response status was <200 or >= 300
+    #
+    # Debug messages are sent to stderr
+    #   
+    local verb="$1"
+    local uri="$2"
+    local auth_user="$3"
+    local auth_pass="$4"
+	shift; shift; shift; shift  # remaining arguments are data
+
+    local url
+    case "$uri" in
+        /* )
+            url="$(nextcloud_url)${uri#/}"
+            ;;
+        http* )
+            url="$uri"
+            ;;
+        * )
+            url="$(carddav_url "$auth_user")${uri#/}"
+            ;;
+    esac
+    
+    local data=()
+	local item output onlydata="false"
+	
+	for item; do
+        case "$item" in
+            -- )
+                onlydata="true"
+                ;;
+            --* )
+                # curl argument
+                if $onlydata; then
+                    data+=("--data" "$item");
+                else
+                    data+=("$item")
+                fi
+                ;;
+            * )
+                onlydata="true"
+                data+=("--data" "$item");
+                ;;
+        esac
+    done
+    
+    local ct
+    case "${data[1]}" in
+        BEGIN:VCARD* )
+            ct="text/vcard"
+            ;;
+        * )
+            ct='text/xml; charset="utf-8"'
+    esac
+
+    local tmp1="/tmp/curl.$$.tmp"
+    
+    echo "spawn: curl -w \"%{http_code}\" -X $verb -H 'Content-Type: $ct' --user \"${auth_user}:xxx\" ${data[@]} \"$url\"" 1>&2
+	output=$(curl -s -S -w "%{http_code}" -X $verb -H "Content-Type: $ct" --user "${auth_user}:${auth_pass}" "${data[@]}" "$url" 2>$tmp1)
+	local code=$?
+
+	# http status is last 3 characters of output, extract it
+	REST_HTTP_CODE=$(awk '{S=substr($0,length($0)-2)} END {print S}' <<<"$output")
+	REST_OUTPUT=$(awk 'BEGIN{L=""}{ if(L!="") print L; L=$0 } END { print substr(L,1,length(L)-3) }' <<<"$output")
+	REST_ERROR=""
+    REST_ERROR_BRIEF=""
+	[ -z "$REST_HTTP_CODE" ] && REST_HTTP_CODE="000"
+
+    if [ $code -ne 0 -o \
+               $REST_HTTP_CODE -lt 200 -o \
+               $REST_HTTP_CODE -ge 300 ]
+    then
+        if [ $code -ne 0 -a "$REST_HTTP_CODE" == "000" ]; then
+            REST_ERROR="exit code $code"
+            REST_ERROR_BRIEF="$REST_ERROR"
+        else
+		    REST_ERROR="REST status $REST_HTTP_CODE: $REST_OUTPUT"
+            REST_ERROR_BRIEF=$(python3 -c "import xml.etree.ElementTree as ET; print(ET.fromstring(r'''$REST_OUTPUT''').find('s:message',{'s':'http://sabredav.org/ns'}).text)" 2>/dev/null)
+            if [ -z "$REST_ERROR_BRIEF" ]; then
+                REST_ERROR_BRIEF="$REST_ERROR"
+            else
+                REST_ERROR_BRIEF="$REST_HTTP_CODE: $REST_ERROR_BRIEF"
+            fi
+            if [ $code -ne 0 ]; then
+                REST_ERROR_BRIEF="exit code $code: $REST_ERROR_BRIEF"
+                REST_ERROR="exit code $code: $REST_ERROR"
+            fi
+        fi
+
+        if [ -s $tmp1 ]; then
+            REST_ERROR="$REST_ERROR: $(cat $tmp1)"
+            REST_ERROR_BRIEF="$REST_ERROR_BRIEF: $(cat $tmp1)"
+        fi
+        rm -f $tmp1
+        
+		echo "${F_DANGER}$REST_ERROR${F_RESET}" 1>&2
+		[ $code -ne 0 ] && return 1
+        return 2
+	fi
+    
+	echo "CURL succeded, HTTP status $REST_HTTP_CODE" 1>&2
+	echo "$output" 1>&2
+    rm -f $tmp1
+	return 0    
+}
+
+
+carddav_ls() {
+    # place all .vcf files into global FILES
+    # debug messages are sent to stderr
+    local user="$1"
+    local pass="$2"
+    shift; shift
+    FILES=()
+    if ! carddav_rest PROPFIND "" "$user" "$pass" $@
+    then
+        return 1
+    fi
+    
+    FILES=( $(python3 -c "import xml.etree.ElementTree as ET; [print(el.find('d:href',{'d':'DAV:'}).text) for el in ET.fromstring(r'''$REST_OUTPUT''').findall('d:response',{'d':'DAV:'}) if el.find('d:href',{'d':'DAV:'}) is not None]") )
+
+    local idx=${#FILES[*]}
+    let idx-=1
+    while [ $idx -ge 0 ]; do
+        # remove non .vcf entries, take basename contact href
+        case "${FILES[$idx]}" in
+            *.vcf )
+                FILES[$idx]=$(basename "${FILES[$idx]}")
+                ;;
+            * )
+                unset "FILES[$idx]"
+                ;;
+        esac
+        let idx-=1
+    done
+}
+
+
+carddav_make_addressbook() {
+    local user="$1"
+    local pass="$2"
+    local name="$3"
+    local desc="${4:-$name}"
+    local xml="<?xml version=\"1.0\" encoding=\"utf-8\" ?>
+<D:mkcol xmlns:D=\"DAV:\"
+         xmlns:C=\"urn:ietf:params:xml:ns:carddav\">
+  <D:set>
+    <D:prop>
+      <D:resourcetype>
+        <D:collection/>
+        <C:addressbook/>
+      </D:resourcetype>
+      <D:displayname>$name</D:displayname>
+      <C:addressbook-description xml:lang=\"en\">$desc</C:addressbook-description>
+    </D:prop>
+  </D:set>
+</D:mkcol>"
+    local url="$(carddav_url "$user" CARDDAV_PATH)"
+    carddav_rest MKCOL "$url" "$user" "$pass" "$xml"
+}
+
+
+carddav_add_contact() {
+    # debug messages are sent to stderr
+    local user="$1"
+    local pass="$2"
+    local c_name="$3"
+    local c_phone="$4"
+    local c_email="$5"
+    local c_uid="${6:-$(generate_uuid)}"
+    shift; shift; shift; shift; shift; shift
+    
+    local vcard="BEGIN:VCARD
+VERSION:3.0
+UID:$c_uid
+REV;VALUE=DATE-AND-OR-TIME:$(date -u +%Y%m%dT%H%M%SZ)
+FN:$c_name
+EMAIL;TYPE=INTERNET,PREF:$c_email
+NOTE:Miab-LDAP QA
+ORG:Miab-LDAP
+TEL;TYPE=WORK,VOICE:$c_phone
+END:VCARD"
+    carddav_rest PUT "$c_uid.vcf" "$user" "$pass" $@ -- "$vcard"
+}
+
+
+carddav_delete_contact() {
+    local user="$1"
+    local pass="$2"
+    local c_uid="$3"
+    shift; shift; shift
+    carddav_rest DELETE "$c_uid.vcf" "$user" "$pass" $@
+}
+
+
+roundcube_force_carddav_refresh() {
+    local user="$1"
+    local pass="$2"
+    local assets_dir="${ASSETS_DIR:-tests/assets}"
+    local code
+    if ! cp "$assets_dir/mail/roundcube/carddav_refresh.sh" $RCM_DIR/bin
+    then
+        return 1
+    fi
+    pushd "$RCM_DIR" >/dev/null
+    bin/carddav_refresh.sh "$user" "$pass"
+    code=$?
+    popd >/dev/null
+    return $code
+}
+
+
+roundcube_carddav_contact_exists() {
+    # returns 0 if contact exists
+    #         1 if contact does not exist
+    #         2 if an error occurred
+    # stderr receives error messages
+    local user="$1"
+    local pass="$2"
+    local c_uid="$3"
+    local db="${4:-$STORAGE_ROOT/mail/roundcube/roundcube.sqlite}"
+    local output
+    output="$(sqlite3 "$db" "select name from carddav_contacts where cuid='$c_uid'")"
+    [ $? -ne 0 ] && return 2
+    if [ -z "$output" ]; then
+        return 1
+    else
+        return 0
+    fi
+}
+
+
+roundcube_dump_contacts() {
+    local db="${1:-$STORAGE_ROOT/mail/roundcube/roundcube.sqlite}"
+    local cols="${2:-name,cuid}"
+    sqlite3 "$db" "select $cols FROM carddav_contacts"
+}
+
diff --git a/tests/lib/color-output.sh b/tests/lib/color-output.sh
index 3e1954f8..3652e98b 100644
--- a/tests/lib/color-output.sh
+++ b/tests/lib/color-output.sh
@@ -27,6 +27,24 @@ warn() {
         * )
             echoarg=""
     esac
-    echo "${F_WARN}$1${F_RESET}"
+    echo $echoarg "${F_WARN}$1${F_RESET}"
+}
+
+H1() {
+    local msg="$1"
+    echo "----------------------------------------------"
+    if [ ! -z "$msg" ]; then
+        echo "           $msg"
+        echo "----------------------------------------------"
+    fi
+}
+
+H2() {
+    local msg="$1"
+    if [ -z "$msg" ]; then
+        echo "***"
+    else
+        echo "*** $msg ***"
+    fi
 }
 
diff --git a/tests/lib/installed-state.sh b/tests/lib/installed-state.sh
new file mode 100644
index 00000000..3efc0411
--- /dev/null
+++ b/tests/lib/installed-state.sh
@@ -0,0 +1,141 @@
+#
+# requires:
+#    scripts: [ colored-output.sh, rest.sh ]
+#
+# these functions are meant for comparing upstream (non-LDAP)
+# installations to a subsequent MiaB-LDAP upgrade
+#
+
+
+installed_state_capture() {
+    # users and aliases
+    # dns zone files
+    # TOOD: tls certificates: expected CN's
+
+    local state_dir="$1"
+    local info="$state_dir/info.txt"
+
+    H1 "Capture installed estate to $state_dir"
+
+    # nuke saved state, if any
+    rm -rf "$state_dir"
+    mkdir -p "$state_dir"
+
+    # create info.json
+    H2 "create info.txt"
+    echo "STATE_VERSION=1" > "$info"
+    echo "GIT_VERSION='$(git describe --abbrev=0)'" >>"$info"
+    echo "MIGRATION_VERSION=$(cat "$STORAGE_ROOT/mailinabox.version")" >>"$info"
+
+    # record users
+    H2 "record users"
+    if ! rest_urlencoded GET "/admin/mail/users?format=json" "$EMAIL_ADDR" "$EMAIL_PW" --insecure 2>/dev/null
+    then
+        echo "Unable to get users: rc=$? err=$REST_ERROR" 1>&2
+        return 1
+    fi
+    echo "$REST_OUTPUT" > "$state_dir/users.json"
+
+    # record aliases
+    H2 "record aliases"
+    if ! rest_urlencoded GET "/admin/mail/aliases?format=json" "$EMAIL_ADDR" "$EMAIL_PW" --insecure 2>/dev/null
+    then
+        echo "Unable to get aliases: rc=$? err=$REST_ERROR" 1>&2
+        return 2
+    fi
+    echo "$REST_OUTPUT" > "$state_dir/aliases.json"
+
+    # record dns config
+    H2 "record dns details"
+    local file
+    mkdir -p "$state_dir/zones"
+    for file in /etc/nsd/zones/*.signed; do
+        if ! cp "$file" "$state_dir/zones"
+        then
+            echo "Copy $file -> $state_dir/zones failed" 1>&2
+            return 3
+        fi
+    done
+    
+    return 0
+}
+
+
+
+installed_state_compare() {
+    local s1="$1"
+    local s2="$2"
+    
+    local output
+    local changed="false"
+
+    H1 "COMPARE STATES: $(basename "$s1") VS $(basename "$2")"
+    H2 "Users"
+    # users
+    output="$(diff "$s1/users.json" "$s2/users.json" 2>&1)"
+    if [ $? -ne 0 ]; then
+        changed="true"
+        echo "USERS ARE DIFFERENT!"
+        echo "$output"
+    else
+        echo "No change"
+    fi
+
+    H2 "Aliases"
+    output="$(diff "$s1/aliases.json" "$s2/aliases.json" 2>&1)"
+    if [ $? -ne 0 ]; then
+        change="true"
+        echo "ALIASES ARE DIFFERENT!"
+        echo "$output"
+    else
+        echo "No change"
+    fi
+
+    H2 "DNS - zones missing"
+    local zone count=0
+    for zone in $(cd "$s1/zones"; ls *.signed); do
+        if [ ! -e "$s2/zones/$zone" ]; then
+            echo "MISSING zone: $zone"
+            changed="true"
+            let count+=1
+        fi
+    done
+    echo "$count missing"
+
+    H2 "DNS - zones added"
+    count=0
+    for zone in $(cd "$s2/zones"; ls *.signed); do
+        if [ ! -e "$s2/zones/$zone" ]; then
+            echo "ADDED zone: $zone"
+            changed="true"
+            let count+=1
+        fi
+    done
+    echo "$count added"
+
+    H2 "DNS - zones changed"
+    count=0
+    for zone in $(cd "$s1/zones"; ls *.signed); do
+        if [ -e "$s2/zones/$zone" ]; then
+            # all the signatures change if we're using self-signed certs
+            local t1="/tmp/s1.$$.txt"
+            local t2="/tmp/s2.$$.txt"
+            awk '$4 == "RRSIG" || $4 == "NSEC3" { next; } $4 == "SOA" { print $1" "$2" "$3" "$4" "$5" "$6" "$8" "$9" "$10" "$11" "$12; next } { print $0 }' "$s1/zones/$zone" > "$t1" 
+            awk '$4 == "RRSIG" || $4 == "NSEC3" { next; } $4 == "SOA" { print $1" "$2" "$3" "$4" "$5" "$6" "$8" "$9" "$10" "$11" "$12; next } { print $0 }' "$s2/zones/$zone" > "$t2" 
+            output="$(diff "$t1" "$t2" 2>&1)"
+            if [ $? -ne 0 ]; then
+                echo "CHANGED zone: $zone"
+                echo "$output"
+                changed="true"
+                let count+=1
+            fi
+        fi
+    done
+    echo "$count zone files had differences"
+
+    if $changed; then
+        return 1
+    else
+        return 0
+    fi
+}
diff --git a/tests/lib/misc.sh b/tests/lib/misc.sh
index 4312d6ae..52a2b8a6 100644
--- a/tests/lib/misc.sh
+++ b/tests/lib/misc.sh
@@ -55,7 +55,11 @@ generate_uuid() {
 }
 
 generate_qa_password() {
-    echo "Test1234."
+    echo "Test$(date +%s)"
+}
+
+static_qa_password() {
+    echo "Test_1234"
 }
 
 sha1() {
diff --git a/tests/lib/populate.sh b/tests/lib/populate.sh
new file mode 100644
index 00000000..8a8b0c5f
--- /dev/null
+++ b/tests/lib/populate.sh
@@ -0,0 +1,99 @@
+
+#
+# requires:
+#   scripts: [ rest.sh, misc.sh ]
+#
+
+populate_miab_users() {
+    local url="$1"
+    local admin_email="${2:-$EMAIL_ADDR}"
+    local admin_pass="${3:-$EMAIL_PW}"
+    shift; shift; shift  # remaining arguments are users to add
+
+    # each "user" argument is in the format "email:password"
+    # if no password is given a "qa" password will be generated
+
+    [ $# -eq 0 ] && return 0
+    
+    #
+    # get the existing users
+    #
+    local current_users=() user
+    if ! rest_urlencoded GET ${url%/}/admin/mail/users "$admin_email" "$admin_pass" --insecure 2>/dev/null; then
+        echo "Unable to enumerate users: rc=$? err=$REST_ERROR" 1>&2
+        return 1
+    fi
+    for user in $REST_OUTPUT; do
+        current_users+=("$user")
+    done
+
+    #
+    # add the new users
+    #
+    local pw="$(generate_qa_password)"
+    
+    for user; do
+        local user_email="$(awk -F: '{print $1}' <<< "$user")"
+        local user_pass="$(awk -F: '{print $2}' <<< "$user")"
+        if array_contains "$user_email" "${current_users[@]}"; then
+            echo "Not adding user $user_email: already exists"
+
+        elif ! rest_urlencoded POST ${url%/}/admin/mail/users/add "$admin_email" "$admin_pass" --insecure -- "email=$user_email" "password=${user_pass:-$pw}" 2>/dev/null
+        then
+            echo "Unable to add user $user_email: rc=$? err=$REST_ERROR" 1>&2
+            return 2
+        else
+            echo "Add: $user"
+        fi
+    done
+
+    return 0
+}
+
+
+
+populate_miab_aliases() {
+    local url="$1"
+    local admin_email="${2:-$EMAIL_ADDR}"
+    local admin_pass="${3:-$EMAIL_PW}"
+    shift; shift; shift  # remaining arguments are aliases to add
+
+    # each "alias" argument is in the format "email-alias > forward-to"
+
+    [ $# -eq 0 ] && return 0
+    
+    #
+    # get the existing aliases
+    #
+    local current_aliases=() alias
+    if ! rest_urlencoded GET ${url%/}/admin/mail/aliases "$admin_email" "$admin_pass" --insecure 2>/dev/null; then
+        echo "Unable to enumerate aliases: rc=$? err=$REST_ERROR" 1>&2
+        return 1
+    fi
+    for alias in $REST_OUTPUT; do
+        current_aliases+=("$alias")
+    done
+
+    #
+    # add the new aliases
+    #
+    local aliasdef
+    for aliasdef; do
+        alias="$(awk -F'[> ]' '{print $1}' <<<"$aliasdef")"
+        local forwards_to="$(sed 's/.*> *\(.*\)/\1/' <<<"$aliasdef")"
+        if array_contains "$alias" "${current_aliases[@]}"; then
+            echo "Not adding alias $aliasdef: already exists"
+            
+        elif ! rest_urlencoded POST ${url%/}/admin/mail/aliases/add "$admin_email" "$admin_pass" --insecure -- "address=$alias" "forwards_to=$forwards_to" 2>/dev/null
+        then
+            echo "Unable to add alias $alias: rc=$? err=$REST_ERROR" 1>&2
+            return 2
+        else
+            echo "Add: $aliasdef"
+        fi
+    done
+
+    return 0
+}
+
+
diff --git a/tests/lib/rest.sh b/tests/lib/rest.sh
index e20a142f..2c8fd39d 100644
--- a/tests/lib/rest.sh
+++ b/tests/lib/rest.sh
@@ -15,7 +15,7 @@ rest_urlencoded() {
     # (https://host/mail/users/add), PRIMARY_HOSTNAME must be set!
     #
     # The function will set the following global variables regardless
-    # of exit c ode:
+    # of exit code:
     #     REST_HTTP_CODE
     #     REST_OUTPUT
     #     REST_ERROR
@@ -26,7 +26,7 @@ rest_urlencoded() {
     #   1 curl returned with non-zero code that indicates and error
     #   2 the response status was <200 or >= 300
     #
-    # Messages and errors are sent to stderr
+    # Debug messages are sent to stderr
     #   
 	local verb="$1" # eg "POST"
 	local uri="$2"  # eg "/mail/users/add"
diff --git a/tests/runner.sh b/tests/runner.sh
index 8119d462..a6fe566c 100755
--- a/tests/runner.sh
+++ b/tests/runner.sh
@@ -23,6 +23,7 @@ default_suites=(
 
 extra_suites=(
 	remote-nextcloud
+	"upgrade-<name>"
 )
 
 usage() {
@@ -41,6 +42,7 @@ usage() {
 	echo "Extra test suites:"
 	echo "------------------"
 	echo "  remote-nextcloud   : test the setup mod for remote Nextcloud"
+	echo "  upgrade-<name>     : verify an upgrade using named populate data"
 	echo ""
 
 	echo "If no suite-name(s) are given, all default suites are run"
@@ -78,23 +80,30 @@ while [ $# -gt 0 ]; do
 			if [ $OVERALL_COUNT_SUITES -eq 0 ]; then
 				rm -rf "${BASE_OUTPUTDIR}"
 			fi
-			
-			if [ "$1" == "default" ]
-			then
-				# run all default suites
-				for suite in ${default_suites[@]}; do
-					. suites/$suite.sh
-				done
-			elif array_contains "$1" ${default_suites[@]} || \
-					array_contains "$1" ${extra_suites[@]}
-			then
-				# run specified suite
-				. "suites/$1.sh"
-			else
-				echo "Unknown suite '$1'" 1>&2
-				usage
-			fi
-			;;
+
+			case "$1" in
+				default )
+					# run all default suites
+					for suite in ${default_suites[@]}; do
+						. suites/$suite.sh
+					done
+					;;
+				upgrade-* )
+					# run upgrade suite with named populate data
+					. "suites/upgrade.sh" "$(awk -F- '{print $2}' <<< "$1")"
+					;;
+				* )
+					if array_contains "$1" "${default_suites[@]}" || \
+							array_contains "$1" "${extra_suites[@]}"
+					then
+						# run specified suite
+						. "suites/$1.sh"
+					else
+						echo "Unknown suite '$1'" 1>&2
+						usage
+					fi
+					;;
+			esac
 	esac
 	shift
 done
diff --git a/tests/suites/_init.sh b/tests/suites/_init.sh
index 829db86d..2a116001 100644
--- a/tests/suites/_init.sh
+++ b/tests/suites/_init.sh
@@ -12,6 +12,8 @@ set +eu
 . suites/_mgmt-functions.sh || exit 1
 
 # globals - all global variables are UPPERCASE
+ASSETS_DIR="assets"
+MIAB_DIR=".."
 BASE_OUTPUTDIR="$(realpath out)"
 PYMAIL="./test_mail.py"
 declare -i OVERALL_SUCCESSES=0
diff --git a/tests/suites/remote-nextcloud.sh b/tests/suites/remote-nextcloud.sh
index 825c3df5..5a4533ba 100644
--- a/tests/suites/remote-nextcloud.sh
+++ b/tests/suites/remote-nextcloud.sh
@@ -35,207 +35,26 @@ assert_is_configured() {
 }
 
 
-nextcloud_url() {
-    # eg: http://localhost/cloud/
-    carddav_url | sed 's|\(.*\)/remote.php/.*|\1/|'
-}
-
-carddav_url() {
-    # get the carddav url as configured in z-push for the user specified
-    # eg: http://localhost/cloud/remote.php/dav/addressbooks/users/admin/contacts/
-    local user="${1:-%u}"
-    local path="${2:-CARDDAV_DEFAULT_PATH}"
-    local php='include "/usr/local/lib/z-push/backend/carddav/config.php"; print CARDDAV_PROTOCOL . "://" . CARDDAV_SERVER . ":" . CARDDAV_PORT . '
-    php="$php$path;"
-    local url
-    url="$(php -n -r "$php")"
-    [ $? -ne 0 ] && die "Unable to run php to extract carddav url from z-push"
-    sed "s/%u/$user/" <<< "$url"
-}
-
-
-carddav_rest() {
-    # issue a CardDAV rest call to Nextcloud
-    # SEE: https://tools.ietf.org/html/rfc6352
-    local verb="$1"
-    local uri="$2"
-    local auth_user="$3"
-    local auth_pass="$4"
-	shift; shift; shift; shift  # remaining arguments are data
-
-    local url
-    case "$uri" in
-        /* )
-            url="$(nextcloud_url)${uri#/}"
-            ;;
-        http*)
-            url="$uri"
-            ;;
-        * )
-            url="$(carddav_url "$auth_user")${uri#/}"
-            ;;
-    esac
-    
-    local data=()
-	local item output
-
-    for item; do data+=("--data" "$item"); done
-
-    local ct
-    case "${data[1]}" in
-        BEGIN:VCARD* )
-            ct="text/vcard"
-            ;;
-        * )
-            ct='text/xml; charset="utf-8"'
-    esac
-    
-    record "spawn: curl -w \"%{http_code}\" -X $verb -H 'Content-Type: $ct' --user \"${auth_user}:xxx\" ${data[@]} \"$url\""
-	output=$(curl -s -S -w "%{http_code}" -X $verb -H "Content-Type: $ct" --user "${auth_user}:${auth_pass}" "${data[@]}" "$url" 2>>$TEST_OF)
-	local code=$?
-
-	# http status is last 3 characters of output, extract it
-	REST_HTTP_CODE=$(awk '{S=substr($0,length($0)-2)} END {print S}' <<<"$output")
-	REST_OUTPUT=$(awk 'BEGIN{L=""}{ if(L!="") print L; L=$0 } END { print substr(L,1,length(L)-3) }' <<<"$output")
-	REST_ERROR=""
-	[ -z "$REST_HTTP_CODE" ] && REST_HTTP_CODE="000"
-
-    if [ $code -ne 0 -o \
-               $REST_HTTP_CODE -lt 200 -o \
-               $REST_HTTP_CODE -ge 300 ]
-    then
-		REST_ERROR="REST status $REST_HTTP_CODE: $REST_OUTPUT"
-        REST_ERROR_BRIEF=$(python3 -c "import xml.etree.ElementTree as ET; print(ET.fromstring(r'''$REST_OUTPUT''').find('s:message',{'s':'http://sabredav.org/ns'}).text)" 2>/dev/null)
-        if [ -z "$REST_ERROR_BRIEF" ]; then
-            REST_ERROR_BRIEF="$REST_ERROR"
-        else
-            REST_ERROR_BRIEF="$REST_HTTP_CODE: $REST_ERROR_BRIEF"
-        fi
-        if [ $code -ne 0 ]; then
-            REST_ERROR_BRIEF="curl exit code $code: $REST_ERROR_BRIEF"
-            REST_ERROR="curl exit code $code: $REST_ERROR"
-        fi
-		record "${F_DANGER}$REST_ERROR${F_RESET}"
-		return 2
-	fi
-	record "CURL succeded, HTTP status $REST_HTTP_CODE"
-	record "$output"
-	return 0    
-}
-
-
-carddav_ls() {
-    # return all .vcf files in array 'FILES'
-    local user="$1"
-    local pass="$2"
-    carddav_rest PROPFIND "" "$user" "$pass" || return $?
-    local file FILES=()
-    python3 -c "import xml.etree.ElementTree as ET; [print(el.find('d:href',{'d':'DAV:'}).text) for el in ET.fromstring(r'''$REST_OUTPUT''').findall('d:response',{'d':'DAV:'}) if el.find('d:href',{'d':'DAV:'}) is not None]" |
-        while read file; do
-            # skip non .vcf entries
-            case "$file" in
-                *.vcf )
-                    FILES+=( "$(basename "$file")" )
-                    ;;
-                * )
-                    ;;
-            esac
-        done
-}
-
-
-make_collection() {
-    local user="$1"
-    local pass="$2"
-    local name="$3"
-    local desc="${4:-$name}"
-    local xml="<?xml version=\"1.0\" encoding=\"utf-8\" ?>
-<D:mkcol xmlns:D=\"DAV:\"
-         xmlns:C=\"urn:ietf:params:xml:ns:carddav\">
-  <D:set>
-    <D:prop>
-      <D:resourcetype>
-        <D:collection/>
-        <C:addressbook/>
-      </D:resourcetype>
-      <D:displayname>$name</D:displayname>
-      <C:addressbook-description xml:lang=\"en\">$desc</C:addressbook-description>
-    </D:prop>
-  </D:set>
-</D:mkcol>"
-    record "[create address book '$name' for $user]"
-    local url="$(carddav_url "$user" CARDDAV_PATH)"
-    carddav_rest MKCOL "$url" "$user" "$pass" "$xml"
-}
-
-
-
-add_contact() {
-    local user="$1"
-    local pass="$2"
-    local c_name="$3"
-    local c_phone="$4"
-    local c_email="$5"
-    local c_uid="${6:-$(generate_uuid)}"
-    local file_name="$c_uid.vcf"
-    
-    local vcard="BEGIN:VCARD
-VERSION:3.0
-UID:$c_uid
-REV;VALUE=DATE-AND-OR-TIME:$(date -u +%Y%m%dT%H%M%SZ)
-FN:$c_name
-EMAIL;TYPE=INTERNET,PREF:$c_email
-NOTE:Miab-LDAP QA
-ORG:Miab-LDAP
-TEL;TYPE=WORK,VOICE:$c_phone
-END:VCARD"
-    record "[add contact '$c_name' to $user]"
-    carddav_rest PUT "$file_name" "$user" "$pass" "$vcard"
-}
-
-delete_contact() {
-    local user="$1"
-    local pass="$2"
-    local c_uid="$3"
-    local file_name="$c_uid.vcf"
-    record "[delete contact with vcard uid '$c_uid' from $user]"
-    carddav_rest DELETE "$file_name" "$user" "$pass"
-}
-
-force_roundcube_carddav_refresh() {
-    local user="$1"
-    local pass="$2"
-    local code
-    record "[forcing refresh of roundcube contact for $user]"
-    copy_or_die assets/mail/roundcube/carddav_refresh.sh $RCM_DIR/bin
-    pushd "$RCM_DIR" >/dev/null
-    bin/carddav_refresh.sh "$user" "$pass" >>$TEST_OF 2>&1
-    code=$?
-    popd >/dev/null
-    return $code
-}
-
 assert_roundcube_carddav_contact_exists() {
     local user="$1"
     local pass="$2"
     local c_uid="$3"
     local output
     record "[checking that roundcube contact with vcard UID=$c_uid exists]"
-    output="$(sqlite3 $STORAGE_ROOT/mail/roundcube/roundcube.sqlite "select name from carddav_contacts where cuid='$c_uid'" 2>>$TEST_OF)"
-    if [ $? -ne 0 ]; then
-        test_failure "Error querying roundcube sqlite database"
-        return 1
-    fi
-    if [ -z "$output" ]; then
+	roundcube_carddav_contact_exists "$user" "$pass" "$c_uid" 2>>$TEST_OF
+	local rc=$?
+	
+    if [ $rc -eq 0 ]; then
+		return
+	elif [ $rc -eq 1 ]; then
         test_failure "Contact not found in Roundcube"
         record "Not found"
-        record "Existing entries (name,vcard-uid):"
-        output="$(sqlite3 $STORAGE_ROOT/mail/roundcube/roundcube.sqlite "select name,cuid FROM carddav_contacts" 2>>$TEST_OF)"
-        return 1
-    else
-        record "$output"
+        record "Existing entries:"
+		roundcube_dump_contacts >>$TEST_OF 2>&1
+	else
+		test_failure "Error querying roundcube contacts"
+		return
     fi
-    return 0
 }
 
 
@@ -245,9 +64,12 @@ test_mail_from_nextcloud() {
 }
 
 test_nextcloud_contacts() {
-    test_start "nextcloud_contacts"
+    test_start "nextcloud-contacts"
 
-    assert_is_configured || test_end && return
+    if ! assert_is_configured; then
+		test_end
+		return
+	fi
 
     local alice="alice.nc@somedomain.com"
     local alice_pw="$(generate_password 16)"
@@ -263,28 +85,37 @@ test_nextcloud_contacts() {
     # LDAP and that Roundcube is able to reach Nextcloud for contacts
     #
 
-    #make_collection "$alice" "$alice_pw" "contacts"
+	#record "[create address book 'contacts' for $alice]"
+	#carddav_make_addressbook "$alice" "$alice_pw" "contacts" 2>>$TEST_OF
 
     # add new contact to alice's Nextcloud account using CardDAV API
     local c_uid="$(generate_uuid)"
-    add_contact \
+    record "[add contact 'JimIno' to $alice]"
+    if ! carddav_add_contact \
         "$alice" \
         "$alice_pw" \
         "JimIno" \
         "555-1212" \
         "jim@ino.com" \
         "$c_uid" \
-        || test_failure "Could not add contact for $alice in Nextcloud: $REST_ERROR_BRIEF"
+		2>>$TEST_OF
+	then
+		test_failure "Could not add contact for $alice in Nextcloud: $REST_ERROR_BRIEF"
+		test_end
+		return
+	fi
     
     # force a refresh/sync of the contacts in Roundcube
-    force_roundcube_carddav_refresh "$alice" "$alice_pw" || \
+    record "[forcing refresh of roundcube contact for $alice]"
+    roundcube_force_carddav_refresh "$alice" "$alice_pw" >>$TEST_OF 2>&1 || \
         test_failure "Could not refresh roundcube contacts for $alice"
 
     # query the roundcube sqlite database for the new contact
     assert_roundcube_carddav_contact_exists "$alice" "$alice_pw" "$c_uid"
     
     # delete the contact
-    delete_contact "$alice" "$alice_pw" "$c_uid" || \
+    record "[delete contact with vcard uid '$c_uid' from $alice]"
+    carddav_delete_contact "$alice" "$alice_pw" "$c_uid" 2>>$TEST_OF || \
         test_failure "Unable to delete contact for $alice in Nextcloud"
     
 
diff --git a/tests/suites/upgrade.sh b/tests/suites/upgrade.sh
new file mode 100644
index 00000000..a8586f70
--- /dev/null
+++ b/tests/suites/upgrade.sh
@@ -0,0 +1,48 @@
+
+#
+# the system must have been populated proir to any upgrade with one of
+# the tests/system-setup/populate scripts to use this suite
+#
+# supply the name of the populate script that was used as an argument
+# eg. if basic-populate.sh was used to populate, supply "basic" to the
+# script as an argument
+#
+
+
+verify_populate() {    
+    local populate_name="$1"
+    local verify_script="system-setup/populate/${populate_name}-verify.sh"
+
+    test_start "verify '$populate_name' population set"
+
+    if [ ! -e "$verify_script" ]; then
+        test_failure "Verify script $(basename "$verify_script") does not exist"
+
+    else
+        record "[run verify-upgrade script $verify_script]"
+        local output rc
+        output=$("$verify_script" 2>>$TEST_OF)
+        rc=$?
+        if [ $rc -ne 0 ]
+        then
+           if [ $rc -eq 127 ]; then
+               test_failure "verify script would not run (wd=$(pwd))"
+           else
+               test_failure "verify script exited with $rc: $output"
+           fi
+        fi
+    fi
+
+    test_end
+}
+
+
+
+suite_start "upgrade"
+
+export ASSETS_DIR
+export MIAB_DIR
+
+verify_populate "$1"
+
+suite_end
diff --git a/tests/system-setup/populate/README.txt b/tests/system-setup/populate/README.txt
new file mode 100644
index 00000000..095f65dc
--- /dev/null
+++ b/tests/system-setup/populate/README.txt
@@ -0,0 +1,31 @@
+This directory contains scripts used to populate a MiaB installation
+with known values, and then subsequently verify that MiaB continues to
+operate poperly after an upgrade or setup mod change.
+
+Each "named" populate set of scripts should contain at least two
+shell scripts:
+
+   1. <name>-populate.sh  : populates the installation
+   2. <name>-verify.sh    : verifies operation after upgrade
+
+The system-setup/* scripts run the populate script, and the test
+runner's 'upgrade' test suite runs the verify script.
+
+These scripts are run, not sourced.
+
+
+Expected script output and return value:
+
+   1. All debug output must go to stderr
+   2. Result messages must be sent to stdout (a single line, preferrably)
+   3. Return 0 if successfully passed verification
+   4. Return non-zero if failed verification
+
+The working directory for <name>-populate.sh is the Mail-in-a-Box root
+directory.
+
+The working directory for <name>-verify.sh is 'tests' (because the
+test runner always changes the working directory there to limit
+contamination of the source tree). Use MIAB_DIR and ASSETS_DIR, if
+needed.
+
diff --git a/tests/system-setup/populate/basic-data.sh b/tests/system-setup/populate/basic-data.sh
new file mode 100755
index 00000000..bf41b0a7
--- /dev/null
+++ b/tests/system-setup/populate/basic-data.sh
@@ -0,0 +1,10 @@
+#
+# requires:
+#    lib scripts: [ misc.sh ]
+#    system-setup scripts: [ setup-defaults.sh ]
+#
+
+TEST_USER="anna@$(email_domainpart "$EMAIL_ADDR")"
+TEST_USER_PASS="$(static_qa_password)"
+TEST_USER_CONTACT_UUID="e0642b47-9104-4adb-adfd-5f907d04216a"
+TEST_USER_CONTACT_EMAIL="sam@bff.org"
diff --git a/tests/system-setup/populate/basic-populate.sh b/tests/system-setup/populate/basic-populate.sh
new file mode 100755
index 00000000..d4f3d9dc
--- /dev/null
+++ b/tests/system-setup/populate/basic-populate.sh
@@ -0,0 +1,48 @@
+#!/bin/bash
+
+. "$(dirname "$0")/../setup-defaults.sh" || exit 1
+. "$(dirname "$0")/../../lib/all.sh" "$(dirname "$0")/../../lib" || exit 1
+. "$(dirname "$0")/basic-data.sh" || exit 1
+
+
+#
+# Add user
+#
+if ! populate_miab_users "" "" "" "${TEST_USER}:${TEST_USER_PASS}"
+then
+    echo "Unable to add user"
+    exit 1
+fi
+
+#
+# Add Nextcloud contact and force Roundcube contact sync to ensure the
+# roundcube carddav addressbooks and contacts tables are populated in
+# case a remote nextcloud is subsequently configured and the
+# syncronization disabled.
+#
+if ! carddav_ls "$TEST_USER" "$TEST_USER_PASS" --insecure 2>/dev/null
+then
+    echo "Could not enumerate contacts: $REST_ERROR"
+    exit 1
+fi
+echo "Current contacts count: ${#FILES[@]}"
+    
+if array_contains "$TEST_USER_CONTACT_UUID.vcf" "${FILES[@]}"; then
+    echo "Contact $TEST_USER_CONTACT_UUID already present"
+else
+    if ! carddav_add_contact "$TEST_USER" "$TEST_USER_PASS" "Anna" "666-1111" "$TEST_USER_CONTACT_EMAIL" "$TEST_USER_CONTACT_UUID" --insecure 2>/dev/null
+    then
+        echo "Could not add contact: $REST_ERROR"
+        exit 1
+    fi
+    
+    echo "Force Roundcube contact sync"
+    if ! roundcube_force_carddav_refresh "$TEST_USER" "$TEST_USER_PASS"
+    then
+        echo "Roundcube <-> Nextcloud contact sync failed"
+        exit 1
+    fi
+fi
+
+exit 0
+
diff --git a/tests/system-setup/populate/basic-verify.sh b/tests/system-setup/populate/basic-verify.sh
new file mode 100755
index 00000000..1165e7f6
--- /dev/null
+++ b/tests/system-setup/populate/basic-verify.sh
@@ -0,0 +1,53 @@
+#!/bin/bash
+
+. "$(dirname "$0")/../setup-defaults.sh" || exit 1
+. "$(dirname "$0")/../../lib/all.sh" "$(dirname "$0")/../../lib" || exit 1
+. "$(dirname "$0")/basic-data.sh" || exit 1
+. /etc/mailinabox.conf || exit 1
+
+
+# 1. the test user can still log in and send mail
+
+echo "[User can still log in with their old passwords and send mail]" 1>&2
+echo "python3 test_mail.py $PRIVATE_IP $TEST_USER $TEST_USER_PASS" 1>&2
+python3 test_mail.py "$PRIVATE_IP" "$TEST_USER" "$TEST_USER_PASS" 1>&2
+if [ $? -ne 0 ]; then
+    echo "Basic mail functionality test failed"
+    exit 1
+fi
+
+
+# 2. the test user's contact is still accessible in Roundcube
+
+echo "[Force Roundcube contact sync]" 1>&2
+# if MiaB's Nextcloud carddav configuration was removed all the
+# contacts for it will be removed in the Roundcube database after the
+# sync
+
+if ! roundcube_force_carddav_refresh "$TEST_USER" "$TEST_USER_PASS" 1>&2
+then
+    echo "Roundcube <-> Nextcloud contact sync failed ($?)"
+    exit 1
+fi
+
+echo "[Ensure old Nextcloud contacts are still present]" 1>&2
+echo "sqlite3 $STORAGE_ROOT/mail/roundcube/roundcube.sqlite \"select email from carddav_contacts where cuid='$TEST_USER_CONTACT_UUID'\"" 1>&2
+output=$(sqlite3 "$STORAGE_ROOT/mail/roundcube/roundcube.sqlite" "select email from carddav_contacts where cuid='$TEST_USER_CONTACT_UUID'")
+rc=$?
+if [ $rc -ne 0 ]
+then
+    echo "Querying Roundcube's sqlite database failed ($rc)"
+    exit 1
+else
+    echo "Success, found $output" 1>&2
+fi
+
+if [ "$output" != "$TEST_USER_CONTACT_EMAIL" ]
+then
+    echo "Unexpected email for contact uuid: got '$output', expected '$TEST_USER_CONTACT_EMAIL'"
+    exit 1
+fi
+
+echo "OK basic-verify passed"
+exit 0
+
diff --git a/tests/system-setup/remote-nextcloud-docker.sh b/tests/system-setup/remote-nextcloud-docker.sh
index e24900e4..c68470e3 100755
--- a/tests/system-setup/remote-nextcloud-docker.sh
+++ b/tests/system-setup/remote-nextcloud-docker.sh
@@ -26,13 +26,6 @@
 #
 
 
-usage() {
-    echo "Usage: $(basename "$0")"
-    echo "Install MiaB-LDAP and a remote Nextcloud running under docker"
-    echo "Nextcloud is exposed as http://localhost:8000"
-    exit 1
-}
-
 # ensure working directory
 if [ ! -d "tests/system-setup" ]; then
     echo "This script must be run from the MiaB root directory"
@@ -51,49 +44,26 @@ fi
 
 
 
-before_miab_install() {
-    H1 "BEFORE MIAB-LDAP INSTALL"
-    system_init
-    miab_testing_init || die "Initialization failed"
-    
-    # enable the remote Nextcloud setup mod, which tells MiaB-LDAP to use
-    # the remote Nextcloud for calendar and contacts instead of the
-    # MiaB-installed one
-    H2 "Enable local mod remote-nextcloud"
-    enable_miab_mod "remote-nextcloud" \
-        || die "Could not enable remote-nextcloud mod"
-    
+init() {
+    H1 "INIT"
+    init_test_system
+    init_miab_testing || die "Initialization failed"    
+}
+
+
+install_nextcloud_docker() {
+    H1 "INSTALL NEXTCLOUD ON DOCKER"
+
     # install Docker
     H2 "Install Docker"
     install_docker || die "Could not install Docker! ($?)"
-}
 
-
-miab_install() {
-    H1 "MIAB-LDAP INSTALL"
-    if ! setup/start.sh; then
-        H1 "OUTPUT OF SELECT FILES"
-        dump_file "/var/log/syslog" 100
-        dump_conf_files "$TRAVIS"
-        H2; H2 "End"; H2
-        die "setup/start.sh failed!"
-    fi
-    H1 "OUTPUT OF SELECT FILES"
-    dump_conf_files "$TRAVIS"
-    H2; H2 "End"; H2
-}
-
-
-after_miab_install() {
-    H1 "AFTER MIAB-LDAP INSTALL"
-    
-    . /etc/mailinabox.conf || die "Could not load /etc/mailinabox.conf"
-    
     # run Nextcloud docker image
     H2 "Start Nextcloud docker container"
     local container_started="true"
     if [ -z "$(docker ps -f NAME=NC -q)" ]; then
         docker run -d --name NC -p 8000:80 \
+               --add-host "$PRIMARY_HOSTNAME:$PRIVATE_IP" \
                --env SQLITE_DATABASE=nextclouddb.sqlite \
                --env NEXTCLOUD_ADMIN_USER="$NC_ADMIN_USER" \
                --env NEXTCLOUD_ADMIN_PASSWORD="$NC_ADMIN_PASSWORD" \
@@ -114,10 +84,10 @@ after_miab_install() {
         container_started="false"
     fi
 
-    H2 "docker: Update /etc/hosts so it can find MiaB-LDAP by name"
-    echo "$PRIVATE_IP $PRIMARY_HOSTNAME" | \
-        docker exec -i NC bash -c 'cat >>/etc/hosts' \
-        || die "docker: could not update /etc/hosts"
+    # H2 "docker: Update /etc/hosts so it can find MiaB-LDAP by name"
+    # echo "$PRIVATE_IP $PRIMARY_HOSTNAME" | \
+    #     docker exec -i NC bash -c 'cat >>/etc/hosts' \
+    #     || die "docker: could not update /etc/hosts"
     
     # apt-get update
     H2 "docker: apt-get update"
@@ -171,26 +141,87 @@ after_miab_install() {
 }
 
 
-#
-# Main
-#
-case "${1:-all}" in
-    before-install )
-        before_miab_install
+
+
+do_upgrade() {
+    local populate_name="$1"
+
+    if [ -e "local/remote-nextcloud.sh" ]; then
+        # we install w/o remote nextcloud first so we can add
+        # a user w/contacts and ensure the contact exists in the
+        # new system
+        if [ ! -L "local/remote-nextcloud.sh" ]; then
+            echo "Warning: local/remote-nextcloud.sh is a regular file - should be a symlink"
+        fi
+        die "Error: local/remote-nextcloud.sh exists - delete it and try again"
+    fi
+
+    # initialize test system
+    init
+
+    # install w/o remote Nextcloud
+    miab_ldap_install
+    
+    # populate some data
+    [ ! -z "$populate_name" ] && populate_by_name "$populate_name"
+    
+    # install Nextcloud in a Docker container (MiaB must be available)
+    install_nextcloud_docker
+    
+    H1 "Enable remote-nextcloud mod"
+    enable_miab_mod "remote-nextcloud" \
+        || die "Could not enable remote-nextcloud mod"
+
+    # re-run setup to use the remote Nextcloud
+    miab_ldap_install
+}
+
+
+do_default() {
+    # initialize test system
+    init
+
+    H1 "Enable remote-nextcloud mod"
+    enable_miab_mod "remote-nextcloud" \
+        || die "Could not enable remote-nextcloud mod"
+    
+    # run setup to use the remote Nextcloud (doesn't need to be available)
+    miab_ldap_install
+
+    # install Nextcloud in a Docker container (MiaB must be available)
+    install_nextcloud_docker    
+}
+
+
+
+
+
+case "$1" in
+    upgrade )
+        # Runs this sequence:
+        #   1. setup w/o remote nextcloud
+        #   2. if an additional argument is given, populate the MiaB
+        #      installation
+        #   3. install a remote nextcloud
+        #   4. enable remote-nextcloud mod
+        #   5. re-run setup
+        #
+
+        shift
+        do_upgrade "$@"
         ;;
-    install )
-        miab_install
-        ;;
-    after-install )
-        after_miab_install
-        ;;
-    all )
-        before_miab_install
-        miab_install
-        after_miab_install
+
+    "" | default )
+        # Runs this sequence:
+        #   1. setup w/remote nextcloud
+        #   2. install and connect the remote nextcloud
+        do_default
         ;;
+    
     * )
-        usage
+        echo "Unknown option $1"
+        exit 1
         ;;
 esac
 
+
diff --git a/tests/system-setup/setup-defaults.sh b/tests/system-setup/setup-defaults.sh
index b415e7a0..7d44f93a 100755
--- a/tests/system-setup/setup-defaults.sh
+++ b/tests/system-setup/setup-defaults.sh
@@ -7,7 +7,7 @@ export STORAGE_USER="${STORAGE_USER:-user-data}"
 export STORAGE_ROOT="${STORAGE_ROOT:-/home/$STORAGE_USER}"
 export EMAIL_ADDR="${EMAIL_ADDR:-qa@abc.com}"
 export EMAIL_PW="${EMAIL_PW:-Test_1234}"
-export PUBLIC_IP="${PUBLIC_IP:-$(source setup/functions.sh; get_default_privateip 4)}"
+export PUBLIC_IP="${PUBLIC_IP:-$(source ${MIAB_DIR:-.}/setup/functions.sh; get_default_privateip 4)}"
 
 if [ "$TRAVIS" == "true" ]; then
     export PRIMARY_HOSTNAME=${PRIMARY_HOSTNAME:-box.abc.com}
diff --git a/tests/system-setup/setup-funcs.sh b/tests/system-setup/setup-funcs.sh
index c04a4fa9..3f6db83e 100755
--- a/tests/system-setup/setup-funcs.sh
+++ b/tests/system-setup/setup-funcs.sh
@@ -5,6 +5,7 @@
 #   test scripts: [ lib/misc.sh, lib/system.sh ]
 #
 
+
 die() {
     local msg="$1"
     echo "$msg" 1>&2
@@ -12,25 +13,6 @@ die() {
 }
 
 
-H1() {
-    local msg="$1"
-    echo "----------------------------------------------"
-    if [ ! -z "$msg" ]; then
-        echo "           $msg"
-        echo "----------------------------------------------"
-    fi
-}
-
-H2() {
-    local msg="$1"
-    if [ -z "$msg" ]; then
-        echo "***"
-    else
-        echo "*** $msg ***"
-    fi
-}
-
-
 wait_for_docker_nextcloud() {
     local container="$1"
     local config_key="$2"
@@ -83,7 +65,9 @@ dump_conf_files() {
 # Initialize the test system
 #   hostname, time, apt update/upgrade, etc
 #
-system_init() {
+# Errors are fatal
+#
+init_test_system() {
     H2 "Update /etc/hosts"
     set_system_hostname || die "Could not set hostname"
 
@@ -113,7 +97,7 @@ system_init() {
 # Anything needed to use the test runner, speed up the installation,
 # etc
 #
-miab_testing_init() {
+init_miab_testing() {
     [ -z "$STORAGE_ROOT" ] \
         && echo "Error: STORAGE_ROOT not set" 1>&2 \
         && return 1
@@ -152,8 +136,12 @@ miab_testing_init() {
 enable_miab_mod() {
     local name="${1}.sh"
     if [ ! -e "local/$name" ]; then
-        mkdir -p local
-        ln -s "../setup/mods.available/$name" "local/$name"
+        mkdir -p "local"
+        if ! ln -s "../setup/mods.available/$name" "local/$name"
+        then
+            echo "Warning: copying instead of symlinking local/$name"
+            cp "setup/mods.available/$name" "local/$name"
+        fi
     fi
 }
 
@@ -166,3 +154,34 @@ tag_from_readme() {
     return 0
 }
 
+
+miab_ldap_install() {
+    H1 "MIAB-LDAP INSTALL"
+    # ensure we're in a MiaB-LDAP working directory
+    if [ ! -e setup/ldap.sh ]; then
+        die "Cannot install: the working directory is not MiaB-LDAP!"
+    fi
+    
+    if ! setup/start.sh; then
+        H1 "OUTPUT OF SELECT FILES"
+        dump_file "/var/log/syslog" 100
+        dump_conf_files "$TRAVIS"
+        H2; H2 "End"; H2
+        die "MiaB-LDAP setup/start.sh failed!"
+    fi
+
+    # set actual STORAGE_ROOT, STORAGE_USER, PRIVATE_IP, etc
+    . /etc/mailinabox.conf || die "Could not source /etc/mailinabox.conf"
+}
+
+
+populate_by_name() {
+    local populate_name="$1"
+
+    H1 "Populate Mail-in-a-Box ($populate_name)"
+    local populate_script="tests/system-setup/populate/${populate_name}-populate.sh"
+    if [ ! -e "$populate_script" ]; then
+        die "Does not exist: $populate_script"
+    fi
+    "$populate_script" || die "Failed: $populate_script"
+}
diff --git a/tests/system-setup/upgrade-from-upstream.sh b/tests/system-setup/upgrade-from-upstream.sh
index aba6c6d2..721d7c11 100755
--- a/tests/system-setup/upgrade-from-upstream.sh
+++ b/tests/system-setup/upgrade-from-upstream.sh
@@ -32,10 +32,10 @@ if [ "$EUID" != "0" ]; then
 fi
 
 
-before_install() {
+init() {
     H1 "INIT"
-    system_init
-    miab_testing_init || die "Initialization failed"
+    init_test_system
+    init_miab_testing || die "Initialization failed"
 }
 
 upstream_install() {
@@ -94,217 +94,60 @@ upstream_install() {
 }
 
 
-add_data() {
+populate() {
+    local pw="$(static_qa_password)"
+    
     H1 "Add some Mail-in-a-Box data"
     local users=()
-    users+=("betsy@$(email_domainpart "$EMAIL_ADDR")")
+    users+=("betsy@$(email_domainpart "$EMAIL_ADDR"):$pw")
 
     local alises=()
-    aliases+=("goalias@testdom.com > ${users[0]}")
+    aliases+=("goalias@testdom.com > $(awk -F: {'print $1'} <<<"${users[0]}")")
     aliases+=("nested@testdom.com > goalias@testdom.com")
 
-    local pw="$(generate_qa_password)"
-
-
-    #
-    # get the existing users and aliases
-    #
-    local current_users=() current_aliases=()
-    local user alias
-    if ! rest_urlencoded GET /admin/mail/users "$EMAIL_ADDR" "$EMAIL_PW" --insecure >/dev/null 2>&1; then
-        die "Unable to enumerate users: rc=$? err=$REST_ERROR"
-    fi
-    for user in $REST_OUTPUT; do
-        current_users+=("$user")
-    done
-
-    if ! rest_urlencoded GET /admin/mail/aliases "$EMAIL_ADDR" "$EMAIL_PW" --insecure 2>/dev/null; then
-        die "Unable to enumerate aliases: rc=$? err=$REST_ERROR"
-    fi
-    for alias in $REST_OUTPUT; do
-        current_aliases+=("$alias")
-    done
-
-    
-    #
-    # add users
-    #
     H2 "Add users"
-    for user in "${users[@]}"; do
-        if array_contains "$user" "${current_users[@]}"; then
-            echo "Not adding user $user: already exists"
+    if ! populate_miab_users "" "" "" "${users[@]}"
+    then
+        die "Unable to add users"
+    fi
 
-        elif ! rest_urlencoded POST /admin/mail/users/add "$EMAIL_ADDR" "$EMAIL_PW" --insecure -- "email=$user" "password=$pw" 2>/dev/null
-        then
-            die "Unable to add user $user: rc=$? err=$REST_ERROR"
-        else
-            echo "Add: $user"
-        fi
-    done
-
-    #
-    # add aliases
-    #
     H2 "Add aliases"
-    local aliasdef
-    for aliasdef in "${aliases[@]}"; do
-        alias="$(awk -F'[> ]' '{print $1}' <<<"$aliasdef")"
-        local forwards_to="$(sed 's/.*> *\(.*\)/\1/' <<<"$aliasdef")"
-        if array_contains "$alias" "${current_aliases[@]}"; then
-            echo "Not adding alias $alias: already exists"
-            
-        elif ! rest_urlencoded POST /admin/mail/aliases/add "$EMAIL_ADDR" "$EMAIL_PW" --insecure -- "address=$alias" "forwards_to=$forwards_to" 2>/dev/null
-        then
-            die "Unable to add alias $alias: rc=$? err=$REST_ERROR"
-        else
-            echo "Add: $aliasdef"
-        fi
-    done
-}
-
-capture_state() {
-    # users and aliases lists
-    # dns zone files
-    # tls certificates: expected CN's
-
-    local state_dir="$1"
-    local info="$state_dir/info.txt"
-
-    H1 "Capture server state to $state_dir"
-    
-    # nuke saved state, if any
-    rm -rf "$state_dir"
-    mkdir -p "$state_dir"
-
-    # create info.json
-    H2 "create info.txt"
-    echo "VERSION='$(git describe --abbrev=0)'" >"$info"
-    echo "MIGRATION_VERSION=$(cat "$STORAGE_ROOT/mailinabox.version")" >>"$info"
-
-    # record users
-    H2 "record users"
-    rest_urlencoded GET "/admin/mail/users?format=json" "$EMAIL_ADDR" "$EMAIL_PW" --insecure 2>/dev/null \
-        || die "Unable to get users: rc=$? err=$REST_ERROR"
-    echo "$REST_OUTPUT" > "$state_dir/users.json"
-
-    # record aliases
-    H2 "record aliases"
-    rest_urlencoded GET "/admin/mail/aliases?format=json" "$EMAIL_ADDR" "$EMAIL_PW" --insecure 2>/dev/null \
-        || die "Unable to get aliases: rc=$? err=$REST_ERROR"
-    echo "$REST_OUTPUT" > "$state_dir/aliases.json"
-
-    # record dns config
-    H2 "record dns details"
-    local file
-    mkdir -p "$state_dir/zones"
-    for file in /etc/nsd/zones/*.signed; do
-        cp "$file" "$state_dir/zones"
-    done    
-}
-
-miab_ldap_install() {
-    H1 "INSTALL MIAB-LDAP"
-    # ensure we're in a MiaB-LDAP working directory
-    if [ ! -e setup/ldap.sh ]; then
-        die "The working directory is not MiaB-LDAP!"
-    fi
-    setup/start.sh -v || die "Upgrade to MiaB-LDAP failed !!!!!!"
-}
-
-compare_state() {
-    local s1="$1"
-    local s2="$2"
-    
-    local output
-    local changed="false"
-
-    H1 "COMPARE STATES: $(basename "$s1") VS $(basename "$2")"
-    H2 "Users"
-    # users
-    output="$(diff "$s1/users.json" "$s2/users.json" 2>&1)"
-    if [ $? -ne 0 ]; then
-        changed="true"
-        echo "USERS ARE DIFFERENT!"
-        echo "$output"
-    else
-        echo "No change"
-    fi
-
-    H2 "Aliases"
-    output="$(diff "$s1/aliases.json" "$s2/aliases.json" 2>&1)"
-    if [ $? -ne 0 ]; then
-        change="true"
-        echo "ALIASES ARE DIFFERENT!"
-        echo "$output"
-    else
-        echo "No change"
-    fi
-
-    H2 "DNS - zones missing"
-    local zone count=0
-    for zone in $(cd "$s1/zones"; ls *.signed); do
-        if [ ! -e "$s2/zones/$zone" ]; then
-            echo "MISSING zone: $zone"
-            changed="true"
-            let count+=1
-        fi
-    done
-    echo "$count missing"
-
-    H2 "DNS - zones added"
-    count=0
-    for zone in $(cd "$s2/zones"; ls *.signed); do
-        if [ ! -e "$s2/zones/$zone" ]; then
-            echo "ADDED zone: $zone"
-            changed="true"
-            let count+=1
-        fi
-    done
-    echo "$count added"
-
-    H2 "DNS - zones changed"
-    count=0
-    for zone in $(cd "$s1/zones"; ls *.signed); do
-        if [ -e "$s2/zones/$zone" ]; then
-            # all the signatures change if we're using self-signed certs
-            local t1="/tmp/s1.$$.txt"
-            local t2="/tmp/s2.$$.txt"
-            awk '$4 == "RRSIG" || $4 == "NSEC3" { next; } $4 == "SOA" { print $1" "$2" "$3" "$4" "$5" "$6" "$8" "$9" "$10" "$11" "$12; next } { print $0 }' "$s1/zones/$zone" > "$t1" 
-            awk '$4 == "RRSIG" || $4 == "NSEC3" { next; } $4 == "SOA" { print $1" "$2" "$3" "$4" "$5" "$6" "$8" "$9" "$10" "$11" "$12; next } { print $0 }' "$s2/zones/$zone" > "$t2" 
-            output="$(diff "$t1" "$t2" 2>&1)"
-            if [ $? -ne 0 ]; then
-                echo "CHANGED zone: $zone"
-                echo "$output"
-                changed="true"
-                let count+=1
-            fi
-        fi
-    done
-    echo "$count zone files had differences"
-
-    if $changed; then
-        return 1
-    else
-        return 0
+    if ! populate_miab_aliases "" "" "" "${aliases[@]}"
+    then
+        die "Unable to add aliases"
     fi
 }
 
 
 
-if [ "$1" == "cap" ]; then
-    capture_state "tests/system-setup/state/miab-ldap"
-    exit $?
-elif [ "$1" == "compare" ]; then
-    compare_state "tests/system-setup/state/upstream" "tests/system-setup/state/miab-ldap"
-    exit $?
-fi
+
+# these are for debugging/testing
+case "$1" in
+    capture )
+        . /etc/mailinabox.conf
+        installed_state_capture "tests/system-setup/state/miab-ldap"
+        exit $?
+        ;;
+    compare )
+        . /etc/mailinabox.conf
+        installed_state_compare "tests/system-setup/state/upstream" "tests/system-setup/state/miab-ldap"
+        exit $?
+        ;;
+    populate )
+        . /etc/mailinabox.conf
+        populate_by_name "${1:-basic}"
+        exit $?
+        ;;
+esac
+
 
 
 
 # install basic stuff, set the hostname, time, etc
-before_install
+init
 
 # if MiaB-LDAP is already migrated, do not run upstream setup
+[ -e /etc/mailinabox.conf ] && . /etc/mailinabox.conf
 if [ -e "$STORAGE_ROOT/mailinabox.version" ] &&
        [ $(cat "$STORAGE_ROOT/mailinabox.version") -ge 13 ]
 then
@@ -312,16 +155,21 @@ then
 else
     # install upstream
     upstream_install
-    add_data
-    capture_state "tests/system-setup/state/upstream"
+    . /etc/mailinabox.conf
+    
+    # populate some data
+    populate_by_name "${1:-basic}"
+
+    # capture upstream state
+    installed_state_capture "tests/system-setup/state/upstream"
 fi
 
-# install miab-ldap
+# install miab-ldap and capture state
 miab_ldap_install
-capture_state "tests/system-setup/state/miab-ldap"
+installed_state_capture "tests/system-setup/state/miab-ldap"
 
 # compare states
-if ! compare_state "tests/system-setup/state/upstream" "tests/system-setup/state/miab-ldap"; then
+if ! installed_state_compare "tests/system-setup/state/upstream" "tests/system-setup/state/miab-ldap"; then
     die "Upstream and upgraded states are different !"
 fi