From a8b1e0c5b7145a5e0d487d5b174c323ac501a347 Mon Sep 17 00:00:00 2001
From: downtownallday <downtownallday@gmail.com>
Date: Sat, 6 Jun 2020 11:17:42 -0400
Subject: [PATCH 01/65] Minor wording changes

---
 README.md | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/README.md b/README.md
index 00487994..b7730fc7 100644
--- a/README.md
+++ b/README.md
@@ -36,12 +36,12 @@ To perform general command-line searches against your LDAP database, run `setup/
   * `setup/ldap.sh -search "(objectClass=mailuser)"` (show all users)
   * etc.
 
-This is a convenient way to run ldapsearch to with all the correct command line arguments.
+This is a convenient way to run ldapsearch having all the correct command line arguments.
 
-Caution: do not make LDAP database changes, such as adding users or groups directly using ldapmodify or any other LDAP database tools. Use the MiaB admin interface or REST API! Adding or removing a user or group with the admin interface may trigger additional database and system changes by the management daemon, such as updating DNS zones for new email domains, updating group memberships, etc.
+Caution: do not make direct LDAP database changes, such as adding users or groups using ldapmodify or other LDAP database tool. Instead, use the MiaB admin interface or REST API. Adding or removing a user or group with the admin interface may trigger additional database and system changes by the management daemon, such as updating DNS zones for new email domains, updating group memberships, etc, that would not be performed with a direct change.
 
 
 Migration
 ---------
-When installing MiaB-LDAP by running any of the setup scripts (`miab`, `setup/bootstrap.sh`, `setup/start.sh`, etc) will automatically migrate your current installation to LDAP. Make a backup before running!
+Running any of the setup scripts to install MiaB-LDAP (`miab`, `setup/bootstrap.sh`, `setup/start.sh`, etc) will automatically migrate your current installation from sqlite to LDAP. Make a full MiaB backup before running!
 

From 81950592a758da95e56dcdab813583cb23c1e4ca Mon Sep 17 00:00:00 2001
From: downtownallday <downtownallday@gmail.com>
Date: Sat, 6 Jun 2020 14:06:15 -0400
Subject: [PATCH 02/65] Initial remote Nextcloud integration support

---
 .../conf/zpush/backend_caldav.php             |  16 +
 .../conf/zpush/backend_carddav.php            |  31 ++
 .../remote-nextcloud-use-miab.sh              | 347 ++++++++++++++++++
 setup/mods.available/remote-nextcloud.sh      | 123 +++++++
 setup/mods.d/.gitignore                       |   2 +
 setup/ssl.sh                                  |   8 +-
 setup/start.sh                                |   7 +
 7 files changed, 533 insertions(+), 1 deletion(-)
 create mode 100644 setup/mods.available/conf/zpush/backend_caldav.php
 create mode 100644 setup/mods.available/conf/zpush/backend_carddav.php
 create mode 100755 setup/mods.available/remote-nextcloud-use-miab.sh
 create mode 100755 setup/mods.available/remote-nextcloud.sh
 create mode 100644 setup/mods.d/.gitignore

diff --git a/setup/mods.available/conf/zpush/backend_caldav.php b/setup/mods.available/conf/zpush/backend_caldav.php
new file mode 100644
index 00000000..ada97462
--- /dev/null
+++ b/setup/mods.available/conf/zpush/backend_caldav.php
@@ -0,0 +1,16 @@
+<?php
+/***********************************************
+* File      :   config.php
+* Project   :   Z-Push
+* Descr     :   CalDAV backend configuration file
+************************************************/
+
+define('CALDAV_PROTOCOL', 'https');
+define('CALDAV_SERVER', '127.0.0.1');
+define('CALDAV_PORT', '443');
+define('CALDAV_PATH', 'NC_PREFIX/remote.php/dav/calendars/%u/');
+define('CALDAV_PERSONAL', 'PRINCIPAL');
+define('CALDAV_SUPPORTS_SYNC', false);
+define('CALDAV_MAX_SYNC_PERIOD', 2147483647);
+
+?>
diff --git a/setup/mods.available/conf/zpush/backend_carddav.php b/setup/mods.available/conf/zpush/backend_carddav.php
new file mode 100644
index 00000000..6445bc9c
--- /dev/null
+++ b/setup/mods.available/conf/zpush/backend_carddav.php
@@ -0,0 +1,31 @@
+<?php
+/***********************************************
+* File      :   config.php
+* Project   :   Z-Push
+* Descr     :   CardDAV backend configuration file
+************************************************/
+
+
+define('CARDDAV_PROTOCOL', 'https'); /* http or https */
+define('CARDDAV_SERVER', '127.0.0.1');
+define('CARDDAV_PORT', '443');
+define('CARDDAV_PATH', 'NC_PREFIX/remote.php/dav/addressbooks/users/%u/');
+define('CARDDAV_DEFAULT_PATH', 'NC_PREFIX/remote.php/dav/addressbooks/users/%u/contacts/'); /* subdirectory of the main path */
+define('CARDDAV_GAL_PATH', ''); /* readonly, searchable, not syncd */
+define('CARDDAV_GAL_MIN_LENGTH', 5);
+define('CARDDAV_CONTACTS_FOLDER_NAME', '%u Addressbook');
+define('CARDDAV_SUPPORTS_SYNC', false);
+
+// If the CardDAV server supports the FN attribute for searches
+// DAViCal supports it, but SabreDav, Nextcloud and SOGo don't
+// Setting this to true will search by FN. If false will search by sn, givenName and email
+// It's safe to leave it as false
+define('CARDDAV_SUPPORTS_FN_SEARCH', false);
+
+
+// If your carddav server needs to use file extension to recover a vcard.
+//    Davical needs it
+//    SOGo official demo online needs it, but some SOGo installation don't need it, so test it
+define('CARDDAV_URL_VCARD_EXTENSION', '.vcf');
+
+?>
diff --git a/setup/mods.available/remote-nextcloud-use-miab.sh b/setup/mods.available/remote-nextcloud-use-miab.sh
new file mode 100755
index 00000000..e144dd18
--- /dev/null
+++ b/setup/mods.available/remote-nextcloud-use-miab.sh
@@ -0,0 +1,347 @@
+#!/bin/bash
+
+#
+# Run this script on your remote Nextcloud to configure it to use
+# Mail-in-a-Box-LDAP.
+#
+# The script will:
+#   1. enable the "LDAP user and group backend" in Nextcloud
+#   2. configure Nextcloud to access MiaB-LDAP for users and groups
+#   3. optionally install and configure ssmtp so system mail is
+#      sent to MiaB-LDAP
+#
+VERBOSE=0
+
+usage() {
+    cat <<EOF
+Usage: $0 <NCDIR> <NC_ADMIN_USER> <NC_ADMIN_PASSWORD> <MIAB_HOSTNAME> <LDAP_NEXTCLOUD_PASS> [ <SSMTP_ALERTS_EMAIL> <SSMTP_AUTH_USER> <SSMTP_AUTH_PASS> ]
+Configure Nextcloud to use MiaB-LDAP for users and groups
+Optionally configure a mail relay to MiaB-LDAP
+
+Arguments:
+    NCDIR               
+        the path to the local Nextcloud installation directory
+    NC_ADMIN_USER
+        a current Nextcloud username that has ADMIN rights
+    NC_ADMIN_PASSWORD         
+        the password for NC_ADMIN
+    MIAB_HOSTNAME       
+        the fully-qualified host name of MiaB-LDAP
+    LDAP_NEXTCLOUD_PASS 
+        supply the password for the LDAP service account Nextcloud
+        uses to locate and enumerate users and groups. A MiaB-LDAP
+        installation automatically creates this limited-access service
+        account with a long random password. Open
+        /home/user-data/ldap/miab_ldap.conf on your MiaB-LDAP box,
+        then paste the password for "$LDAP_NEXTCLOUD_DN" as a script
+        argument. It will be the value of the LDAP_NEXTCLOUD_PASSWORD
+        key.
+    SSMTP_ALERTS_EMAIL / SSMTP_AUTH_USER / SSMTP_AUTH_PASS
+        OPTIONAL. Supplying these arguments will setup ssmtp on your
+        system and configure it to use MiaB-LDAP as its mail relay.
+        Email sent with sendmail or ssmtp will be relayed to
+        MiaB-LDAP. SSMTP_ALERTS_EMAIL is the email address that will
+        receive messages for all userids less than
+        1000. SSMTP_AUTH_USER / SSMTP_AUTH_PASS is the email address
+        that will be used to authenticate with MiaB-LDAP (the sender
+        or envelope FROM address). You probably want a new/dedicated
+        email address for this - create on in the MiaB-LDAP admin
+        interface. More information on ssmtp is available at
+        https://help.ubuntu.com/community/EmailAlerts.
+
+The script must be run as root.
+
+EOF
+    exit 1
+}
+
+if [ "$1" == "-v" ]; then
+    VERBOSE=1
+    shift
+fi
+
+# Directory where Nextcloud is installed (must contain occ)
+NCDIR="$1"
+
+# Nextcloud admin credentials for making user-ldap API calls via curl
+NC_ADMIN_USER="$2"
+NC_ADMIN_PASSWORD="$3"
+
+# Hostname of the remote MiaB-LDAP
+MAILINABOX_HOSTNAME="$4"
+
+# LDAP service account Nextcloud uses to perform ldap searches.
+# Values are found in mailinabox:/home/user-data/ldap/miab_ldap.conf
+LDAP_NEXTCLOUD_DN="cn=nextcloud,ou=Services,dc=mailinabox"
+LDAP_NEXTCLOUD_PASSWORD="$5"
+
+# ssmtp: the person who gets all emails for userids < 1000
+SSMTP_ALERTS_EMAIL="$6"
+SSMTP_AUTH_USER="$7"
+SSMTP_AUTH_PASS="$8"
+
+#
+# validate arguments
+#
+if [ -z "$NCDIR" -o "$1" == "-h" -o "$1" == "--help" ]
+then
+    usage
+fi
+
+if [ -z "$NCDIR" -o ! -d "$NCDIR" ]
+then
+    echo "Invalid directory: $NCDIR" 1>&2
+    exit 1
+fi
+
+if [ ! -e "$NCDIR/occ" ]; then
+    echo "OCC not found at: $NCDIR/occ !" 1>&2
+    exit 1
+fi
+
+if [ -z "$NC_ADMIN_USER" -o \
+     -z "$MAILINABOX_HOSTNAME" -o \
+     -z "$LDAP_NEXTCLOUD_PASSWORD" ]
+then
+    usage
+fi
+
+if [ ! -z "$SSMTP_ALERTS_EMAIL" ]; then
+    if [ -z "$(awk -F@ '{print $2}' <<< "$SSMTP_ALERTS_EMAIL")" ]; then
+        echo "Invalid email address: $SSMTP_ALERTS_EMAIL" 1>&2
+        exit 1
+    fi
+    if [ -z "$(awk -F@ '{print $2}' <<< "$SSMTP_AUTH_USER")" ]; then
+        echo "Invalid email address: $SSMTP_AUTH_USER" 1>&2
+        exit 1
+    fi
+fi
+
+if [ -s /etc/mailinabox.conf ]; then
+    echo "Run on your remote Nextcloud, not on Mail-in-a-Box !!" 1>&2
+    exit 1
+fi
+
+if [ "$EUID" != "0" ]; then
+    echo "The script must be run as root (sudo)" 1>&2
+    exit 1
+fi
+
+
+#
+# other constants
+#
+
+LDAP_URL="ldaps://$MAILINABOX_HOSTNAME"
+LDAP_SERVER="$MAILINABOX_HOSTNAME"
+LDAP_SERVER_PORT="636"
+LDAP_SERVER_STARTTLS="no"
+LDAP_BASE="dc=mailinabox"
+LDAP_USERS_BASE="ou=Users,dc=mailinabox"
+PRIMARY_HOSTNAME="$(hostname --fqdn)"
+
+
+say() {
+    echo "$@"
+}
+
+say_verbose() {
+    if [ $VERBOSE -gt 0 ]; then
+        echo "$@"
+    fi
+}
+
+die() {
+    echo "$@" 1>&2
+    exit 2
+}
+
+
+
+#
+# configure nextcloud's user-ldap to access the local ldap server
+#
+# See: https://docs.nextcloud.com/server/17/admin_manual/configuration_user/user_auth_ldap_api.html
+#
+config_user_ldap() {
+    local id="${1:-s01}"
+    local first_call="${2:-yes}"
+    local starttls=0
+    [ "$LDAP_SERVER_STARTTLS" == "yes" ] && starttls=1
+    
+    local c=(
+        "--data-urlencode configData[ldapHost]=$LDAP_URL"
+        "--data-urlencode configData[ldapPort]=$LDAP_SERVER_PORT"
+        "--data-urlencode configData[ldapBase]=$LDAP_USERS_BASE"
+        "--data-urlencode configData[ldapTLS]=$starttls"
+        
+        "--data-urlencode configData[ldapAgentName]=$LDAP_NEXTCLOUD_DN"
+        "--data-urlencode configData[ldapAgentPassword]=$LDAP_NEXTCLOUD_PASSWORD"
+        
+        "--data-urlencode configData[ldapUserDisplayName]=cn"
+        "--data-urlencode configData[ldapUserDisplayName2]="
+        "--data-urlencode configData[ldapUserFilter]=(&(objectClass=inetOrgPerson)(objectClass=mailUser))"
+        "--data-urlencode configData[ldapLoginFilter]=(&(objectClass=inetOrgPerson)(objectClass=mailUser)(|(mail=%uid)(uid=%uid)))"
+        "--data-urlencode configData[ldapEmailAttribute]=mail"
+        
+        "--data-urlencode configData[ldapGroupFilter]=(objectClass=mailGroup)"
+        "--data-urlencode configData[ldapGroupMemberAssocAttr]=member"
+        "--data-urlencode configData[ldapGroupDisplayName]=mail"
+        "--data-urlencode configData[ldapNestedGroups]=1"
+        "--data-urlencode configData[turnOnPasswordChange]=1"
+        
+        "--data-urlencode configData[ldapExpertUsernameAttr]=maildrop"
+        "--data-urlencode configData[ldapExpertUUIDUserAttr]=uid"
+        "--data-urlencode configData[ldapExpertUUIDGroupAttr]=entryUUID"
+
+        "--data-urlencode configData[ldapConfigurationActive]=1"
+    )
+
+    # apply the settings - note: we can't use localhost because nginx
+    # will route to the wrong virtual host
+    local xml
+    say_verbose "curl \"https://${NC_ADMIN_USER}:${NC_ADMIN_PASSWORD}@$PRIMARY_HOSTNAME/ocs/v2.php/apps/user_ldap/api/v1/config/$id\""
+    xml="$(curl -s -S --insecure -X PUT "https://${NC_ADMIN_USER}:${NC_ADMIN_PASSWORD}@$PRIMARY_HOSTNAME/ocs/v2.php/apps/user_ldap/api/v1/config/$id" -H "OCS-APIREQUEST: true" ${c[@]})"
+    [ $? -ne 0 ] &&
+        die "Unable to issue a REST call as $NC_ADMIN_USER to nextcloud"
+
+    # did it work?
+    local statuscode
+    statuscode=$(python3 -c "import xml.etree.ElementTree as ET; print(ET.fromstring('''$xml''').findall('meta')[0].findall('statuscode')[0].text)")
+    
+    if [ "$statuscode" == "404" -a "$first_call" == "yes" ]; then
+        # got a 404 so maybe this is the first time -- we have to create
+        # an initial blank ldap configuration and try again
+        xml="$(curl -s -S --insecure -X POST "https://${NC_ADMIN_USER}:${NC_ADMIN_PASSWORD}@$PRIMARY_HOSTNAME/ocs/v2.php/apps/user_ldap/api/v1/config" -H "OCS-APIREQUEST: true")"
+        [ $? -ne 0 ] &&
+            die "Unable to issue a REST call as $NC_ADMIN_USER to nextcloud: $xml"
+        statuscode=$(python3 -c "import xml.etree.ElementTree as ET; print(ET.fromstring('''$xml''').findall('meta')[0].findall('statuscode')[0].text)")
+        [ $? -ne 0 -o "$statuscode" != "200" ] &&
+            die "Error creating initial ldap configuration: $xml"
+        
+        id=$(python3 -c "import xml.etree.ElementTree as ET; print(ET.fromstring('''$xml''').findall('data')[0].findall('configID')[0].text)" 2>/dev/null)
+        [ $? -ne 0 ] &&
+            die "Error creating initial ldap configuration: $xml"
+        
+        config_user_ldap "$id" no
+
+    elif [ "$statuscode" == "997" -a "$first_call" == "yes" ]; then
+        # could not log in
+        die "Could not authenticate as $NC_ADMIN_USER to perform user-ldap API call. statuscode=$statuscode: $xml"
+        
+    elif [ "$statuscode" != "200" ]; then
+        die "Unable to apply ldap configuration to nextcloud: id=$id first_call=$first_call statuscode=$statuscode: $xml"
+    fi
+    return 0
+}
+
+
+
+enable_user_ldap() {
+    # install prerequisite package
+    say_verbose "Installing system package php-ldap"
+    apt-get install -y -qq php-ldap || die "Could not install php-ldap package"
+    #restart_service php7.2-fpm
+    
+    # enable user_ldap
+    sudo -E -u www-data php $NCDIR/occ app:enable user_ldap -q
+    [ $? -ne 0 ] && die "Unable to enable user_ldap nextcloud app"
+}
+
+install_app() {
+    local app="$1"
+    if [ -z "$(sudo -E -u www-data php $NCDIR/occ app:list | grep $app)" ]; then
+        sudo -E -u www-data php $NCDIR/occ app:install $app
+        [ $? -ne 0 ] && die "Unable to install Nextcloud app '$app'"
+    fi
+}
+
+
+setup_ssmtp() {
+    # sendmail-like mailer with a mailhub to remote mail-in-a-box
+    # see: https://help.ubuntu.com/community/EmailAlerts
+    say_verbose "Installing system package ssmtp"
+    apt-get install -y -qq ssmtp
+
+    if [ ! -e /etc/ssmtp/ssmtp.conf.orig ]; then
+        cp /etc/ssmtp/ssmtp.conf /etc/ssmtp/ssmtp.conf.orig
+    fi
+    
+    cat <<EOF >/etc/ssmtp/ssmtp.conf
+# Generated by MiaB-LDAP integration script on $(date)
+
+# The person who gets all mail for userids < 1000
+root=${SSMTP_ALERTS_EMAIL}
+
+# The place where mail goes
+mailhub=${MAILINABOX_HOSTNAME}:587
+AuthUser=${SSMTP_AUTH_USER}
+AuthPass=${SSMTP_AUTH_PASS}
+UseTLS=YES
+UseSTARTTLS=YES
+
+# The full hostname
+hostname=${PRIMARY_HOSTNAME}
+
+# Are users allowed to set their own From address?
+FromLineOverride=YES
+EOF
+}
+
+
+
+remote_mailinabox_handler() {
+
+    say_verbose "Installing system package ldap-utils"
+    apt-get install -y -qq ldap-utils || die "Could not install ldap-utils package"
+    
+    while /bin/true; do
+        # ensure we can search
+        local output
+        say ""
+        say "Testing MiaB-LDAP connection..."
+        output="$(ldapsearch -H $LDAP_URL -x -D "$LDAP_NEXTCLOUD_DN" -w "$LDAP_NEXTCLOUD_PASSWORD" -b "$LDAP_BASE" -s base 2>&1)"
+        local code=$?
+        if [ $code -ne 0 ]; then
+            say "Unable to contact $LDAP_URL"
+            say "   base=$LDAP_BASE"
+            say "   user=$LDAP_NEXTCLOUD_DN"
+            say "   error code=$code"
+            say "   msg= $output"
+            say ""
+            say "You may need to permit access to the ldap server running on $LDAP_SERVER"
+            say "On $LDAP_SERVER execute:"
+            local ip
+            for ip in $(hostname -I); do
+                say "   \$ ufw allow proto tcp from $ip to any port ldaps"
+            done
+            say ""
+            read -p "Press [enter] when ready, or \"no\" to quit: " ans
+            [ "$ans" == "no" ] && die "Quit"
+            
+        else
+            break
+        fi
+    done
+        
+    enable_user_ldap
+    config_user_ldap
+
+    return 0
+}
+
+
+
+echo "Integrating Nextcloud with Mail-in-a-box LDAP"
+remote_mailinabox_handler
+
+# contacts and calendar are required for Roundcube and Z-Push
+install_app "calendar"
+install_app "contacts"
+
+if [ ! -z "${SSMTP_ALERTS_EMAIL}" ]; then
+    setup_ssmtp
+fi
+
+say ""
+say "Done!"
diff --git a/setup/mods.available/remote-nextcloud.sh b/setup/mods.available/remote-nextcloud.sh
new file mode 100755
index 00000000..733d349d
--- /dev/null
+++ b/setup/mods.available/remote-nextcloud.sh
@@ -0,0 +1,123 @@
+#!/bin/bash
+
+source setup/functions.sh # load our functions
+source /etc/mailinabox.conf # load global vars
+
+# maintain a separate conf file because setup rewrites mailinabox.conf
+touch /etc/mailinabox_mods.conf
+. /etc/mailinabox_mods.conf
+
+# where webmail.sh installs roundcube
+RCM_DIR=/usr/local/lib/roundcubemail
+RCM_PLUGIN_DIR=${RCM_DIR}/plugins
+
+# where zpush.sh installs z-push
+ZPUSH_DIR=/usr/local/lib/z-push
+
+
+configure_zpush() {
+    # have zpush use the remote nextcloud for carddav/caldav
+    # instead of the nextcloud that comes with mail-in-a-box
+    local nc_host="$1"
+    local nc_prefix="$2"
+    [ "$nc_prefix" == "/" ] && nc_prefix=""
+    
+    # Configure CardDav
+    if [ ! -z "$nc_host" ]
+    then
+        cp setup/mods.available/conf/zpush/backend_carddav.php $ZPUSH_DIR/backend/carddav/config.php
+        cp setup/mods.available/conf/zpush/backend_caldav.php $ZPUSH_DIR/backend/caldav/config.php
+        sed -i "s/127\.0\.0\.1/$nc_host/g" $ZPUSH_DIR/backend/carddav/config.php
+        sed -i "s^NC_PREFIX^$nc_prefix^g" $ZPUSH_DIR/backend/carddav/config.php
+        sed -i "s/127\.0\.0\.1/$nc_host/g" $ZPUSH_DIR/backend/caldav/config.php
+        sed -i "s^NC_PREFIX^$nc_prefix^g" $ZPUSH_DIR/backend/caldav/config.php
+    fi
+}
+
+
+configure_roundcube() {
+    # replace the plugin configuration from the default Mail-In-A-Box
+    local name="$1"
+    local nc_host="$2"
+    local nc_prefix="$3"
+    [ "$nc_prefix" == "/" ] && nc_prefix=""
+    
+    # Configure CardDav
+    cat > ${RCM_PLUGIN_DIR}/carddav/config.inc.php <<EOF
+<?php
+/* Do not edit. Written by Mail-in-a-Box-LDAP mods. Regenerated on updates. */
+\$prefs['_GLOBAL']['hide_preferences'] = true;
+\$prefs['_GLOBAL']['suppress_version_warning'] = true;
+\$prefs['cloud'] = array(
+	 'name'         =>  '$name',
+	 'username'     =>  '%u', // login username
+	 'password'     =>  '%p', // login password
+	 'url'          =>  'https://${nc_host}${nc_prefix}/remote.php/carddav/addressbooks/%u/contacts',
+	 'active'       =>  true,
+	 'readonly'     =>  false,
+	 'refresh_time' => '02:00:00',
+	 'fixed'        =>  array('username','password'),
+	 'preemptive_auth' => '1',
+	 'hide'        =>  false,
+);
+?>
+EOF
+}
+
+
+
+remote_nextcloud_handler() {
+    echo "Configure a remote Nextcloud"
+    echo "============================"
+    echo 'Enter the hostname and web prefix of your remote Nextcloud'
+    echo 'For example:'
+    echo '    "cloud.mydomain.com/" - Nextcloud server with no prefix'
+    echo '    "cloud.mydomain.com"  - same as above'
+    echo '    "www.mydomain.com/cloud"  - a Nextcloud server having a prefix /cloud'
+    echo ''
+    
+    local ans_hostname
+    local ans_prefix
+        
+    if [ -z "${NC_HOST:-}" ]; then
+        if [ -z "${NONINTERACTIVE:-}" ]; then
+            read -p "[your Nextcloud's hostname/prefix] " ans_hostname
+        fi
+        [ -z "$ans_hostname" ] && return 0
+    else
+        if [ -z "${NONINTERACTIVE:-}" ]; then
+            read -p "[$NC_HOST/$NC_PREFIX] " ans_hostname
+            if [ -z "$ans_hostname" ]; then
+                ans_hostname="$NC_HOST/$NC_PREFIX"
+            
+            elif [ "$ans_hostname" == "none" ]; then
+                ans_hostname=""
+            fi
+        else
+            ans_hostname="${NC_HOST}${NC_PREFIX}"
+        fi
+    fi
+
+    ans_prefix="/$(awk -F/ '{print substr($0,length($1)+2)}' <<< "$ans_hostname")"
+    ans_hostname="$(awk -F/ '{print $1}' <<< "$ans_hostname")"
+    
+
+    if [ ! -z "$ans_hostname" ]; then
+        echo "Using Nextcloud ${ans_hostname}${ans_prefix}"
+
+        # configure roundcube contacts
+        configure_roundcube "$ans_hostname" "$ans_hostname" "$ans_prefix"
+        
+        # configure zpush (which links to contacts & calendar)
+        configure_zpush "$ans_hostname" "$ans_prefix"
+        
+        # prevent nginx from serving any miab-installed nextcloud files
+        chmod 000 /usr/local/lib/owncloud
+    fi
+    
+    tools/editconf.py /etc/mailinabox_mods.conf \
+                      "NC_HOST=$ans_hostname" \
+                      "NC_PREFIX=$ans_prefix"
+}
+
+remote_nextcloud_handler
diff --git a/setup/mods.d/.gitignore b/setup/mods.d/.gitignore
new file mode 100644
index 00000000..d6b7ef32
--- /dev/null
+++ b/setup/mods.d/.gitignore
@@ -0,0 +1,2 @@
+*
+!.gitignore
diff --git a/setup/ssl.sh b/setup/ssl.sh
index 3d46f9a3..c8681df4 100755
--- a/setup/ssl.sh
+++ b/setup/ssl.sh
@@ -123,15 +123,21 @@ if [ ! -f $STORAGE_ROOT/ssl/ca_certificate.pem ]; then
 	  -passin 'pass:SECRET-PASSWORD' \
 	  -out $CERT \
 	  -subj '/CN=Temporary-Mail-In-A-Box-CA'
+fi
 
-	# add the certificate to the system's trusted root ca list
+if [ ! -e /usr/local/share/ca-certificates/mailinabox.crt ]; then
+	# add the CA certificate to the system's trusted root ca list
 	# this is required for openldap's TLS implementation
+    # do this as a separate step in case a CA certificate is manually
+    # copied onto the machine for QA/test
+	CERT=$STORAGE_ROOT/ssl/ca_certificate.pem
 	hide_output \
 	  cp $CERT /usr/local/share/ca-certificates/mailinabox.crt
 	hide_output \
 	  update-ca-certificates
 fi
 
+
 # Generate a signed SSL certificate because things like nginx, dovecot,
 # etc. won't even start without some certificate in place, and we need nginx
 # so we can offer the user a control panel to install a better certificate.
diff --git a/setup/start.sh b/setup/start.sh
index f686afbe..cfe239b8 100755
--- a/setup/start.sh
+++ b/setup/start.sh
@@ -147,6 +147,13 @@ echo
 certbot register --register-unsafely-without-email --agree-tos --config-dir $STORAGE_ROOT/ssl/lets_encrypt
 fi
 
+#
+# Run setup mods 
+#
+for mod in $(ls setup/mods.d | grep -v '~$'); do
+    setup/mods.d/$mod
+done
+
 # Done.
 echo
 echo "-----------------------------------------------"

From cfc8fb484cfdb3ee581630a869fd93d4e1b3cb03 Mon Sep 17 00:00:00 2001
From: Marcus Bointon <marcus@synchromedia.co.uk>
Date: Sun, 7 Jun 2020 15:47:51 +0200
Subject: [PATCH 03/65] Add rate limiting of SSH in the firewall (#1770)

See #1767.
---
 setup/functions.sh     | 9 ++++++++-
 setup/system.sh        | 4 ++--
 tools/readable_bash.py | 8 ++++++--
 3 files changed, 16 insertions(+), 5 deletions(-)

diff --git a/setup/functions.sh b/setup/functions.sh
index b36d14bc..90c4c55d 100644
--- a/setup/functions.sh
+++ b/setup/functions.sh
@@ -136,7 +136,14 @@ function get_default_privateip {
 function ufw_allow {
 	if [ -z "${DISABLE_FIREWALL:-}" ]; then
 		# ufw has completely unhelpful output
-		ufw allow $1 > /dev/null;
+		ufw allow "$1" > /dev/null;
+	fi
+}
+
+function ufw_limit {
+	if [ -z "${DISABLE_FIREWALL:-}" ]; then
+		# ufw has completely unhelpful output
+		ufw limit "$1" > /dev/null;
 	fi
 }
 
diff --git a/setup/system.sh b/setup/system.sh
index 28043b16..4d33deb6 100755
--- a/setup/system.sh
+++ b/setup/system.sh
@@ -256,7 +256,7 @@ if [ -z "${DISABLE_FIREWALL:-}" ]; then
 	apt_install ufw
 
 	# Allow incoming connections to SSH.
-	ufw_allow ssh;
+	ufw_limit ssh;
 
 	# ssh might be running on an alternate port. Use sshd -T to dump sshd's #NODOC
 	# settings, find the port it is supposedly running on, and open that port #NODOC
@@ -266,7 +266,7 @@ if [ -z "${DISABLE_FIREWALL:-}" ]; then
 	if [ "$SSH_PORT" != "22" ]; then
 
 	echo Opening alternate SSH port $SSH_PORT. #NODOC
-	ufw_allow $SSH_PORT #NODOC
+	ufw_limit $SSH_PORT #NODOC
 
 	fi
 	fi
diff --git a/tools/readable_bash.py b/tools/readable_bash.py
index 5207a78a..1fcdd5cd 100644
--- a/tools/readable_bash.py
+++ b/tools/readable_bash.py
@@ -58,7 +58,7 @@ def generate_documentation():
 	    	}
 
 	    	.prose {
-	    		padding-top: 1em;    	
+	    		padding-top: 1em;
 	    		padding-bottom: 1em;
 	    	}
 	    	.terminal {
@@ -261,6 +261,10 @@ class UfwAllow(Grammar):
 	grammar = (ZERO_OR_MORE(SPACE), L("ufw_allow "), REST_OF_LINE, EOL)
 	def value(self):
 		return shell_line("ufw allow " + self[2].string)
+class UfwLimit(Grammar):
+	grammar = (ZERO_OR_MORE(SPACE), L("ufw_limit "), REST_OF_LINE, EOL)
+	def value(self):
+		return shell_line("ufw limit " + self[2].string)
 class RestartService(Grammar):
 	grammar = (ZERO_OR_MORE(SPACE), L("restart_service "), REST_OF_LINE, EOL)
 	def value(self):
@@ -275,7 +279,7 @@ class OtherLine(Grammar):
 		return "<pre class='shell'><div>" + recode_bash(self.string.strip()) + "</div></pre>\n"
 
 class BashElement(Grammar):
-	grammar = Comment | CatEOF | EchoPipe | EchoLine | HideOutput | EditConf | SedReplace | AptGet | UfwAllow | RestartService | OtherLine
+	grammar = Comment | CatEOF | EchoPipe | EchoLine | HideOutput | EditConf | SedReplace | AptGet | UfwAllow | UfwLimit | RestartService | OtherLine
 	def value(self):
 		return self[0].value()
 

From 339c330b4ff61e6bf116d98947e4a8e93e1b72f8 Mon Sep 17 00:00:00 2001
From: Faye Duxovni <duxovni@duxovni.org>
Date: Sun, 7 Jun 2020 09:50:04 -0400
Subject: [PATCH 04/65] Fix roundcube error log file path in setup script
 (#1775)

---
 setup/webmail.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/setup/webmail.sh b/setup/webmail.sh
index 20d43c57..bd31e221 100755
--- a/setup/webmail.sh
+++ b/setup/webmail.sh
@@ -160,7 +160,7 @@ mkdir -p /var/log/roundcubemail /var/tmp/roundcubemail $STORAGE_ROOT/mail/roundc
 chown -R www-data.www-data /var/log/roundcubemail /var/tmp/roundcubemail $STORAGE_ROOT/mail/roundcube
 
 # Ensure the log file monitored by fail2ban exists, or else fail2ban can't start.
-sudo -u www-data touch /var/log/roundcubemail/errors
+sudo -u www-data touch /var/log/roundcubemail/errors.log
 
 # Password changing plugin settings
 # The config comes empty by default, so we need the settings

From df9bb263dc7983b71b5a1ecd400f5ae10ab16fbe Mon Sep 17 00:00:00 2001
From: Vasek Sraier <vakabus@users.noreply.github.com>
Date: Sun, 7 Jun 2020 15:56:45 +0200
Subject: [PATCH 05/65] daily_tasks.sh: redirect stderr to stdout (#1768)

When the management commands fail, they can print something to the standard error output.
The administrator would never notice, because it wouldn't be send to him with the usual emails.
Fixes #1763
---
 management/daily_tasks.sh | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/management/daily_tasks.sh b/management/daily_tasks.sh
index 2f723352..db496399 100755
--- a/management/daily_tasks.sh
+++ b/management/daily_tasks.sh
@@ -16,10 +16,10 @@ if [ `date "+%u"` -eq 1 ]; then
 fi
 
 # Take a backup.
-management/backup.py | management/email_administrator.py "Backup Status"
+management/backup.py 2>&1 | management/email_administrator.py "Backup Status"
 
 # Provision any new certificates for new domains or domains with expiring certificates.
-management/ssl_certificates.py -q | management/email_administrator.py "TLS Certificate Provisioning Result"
+management/ssl_certificates.py -q  2>&1 | management/email_administrator.py "TLS Certificate Provisioning Result"
 
 # Run status checks and email the administrator if anything changed.
-management/status_checks.py --show-changes | management/email_administrator.py "Status Checks Change Notice"
+management/status_checks.py --show-changes  2>&1 | management/email_administrator.py "Status Checks Change Notice"

From d2f418a3632955ce07575155c4ccfbb3cc157396 Mon Sep 17 00:00:00 2001
From: downtownallday <downtownallday@gmail.com>
Date: Tue, 9 Jun 2020 20:24:46 -0400
Subject: [PATCH 06/65] Use sha1 hash of maildrop instead of a generated UUID

---
 management/mailconfig.py | 10 +++++++---
 setup/migration_13.py    | 12 ++++++++----
 2 files changed, 15 insertions(+), 7 deletions(-)

diff --git a/management/mailconfig.py b/management/mailconfig.py
index b9852def..fbdc5d4e 100755
--- a/management/mailconfig.py
+++ b/management/mailconfig.py
@@ -10,7 +10,7 @@
 # Python 3 in setup/questions.sh to validate the email
 # address entered by the user.
 
-import subprocess, shutil, os, sqlite3, re, ldap3, uuid
+import subprocess, shutil, os, sqlite3, re, ldap3, uuid, hashlib
 import utils, backend
 from email_validator import validate_email as validate_email_, EmailNotValidError
 import idna
@@ -637,8 +637,12 @@ def add_mail_user(email, pw, privs, env):
 	if conn.wait(id).count() > 0:
 		return ("An alias exists with that address.", 400)
 	
-	# Generate a unique id for uid
-	uid = '%s' % uuid.uuid4()
+	## Generate a unique id for uid
+	#uid = '%s' % uuid.uuid4()
+	# use a sha-1 hash of maildrop for uid
+	m = hashlib.sha1()
+	m.update(bytearray(email.lower(),'utf-8'))
+	uid = m.hexdigest()
 
 	# choose a common name and surname (required attributes)
 	cn = email.split("@")[0].replace('.',' ').replace('_',' ')
diff --git a/setup/migration_13.py b/setup/migration_13.py
index e91bc3b7..81cdd9b4 100644
--- a/setup/migration_13.py
+++ b/setup/migration_13.py
@@ -5,7 +5,7 @@
 # helper functions for migration #13
 #
 
-import uuid, os, sqlite3, ldap3
+import uuid, os, sqlite3, ldap3, hashlib
 
 
 def add_user(env, ldapconn, search_base, users_base, domains_base, email, password, privs, cn=None):
@@ -29,9 +29,13 @@ def add_user(env, ldapconn, search_base, users_base, domains_base, email, passwo
 		print("user already exists: %s" % email)
 		return ldapconn.response[0]['dn']
 
-	# Generate a unique id for uid
-	uid = '%s' % uuid.uuid4()
-
+	## Generate a unique id for uid
+	#uid = '%s' % uuid.uuid4()
+	# use a sha-1 hash of the email address for uid
+	m = hashlib.sha1()
+	m.update(bytearray(email.lower(),'utf-8'))
+	uid = m.hexdigest()
+	
 	# Attributes to apply to the new ldap entry
 	attrs = {
 		"mail" : email,

From 8f2e4d12477d1c61b19e7972069a1a003d13f3bb Mon Sep 17 00:00:00 2001
From: downtownallday <downtownallday@gmail.com>
Date: Tue, 9 Jun 2020 20:26:40 -0400
Subject: [PATCH 07/65] Set miab_ldap.conf variables only if they don't already
 exist so that the file may be pre-populated

---
 setup/ldap.sh | 54 +++++++++++++++++++++++++++------------------------
 1 file changed, 29 insertions(+), 25 deletions(-)

diff --git a/setup/ldap.sh b/setup/ldap.sh
index 0d52aded..78e2c4cb 100755
--- a/setup/ldap.sh
+++ b/setup/ldap.sh
@@ -71,42 +71,46 @@ wait_slapd_start() {
 	say_verbose "...ok"
 }
 
+_add_if_missing() {
+	local var="$1"
+	local val="$2"
+	local conf="$MIAB_INTERNAL_CONF_FILE"
+	if [ $(grep -c "^${var}=" "$conf") -eq 0 ]; then
+		echo "${var}=\"${val}\"" >> "$conf"
+	fi
+}
+
 create_miab_conf() {
 	# create (if non-existing) or load (existing) ldap/miab_ldap.conf
 	if [ ! -e "$MIAB_INTERNAL_CONF_FILE" ]; then
 		say_verbose "Generating a new $MIAB_INTERNAL_CONF_FILE"
 		mkdir -p "$(dirname $MIAB_INTERNAL_CONF_FILE)"
-		
-		# Use 64-character secret keys of safe characters
-		cat > "$MIAB_INTERNAL_CONF_FILE" <<EOF
-LDAP_SERVER=127.0.0.1
-LDAP_SERVER_PORT=389
-LDAP_SERVER_STARTTLS=no
-LDAP_SERVER_TLS=no
-LDAP_URL=ldap://127.0.0.1/
-LDAP_BASE="${LDAP_BASE}"
-LDAP_SERVICES_BASE="${LDAP_SERVICES_BASE}"
-LDAP_CONFIG_BASE="${LDAP_CONFIG_BASE}"
-LDAP_DOMAINS_BASE="${LDAP_DOMAINS_BASE}"
-LDAP_PERMITTED_SENDERS_BASE="${LDAP_PERMITTED_SENDERS_BASE}"
-LDAP_USERS_BASE="${LDAP_USERS_BASE}"
-LDAP_ALIASES_BASE="${LDAP_ALIASES_BASE}"
-LDAP_ADMIN_DN="${LDAP_ADMIN_DN}"
-LDAP_ADMIN_PASSWORD="$(generate_password 64)"
-EOF
+		touch "$MIAB_INTERNAL_CONF_FILE"
 	fi
 
+	# ensure all required values exist, and if not set to default values
+	_add_if_missing LDAP_SERVER 127.0.0.1
+	_add_if_missing LDAP_SERVER_PORT 389
+	_add_if_missing LDAP_SERVER_STARTTLS no
+	_add_if_missing LDAP_SERVER_TLS no
+	_add_if_missing LDAP_URL ldap://127.0.0.1/
+	_add_if_missing LDAP_BASE "${LDAP_BASE}"
+	_add_if_missing LDAP_SERVICES_BASE "${LDAP_SERVICES_BASE}"
+	_add_if_missing LDAP_CONFIG_BASE "${LDAP_CONFIG_BASE}"
+	_add_if_missing LDAP_DOMAINS_BASE "${LDAP_DOMAINS_BASE}"
+	_add_if_missing LDAP_PERMITTED_SENDERS_BASE "${LDAP_PERMITTED_SENDERS_BASE}"
+	_add_if_missing LDAP_USERS_BASE "${LDAP_USERS_BASE}"
+	_add_if_missing LDAP_ALIASES_BASE "${LDAP_ALIASES_BASE}"
+	_add_if_missing LDAP_ADMIN_DN "${LDAP_ADMIN_DN}"
+	_add_if_missing LDAP_ADMIN_PASSWORD "$(generate_password 64)"
+
 	# add service account credentials
 	local prefix
 	for prefix in ${SERVICE_ACCOUNTS[*]}
 	do
-		if [ $(grep -c "^$prefix" "$MIAB_INTERNAL_CONF_FILE") -eq 0 ]; then
-			local cn=$(awk -F_ '{print tolower($2)}' <<< $prefix)
-			cat >>"$MIAB_INTERNAL_CONF_FILE" <<EOF
-${prefix}_DN="cn=$cn,$LDAP_SERVICES_BASE"
-${prefix}_PASSWORD="$(generate_password 64)"
-EOF
-		fi
+		local cn=$(awk -F_ '{print tolower($2)}' <<< $prefix)
+		_add_if_missing "${prefix}_DN" "cn=$cn,$LDAP_SERVICES_BASE"
+		_add_if_missing "${prefix}_PASSWORD" "$(generate_password 64)"
 	done
 	
 	chmod 0640 "$MIAB_INTERNAL_CONF_FILE"

From 8082b06bd25ff335ae9635f9dd0bc68a72435ec2 Mon Sep 17 00:00:00 2001
From: downtownallday <downtownallday@gmail.com>
Date: Tue, 9 Jun 2020 20:37:08 -0400
Subject: [PATCH 08/65] Ensure owncloud directory is accessible

---
 setup/nextcloud.sh | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/setup/nextcloud.sh b/setup/nextcloud.sh
index 76c04800..6bf7899c 100755
--- a/setup/nextcloud.sh
+++ b/setup/nextcloud.sh
@@ -100,6 +100,10 @@ nextcloud_hash=50b98d2c2f18510b9530e558ced9ab51eb4f11b0
 # version.php since the restore procedure can leave the system in a state where you have a newer Nextcloud
 # application version than the database.
 
+# ensure directory is accessible
+if [ -d "/usr/local/lib/owncloud" ]; then
+    chmod u+rx /usr/local/lib/owncloud
+fi
 # If config.php exists, get version number, otherwise CURRENT_NEXTCLOUD_VER is empty.
 if [ -f "$STORAGE_ROOT/owncloud/config.php" ]; then
 	CURRENT_NEXTCLOUD_VER=$(php -r "include(\"$STORAGE_ROOT/owncloud/config.php\"); echo(\$CONFIG['version']);")

From 844ea0884553747a9d06fa87b3f1819e0d5da501 Mon Sep 17 00:00:00 2001
From: downtownallday <downtownallday@gmail.com>
Date: Tue, 9 Jun 2020 20:44:22 -0400
Subject: [PATCH 09/65] Additional remote Nextcloud support for port and prefix
 plus z-push

---
 .../conf/zpush/backend_caldav.php             |  6 +-
 .../conf/zpush/backend_carddav.php            |  6 +-
 .../remote-nextcloud-use-miab.sh              | 90 +++++++++++++----
 setup/mods.available/remote-nextcloud.sh      | 97 +++++++++++--------
 4 files changed, 135 insertions(+), 64 deletions(-)

diff --git a/setup/mods.available/conf/zpush/backend_caldav.php b/setup/mods.available/conf/zpush/backend_caldav.php
index ada97462..1990539a 100644
--- a/setup/mods.available/conf/zpush/backend_caldav.php
+++ b/setup/mods.available/conf/zpush/backend_caldav.php
@@ -5,9 +5,9 @@
 * Descr     :   CalDAV backend configuration file
 ************************************************/
 
-define('CALDAV_PROTOCOL', 'https');
-define('CALDAV_SERVER', '127.0.0.1');
-define('CALDAV_PORT', '443');
+define('CALDAV_PROTOCOL', 'NC_PROTO');
+define('CALDAV_SERVER', 'NC_HOST');
+define('CALDAV_PORT', 'NC_PORT');
 define('CALDAV_PATH', 'NC_PREFIX/remote.php/dav/calendars/%u/');
 define('CALDAV_PERSONAL', 'PRINCIPAL');
 define('CALDAV_SUPPORTS_SYNC', false);
diff --git a/setup/mods.available/conf/zpush/backend_carddav.php b/setup/mods.available/conf/zpush/backend_carddav.php
index 6445bc9c..fd328dbe 100644
--- a/setup/mods.available/conf/zpush/backend_carddav.php
+++ b/setup/mods.available/conf/zpush/backend_carddav.php
@@ -6,9 +6,9 @@
 ************************************************/
 
 
-define('CARDDAV_PROTOCOL', 'https'); /* http or https */
-define('CARDDAV_SERVER', '127.0.0.1');
-define('CARDDAV_PORT', '443');
+define('CARDDAV_PROTOCOL', 'NC_PROTO'); /* http or https */
+define('CARDDAV_SERVER', 'NC_HOST');
+define('CARDDAV_PORT', 'NC_PORT');
 define('CARDDAV_PATH', 'NC_PREFIX/remote.php/dav/addressbooks/users/%u/');
 define('CARDDAV_DEFAULT_PATH', 'NC_PREFIX/remote.php/dav/addressbooks/users/%u/contacts/'); /* subdirectory of the main path */
 define('CARDDAV_GAL_PATH', ''); /* readonly, searchable, not syncd */
diff --git a/setup/mods.available/remote-nextcloud-use-miab.sh b/setup/mods.available/remote-nextcloud-use-miab.sh
index e144dd18..fd0277bf 100755
--- a/setup/mods.available/remote-nextcloud-use-miab.sh
+++ b/setup/mods.available/remote-nextcloud-use-miab.sh
@@ -12,6 +12,7 @@
 #
 VERBOSE=0
 
+
 usage() {
     cat <<EOF
 Usage: $0 <NCDIR> <NC_ADMIN_USER> <NC_ADMIN_PASSWORD> <MIAB_HOSTNAME> <LDAP_NEXTCLOUD_PASS> [ <SSMTP_ALERTS_EMAIL> <SSMTP_AUTH_USER> <SSMTP_AUTH_PASS> ]
@@ -45,8 +46,8 @@ Arguments:
         1000. SSMTP_AUTH_USER / SSMTP_AUTH_PASS is the email address
         that will be used to authenticate with MiaB-LDAP (the sender
         or envelope FROM address). You probably want a new/dedicated
-        email address for this - create on in the MiaB-LDAP admin
-        interface. More information on ssmtp is available at
+        email address for this - create a new account in the MiaB-LDAP
+        admin interface. More information on ssmtp is available at
         https://help.ubuntu.com/community/EmailAlerts.
 
 The script must be run as root.
@@ -140,6 +141,25 @@ LDAP_BASE="dc=mailinabox"
 LDAP_USERS_BASE="ou=Users,dc=mailinabox"
 PRIMARY_HOSTNAME="$(hostname --fqdn)"
 
+#
+# get the url used to access nextcloud as NC_ADMIN_USER
+#
+NC_CONFIG_CLI_URL="$(cd "$NCDIR/config"; php -n -r 'include "config.php"; print $CONFIG["overwrite.cli.url"];')"
+case "$NC_CONFIG_CLI_URL" in
+    http:* | https:* )
+        urlproto=$(awk -F/ '{print $1}' <<< "$NC_CONFIG_CLI_URL")
+        urlhost=$(awk -F/ '{print $3}'  <<< "$NC_CONFIG_CLI_URL")
+        urlprefix=$(awk -F/ "{ print substr(\$0,length(\"$urlproto\")+length(\"\
+$urlhost\")+4) }" <<<"$NC_CONFIG_CLI_URL")
+        NC_AUTH_URL="$urlproto//${NC_ADMIN_USER}:${NC_ADMIN_PASSWORD}@$urlhost/\
+$urlprefix"
+        ;;
+    * )
+        NC_AUTH_URL="https://${NC_ADMIN_USER}:${NC_ADMIN_PASSWORD}@$PRIMARY_HOS\
+TNAME${NC_CONFIG_CLI_URL:-/}"
+        ;;
+esac
+
 
 say() {
     echo "$@"
@@ -159,7 +179,7 @@ die() {
 
 
 #
-# configure nextcloud's user-ldap to access the local ldap server
+# configure Nextcloud's user-ldap for MiaB-LDAP
 #
 # See: https://docs.nextcloud.com/server/17/admin_manual/configuration_user/user_auth_ldap_api.html
 #
@@ -181,6 +201,7 @@ config_user_ldap() {
         "--data-urlencode configData[ldapUserDisplayName]=cn"
         "--data-urlencode configData[ldapUserDisplayName2]="
         "--data-urlencode configData[ldapUserFilter]=(&(objectClass=inetOrgPerson)(objectClass=mailUser))"
+        "--data-urlencode configData[ldapUserFilterMode]=1"
         "--data-urlencode configData[ldapLoginFilter]=(&(objectClass=inetOrgPerson)(objectClass=mailUser)(|(mail=%uid)(uid=%uid)))"
         "--data-urlencode configData[ldapEmailAttribute]=mail"
         
@@ -200,26 +221,30 @@ config_user_ldap() {
     # apply the settings - note: we can't use localhost because nginx
     # will route to the wrong virtual host
     local xml
-    say_verbose "curl \"https://${NC_ADMIN_USER}:${NC_ADMIN_PASSWORD}@$PRIMARY_HOSTNAME/ocs/v2.php/apps/user_ldap/api/v1/config/$id\""
-    xml="$(curl -s -S --insecure -X PUT "https://${NC_ADMIN_USER}:${NC_ADMIN_PASSWORD}@$PRIMARY_HOSTNAME/ocs/v2.php/apps/user_ldap/api/v1/config/$id" -H "OCS-APIREQUEST: true" ${c[@]})"
+    say_verbose "curl \"${NC_AUTH_URL%/}/ocs/v2.php/apps/user_ldap/api/v1/config/$id\""
+    xml="$(curl -s -S --insecure -X PUT "${NC_AUTH_URL%/}/ocs/v2.php/apps/user_ldap/api/v1/config/$id" -H "OCS-APIREQUEST: true" ${c[@]})"
     [ $? -ne 0 ] &&
-        die "Unable to issue a REST call as $NC_ADMIN_USER to nextcloud"
+        die "Unable to issue a REST call as $NC_ADMIN_USER to nextcloud. url=$NC_AUTH_URL/ocs/v2.php/apps/user_ldap/api/v1/config/$id"
 
     # did it work?
+    if [ -z "$xml" ]; then
+        die "Invalid response from Nextcloud using url '$NC_AUTH_URL'. reponse was '$xml'. Cannot continue."
+    fi
+        
     local statuscode
-    statuscode=$(python3 -c "import xml.etree.ElementTree as ET; print(ET.fromstring('''$xml''').findall('meta')[0].findall('statuscode')[0].text)")
+    statuscode=$(python3 -c "import xml.etree.ElementTree as ET; print(ET.fromstring(r'''$xml''').findall('meta')[0].findall('statuscode')[0].text)")
     
     if [ "$statuscode" == "404" -a "$first_call" == "yes" ]; then
         # got a 404 so maybe this is the first time -- we have to create
         # an initial blank ldap configuration and try again
-        xml="$(curl -s -S --insecure -X POST "https://${NC_ADMIN_USER}:${NC_ADMIN_PASSWORD}@$PRIMARY_HOSTNAME/ocs/v2.php/apps/user_ldap/api/v1/config" -H "OCS-APIREQUEST: true")"
+        xml="$(curl -s -S --insecure -X POST "${NC_AUTH_URL%/}/ocs/v2.php/apps/user_ldap/api/v1/config" -H "OCS-APIREQUEST: true")"
         [ $? -ne 0 ] &&
             die "Unable to issue a REST call as $NC_ADMIN_USER to nextcloud: $xml"
-        statuscode=$(python3 -c "import xml.etree.ElementTree as ET; print(ET.fromstring('''$xml''').findall('meta')[0].findall('statuscode')[0].text)")
+        statuscode=$(python3 -c "import xml.etree.ElementTree as ET; print(ET.fromstring(r'''$xml''').findall('meta')[0].findall('statuscode')[0].text)")
         [ $? -ne 0 -o "$statuscode" != "200" ] &&
             die "Error creating initial ldap configuration: $xml"
         
-        id=$(python3 -c "import xml.etree.ElementTree as ET; print(ET.fromstring('''$xml''').findall('data')[0].findall('configID')[0].text)" 2>/dev/null)
+        id=$(python3 -c "import xml.etree.ElementTree as ET; print(ET.fromstring(r'''$xml''').findall('data')[0].findall('configID')[0].text)" 2>/dev/null)
         [ $? -ne 0 ] &&
             die "Error creating initial ldap configuration: $xml"
         
@@ -238,19 +263,31 @@ config_user_ldap() {
 
 
 enable_user_ldap() {
-    # install prerequisite package
-    say_verbose "Installing system package php-ldap"
-    apt-get install -y -qq php-ldap || die "Could not install php-ldap package"
-    #restart_service php7.2-fpm
+    # install prerequisite package php-ldap
+    # if using Docker Hub's php image, don't install at all
+    if [ ! -e /etc/apt/preferences.d/no-debian-php ]; then
+        say_verbose "Installing system package php-ldap"
+        apt-get install -y -qq php-ldap || die "Could not install php-ldap package"
+        #restart_service php7.2-fpm
+    fi
     
     # enable user_ldap
-    sudo -E -u www-data php $NCDIR/occ app:enable user_ldap -q
-    [ $? -ne 0 ] && die "Unable to enable user_ldap nextcloud app"
+    if [ ! -x /usr/bin/sudo ]; then
+        say "WARNING: sudo is not installed: Unable to run occ to check and/or enable Nextcloud app \"user-ldap\"."
+    else
+        say_verbose "Enable user-ldap"
+        sudo -E -u www-data php $NCDIR/occ app:enable user_ldap -q
+        [ $? -ne 0 ] && die "Unable to enable user_ldap nextcloud app"
+    fi
 }
 
 install_app() {
     local app="$1"
-    if [ -z "$(sudo -E -u www-data php $NCDIR/occ app:list | grep $app)" ]; then
+    if [ ! -x /usr/bin/sudo ]; then
+        say "WARNING: sudo is not installed: Unable to run occ to check and/or install Nextcloud app \"$app\"."
+        
+    elif [ -z "$(sudo -E -u www-data php $NCDIR/occ app:list | grep $app)" ]; then
+        say_verbose "Install app '$app'"
         sudo -E -u www-data php $NCDIR/occ app:install $app
         [ $? -ne 0 ] && die "Unable to install Nextcloud app '$app'"
     fi
@@ -260,6 +297,11 @@ install_app() {
 setup_ssmtp() {
     # sendmail-like mailer with a mailhub to remote mail-in-a-box
     # see: https://help.ubuntu.com/community/EmailAlerts
+
+    if [ "$(. /etc/os-release; echo $NAME)" != "Ubuntu" ]; then
+        die "Sorry, ssmtp is only supported on Ubuntu"
+    fi
+    
     say_verbose "Installing system package ssmtp"
     apt-get install -y -qq ssmtp
 
@@ -291,16 +333,18 @@ EOF
 
 
 remote_mailinabox_handler() {
-
     say_verbose "Installing system package ldap-utils"
-    apt-get install -y -qq ldap-utils || die "Could not install ldap-utils package"
+    apt-get install -y -qq ldap-utils python3 || die "Could not install required packages"
+    
+    local count=0
+    local ldap_debug=""
     
     while /bin/true; do
         # ensure we can search
         local output
         say ""
         say "Testing MiaB-LDAP connection..."
-        output="$(ldapsearch -H $LDAP_URL -x -D "$LDAP_NEXTCLOUD_DN" -w "$LDAP_NEXTCLOUD_PASSWORD" -b "$LDAP_BASE" -s base 2>&1)"
+        output="$(ldapsearch $ldap_debug -v -H $LDAP_URL -x -D "$LDAP_NEXTCLOUD_DN" -w "$LDAP_NEXTCLOUD_PASSWORD" -b "$LDAP_BASE" -s base 2>&1)"
         local code=$?
         if [ $code -ne 0 ]; then
             say "Unable to contact $LDAP_URL"
@@ -316,10 +360,16 @@ remote_mailinabox_handler() {
                 say "   \$ ufw allow proto tcp from $ip to any port ldaps"
             done
             say ""
+            let count+=1
+            if [ $count -gt 5 ]; then
+                die "Giving up"
+            fi
             read -p "Press [enter] when ready, or \"no\" to quit: " ans
             [ "$ans" == "no" ] && die "Quit"
+            ldap_debug="-d 9"
             
         else
+            say "Test successful - able to bind and search as $LDAP_NEXTCLOUD_DN"
             break
         fi
     done
diff --git a/setup/mods.available/remote-nextcloud.sh b/setup/mods.available/remote-nextcloud.sh
index 733d349d..c09e2130 100755
--- a/setup/mods.available/remote-nextcloud.sh
+++ b/setup/mods.available/remote-nextcloud.sh
@@ -18,29 +18,22 @@ ZPUSH_DIR=/usr/local/lib/z-push
 configure_zpush() {
     # have zpush use the remote nextcloud for carddav/caldav
     # instead of the nextcloud that comes with mail-in-a-box
-    local nc_host="$1"
-    local nc_prefix="$2"
-    [ "$nc_prefix" == "/" ] && nc_prefix=""
     
-    # Configure CardDav
-    if [ ! -z "$nc_host" ]
-    then
-        cp setup/mods.available/conf/zpush/backend_carddav.php $ZPUSH_DIR/backend/carddav/config.php
-        cp setup/mods.available/conf/zpush/backend_caldav.php $ZPUSH_DIR/backend/caldav/config.php
-        sed -i "s/127\.0\.0\.1/$nc_host/g" $ZPUSH_DIR/backend/carddav/config.php
-        sed -i "s^NC_PREFIX^$nc_prefix^g" $ZPUSH_DIR/backend/carddav/config.php
-        sed -i "s/127\.0\.0\.1/$nc_host/g" $ZPUSH_DIR/backend/caldav/config.php
-        sed -i "s^NC_PREFIX^$nc_prefix^g" $ZPUSH_DIR/backend/caldav/config.php
-    fi
+    cp setup/mods.available/conf/zpush/backend_carddav.php $ZPUSH_DIR/backend/carddav/config.php
+    cp setup/mods.available/conf/zpush/backend_caldav.php $ZPUSH_DIR/backend/caldav/config.php
+    local var val
+    for var in NC_PROTO NC_HOST NC_PORT NC_PREFIX; do
+        eval "val=\$$var"
+        sed -i "s^$var^${val%/}^g" $ZPUSH_DIR/backend/carddav/config.php
+        sed -i "s^$var^${val%/}^g" $ZPUSH_DIR/backend/caldav/config.php
+    done    
 }
 
 
 configure_roundcube() {
     # replace the plugin configuration from the default Mail-In-A-Box
-    local name="$1"
-    local nc_host="$2"
-    local nc_prefix="$3"
-    [ "$nc_prefix" == "/" ] && nc_prefix=""
+    local name="${1:-$NC_HOST}"
+    local baseurl="$NC_PROTO://$NC_HOST:$NC_PORT$NC_PREFIX"
     
     # Configure CardDav
     cat > ${RCM_PLUGIN_DIR}/carddav/config.inc.php <<EOF
@@ -52,7 +45,7 @@ configure_roundcube() {
 	 'name'         =>  '$name',
 	 'username'     =>  '%u', // login username
 	 'password'     =>  '%p', // login password
-	 'url'          =>  'https://${nc_host}${nc_prefix}/remote.php/carddav/addressbooks/%u/contacts',
+	 'url'          =>  '${baseurl%/}/remote.php/carddav/addressbooks/%u/contacts',
 	 'active'       =>  true,
 	 'readonly'     =>  false,
 	 'refresh_time' => '02:00:00',
@@ -67,57 +60,85 @@ EOF
 
 
 remote_nextcloud_handler() {
+    echo ""
+    echo "============================"
     echo "Configure a remote Nextcloud"
     echo "============================"
-    echo 'Enter the hostname and web prefix of your remote Nextcloud'
+    echo 'Enter the url or hostname and web prefix of your remote Nextcloud'
     echo 'For example:'
     echo '    "cloud.mydomain.com/" - Nextcloud server with no prefix'
     echo '    "cloud.mydomain.com"  - same as above'
     echo '    "www.mydomain.com/cloud"  - a Nextcloud server having a prefix /cloud'
     echo ''
+
+    local ans
+    local current_url=""
     
-    local ans_hostname
-    local ans_prefix
-        
     if [ -z "${NC_HOST:-}" ]; then
         if [ -z "${NONINTERACTIVE:-}" ]; then
-            read -p "[your Nextcloud's hostname/prefix] " ans_hostname
+            read -p "[your Nextcloud's hostname/prefix] " ans
         fi
-        [ -z "$ans_hostname" ] && return 0
+        [ -z "$ans" ] && return 0
     else
+        current_url="$NC_PROTO://$NC_HOST:$NC_PORT$NC_PREFIX"
         if [ -z "${NONINTERACTIVE:-}" ]; then
-            read -p "[$NC_HOST/$NC_PREFIX] " ans_hostname
-            if [ -z "$ans_hostname" ]; then
-                ans_hostname="$NC_HOST/$NC_PREFIX"
+            read -p "[$current_url] " ans
+            if [ -z "$ans" ]; then
+                ans="$current_url"
             
-            elif [ "$ans_hostname" == "none" ]; then
-                ans_hostname=""
+            elif [ "$ans" == "none" ]; then
+                ans=""
             fi
         else
-            ans_hostname="${NC_HOST}${NC_PREFIX}"
+            ans="$current_url"
         fi
     fi
 
-    ans_prefix="/$(awk -F/ '{print substr($0,length($1)+2)}' <<< "$ans_hostname")"
-    ans_hostname="$(awk -F/ '{print $1}' <<< "$ans_hostname")"
+    case "$ans" in
+        https://* )
+            NC_PROTO="https"
+            NC_PORT="443"
+            ans="$(awk -F: '{print substr($0,9)}' <<< "$ans")"
+            ;;
+        http://* )
+            NC_PROTO="http"
+            NC_PORT="80"
+            ans="$(awk -F: '{print substr($0,8)}' <<< "$ans")"
+            ;;
+        * )
+            NC_PROTO="https"
+            NC_PORT="443"
+            ;;
+    esac
     
+    NC_PREFIX="/$(awk -F/ '{print substr($0,length($1)+2)}' <<< "$ans")"
+    NC_HOST="$(awk -F/ '{print $1}' <<< "$ans")"
+    
+    if grep ":" <<< "$NC_HOST" >/dev/null; then
+        NC_PORT="$(awk -F: '{print $2}'  <<< "$NC_HOST")"
+        NC_HOST="$(awk -F: '{print $1}' <<< "$NC_HOST")"
+    fi
+    
+    local new_url="$NC_PROTO://$NC_HOST:$NC_PORT$NC_PREFIX"
 
-    if [ ! -z "$ans_hostname" ]; then
-        echo "Using Nextcloud ${ans_hostname}${ans_prefix}"
+    if [ ! -z "$NC_HOST" ]; then
+        echo "Using Nextcloud ${new_url}"
 
         # configure roundcube contacts
-        configure_roundcube "$ans_hostname" "$ans_hostname" "$ans_prefix"
+        configure_roundcube "$NC_HOST"
         
         # configure zpush (which links to contacts & calendar)
-        configure_zpush "$ans_hostname" "$ans_prefix"
+        configure_zpush
         
         # prevent nginx from serving any miab-installed nextcloud files
         chmod 000 /usr/local/lib/owncloud
     fi
     
     tools/editconf.py /etc/mailinabox_mods.conf \
-                      "NC_HOST=$ans_hostname" \
-                      "NC_PREFIX=$ans_prefix"
+                      "NC_PROTO=$NC_PROTO" \
+                      "NC_HOST=$NC_HOST" \
+                      "NC_PORT=$NC_PORT" \
+                      "NC_PREFIX=$NC_PREFIX"
 }
 
 remote_nextcloud_handler

From 83cb7cbcbeb71770a2a4979e72e64a7be0455255 Mon Sep 17 00:00:00 2001
From: downtownallday <downtownallday@gmail.com>
Date: Tue, 9 Jun 2020 20:46:59 -0400
Subject: [PATCH 10/65] Automated QA tests for remote nextcloud

---
 .../assets/mail/roundcube/carddav_refresh.sh  | 136 ++++++++
 .../system-setup/remote-nextcloud-docker.sh   | 200 +++++++++++
 tests/assets/system-setup/setup-defaults.sh   |  33 ++
 tests/assets/system-setup/setup-funcs.sh      |  97 ++++++
 tests/runner.sh                               |  53 ++-
 tests/suites/_init.sh                         |  22 +-
 tests/suites/_ldap-functions.sh               |  10 +-
 tests/suites/_locations.sh                    |   8 +
 tests/suites/remote-nextcloud.sh              | 313 ++++++++++++++++++
 9 files changed, 852 insertions(+), 20 deletions(-)
 create mode 100755 tests/assets/mail/roundcube/carddav_refresh.sh
 create mode 100755 tests/assets/system-setup/remote-nextcloud-docker.sh
 create mode 100755 tests/assets/system-setup/setup-defaults.sh
 create mode 100755 tests/assets/system-setup/setup-funcs.sh
 create mode 100644 tests/suites/_locations.sh
 create mode 100644 tests/suites/remote-nextcloud.sh

diff --git a/tests/assets/mail/roundcube/carddav_refresh.sh b/tests/assets/mail/roundcube/carddav_refresh.sh
new file mode 100755
index 00000000..e399c9bc
--- /dev/null
+++ b/tests/assets/mail/roundcube/carddav_refresh.sh
@@ -0,0 +1,136 @@
+#!/usr/bin/env php
+<?php
+
+define('INSTALL_PATH', realpath(__DIR__ . '/..') . '/' );
+
+require_once INSTALL_PATH.'program/include/clisetup.php';
+ini_set('memory_limit', -1);
+
+
+
+function usage()
+{
+    print "Usage: carddav_refresh.sh [-id <number>] username password\n";
+    print "Force a sync of a user's addressbook with the remote server\n";
+    print "Place this script in /path/to/roundcubemail/bin, then change the working directory to /path/to/roundcubemail, then run ./bin/cardav_refresh.sh"; 
+    exit(1);
+}
+
+function _die($msg)
+{
+    fwrite(STDERR, $msg . "\n");
+    exit(1);
+}
+
+$args = rcube_utils::get_opt(array('id' => 'dbid'));
+
+$dbid = 0;
+if (!empty($args['dbid'])) {
+    $dbid = intval($args['dbid']);
+}
+   
+$username = trim($args[0]);
+if (empty($username)) {
+    print "Missing username";
+    usage();
+}
+$password = trim($args[1]);
+if (empty($password)) {
+    usage();
+}
+
+
+// -----
+// From index.php -- initialization and login
+// -----
+
+// init application, start session, init output class, etc.
+$RCMAIL = rcmail::get_instance(0, $GLOBALS['env']);
+
+// trigger startup plugin hook
+$startup = $RCMAIL->plugins->exec_hook('startup', array('task' => $RCMAIL->task, 'action' => $RCMAIL->action));
+$RCMAIL->set_task($startup['task']);
+$RCMAIL->action = $startup['action'];
+$auth = $RCMAIL->plugins->exec_hook('authenticate', array(
+            'host'  => $RCMAIL->autoselect_host(),
+            'user'  => $username,
+            'pass'  => $password,
+            'valid' => true,
+            'cookiecheck' => false,));
+
+// Login
+if ($auth['valid'] && !$auth['abort']
+    && $RCMAIL->login($auth['user'], $auth['pass'], $auth['host'], $auth['cookiecheck']))
+  {
+    print "login ok\n";
+  }
+ else
+   {
+     _die("login failed");
+   }
+
+ 
+// ----------------------------------------------------
+// ensure the carddav tables are created and populated
+// ----------------------------------------------------
+
+require_once('plugins/carddav/carddav_backend.php');
+require_once('plugins/carddav/carddav.php');
+
+$c = new carddav(rcube_plugin_api::get_instance());
+$c->task .= "|cli";
+$c->init();
+print "done: init\n";
+// this ensures the carddav tables are created
+$c->checkMigrations();
+print "done: init tables\n";
+// this populates carddav_addressbooks from config
+$c->init_presets();
+print "done: init addressbooks\n";
+
+// -------------------------------------------------------------
+// Set the last_updated field for addressbooks to an old date.
+// That will force a sync/update
+// -------------------------------------------------------------
+$db = $rcmail->get_dbh();
+$db->db_connect('w');
+if (!$db->is_connected() || $db->is_error()) {
+  _die("No DB connection\n" . $db->is_error());
+}
+print "db connected\n";
+
+$db->query("update " . $db->table_name('carddav_addressbooks') . " set last_updated=? WHERE active=1", '2000-01-01 00:00:00');
+print "update made\n";
+if ($db->is_error()) {
+  _die("DB error occurred: " . $db->is_error());
+}
+
+
+// ------------------------------------------------------
+// Update/sync all active address books
+// ------------------------------------------------------
+
+// first get all row ids
+$dbid=array();
+$sql_result = $db->query('SELECT id FROM ' .
+           $db->table_name('carddav_addressbooks') .
+           ' WHERE active=1');
+if ($db->is_error()) {
+  _die("DB error occurred: " . $db->is_error());
+}
+   
+while ($row = $db->fetch_assoc($sql_result)) {
+  $dbid += array(intval($row['id']));
+  print "carddav_addressbooks id: " . $row['id'] . "\n";
+}
+
+// instantiating carddav_backend causes the update/sync
+foreach($dbid as $id) {
+  $config = carddav_backend::carddavconfig($id);
+  if ($config['needs_update']) {
+    print "instantiating carddav_backend: " . $id . "\n";
+    $b = new carddav_backend($id);
+    print("success\n");
+  }
+}
+
diff --git a/tests/assets/system-setup/remote-nextcloud-docker.sh b/tests/assets/system-setup/remote-nextcloud-docker.sh
new file mode 100755
index 00000000..c9418ae4
--- /dev/null
+++ b/tests/assets/system-setup/remote-nextcloud-docker.sh
@@ -0,0 +1,200 @@
+#!/bin/bash
+
+# setup MiaB-LDAP with a remote Nextcloud running on the same
+# host under Docker exposed as localhost:8000
+#
+# this script must be run with the working directory set to 'tests'
+
+usage() {
+    echo "Usage: $(basename "$0") [\"before-miab-install\"|\"miab-install\"|\"after-miab-install\"]"
+    echo "Install MiaB-LDAP and a remote Nextcloud running under docker exposed as localhost:8000"
+    echo "With no arguments, all three stages are run."
+    exit 1
+}
+
+# ensure working directory
+if [ ! -d "tests/assets/system-setup" ]; then
+    echo "This script must be run from the MiaB root directory"
+    exit 1
+fi
+
+# load helper scripts
+. "tests/assets/system-setup/setup-defaults.sh" \
+    || die "Could not load setup-defaults"
+. "tests/assets/system-setup/setup-funcs.sh" \
+    || die "Could not load setup-funcs"
+
+# ensure running as root
+if [ "$EUID" != "0" ]; then
+    die "This script must be run as root (sudo)"
+fi
+
+
+
+before_miab_install() {
+    H1 "BEFORE MIAB-LDAP INSTALL"
+
+    # create /etc/hosts entry for PRIVATE_IP
+    H2 "Update /etc/hosts"
+    update_hosts_for_private_ip || die "Could not update /etc/hosts"
+
+    # install prerequisites
+    H2 "QA prerequisites"
+    install_qa_prerequisites || die "Error installing QA prerequisites"
+
+    # update system time (ignore errors)
+    H2 "Set system time"
+    update_system_time
+
+    # copy in pre-built MiaB-LDAP ssl files
+    #   1. avoid the lengthy generation of DH params
+    H2 "Install QA pre-built"
+    mkdir -p $STORAGE_ROOT/ssl || die "Unable to create $STORAGE_ROOT/ssl"
+    cp tests/assets/ssl/dh2048.pem $STORAGE_ROOT/ssl \
+        || die "Copy dhparams failed"
+
+    # create miab_ldap.conf to specify what the Nextcloud LDAP service
+    # account password will be to avoid a random one created by start.sh
+    mkdir -p $STORAGE_ROOT/ldap
+    [ -e $STORAGE_ROOT/ldap/miab_ldap.conf ] && \
+        echo "Warning: exists: $STORAGE_ROOT/ldap/miab_ldap.conf" 1>&2
+    echo "LDAP_NEXTCLOUD_PASSWORD=\"$LDAP_NEXTCLOUD_PASSWORD\"" >> $STORAGE_ROOT/ldap/miab_ldap.conf
+
+    # enable the remote Nextcloud setup mod, which tells MiaB-LDAP to use
+    # the remote Nextcloud for calendar and contacts instead of the
+    # MiaB-installed one
+    if [ ! -e "setup/mods.d/remote-nextcloud.sh" ]; then
+        ln -s "../mods.available/remote-nextcloud.sh" "setup/mods.d/remote-nextcloud.sh" || die "Could not create remote-nextcloud.sh symlink"
+    fi
+
+    
+    # install Docker
+    H2 "Install Docker"
+    install_docker || die "Could not install Docker! ($?)"
+}
+
+
+miab_install() {
+    H1 "MIAB-LDAP INSTALL"
+    setup/start.sh -v || die "setup/start.sh failed!"
+}
+
+
+after_miab_install() {
+    H1 "AFTER MIAB-LDAP INSTALL"
+    
+    . /etc/mailinabox.conf || die "Could not load /etc/mailinabox.conf"
+
+    # TRAVIS: fix nsd startup problem
+    H2 "Apply Travis-CI nsd fix"
+    travis_fix_nsd || die "Could not fix NSD startup issue for TRAVIS-CI"
+    
+    # run Nextcloud docker image
+    H2 "Start Nextcloud docker container"
+    docker run -d --name NC -p 8000:80 \
+           --env SQLITE_DATABASE=nextclouddb.sqlite \
+           --env NEXTCLOUD_ADMIN_USER="$NC_ADMIN_USER" \
+           --env NEXTCLOUD_ADMIN_PASSWORD="$NC_ADMIN_PASSWORD" \
+           --env NEXTCLOUD_TRUSTED_DOMAINS="127.0.0.1 ::1" \
+           --env NEXTCLOUD_UPDATE=1 \
+           --env SMTP_HOST="$PRIMARY_HOSTNAME" \
+           --env SMTP_SECURE="tls" \
+           --env SMTP_PORT=587 \
+           --env SMTP_AUTHTYPE="LOGIN" \
+           --env SMTP_NAME="$EMAIL_ADDR" \
+           --env SMTP_PASSWORD="$EMAIL_PW" \
+           --env SMTP_FROM_ADDRESS="$(awk -F@ '{print $1}' <<< "$EMAIL_ADDR")" \
+           --env MAIL_DOMAIN="$(awk -F@ '{print $2}' <<< "$EMAIL_ADDR")" \
+           nextcloud:latest \
+        || die "Docker run failed!"
+
+    H2 "docker: Update /etc/hosts so it can find MiaB-LDAP by name"
+    echo "$PRIVATE_IP $PRIMARY_HOSTNAME" | \
+        docker exec -i NC bash -c 'cat >>/etc/hosts' \
+        || die "docker: could not update /etc/hosts"
+    
+    # apt-get update
+    H2 "docker: apt-get update"
+    docker exec NC apt-get update || die "docker: apt-get update failed"
+
+    # allow LDAP access from docker image
+    H2 "Allow ldaps through firewall so Nextcloud can perform LDAP searches"
+    ufw allow ldaps || die "Unable to modify firewall to permit ldaps"
+
+    # add MiaB-LDAP's ca_certificate.pem to docker's trusted cert list
+    H2 "docker: update trusted CA list"
+    docker cp \
+           $STORAGE_ROOT/ssl/ca_certificate.pem \
+           NC:/usr/local/share/ca-certificates/mailinabox.crt \
+        || die "docker: copy ca_certificate.pem failed"
+    docker exec NC update-ca-certificates \
+        || die "docker: update-ca-certificates failed"
+
+    # wait for Nextcloud installation to complete
+    H2 "Wait for Nextcloud installation to complete"
+    echo -n "Waiting ..."
+    local count=0
+    while true; do
+        if [ $count -ge 10 ]; then
+            echo "FAILED"
+            die "Giving up"
+        fi
+        sleep 6
+        let count+=1
+        if [ $(docker exec NC php -n -r "include 'config/config.php'; print \$CONFIG['installed']?'true':'false';") == "true" ]; then
+            echo "ok"
+            break
+        fi
+        echo -n "${count}..."
+    done
+    
+    # install and enable Nextcloud and apps
+    H2 "docker: install Nextcloud calendar app"
+    docker exec -u www-data NC ./occ app:install calendar \
+        || die "docker: installing calendar app failed"
+    H2 "docker: install Nextcloud contacts app"
+    docker exec -u www-data NC ./occ app:install contacts \
+        || die "docker: installing contacts app failed"
+    H2 "docker: enable user_ldap"
+    docker exec -u www-data NC ./occ app:enable user_ldap \
+        || die "docker: enabling user_ldap failed"
+
+    # integrate Nextcloud with MiaB-LDAP
+    H2 "docker: integrate Nextcloud with MiaB-LDAP"
+    docker cp setup/mods.available/remote-nextcloud-use-miab.sh NC:/tmp \
+        || die "docker: cp remote-nextcloud-use-miab.sh failed"
+    docker exec NC /tmp/remote-nextcloud-use-miab.sh \
+           . \
+           "$NC_ADMIN_USER" \
+           "$NC_ADMIN_PASSWORD" \
+           "$PRIMARY_HOSTNAME" \
+           "$LDAP_NEXTCLOUD_PASSWORD" \
+        || die "docker: error running remote-nextcloud-use-miab.sh"
+}
+
+
+
+#
+# process command line
+#
+
+case "$1" in
+    before-miab-install )
+        before_miab_install
+        ;;
+    after-miab-install )
+        after_miab_install
+        ;;
+    miab-install )
+        miab_install
+        ;;
+    "" )
+        before_miab_install
+        miab_install
+        after_miab_install
+        ;;
+    * )
+        
+        usage
+        ;;
+esac
diff --git a/tests/assets/system-setup/setup-defaults.sh b/tests/assets/system-setup/setup-defaults.sh
new file mode 100755
index 00000000..851f92d6
--- /dev/null
+++ b/tests/assets/system-setup/setup-defaults.sh
@@ -0,0 +1,33 @@
+#!/bin/bash
+
+# Used by MiaB-LDAP setup/start.sh
+export NONINTERACTIVE=${NONINTERACTIVE:-1}
+export SKIP_NETWORK_CHECKS=${SKIP_NETWORK_CHECKS:-1}
+export STORAGE_USER="${STORAGE_USER:-user-data}"
+export STORAGE_ROOT="${STORAGE_ROOT:-/home/$STORAGE_USER}"
+export EMAIL_ADDR="${EMAIL_ADDR:-qa@abc.com}"
+export EMAIL_PW="${EMAIL_PW:-Test_1234}"
+export PUBLIC_IP="${PUBLIC_IP:-$(source setup/functions.sh; get_default_privateip 4)}"
+
+if [ "$TRAVIS" == "true" ]; then
+    export PRIMARY_HOSTNAME=${PRIMARY_HOSTNAME:-box.abc.com}
+elif [ -z "$PRIMARY_HOSTNAME" ]; then
+    export PRIMARY_HOSTNAME=$(hostname --fqdn || hostname)
+fi
+
+
+# Placing this var in STORAGE_ROOT/ldap/miab_ldap.conf before running
+# setup/start.sh will avoid a random password from being used for the
+# Nextcloud LDAP service account
+export LDAP_NEXTCLOUD_PASSWORD=${LDAP_NEXTCLOUD_PASSWORD:-Test_LDAP_1234}
+
+# Used by setup/mods.available/remote-nextcloud.sh. These define to
+# MiaB-LDAP the remote Nextcloud that serves calendar and contacts
+export NC_PROTO=${NC_PROTO:-http}
+export NC_HOST=${NC_HOST:-127.0.0.1}
+export NC_PORT=${NC_PORT:-8000}
+export NC_PREFIX=${NC_PREFIX:-/}
+
+# For setup scripts that are installing a remote Nextcloud
+export NC_ADMIN_USER="${NC_ADMIN_USER:-admin}"
+export NC_ADMIN_PASSWORD="${NC_ADMIN_PASSWORD:-Test_1234}"
diff --git a/tests/assets/system-setup/setup-funcs.sh b/tests/assets/system-setup/setup-funcs.sh
new file mode 100755
index 00000000..b5768b1a
--- /dev/null
+++ b/tests/assets/system-setup/setup-funcs.sh
@@ -0,0 +1,97 @@
+
+die() {
+    local msg="$1"
+    echo "$msg" 1>&2
+    exit 1
+}
+
+H1() {
+    local msg="$1"
+    echo "----------------------------------------------"
+    echo "           $msg"
+    echo "----------------------------------------------"
+}
+
+H2() {
+    local msg="$1"
+    echo "*** $msg ***"
+}
+
+install_qa_prerequisites() {
+    # python3-dnspython: is used by the python scripts in 'tests' and is
+    #   not installed by setup
+    # ntpdate: is used by this script
+    apt-get install -y \
+            ntpdate \
+            python3-dnspython
+}
+
+update_system_time() {
+    ntpdate -s ntp.ubuntu.com && echo "System time updated"
+}
+
+update_hosts() {
+    local host="$1"
+    local ip="$2"
+    local line="$ip $host"
+    if ! grep -F "$line" /etc/hosts 1>/dev/null; then
+        echo "$line" >>/etc/hosts
+    fi
+}
+
+update_hosts_for_private_ip() {
+    # create /etc/hosts entry for PRIVATE_IP
+    # PRIMARY_HOSTNAME must already be set
+    local ip=$(source setup/functions.sh; get_default_privateip 4)
+    [ -z "$ip" ] && return 1
+    update_hosts "$PRIMARY_HOSTNAME" "$ip" || return 1
+}
+
+install_docker() {
+    if [ -x /usr/bin/docker ]; then
+        echo "Docker already installed"
+        return 0
+    fi
+    
+    apt-get install -y \
+            apt-transport-https \
+            ca-certificates \
+            curl \
+            gnupg-agent \
+            software-properties-common \
+        || return 1
+       
+    curl -fsSL https://download.docker.com/linux/ubuntu/gpg | apt-key add - \
+        || return 2
+    
+    apt-key fingerprint 0EBFCD88 || return 3
+    
+    add-apt-repository -y --update "deb [arch=amd64] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" || return 4
+    
+    apt-get install -y \
+            docker-ce \
+            docker-ce-cli \
+            containerd.io \
+        || return 5
+}
+
+
+travis_fix_nsd() {
+    if [ "$TRAVIS" != "true" ]; then
+        return 0
+    fi
+    
+    # nsd won't start on Travis-CI without the changes below: ip6 off and
+    # control-enable set to no. Even though the nsd docs says the
+    # default value for control-enable is no, running "nsd-checkconf -o
+    # control-enable /etc/nsd/nsd.conf" returns "yes", so we explicitly
+    # set it here.
+    #
+    # we're assuming that the "ip-address" line is the last line in the
+    # "server" section of nsd.conf. if this generated file output
+    # changes, the sed command below may need to be adjusted.
+    sed -i 's/ip-address\(.\)\(.*\)/ip-address\1\2\n  do-ip4\1 yes\n  do-ip6\1 no\n  verbosity\1 3\nremote-control\1\n  control-enable\1 no/' /etc/nsd/nsd.conf || return 1
+    cat /etc/nsd/nsd.conf
+    systemctl reset-failed nsd.service || return 2
+    systemctl restart nsd.service || return 3
+}
diff --git a/tests/runner.sh b/tests/runner.sh
index db0511a5..8119d462 100755
--- a/tests/runner.sh
+++ b/tests/runner.sh
@@ -11,7 +11,7 @@ cd "$(dirname $0)"
 # load global functions and variables
 . suites/_init.sh
 
-runner_suites=(
+default_suites=(
 	ldap-connection
 	ldap-access
 	mail-basic
@@ -21,21 +21,37 @@ runner_suites=(
 	management-users
 )
 
+extra_suites=(
+	remote-nextcloud
+)
+
 usage() {
 	echo ""
-	echo "Usage: $(basename $0) [-failfatal] [suite-name ...]"
-	echo "Valid suite names:"
-	for runner_suite in ${runner_suites[@]}; do
-		echo "	 $runner_suite"
+	echo "Usage: $(basename $0) [options] [suite-name ...]"
+	echo "Run QA tests"
+
+	echo ""
+	echo "Default test suites:"
+	echo "--------------------"
+	for runner_suite in ${default_suites[@]}; do
+		echo "  $runner_suite"
 	done
-	echo "If no suite-name(s) given, all suites are run"
+
+	echo ""
+	echo "Extra test suites:"
+	echo "------------------"
+	echo "  remote-nextcloud   : test the setup mod for remote Nextcloud"
+	echo ""
+
+	echo "If no suite-name(s) are given, all default suites are run"
 	echo ""
 	echo "Options:"
+	echo "--------"
 	echo "  -failfatal	    The runner will stop if any test fails"
 	echo "  -dumpoutput     After all tests have run, dump all failed test output"
 	echo "  -no-smtp-remote Skip tests requiring a remote SMTP server"
 	echo ""
-	echo "Output directory: $(dirname $0)/${base_outputdir}"
+	echo "Output directory: ${BASE_OUTPUTDIR}"
 	echo ""
 	exit 1
 }
@@ -59,7 +75,20 @@ while [ $# -gt 0 ]; do
 			;;
 		* )
 			# run named suite
-			if array_contains "$1" ${runner_suites[@]}; then
+			if [ $OVERALL_COUNT_SUITES -eq 0 ]; then
+				rm -rf "${BASE_OUTPUTDIR}"
+			fi
+			
+			if [ "$1" == "default" ]
+			then
+				# run all default suites
+				for suite in ${default_suites[@]}; do
+					. suites/$suite.sh
+				done
+			elif array_contains "$1" ${default_suites[@]} || \
+					array_contains "$1" ${extra_suites[@]}
+			then
+				# run specified suite
 				. "suites/$1.sh"
 			else
 				echo "Unknown suite '$1'" 1>&2
@@ -70,11 +99,11 @@ while [ $# -gt 0 ]; do
 	shift
 done
 
-# if no suites specified on command line, run all suites
+# if no suites specified on command line, run all default suites
 if [ $OVERALL_COUNT_SUITES -eq 0 ]; then
-	rm -rf "${base_outputdir}"
-	for runner_suite in ${runner_suites[@]}; do
-		. suites/$runner_suite.sh
+	rm -rf "${BASE_OUTPUTDIR}"
+	for suite in ${default_suites[@]}; do
+		. suites/$suite.sh
 	done
 fi
 
diff --git a/tests/suites/_init.sh b/tests/suites/_init.sh
index 2b29e576..bee4fb20 100644
--- a/tests/suites/_init.sh
+++ b/tests/suites/_init.sh
@@ -6,12 +6,13 @@
 set +eu
 
 # load test suite helper functions
+. suites/_locations.sh      || exit 1
 . suites/_ldap-functions.sh || exit 1
 . suites/_mail-functions.sh || exit 1
 . suites/_mgmt-functions.sh || exit 1
 
 # globals - all global variables are UPPERCASE
-BASE_OUTPUTDIR="out"
+BASE_OUTPUTDIR="$(realpath out)"
 PYMAIL="./test_mail.py"
 declare -i OVERALL_SUCCESSES=0
 declare -i OVERALL_FAILURES=0
@@ -45,11 +46,12 @@ suite_start() {
 	mkdir -p "$OUTDIR"
 	echo ""
 	echo "Starting suite: $SUITE_NAME"
-	suite_setup "$2"
+	shift
+	suite_setup "$@"
 }
 
 suite_end() {
-	suite_cleanup "$1"
+	suite_cleanup "$@"
 	echo "Suite $SUITE_NAME finished"
 	let OVERALL_SUCCESSES+=$SUITE_COUNT_SUCCESS
 	let OVERALL_FAILURES+=$SUITE_COUNT_FAILURE
@@ -61,14 +63,16 @@ suite_end() {
 suite_setup() {
 	[ -z "$1" ] && return 0
 	TEST_OF="$OUTDIR/setup"
-	eval "$1"
+	local script
+	for script; do eval "$script";	done
 	TEST_OF=""
 }
 
 suite_cleanup() {
 	[ -z "$1" ] && return 0
 	TEST_OF="$OUTDIR/cleanup"
-	eval "$1"
+	local script
+	for script; do eval "$script"; done
 	TEST_OF=""
 }
 
@@ -105,7 +109,7 @@ test_end() {
 				let idx+=1
 			done
 			echo "$TEST_OF" >>$FAILED_TESTS_MANIFEST
-			echo "	   see: $(dirname $0)/$TEST_OF"
+			echo "	   see: $TEST_OF"
 			let SUITE_COUNT_FAILURE+=1
 			if [ "$FAILURE_IS_FATAL" == "yes" ]; then
 				record "FATAL: failures are fatal option enabled"
@@ -205,6 +209,12 @@ python_error() {
 	[ $? -eq 1 ] && echo "$output"
 }
 
+copy_or_die() {
+	local src="$1"
+	local dst="$2"
+	cp "$src" "$dst" || die "Unable to copy '$src' => '$dst'"
+}
+
 dump_failed_tests_output() {
 	if [ "$DUMP_FAILED_TESTS_OUTPUT" == "yes" ]; then
 		echo ""
diff --git a/tests/suites/_ldap-functions.sh b/tests/suites/_ldap-functions.sh
index 4dcac8cb..77e102fe 100644
--- a/tests/suites/_ldap-functions.sh
+++ b/tests/suites/_ldap-functions.sh
@@ -7,6 +7,11 @@ generate_uuid() {
 	echo "$uuid"
 }
 
+sha1() {
+	local txt="$1"
+	python3 -c "import hashlib; m=hashlib.sha1(); m.update(bytearray(r'''$txt''','utf-8')); print(m.hexdigest());" || die "Unable to generate sha1 hash"
+}
+
 delete_user() {
 	local email="$1"
 	local domainpart="$(awk -F@ '{print $2}' <<< "$email")"
@@ -32,12 +37,13 @@ create_user() {
 	local priv="${3:-test}"
 	local localpart="$(awk -F@ '{print $1}' <<< "$email")"
 	local domainpart="$(awk -F@ '{print $2}' <<< "$email")"
-	local uid="$localpart"
+	#local uid="$localpart"
+	local uid="$(sha1 "$email")"
 	local dn="uid=${uid},${LDAP_USERS_BASE}"
 	
 	delete_user "$email"
 
-	record "[create user $email]"
+	record "[create user $email ($dn)]"
 	delete_dn "$dn"
 	
 	ldapadd -H "$LDAP_URL" -x -D "$LDAP_ADMIN_DN" -w "$LDAP_ADMIN_PASSWORD" >>$TEST_OF 2>&1 <<EOF
diff --git a/tests/suites/_locations.sh b/tests/suites/_locations.sh
new file mode 100644
index 00000000..52a26341
--- /dev/null
+++ b/tests/suites/_locations.sh
@@ -0,0 +1,8 @@
+#
+
+# where webmail.sh installs roundcube
+RCM_DIR=/usr/local/lib/roundcubemail
+RCM_PLUGIN_DIR=${RCM_DIR}/plugins
+
+# where zpush.sh installs z-push
+ZPUSH_DIR=/usr/local/lib/z-push
diff --git a/tests/suites/remote-nextcloud.sh b/tests/suites/remote-nextcloud.sh
new file mode 100644
index 00000000..d2af6c7d
--- /dev/null
+++ b/tests/suites/remote-nextcloud.sh
@@ -0,0 +1,313 @@
+# -*- indent-tabs-mode: t; tab-width: 4; -*-
+#	
+# Test the setup modification script setup/mods.available/remote-nextcloud.sh
+# Prerequisites:
+
+#    - Nextcloud is already installed and MiaB-LDAP is already
+#      configured to use it.
+#
+#      ie. remote-nextcloud.sh was run on MiaB-LDAP.sh by
+#          setup/start.sh because there was a symbolic link from
+#          setup/mods.d/remote-nextcloud.sh to the script in
+#          mods.available
+
+#    - The remote Nextcloud has been configured to use MiaB-LDAP
+#      for users and groups.
+#
+#      eg. remote-nextcloud-use-miab.sh was copied to the remote Nextcloud
+#      server and was run successfully there
+#
+
+is_configured() {
+    . /etc/mailinabox_mods.conf
+    if [ $? -ne 0 -o -z "$NC_HOST" ]; then
+        return 1
+    fi
+    return 0
+}
+
+assert_is_configured() {
+    if ! is_configured; then
+        test_failure "remote-nextcloud is not configured"
+        return 1
+    fi
+    return 0
+}
+
+
+nextcloud_url() {
+    # eg: http://localhost/cloud/
+    carddav_url | sed 's|\(.*\)/remote.php/.*|\1/|'
+}
+
+carddav_url() {
+    # get the carddav url as configured in z-push for the user specified
+    # eg: http://localhost/cloud/remote.php/dav/addressbooks/users/admin/contacts/
+    local user="${1:-%u}"
+    local path="${2:-CARDDAV_DEFAULT_PATH}"
+    local php='include "/usr/local/lib/z-push/backend/carddav/config.php"; print CARDDAV_PROTOCOL . "://" . CARDDAV_SERVER . ":" . CARDDAV_PORT . '
+    php="$php$path;"
+    local url
+    url="$(php -n -r "$php")"
+    [ $? -ne 0 ] && die "Unable to run php to extract carddav url from z-push"
+    sed "s/%u/$user/" <<< "$url"
+}
+
+
+carddav_rest() {
+    # issue a CardDAV rest call to Nextcloud
+    # SEE: https://tools.ietf.org/html/rfc6352
+    local verb="$1"
+    local uri="$2"
+    local auth_user="$3"
+    local auth_pass="$4"
+	shift; shift; shift; shift  # remaining arguments are data
+
+    local url
+    case "$uri" in
+        /* )
+            url="$(nextcloud_url)${uri#/}"
+            ;;
+        http*)
+            url="$uri"
+            ;;
+        * )
+            url="$(carddav_url "$auth_user")${uri#/}"
+            ;;
+    esac
+    
+    local data=()
+	local item output
+
+    for item; do data+=("--data" "$item"); done
+
+    local ct
+    case "${data[1]}" in
+        BEGIN:VCARD* )
+            ct="text/vcard"
+            ;;
+        * )
+            ct='text/xml; charset="utf-8"'
+    esac
+    
+    record "spawn: curl -w \"%{http_code}\" -X $verb -H 'Content-Type: $ct' --user \"${auth_user}:xxx\" ${data[@]} \"$url\""
+	output=$(curl -s -S -w "%{http_code}" -X $verb -H "Content-Type: $ct" --user "${auth_user}:${auth_pass}" "${data[@]}" "$url" 2>>$TEST_OF)
+	local code=$?
+
+	# http status is last 3 characters of output, extract it
+	REST_HTTP_CODE=$(awk '{S=substr($0,length($0)-2)} END {print S}' <<<"$output")
+	REST_OUTPUT=$(awk 'BEGIN{L=""}{ if(L!="") print L; L=$0 } END { print substr(L,1,length(L)-3) }' <<<"$output")
+	REST_ERROR=""
+	[ -z "$REST_HTTP_CODE" ] && REST_HTTP_CODE="000"
+
+    if [ $code -ne 0 -o \
+               $REST_HTTP_CODE -lt 200 -o \
+               $REST_HTTP_CODE -ge 300 ]
+    then
+		REST_ERROR="REST status $REST_HTTP_CODE: $REST_OUTPUT"
+        REST_ERROR_BRIEF=$(python3 -c "import xml.etree.ElementTree as ET; print(ET.fromstring(r'''$REST_OUTPUT''').find('s:message',{'s':'http://sabredav.org/ns'}).text)" 2>/dev/null)
+        if [ -z "$REST_ERROR_BRIEF" ]; then
+            REST_ERROR_BRIEF="$REST_ERROR"
+        else
+            REST_ERROR_BRIEF="$REST_HTTP_CODE: $REST_ERROR_BRIEF"
+        fi
+        if [ $code -ne 0 ]; then
+            REST_ERROR_BRIEF="curl exit code $code: $REST_ERROR_BRIEF"
+            REST_ERROR="curl exit code $code: $REST_ERROR"
+        fi
+		record "${F_DANGER}$REST_ERROR${F_RESET}"
+		return 2
+	fi
+	record "CURL succeded, HTTP status $REST_HTTP_CODE"
+	record "$output"
+	return 0    
+}
+
+
+carddav_ls() {
+    # return all .vcf files in array 'FILES'
+    local user="$1"
+    local pass="$2"
+    carddav_rest PROPFIND "" "$user" "$pass" || return $?
+    local file FILES=()
+    python3 -c "import xml.etree.ElementTree as ET; [print(el.find('d:href',{'d':'DAV:'}).text) for el in ET.fromstring(r'''$REST_OUTPUT''').findall('d:response',{'d':'DAV:'}) if el.find('d:href',{'d':'DAV:'}) is not None]" |
+        while read file; do
+            # skip non .vcf entries
+            case "$file" in
+                *.vcf )
+                    FILES+=( "$(basename "$file")" )
+                    ;;
+                * )
+                    ;;
+            esac
+        done
+}
+
+
+make_collection() {
+    local user="$1"
+    local pass="$2"
+    local name="$3"
+    local desc="${4:-$name}"
+    local xml="<?xml version=\"1.0\" encoding=\"utf-8\" ?>
+<D:mkcol xmlns:D=\"DAV:\"
+         xmlns:C=\"urn:ietf:params:xml:ns:carddav\">
+  <D:set>
+    <D:prop>
+      <D:resourcetype>
+        <D:collection/>
+        <C:addressbook/>
+      </D:resourcetype>
+      <D:displayname>$name</D:displayname>
+      <C:addressbook-description xml:lang=\"en\">$desc</C:addressbook-description>
+    </D:prop>
+  </D:set>
+</D:mkcol>"
+    record "[create address book '$name' for $user]"
+    local url="$(carddav_url "$user" CARDDAV_PATH)"
+    carddav_rest MKCOL "$url" "$user" "$pass" "$xml"
+}
+
+
+
+add_contact() {
+    local user="$1"
+    local pass="$2"
+    local c_name="$3"
+    local c_phone="$4"
+    local c_email="$5"
+    local c_uid="${6:-$(generate_uuid)}"
+    local file_name="$c_uid.vcf"
+    
+    local vcard="BEGIN:VCARD
+VERSION:3.0
+UID:$c_uid
+REV;VALUE=DATE-AND-OR-TIME:$(date -u +%Y%m%dT%H%M%SZ)
+FN:$c_name
+EMAIL;TYPE=INTERNET,PREF:$c_email
+NOTE:Miab-LDAP QA
+ORG:Miab-LDAP
+TEL;TYPE=WORK,VOICE:$c_phone
+END:VCARD"
+    record "[add contact '$c_name' to $user]"
+    carddav_rest PUT "$file_name" "$user" "$pass" "$vcard"
+}
+
+delete_contact() {
+    local user="$1"
+    local pass="$2"
+    local c_uid="$3"
+    local file_name="$c_uid.vcf"
+    record "[delete contact with vcard uid '$c_uid' from $user]"
+    carddav_rest DELETE "$file_name" "$user" "$pass"
+}
+
+force_roundcube_carddav_refresh() {
+    local user="$1"
+    local pass="$2"
+    local code
+    record "[forcing refresh of roundcube contact for $user]"
+    copy_or_die assets/mail/roundcube/carddav_refresh.sh $RCM_DIR/bin
+    pushd "$RCM_DIR" >/dev/null
+    bin/carddav_refresh.sh "$user" "$pass" >>$TEST_OF 2>&1
+    code=$?
+    popd >/dev/null
+    return $code
+}
+
+assert_roundcube_carddav_contact_exists() {
+    local user="$1"
+    local pass="$2"
+    local c_uid="$3"
+    local output
+    record "[checking that roundcube contact with vcard UID=$c_uid exists]"
+    output="$(sqlite3 $STORAGE_ROOT/mail/roundcube/roundcube.sqlite "select name from carddav_contacts where cuid='$c_uid'" 2>>$TEST_OF)"
+    if [ $? -ne 0 ]; then
+        test_failure "Error querying roundcube sqlite database"
+        return 1
+    fi
+    if [ -z "$output" ]; then
+        test_failure "Contact not found in Roundcube"
+        record "Not found"
+        record "Existing entries (name,vcard-uid):"
+        output="$(sqlite3 $STORAGE_ROOT/mail/roundcube/roundcube.sqlite "select name,cuid FROM carddav_contacts" 2>>$TEST_OF)"
+        return 1
+    else
+        record "$output"
+    fi
+    return 0
+}
+
+
+test_mail_from_nextcloud() {
+    test_start "mail_from_nextcloud"
+    test_end    
+}
+
+test_nextcloud_contacts() {
+    test_start "nextcloud_contacts"
+
+    assert_is_configured || (test_end && return)
+
+    local alice="alice3@somedomain.com"
+    local alice_pw="$(generate_password 16)"
+
+	# create local user alice
+	mgmt_assert_create_user "$alice" "$alice_pw"
+
+
+    #
+    # 1. create contact in Nextcloud - ensure it is available in Roundcube
+    #    
+    # this will validate Nextcloud's ability to authenticate users via
+    # LDAP and that Roundcube is able to reach Nextcloud for contacts
+    #
+
+    #make_collection "$alice" "$alice_pw" "contacts"
+
+    # add new contact to alice's Nextcloud account using CardDAV API
+    local c_uid="$(generate_uuid)"
+    add_contact \
+        "$alice" \
+        "$alice_pw" \
+        "JimIno" \
+        "555-1212" \
+        "jim@ino.com" \
+        "$c_uid" \
+        || test_failure "Could not add contact for $alice in Nextcloud: $REST_ERROR_BRIEF"
+    
+    # force a refresh/sync of the contacts in Roundcube
+    force_roundcube_carddav_refresh "$alice" "$alice_pw" || \
+        test_failure "Could not refresh roundcube contacts for $alice"
+
+    # query the roundcube sqlite database for the new contact
+    assert_roundcube_carddav_contact_exists "$alice" "$alice_pw" "$c_uid"
+    
+    # delete the contact
+    delete_contact "$alice" "$alice_pw" "$c_uid" || \
+        test_failure "Unable to delete contact for $alice in Nextcloud"
+    
+
+    #
+    # 2. create contact in Roundcube - ensure contact appears in Nextcloud
+    #
+    # TODO
+
+    
+    # clean up
+    mgmt_assert_delete_user "$alice"
+    
+    test_end
+}
+
+
+suite_start "remote-nextcloud" mgmt_start
+
+#test_mail_from_nextcloud
+test_nextcloud_contacts
+
+suite_end mgmt_end
+
+
+
+

From 29fad6106eb5ea5550467d851e4b1faa62220e75 Mon Sep 17 00:00:00 2001
From: downtownallday <downtownallday@gmail.com>
Date: Tue, 9 Jun 2020 21:32:49 -0400
Subject: [PATCH 11/65] Update comment

---
 .../system-setup/remote-nextcloud-docker.sh   | 23 ++++++++++++++++++-
 1 file changed, 22 insertions(+), 1 deletion(-)

diff --git a/tests/assets/system-setup/remote-nextcloud-docker.sh b/tests/assets/system-setup/remote-nextcloud-docker.sh
index c9418ae4..39ee400a 100755
--- a/tests/assets/system-setup/remote-nextcloud-docker.sh
+++ b/tests/assets/system-setup/remote-nextcloud-docker.sh
@@ -3,7 +3,28 @@
 # setup MiaB-LDAP with a remote Nextcloud running on the same
 # host under Docker exposed as localhost:8000
 #
-# this script must be run with the working directory set to 'tests'
+# to use:
+#   on a fresh Ubuntu:
+#      1. checkout or copy the MiaB-LDAP code to ~/mailinabox
+#      2. cd ~/mailinabox
+#      3. sudo tests/assets/system-setup/remote-nextcloud-docker.sh
+#
+# when complete you should have a working MiaB-LDAP and Nextcloud
+#
+# You can access MiaB-LDAP using your browser to the Ubuntu system in
+# the normal way, (eg: https://<ubuntu-box>/admin).
+#
+# Nextcloud is running under Docker on the ubuntu box, so to access it
+# you'll first need to ssh into the ubuntu box with port-forrwarding
+# enabled.
+#
+# eg: ssh -L 8000:localhost:8000 user@<ubuntu-box>
+#
+# Then, in your browser visit http://localhost:8000/.
+#
+# See setup-defaults.sh for usernames and passwords.
+#
+
 
 usage() {
     echo "Usage: $(basename "$0") [\"before-miab-install\"|\"miab-install\"|\"after-miab-install\"]"

From 46e0d7a0709fb5f2279b5ad97891694d1280945d Mon Sep 17 00:00:00 2001
From: downtownallday <downtownallday@gmail.com>
Date: Tue, 9 Jun 2020 21:33:30 -0400
Subject: [PATCH 12/65] Don't overwrite PRIMARY_HOSTNAME if already set

---
 tests/assets/system-setup/setup-defaults.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/assets/system-setup/setup-defaults.sh b/tests/assets/system-setup/setup-defaults.sh
index 851f92d6..739a0260 100755
--- a/tests/assets/system-setup/setup-defaults.sh
+++ b/tests/assets/system-setup/setup-defaults.sh
@@ -12,7 +12,7 @@ export PUBLIC_IP="${PUBLIC_IP:-$(source setup/functions.sh; get_default_privatei
 if [ "$TRAVIS" == "true" ]; then
     export PRIMARY_HOSTNAME=${PRIMARY_HOSTNAME:-box.abc.com}
 elif [ -z "$PRIMARY_HOSTNAME" ]; then
-    export PRIMARY_HOSTNAME=$(hostname --fqdn || hostname)
+    export PRIMARY_HOSTNAME=${PRIMARY_HOSTNAME:-$(hostname --fqdn || hostname)}
 fi
 
 

From 828f9d0185e84bdd0664c388293c8ab52e07447e Mon Sep 17 00:00:00 2001
From: downtownallday <downtownallday@gmail.com>
Date: Tue, 9 Jun 2020 21:34:22 -0400
Subject: [PATCH 13/65] First attempt at automating tests with a remote
 Nextcloud under Travis-CI

---
 .travis.yml | 40 ++--------------------------------------
 1 file changed, 2 insertions(+), 38 deletions(-)

diff --git a/.travis.yml b/.travis.yml
index 1f0a3ac0..7479bc52 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -1,8 +1,6 @@
 # travisci config
 env:
   global:
-  - NONINTERACTIVE=1
-  - SKIP_NETWORK_CHECKS=1
   - PRIMARY_HOSTNAME=box.abc.com
 
 language: shell
@@ -16,46 +14,12 @@ before_install:
   #
   - echo "==== DUMP AppArmor Status ===="
   - (sudo aa-status; true)
-  #
-  - echo "==== System update ===="
-  # Do not run 'upgrade' - takes too long
-  - sudo apt-get update
-  #
-  - echo "==== Install QA/test prerequisites ===="
-  # python3-dnspython is used by the python scripts in 'tests' and is
-  # not installed by setup
-  - sudo apt-get -y install python3-dnspython
-  # avoid the lengthy generation of DH params by copying in a prebuilt file
-  - sudo mkdir -p /home/user-data/ssl
-  - sudo cp ./tests/assets/ssl/dh2048.pem /home/user-data/ssl
-  #
-  - echo "==== Add the PRIMARY_HOSTNAME to /etc/hosts ===="
-  # The PRIMARY_HOSTNAME should point to the interface address not
-  # loopback.  That is because of the way MiaB resolves - the local
-  # resolver is bind9, which requires valid NS records, which would
-  # point back to the local nsd authoritative name server for the
-  # domain. We don't have those in QA, so we need the hosts entry.
-  - echo "$(source setup/functions.sh; get_default_privateip 4)   $PRIMARY_HOSTNAME" > /tmp/hosts_add.tmp
-  - sudo $SHELL -c 'cat /tmp/hosts_add.tmp >>/etc/hosts'
   
 install:
-  - sudo ./setup/start.sh -v
+  - sudo tests/assets/system-setup/remote-nextcloud-docker.sh
 
 script:
-  # nsd won't start on Travis without the changes below: ip6 off and
-  # control-enable set to no. Even though the nsd docs says the
-  # default value for control-enable is no, running "nsd-checkconf -o
-  # control-enable /etc/nsd/nsd.conf" returns "yes", so we explicitly
-  # set it here.
-  #
-  # we're assuming that the "ip-address" line is the last line in the
-  # "server" section of nsd.conf. if this generated file output
-  # changes, the sed command below may need to be adjusted.
-  - sudo sed -i 's/ip-address\(.\)\(.*\)/ip-address\1\2\n  do-ip4\1 yes\n  do-ip6\1 no\n  verbosity\1 3\nremote-control\1\n  control-enable\1 no/' /etc/nsd/nsd.conf
-  - sudo cat /etc/nsd/nsd.conf
-  - sudo systemctl reset-failed nsd.service
-  - sudo systemctl restart nsd.service
   #
   # launch automated tests, but skip tests that require remote
   # smtp support because Travis-CI blocks outgoing port 25
-  - sudo ./tests/runner.sh -dumpoutput -no-smtp-remote
+  - sudo tests/runner.sh -dumpoutput -no-smtp-remote default remote-nextcloud

From ca33614cbffd792d4d928d9beebe5370e2f0ae12 Mon Sep 17 00:00:00 2001
From: downtownallday <downtownallday@gmail.com>
Date: Tue, 9 Jun 2020 21:51:01 -0400
Subject: [PATCH 14/65] Run apt-get update before installing anything

---
 tests/assets/system-setup/remote-nextcloud-docker.sh | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/tests/assets/system-setup/remote-nextcloud-docker.sh b/tests/assets/system-setup/remote-nextcloud-docker.sh
index 39ee400a..96117767 100755
--- a/tests/assets/system-setup/remote-nextcloud-docker.sh
+++ b/tests/assets/system-setup/remote-nextcloud-docker.sh
@@ -59,6 +59,9 @@ before_miab_install() {
     H2 "Update /etc/hosts"
     update_hosts_for_private_ip || die "Could not update /etc/hosts"
 
+    # update package lists before installing anything
+    apt-get update || die "apt-get update failed!"
+    
     # install prerequisites
     H2 "QA prerequisites"
     install_qa_prerequisites || die "Error installing QA prerequisites"

From e498b47d53a0a291250d4968d6e96a87f539eef7 Mon Sep 17 00:00:00 2001
From: downtownallday <downtownallday@gmail.com>
Date: Tue, 9 Jun 2020 22:06:25 -0400
Subject: [PATCH 15/65] Dump /var/log/syslog when start.sh fails

---
 .../assets/system-setup/remote-nextcloud-docker.sh  |  8 ++++++--
 tests/assets/system-setup/setup-funcs.sh            | 13 +++++++++++++
 2 files changed, 19 insertions(+), 2 deletions(-)

diff --git a/tests/assets/system-setup/remote-nextcloud-docker.sh b/tests/assets/system-setup/remote-nextcloud-docker.sh
index 96117767..d269d49e 100755
--- a/tests/assets/system-setup/remote-nextcloud-docker.sh
+++ b/tests/assets/system-setup/remote-nextcloud-docker.sh
@@ -60,6 +60,7 @@ before_miab_install() {
     update_hosts_for_private_ip || die "Could not update /etc/hosts"
 
     # update package lists before installing anything
+    H2 "apt-get update"
     apt-get update || die "apt-get update failed!"
     
     # install prerequisites
@@ -87,10 +88,10 @@ before_miab_install() {
     # enable the remote Nextcloud setup mod, which tells MiaB-LDAP to use
     # the remote Nextcloud for calendar and contacts instead of the
     # MiaB-installed one
+    H2 "Create setup/mod.d/remote-nextcloud.sh symbolic link"
     if [ ! -e "setup/mods.d/remote-nextcloud.sh" ]; then
         ln -s "../mods.available/remote-nextcloud.sh" "setup/mods.d/remote-nextcloud.sh" || die "Could not create remote-nextcloud.sh symlink"
     fi
-
     
     # install Docker
     H2 "Install Docker"
@@ -100,7 +101,10 @@ before_miab_install() {
 
 miab_install() {
     H1 "MIAB-LDAP INSTALL"
-    setup/start.sh -v || die "setup/start.sh failed!"
+    if ! setup/start.sh; then
+        dump_log "/var/log/syslog" 200
+        die "setup/start.sh failed!"
+    fi
 }
 
 
diff --git a/tests/assets/system-setup/setup-funcs.sh b/tests/assets/system-setup/setup-funcs.sh
index b5768b1a..7f64da49 100755
--- a/tests/assets/system-setup/setup-funcs.sh
+++ b/tests/assets/system-setup/setup-funcs.sh
@@ -17,6 +17,19 @@ H2() {
     echo "*** $msg ***"
 }
 
+dump_log() {
+    local log_file="$1"
+    local lines="$2"
+    local title="DUMP OF $log_file"
+    if [ ! -z "$lines" ]; then
+        H1 "$title (last $lines lines)"
+        tail -$lines "$log_file"
+    else
+        H1 "$title"
+        tail "$log_file"
+    fi
+}
+
 install_qa_prerequisites() {
     # python3-dnspython: is used by the python scripts in 'tests' and is
     #   not installed by setup

From 0cd9e925668bd5fa0d179b82a25036c4de2fb896 Mon Sep 17 00:00:00 2001
From: downtownallday <downtownallday@gmail.com>
Date: Tue, 9 Jun 2020 22:45:12 -0400
Subject: [PATCH 16/65] Try to deal with nsd failure under Travis

---
 .travis.yml                                   |  7 +++++++
 .../system-setup/remote-nextcloud-docker.sh   | 20 ++++++++++++++++---
 tests/assets/system-setup/setup-funcs.sh      |  4 ++--
 3 files changed, 26 insertions(+), 5 deletions(-)

diff --git a/.travis.yml b/.travis.yml
index 7479bc52..879d0a52 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -14,6 +14,13 @@ before_install:
   #
   - echo "==== DUMP AppArmor Status ===="
   - (sudo aa-status; true)
+  #
+  - echo "==== DUMP NETWORK INFO ===="
+  - (hostname -I; true)
+  - (hostname -i; true)
+  - (hostname; true)
+  - (hostname --fqdn; true)
+  - (if add; true)
   
 install:
   - sudo tests/assets/system-setup/remote-nextcloud-docker.sh
diff --git a/tests/assets/system-setup/remote-nextcloud-docker.sh b/tests/assets/system-setup/remote-nextcloud-docker.sh
index d269d49e..301c7efd 100755
--- a/tests/assets/system-setup/remote-nextcloud-docker.sh
+++ b/tests/assets/system-setup/remote-nextcloud-docker.sh
@@ -88,7 +88,7 @@ before_miab_install() {
     # enable the remote Nextcloud setup mod, which tells MiaB-LDAP to use
     # the remote Nextcloud for calendar and contacts instead of the
     # MiaB-installed one
-    H2 "Create setup/mod.d/remote-nextcloud.sh symbolic link"
+    H2 "Create setup/mods.d/remote-nextcloud.sh symbolic link"
     if [ ! -e "setup/mods.d/remote-nextcloud.sh" ]; then
         ln -s "../mods.available/remote-nextcloud.sh" "setup/mods.d/remote-nextcloud.sh" || die "Could not create remote-nextcloud.sh symlink"
     fi
@@ -102,8 +102,22 @@ before_miab_install() {
 miab_install() {
     H1 "MIAB-LDAP INSTALL"
     if ! setup/start.sh; then
-        dump_log "/var/log/syslog" 200
-        die "setup/start.sh failed!"
+        local failure="true"
+        
+        if [ "$TRAVIS" == "true" ] && tail -10 /var/log/syslog | grep "Exception on /mail/users/add" >/dev/null; then
+            dump_log "/etc/mailinabox.conf"
+            H2 "Apply Travis-CI nsd fix"
+            travis_fix_nsd || die "Could not fix NSD startup issue"
+            H2 "Re-run firstuser.sh"
+            if setup/firstuser.sh; then
+                failure="false"
+            fi
+        fi
+        
+        if [ "$failure" == "true" ]; then
+            dump_log "/var/log/syslog" 100
+            die "setup/start.sh failed!"
+        fi
     fi
 }
 
diff --git a/tests/assets/system-setup/setup-funcs.sh b/tests/assets/system-setup/setup-funcs.sh
index 7f64da49..da2f16a3 100755
--- a/tests/assets/system-setup/setup-funcs.sh
+++ b/tests/assets/system-setup/setup-funcs.sh
@@ -26,7 +26,7 @@ dump_log() {
         tail -$lines "$log_file"
     else
         H1 "$title"
-        tail "$log_file"
+        cat "$log_file"
     fi
 }
 
@@ -95,7 +95,7 @@ travis_fix_nsd() {
     fi
     
     # nsd won't start on Travis-CI without the changes below: ip6 off and
-    # control-enable set to no. Even though the nsd docs says the
+    # control-enable set to no. Even though the nsd docs say the
     # default value for control-enable is no, running "nsd-checkconf -o
     # control-enable /etc/nsd/nsd.conf" returns "yes", so we explicitly
     # set it here.

From f731d7444f490eafadd22487d004980beefcbcda Mon Sep 17 00:00:00 2001
From: downtownallday <downtownallday@gmail.com>
Date: Tue, 9 Jun 2020 22:52:04 -0400
Subject: [PATCH 17/65] Fix typo

---
 .travis.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.travis.yml b/.travis.yml
index 879d0a52..e0f22d7f 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -20,7 +20,7 @@ before_install:
   - (hostname -i; true)
   - (hostname; true)
   - (hostname --fqdn; true)
-  - (if add; true)
+  - (ip add; true)
   
 install:
   - sudo tests/assets/system-setup/remote-nextcloud-docker.sh

From 38319c90d5e0b5780e16178d09a0dce2835b704f Mon Sep 17 00:00:00 2001
From: downtownallday <downtownallday@gmail.com>
Date: Tue, 9 Jun 2020 22:52:46 -0400
Subject: [PATCH 18/65] More Travis nsd startup failure issues

---
 tests/assets/system-setup/remote-nextcloud-docker.sh | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tests/assets/system-setup/remote-nextcloud-docker.sh b/tests/assets/system-setup/remote-nextcloud-docker.sh
index 301c7efd..ebf433bb 100755
--- a/tests/assets/system-setup/remote-nextcloud-docker.sh
+++ b/tests/assets/system-setup/remote-nextcloud-docker.sh
@@ -108,8 +108,8 @@ miab_install() {
             dump_log "/etc/mailinabox.conf"
             H2 "Apply Travis-CI nsd fix"
             travis_fix_nsd || die "Could not fix NSD startup issue"
-            H2 "Re-run firstuser.sh"
-            if setup/firstuser.sh; then
+            H2 "Re-run firstuser.sh and mods.d/remote-nextcloud.sh"
+            if setup/firstuser.sh && setup/mods.d/remote-nextcloud.sh; then
                 failure="false"
             fi
         fi

From 81c2b75ee4f07cb98c4729c949b4b1f18df98882 Mon Sep 17 00:00:00 2001
From: downtownallday <downtownallday@gmail.com>
Date: Tue, 9 Jun 2020 23:19:24 -0400
Subject: [PATCH 19/65] Simplify networking info dump

---
 .travis.yml | 11 ++++++-----
 1 file changed, 6 insertions(+), 5 deletions(-)

diff --git a/.travis.yml b/.travis.yml
index e0f22d7f..8f1598ce 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -16,11 +16,12 @@ before_install:
   - (sudo aa-status; true)
   #
   - echo "==== DUMP NETWORK INFO ===="
-  - (hostname -I; true)
-  - (hostname -i; true)
-  - (hostname; true)
-  - (hostname --fqdn; true)
-  - (ip add; true)
+  - hostname -I
+  - hostname -i
+  - hostname
+  - hostname --fqdn
+  - ip add
+  - cat /etc/hosts
   
 install:
   - sudo tests/assets/system-setup/remote-nextcloud-docker.sh

From 4d99e6021ba856337667cbd679e2dd5b4310f8f7 Mon Sep 17 00:00:00 2001
From: downtownallday <downtownallday@gmail.com>
Date: Tue, 9 Jun 2020 23:20:02 -0400
Subject: [PATCH 20/65] Move nsd fix for Travis-CI into setup/dns.sh

---
 setup/dns.sh                                  | 14 +++++++++++
 .../system-setup/remote-nextcloud-docker.sh   | 23 ++++---------------
 tests/assets/system-setup/setup-funcs.sh      |  1 +
 3 files changed, 20 insertions(+), 18 deletions(-)

diff --git a/setup/dns.sh b/setup/dns.sh
index 5d86227a..8281c52e 100755
--- a/setup/dns.sh
+++ b/setup/dns.sh
@@ -62,6 +62,20 @@ for ip in $PRIVATE_IP $PRIVATE_IPV6; do
 	echo "  ip-address: $ip" >> /etc/nsd/nsd.conf;
 done
 
+# Deal with a failure for nsd to start on Travis-CI by disabling ip6
+# and setting control-enable to "no". Even though the nsd docs say the
+# default value for control-enable is no, running "nsd-checkconf -o
+# control-enable /etc/nsd/nsd.conf" returns "yes", so we explicitly
+# set it here.
+if [ -z "$PRIVATE_IPV6" -a "$TRAVIS" == "true" ]; then
+    cat >> /etc/nsd.conf <<EOF
+  do-ip4: yes
+  do-ip6: no
+remote-control:
+  control-enable: no
+EOF
+fi
+
 echo "include: /etc/nsd/zones.conf" >> /etc/nsd/nsd.conf;
 
 # Create DNSSEC signing keys.
diff --git a/tests/assets/system-setup/remote-nextcloud-docker.sh b/tests/assets/system-setup/remote-nextcloud-docker.sh
index ebf433bb..e9bd7fee 100755
--- a/tests/assets/system-setup/remote-nextcloud-docker.sh
+++ b/tests/assets/system-setup/remote-nextcloud-docker.sh
@@ -102,22 +102,9 @@ before_miab_install() {
 miab_install() {
     H1 "MIAB-LDAP INSTALL"
     if ! setup/start.sh; then
-        local failure="true"
-        
-        if [ "$TRAVIS" == "true" ] && tail -10 /var/log/syslog | grep "Exception on /mail/users/add" >/dev/null; then
-            dump_log "/etc/mailinabox.conf"
-            H2 "Apply Travis-CI nsd fix"
-            travis_fix_nsd || die "Could not fix NSD startup issue"
-            H2 "Re-run firstuser.sh and mods.d/remote-nextcloud.sh"
-            if setup/firstuser.sh && setup/mods.d/remote-nextcloud.sh; then
-                failure="false"
-            fi
-        fi
-        
-        if [ "$failure" == "true" ]; then
-            dump_log "/var/log/syslog" 100
-            die "setup/start.sh failed!"
-        fi
+        dump_log "/var/log/syslog" 100
+        dump_log "/etc/mailinabox.conf"
+        die "setup/start.sh failed!"
     fi
 }
 
@@ -128,8 +115,8 @@ after_miab_install() {
     . /etc/mailinabox.conf || die "Could not load /etc/mailinabox.conf"
 
     # TRAVIS: fix nsd startup problem
-    H2 "Apply Travis-CI nsd fix"
-    travis_fix_nsd || die "Could not fix NSD startup issue for TRAVIS-CI"
+    #H2 "Apply Travis-CI nsd fix"
+    #travis_fix_nsd || die "Could not fix NSD startup issue for TRAVIS-CI"
     
     # run Nextcloud docker image
     H2 "Start Nextcloud docker container"
diff --git a/tests/assets/system-setup/setup-funcs.sh b/tests/assets/system-setup/setup-funcs.sh
index da2f16a3..5b453a3a 100755
--- a/tests/assets/system-setup/setup-funcs.sh
+++ b/tests/assets/system-setup/setup-funcs.sh
@@ -28,6 +28,7 @@ dump_log() {
         H1 "$title"
         cat "$log_file"
     fi
+    H1 "END $title"
 }
 
 install_qa_prerequisites() {

From 74b29265864ff36c961b12879c27b98909288068 Mon Sep 17 00:00:00 2001
From: downtownallday <downtownallday@gmail.com>
Date: Tue, 9 Jun 2020 23:27:20 -0400
Subject: [PATCH 21/65] dump nsd.conf

---
 tests/assets/system-setup/remote-nextcloud-docker.sh | 1 +
 1 file changed, 1 insertion(+)

diff --git a/tests/assets/system-setup/remote-nextcloud-docker.sh b/tests/assets/system-setup/remote-nextcloud-docker.sh
index e9bd7fee..c4fd47bc 100755
--- a/tests/assets/system-setup/remote-nextcloud-docker.sh
+++ b/tests/assets/system-setup/remote-nextcloud-docker.sh
@@ -104,6 +104,7 @@ miab_install() {
     if ! setup/start.sh; then
         dump_log "/var/log/syslog" 100
         dump_log "/etc/mailinabox.conf"
+        [ "$TRAVIS" == "true" ] && dump_log "/etc/nsd/nsd.conf"
         die "setup/start.sh failed!"
     fi
 }

From a098992d1ee50380633310fa6af0e77e9f40731a Mon Sep 17 00:00:00 2001
From: downtownallday <downtownallday@gmail.com>
Date: Tue, 9 Jun 2020 23:35:13 -0400
Subject: [PATCH 22/65] fix path to nsd.conf

---
 setup/dns.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/setup/dns.sh b/setup/dns.sh
index 8281c52e..cfc3879c 100755
--- a/setup/dns.sh
+++ b/setup/dns.sh
@@ -68,7 +68,7 @@ done
 # control-enable /etc/nsd/nsd.conf" returns "yes", so we explicitly
 # set it here.
 if [ -z "$PRIVATE_IPV6" -a "$TRAVIS" == "true" ]; then
-    cat >> /etc/nsd.conf <<EOF
+    cat >> /etc/nsd/nsd.conf <<EOF
   do-ip4: yes
   do-ip6: no
 remote-control:

From 6ed1a2ad201ab3a80641cee5160d445d0765e429 Mon Sep 17 00:00:00 2001
From: downtownallday <downtownallday@gmail.com>
Date: Tue, 9 Jun 2020 23:35:46 -0400
Subject: [PATCH 23/65] add foreground color

---
 tests/assets/system-setup/setup-funcs.sh | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/tests/assets/system-setup/setup-funcs.sh b/tests/assets/system-setup/setup-funcs.sh
index 5b453a3a..2627285e 100755
--- a/tests/assets/system-setup/setup-funcs.sh
+++ b/tests/assets/system-setup/setup-funcs.sh
@@ -1,4 +1,9 @@
 
+# ansi escapes for hilighting text
+F_DIMTEXT=$(echo -e "\037[31m")
+F_RESET=$(echo -e "\033[39m")
+
+
 die() {
     local msg="$1"
     echo "$msg" 1>&2
@@ -21,6 +26,7 @@ dump_log() {
     local log_file="$1"
     local lines="$2"
     local title="DUMP OF $log_file"
+    echo -n "$F_DIMTEXT"
     if [ ! -z "$lines" ]; then
         H1 "$title (last $lines lines)"
         tail -$lines "$log_file"
@@ -29,6 +35,7 @@ dump_log() {
         cat "$log_file"
     fi
     H1 "END $title"
+    echo -n "$F_RESET"
 }
 
 install_qa_prerequisites() {

From 2f1082a2906de275271c14c4e7991a0381be2a8a Mon Sep 17 00:00:00 2001
From: downtownallday <downtownallday@gmail.com>
Date: Wed, 10 Jun 2020 00:19:59 -0400
Subject: [PATCH 24/65] More debug output

---
 .../system-setup/remote-nextcloud-docker.sh   | 14 ++++++++++++--
 tests/assets/system-setup/setup-funcs.sh      | 19 ++++++++++---------
 tests/suites/_mail-functions.sh               |  6 +++---
 3 files changed, 25 insertions(+), 14 deletions(-)

diff --git a/tests/assets/system-setup/remote-nextcloud-docker.sh b/tests/assets/system-setup/remote-nextcloud-docker.sh
index c4fd47bc..07d91c0c 100755
--- a/tests/assets/system-setup/remote-nextcloud-docker.sh
+++ b/tests/assets/system-setup/remote-nextcloud-docker.sh
@@ -99,14 +99,24 @@ before_miab_install() {
 }
 
 
+debug_output_conf_files() {
+    dump_log "/etc/mailinabox.conf"
+    dump_log "/etc/nsd/nsd.conf"
+    dump_log "/etc/postfix/main.cf"
+}
+
 miab_install() {
     H1 "MIAB-LDAP INSTALL"
     if ! setup/start.sh; then
+        H1 "OUTPUT OF SELECT FILES"
         dump_log "/var/log/syslog" 100
-        dump_log "/etc/mailinabox.conf"
-        [ "$TRAVIS" == "true" ] && dump_log "/etc/nsd/nsd.conf"
+        debug_output_conf_files
+        H2; H2 "End"; H2
         die "setup/start.sh failed!"
     fi
+    H1 "OUTPUT OF SELECT FILES"
+    debug_output_conf_files
+    H2; H2 "End"; H2
 }
 
 
diff --git a/tests/assets/system-setup/setup-funcs.sh b/tests/assets/system-setup/setup-funcs.sh
index 2627285e..96f2d19d 100755
--- a/tests/assets/system-setup/setup-funcs.sh
+++ b/tests/assets/system-setup/setup-funcs.sh
@@ -1,8 +1,4 @@
 
-# ansi escapes for hilighting text
-F_DIMTEXT=$(echo -e "\037[31m")
-F_RESET=$(echo -e "\033[39m")
-
 
 die() {
     local msg="$1"
@@ -26,16 +22,21 @@ dump_log() {
     local log_file="$1"
     local lines="$2"
     local title="DUMP OF $log_file"
-    echo -n "$F_DIMTEXT"
+    echo ""
+    echo "--------"
+    echo -n "-------- $log_file"
+    if [ -z "$lines" ]; then
+        echo " (last $line lines)"
+    else
+        echo ""
+    fi
+    echo "--------"
+      
     if [ ! -z "$lines" ]; then
-        H1 "$title (last $lines lines)"
         tail -$lines "$log_file"
     else
-        H1 "$title"
         cat "$log_file"
     fi
-    H1 "END $title"
-    echo -n "$F_RESET"
 }
 
 install_qa_prerequisites() {
diff --git a/tests/suites/_mail-functions.sh b/tests/suites/_mail-functions.sh
index 37c33661..302b240d 100644
--- a/tests/suites/_mail-functions.sh
+++ b/tests/suites/_mail-functions.sh
@@ -168,11 +168,11 @@ detect_slapd_log_error() {
 				record "$F_DANGER[ERROR] $line$F_RESET"
 			elif [ $r -eq 2 ]; then
 				let wc+=1
-				record "$F_WARN[WARN ] $line$F_RESET"
+				record "$F_WARN[ WARN] $line$F_RESET"
 			elif [ $r -eq 3 ]; then
 				let ignored+=1
 			else
-				record "[OK   ] $line"
+				record "[   OK] $line"
 			fi
 		done
 		record "$ignored unreported/ignored log lines"
@@ -214,7 +214,7 @@ detect_dovecot_log_error() {
 			elif [ $r -eq 2 ]; then
 				let ignored+=1
 			else
-				record "[	OK] $line"
+				record "[   OK] $line"
 			fi
 		done
 		record "$ignored unreported/ignored log lines"

From 8be3011c1918a3d7359f5c8c92b772e5a4d6e2d5 Mon Sep 17 00:00:00 2001
From: downtownallday <downtownallday@gmail.com>
Date: Wed, 10 Jun 2020 07:21:26 -0400
Subject: [PATCH 25/65] Set smtp_address_preference to any in main.cf

---
 setup/mail-postfix.sh | 1 +
 1 file changed, 1 insertion(+)

diff --git a/setup/mail-postfix.sh b/setup/mail-postfix.sh
index 88d884b7..bb0678b8 100755
--- a/setup/mail-postfix.sh
+++ b/setup/mail-postfix.sh
@@ -56,6 +56,7 @@ apt_install postfix postfix-sqlite postfix-pcre postgrey ca-certificates postfix
 # * Extend the SPF time limit to avoid timeouts chasing SPF records
 tools/editconf.py /etc/postfix/main.cf \
 	inet_interfaces=all \
+	smtp_address_preference=any \
 	smtp_bind_address=$PRIVATE_IP \
 	smtp_bind_address6=$PRIVATE_IPV6 \
 	myhostname=$PRIMARY_HOSTNAME\

From f4fda417053923d4b4c5a5b1264332939c3a4e4b Mon Sep 17 00:00:00 2001
From: downtownallday <downtownallday@gmail.com>
Date: Wed, 10 Jun 2020 07:21:47 -0400
Subject: [PATCH 26/65] Output formatting changes

---
 .../system-setup/remote-nextcloud-docker.sh   | 12 +++--------
 tests/assets/system-setup/setup-funcs.sh      | 20 +++++++++++++++----
 tests/suites/_mail-functions.sh               |  2 +-
 3 files changed, 20 insertions(+), 14 deletions(-)

diff --git a/tests/assets/system-setup/remote-nextcloud-docker.sh b/tests/assets/system-setup/remote-nextcloud-docker.sh
index 07d91c0c..1d8d9401 100755
--- a/tests/assets/system-setup/remote-nextcloud-docker.sh
+++ b/tests/assets/system-setup/remote-nextcloud-docker.sh
@@ -61,7 +61,7 @@ before_miab_install() {
 
     # update package lists before installing anything
     H2 "apt-get update"
-    apt-get update || die "apt-get update failed!"
+    apt-get update -qq || die "apt-get update failed!"
     
     # install prerequisites
     H2 "QA prerequisites"
@@ -99,23 +99,17 @@ before_miab_install() {
 }
 
 
-debug_output_conf_files() {
-    dump_log "/etc/mailinabox.conf"
-    dump_log "/etc/nsd/nsd.conf"
-    dump_log "/etc/postfix/main.cf"
-}
-
 miab_install() {
     H1 "MIAB-LDAP INSTALL"
     if ! setup/start.sh; then
         H1 "OUTPUT OF SELECT FILES"
         dump_log "/var/log/syslog" 100
-        debug_output_conf_files
+        dump_conf_files
         H2; H2 "End"; H2
         die "setup/start.sh failed!"
     fi
     H1 "OUTPUT OF SELECT FILES"
-    debug_output_conf_files
+    dump_conf_files
     H2; H2 "End"; H2
 }
 
diff --git a/tests/assets/system-setup/setup-funcs.sh b/tests/assets/system-setup/setup-funcs.sh
index 96f2d19d..7eb66b81 100755
--- a/tests/assets/system-setup/setup-funcs.sh
+++ b/tests/assets/system-setup/setup-funcs.sh
@@ -9,13 +9,19 @@ die() {
 H1() {
     local msg="$1"
     echo "----------------------------------------------"
-    echo "           $msg"
-    echo "----------------------------------------------"
+    if [ ! -z "$msg" ]; then
+        echo "           $msg"
+        echo "----------------------------------------------"
+    fi
 }
 
 H2() {
     local msg="$1"
-    echo "*** $msg ***"
+    if [ -z "$msg" ]; then
+        echo "***"
+    else
+        echo "*** $msg ***"
+    fi
 }
 
 dump_log() {
@@ -25,7 +31,7 @@ dump_log() {
     echo ""
     echo "--------"
     echo -n "-------- $log_file"
-    if [ -z "$lines" ]; then
+    if [ ! -z "$lines" ]; then
         echo " (last $line lines)"
     else
         echo ""
@@ -39,6 +45,12 @@ dump_log() {
     fi
 }
 
+dump_conf_files() {
+    dump_log "/etc/mailinabox.conf"
+    dump_log "/etc/nsd/nsd.conf"
+    dump_log "/etc/postfix/main.cf"
+}
+
 install_qa_prerequisites() {
     # python3-dnspython: is used by the python scripts in 'tests' and is
     #   not installed by setup
diff --git a/tests/suites/_mail-functions.sh b/tests/suites/_mail-functions.sh
index 302b240d..a9a64bca 100644
--- a/tests/suites/_mail-functions.sh
+++ b/tests/suites/_mail-functions.sh
@@ -113,7 +113,7 @@ detect_syslog_error() {
 				let ec+=1
 				record "$F_DANGER[ERROR] $line$F_RESET"
 			else
-				record "[	OK] $line"
+				record "[   OK] $line"
 			fi
 		done
 		[ $ec -gt 0 ] && exit 0

From 64d24566b2b67dfdbb9181a40da4968111bd7f35 Mon Sep 17 00:00:00 2001
From: downtownallday <downtownallday@gmail.com>
Date: Wed, 10 Jun 2020 07:49:17 -0400
Subject: [PATCH 27/65] Dump /etc/hosts on travis

---
 .travis.yml                                    |  7 +++----
 .../system-setup/remote-nextcloud-docker.sh    |  4 ++--
 tests/assets/system-setup/setup-funcs.sh       | 18 +++++++++++++++---
 3 files changed, 20 insertions(+), 9 deletions(-)

diff --git a/.travis.yml b/.travis.yml
index 8f1598ce..5fb6275c 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -8,20 +8,19 @@ os: linux
 dist: bionic
   
 before_install:
-  - echo "==== DUMP ENVIRONMENT ===="
+  - echo "==== ENVIRONMENT ===="
   - env | sort
   - echo "UMASK=$(umask)"
   #
-  - echo "==== DUMP AppArmor Status ===="
+  - echo "==== AppArmor Status ===="
   - (sudo aa-status; true)
   #
-  - echo "==== DUMP NETWORK INFO ===="
+  - echo "==== NETWORK INFO ===="
   - hostname -I
   - hostname -i
   - hostname
   - hostname --fqdn
   - ip add
-  - cat /etc/hosts
   
 install:
   - sudo tests/assets/system-setup/remote-nextcloud-docker.sh
diff --git a/tests/assets/system-setup/remote-nextcloud-docker.sh b/tests/assets/system-setup/remote-nextcloud-docker.sh
index 1d8d9401..930b7879 100755
--- a/tests/assets/system-setup/remote-nextcloud-docker.sh
+++ b/tests/assets/system-setup/remote-nextcloud-docker.sh
@@ -104,12 +104,12 @@ miab_install() {
     if ! setup/start.sh; then
         H1 "OUTPUT OF SELECT FILES"
         dump_log "/var/log/syslog" 100
-        dump_conf_files
+        dump_conf_files "$TRAVIS"
         H2; H2 "End"; H2
         die "setup/start.sh failed!"
     fi
     H1 "OUTPUT OF SELECT FILES"
-    dump_conf_files
+    dump_conf_files "$TRAVIS"
     H2; H2 "End"; H2
 }
 
diff --git a/tests/assets/system-setup/setup-funcs.sh b/tests/assets/system-setup/setup-funcs.sh
index 7eb66b81..9154ec55 100755
--- a/tests/assets/system-setup/setup-funcs.sh
+++ b/tests/assets/system-setup/setup-funcs.sh
@@ -46,9 +46,21 @@ dump_log() {
 }
 
 dump_conf_files() {
-    dump_log "/etc/mailinabox.conf"
-    dump_log "/etc/nsd/nsd.conf"
-    dump_log "/etc/postfix/main.cf"
+    local skip
+    if [ $# -eq 0 ]; then
+        skip="false"
+    else
+        skip="true"
+        for item; do
+            [ "$item" == "true" ] skip="false"
+        done
+    fi
+    if [ "$skip" == "false" ]; then
+        dump_log "/etc/mailinabox.conf"
+        dump_log "/etc/hosts"
+        dump_log "/etc/nsd/nsd.conf"
+        dump_log "/etc/postfix/main.cf"
+    fi
 }
 
 install_qa_prerequisites() {

From 7a12b52f8feef0c9dfd04a20b781590d3aeb89f4 Mon Sep 17 00:00:00 2001
From: downtownallday <downtownallday@gmail.com>
Date: Wed, 10 Jun 2020 07:50:23 -0400
Subject: [PATCH 28/65] Back out prior change, did not fix delivery problem:
 "status=bounced (Host or domain name not found. Name service error for
 name=box.abc.com type=AAAA: Host not found"

---
 setup/mail-postfix.sh | 1 -
 1 file changed, 1 deletion(-)

diff --git a/setup/mail-postfix.sh b/setup/mail-postfix.sh
index bb0678b8..88d884b7 100755
--- a/setup/mail-postfix.sh
+++ b/setup/mail-postfix.sh
@@ -56,7 +56,6 @@ apt_install postfix postfix-sqlite postfix-pcre postgrey ca-certificates postfix
 # * Extend the SPF time limit to avoid timeouts chasing SPF records
 tools/editconf.py /etc/postfix/main.cf \
 	inet_interfaces=all \
-	smtp_address_preference=any \
 	smtp_bind_address=$PRIVATE_IP \
 	smtp_bind_address6=$PRIVATE_IPV6 \
 	myhostname=$PRIMARY_HOSTNAME\

From 96da15ad2752aee74bd3e34df19fca9c54772bb0 Mon Sep 17 00:00:00 2001
From: downtownallday <downtownallday@gmail.com>
Date: Wed, 10 Jun 2020 08:21:45 -0400
Subject: [PATCH 29/65] Add ipv6 address to /etc/hosts

---
 tests/assets/system-setup/setup-funcs.sh | 46 ++++++++++++++++++------
 1 file changed, 36 insertions(+), 10 deletions(-)

diff --git a/tests/assets/system-setup/setup-funcs.sh b/tests/assets/system-setup/setup-funcs.sh
index 9154ec55..9e89ac95 100755
--- a/tests/assets/system-setup/setup-funcs.sh
+++ b/tests/assets/system-setup/setup-funcs.sh
@@ -45,6 +45,22 @@ dump_log() {
     fi
 }
 
+is_true() {
+    if [ "$1" == "true" \
+              -o "$1" == "TRUE" \
+              -o "$1" == "True" \
+              -o "$1" == "yes" \
+              -o "$1" == "YES" \
+              -o "$1" == "Yes" \
+              -o "$1" == "1" ]
+    then
+        return 0
+    else
+        return 1
+    fi
+}
+    
+
 dump_conf_files() {
     local skip
     if [ $# -eq 0 ]; then
@@ -52,7 +68,10 @@ dump_conf_files() {
     else
         skip="true"
         for item; do
-            [ "$item" == "true" ] skip="false"
+            if is_true "$item"; then
+                skip="false"
+                break
+            fi
         done
     fi
     if [ "$skip" == "false" ]; then
@@ -78,19 +97,26 @@ update_system_time() {
 
 update_hosts() {
     local host="$1"
-    local ip="$2"
-    local line="$ip $host"
-    if ! grep -F "$line" /etc/hosts 1>/dev/null; then
-        echo "$line" >>/etc/hosts
-    fi
+    shift
+    local ip
+    for ip; do
+        if [ ! -z "$ip" ]; then
+            local line="$ip $host"
+            if ! grep -F "$line" /etc/hosts 1>/dev/null; then
+                echo "$line" >>/etc/hosts
+            fi
+        fi
+    done
 }
 
 update_hosts_for_private_ip() {
-    # create /etc/hosts entry for PRIVATE_IP
+    # create /etc/hosts entry for PRIVATE_IP and PRIVATE_IPV6
     # PRIMARY_HOSTNAME must already be set
-    local ip=$(source setup/functions.sh; get_default_privateip 4)
-    [ -z "$ip" ] && return 1
-    update_hosts "$PRIMARY_HOSTNAME" "$ip" || return 1
+    local ip4=$(source setup/functions.sh; get_default_privateip 4)
+    local ip6=$(source setup/functions.sh; get_default_privateip 6)
+    [ -z "$ip4" -a -z "$ip6" ] && return 1
+    [ -z "$ip6" ] && ip6="::1"
+    update_hosts "$PRIMARY_HOSTNAME" "$ip4" "$ip6" || return 1
 }
 
 install_docker() {

From ab087365c4666286e77c7ebdcdcfbacdf6384feb Mon Sep 17 00:00:00 2001
From: downtownallday <downtownallday@gmail.com>
Date: Wed, 10 Jun 2020 11:40:44 -0400
Subject: [PATCH 30/65] Move system-setup directory to tests Set the hosts FQDN
 instead of adding hosts entry for private ip

---
 .travis.yml                                   |  2 +-
 .../system-setup/remote-nextcloud-docker.sh   | 49 +++++-----
 .../system-setup/setup-defaults.sh            |  0
 .../{assets => }/system-setup/setup-funcs.sh  | 93 ++++++++++++++++---
 4 files changed, 104 insertions(+), 40 deletions(-)
 rename tests/{assets => }/system-setup/remote-nextcloud-docker.sh (84%)
 rename tests/{assets => }/system-setup/setup-defaults.sh (100%)
 rename tests/{assets => }/system-setup/setup-funcs.sh (62%)

diff --git a/.travis.yml b/.travis.yml
index 5fb6275c..b9bed2b8 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -23,7 +23,7 @@ before_install:
   - ip add
   
 install:
-  - sudo tests/assets/system-setup/remote-nextcloud-docker.sh
+  - sudo tests/system-setup/remote-nextcloud-docker.sh
 
 script:
   #
diff --git a/tests/assets/system-setup/remote-nextcloud-docker.sh b/tests/system-setup/remote-nextcloud-docker.sh
similarity index 84%
rename from tests/assets/system-setup/remote-nextcloud-docker.sh
rename to tests/system-setup/remote-nextcloud-docker.sh
index 930b7879..befcd452 100755
--- a/tests/assets/system-setup/remote-nextcloud-docker.sh
+++ b/tests/system-setup/remote-nextcloud-docker.sh
@@ -7,7 +7,7 @@
 #   on a fresh Ubuntu:
 #      1. checkout or copy the MiaB-LDAP code to ~/mailinabox
 #      2. cd ~/mailinabox
-#      3. sudo tests/assets/system-setup/remote-nextcloud-docker.sh
+#      3. sudo tests/system-setup/remote-nextcloud-docker.sh
 #
 # when complete you should have a working MiaB-LDAP and Nextcloud
 #
@@ -34,15 +34,15 @@ usage() {
 }
 
 # ensure working directory
-if [ ! -d "tests/assets/system-setup" ]; then
+if [ ! -d "tests/system-setup" ]; then
     echo "This script must be run from the MiaB root directory"
     exit 1
 fi
 
 # load helper scripts
-. "tests/assets/system-setup/setup-defaults.sh" \
+. "tests/system-setup/setup-defaults.sh" \
     || die "Could not load setup-defaults"
-. "tests/assets/system-setup/setup-funcs.sh" \
+. "tests/system-setup/setup-funcs.sh" \
     || die "Could not load setup-funcs"
 
 # ensure running as root
@@ -55,36 +55,33 @@ fi
 before_miab_install() {
     H1 "BEFORE MIAB-LDAP INSTALL"
 
-    # create /etc/hosts entry for PRIVATE_IP
     H2 "Update /etc/hosts"
-    update_hosts_for_private_ip || die "Could not update /etc/hosts"
+    #update_hosts_for_private_ip || die "Could not update /etc/hosts"
+    set_system_hostname || die "Could not set hostname"
 
+    # update system time
+    H2 "Set system time"
+    update_system_time || echo "Ignoring error..."
+    
     # update package lists before installing anything
     H2 "apt-get update"
+    wait_for_apt
     apt-get update -qq || die "apt-get update failed!"
+
+    # upgrade packages - if we don't do this and something like bind
+    # is upgraded through automatic upgrades (because maybe MiaB was
+    # previously installed), it may cause problems with the rest of
+    # the setup, such as with name resolution failures
+    if is_false "$TRAVIS"; then
+        H2 "apt-get upgrade"
+        wait_for_apt
+        apt-get upgrade -qq || die "apt-get upgrade failed!"
+    fi
     
     # install prerequisites
     H2 "QA prerequisites"
     install_qa_prerequisites || die "Error installing QA prerequisites"
 
-    # update system time (ignore errors)
-    H2 "Set system time"
-    update_system_time
-
-    # copy in pre-built MiaB-LDAP ssl files
-    #   1. avoid the lengthy generation of DH params
-    H2 "Install QA pre-built"
-    mkdir -p $STORAGE_ROOT/ssl || die "Unable to create $STORAGE_ROOT/ssl"
-    cp tests/assets/ssl/dh2048.pem $STORAGE_ROOT/ssl \
-        || die "Copy dhparams failed"
-
-    # create miab_ldap.conf to specify what the Nextcloud LDAP service
-    # account password will be to avoid a random one created by start.sh
-    mkdir -p $STORAGE_ROOT/ldap
-    [ -e $STORAGE_ROOT/ldap/miab_ldap.conf ] && \
-        echo "Warning: exists: $STORAGE_ROOT/ldap/miab_ldap.conf" 1>&2
-    echo "LDAP_NEXTCLOUD_PASSWORD=\"$LDAP_NEXTCLOUD_PASSWORD\"" >> $STORAGE_ROOT/ldap/miab_ldap.conf
-
     # enable the remote Nextcloud setup mod, which tells MiaB-LDAP to use
     # the remote Nextcloud for calendar and contacts instead of the
     # MiaB-installed one
@@ -118,10 +115,6 @@ after_miab_install() {
     H1 "AFTER MIAB-LDAP INSTALL"
     
     . /etc/mailinabox.conf || die "Could not load /etc/mailinabox.conf"
-
-    # TRAVIS: fix nsd startup problem
-    #H2 "Apply Travis-CI nsd fix"
-    #travis_fix_nsd || die "Could not fix NSD startup issue for TRAVIS-CI"
     
     # run Nextcloud docker image
     H2 "Start Nextcloud docker container"
diff --git a/tests/assets/system-setup/setup-defaults.sh b/tests/system-setup/setup-defaults.sh
similarity index 100%
rename from tests/assets/system-setup/setup-defaults.sh
rename to tests/system-setup/setup-defaults.sh
diff --git a/tests/assets/system-setup/setup-funcs.sh b/tests/system-setup/setup-funcs.sh
similarity index 62%
rename from tests/assets/system-setup/setup-funcs.sh
rename to tests/system-setup/setup-funcs.sh
index 9e89ac95..37f78afd 100755
--- a/tests/assets/system-setup/setup-funcs.sh
+++ b/tests/system-setup/setup-funcs.sh
@@ -6,6 +6,7 @@ die() {
     exit 1
 }
 
+
 H1() {
     local msg="$1"
     echo "----------------------------------------------"
@@ -46,6 +47,7 @@ dump_log() {
 }
 
 is_true() {
+    # empty string is not true
     if [ "$1" == "true" \
               -o "$1" == "TRUE" \
               -o "$1" == "True" \
@@ -59,7 +61,28 @@ is_true() {
         return 1
     fi
 }
-    
+
+is_false() {
+    if is_true $@; then return 1; fi
+    return 0
+}
+
+wait_for_apt() {
+    local count=0
+    while fuser /var/lib/dpkg/lock >/dev/null 2>&1 || fuser /var/lib/dpkg/lock-frontend >/dev/null 2>&1; do
+        sleep 6
+        let count+=1
+        if [ $count -eq 1 ]; then
+            echo -n "Waiting for other package manager to finish..."
+        elif [ $count -gt 100 ]; then
+            echo -n "FAILED"
+            return 1
+        else
+            echo -n "${count}.."
+        fi
+    done
+    [ $count -ge 1 ] && echo ""
+}
 
 dump_conf_files() {
     local skip
@@ -82,16 +105,12 @@ dump_conf_files() {
     fi
 }
 
-install_qa_prerequisites() {
-    # python3-dnspython: is used by the python scripts in 'tests' and is
-    #   not installed by setup
-    # ntpdate: is used by this script
-    apt-get install -y \
-            ntpdate \
-            python3-dnspython
-}
 
 update_system_time() {
+    if [ ! -x /usr/sbin/ntpdate ]; then
+        wait_for_apt
+        apt-get install -y -qq ntpdate || return 1
+    fi
     ntpdate -s ntp.ubuntu.com && echo "System time updated"
 }
 
@@ -119,13 +138,26 @@ update_hosts_for_private_ip() {
     update_hosts "$PRIMARY_HOSTNAME" "$ip4" "$ip6" || return 1
 }
 
+set_system_hostname() {
+    # set the system hostname to the FQDN specified or
+    # PRIMARY_HOSTNAME if no FQDN was given
+    local fqdn="${1:-$PRIMARY_HOSTNAME}"
+    local host="$(awk -F. '{print $1}' <<< "$fqdn")"
+    sed -i 's/^127\.0\.1\.1[ \t].*/127.0.1.1 '"$fqdn $host ip4-loopback/" /etc/hosts || return 1
+    #hostname "$host" || return 1
+    #echo "$host" > /etc/hostname
+    return 0
+}
+
+
 install_docker() {
     if [ -x /usr/bin/docker ]; then
         echo "Docker already installed"
         return 0
     fi
     
-    apt-get install -y \
+    wait_for_apt
+    apt-get install -y -qq \
             apt-transport-https \
             ca-certificates \
             curl \
@@ -133,14 +165,18 @@ install_docker() {
             software-properties-common \
         || return 1
        
+    wait_for_apt
     curl -fsSL https://download.docker.com/linux/ubuntu/gpg | apt-key add - \
         || return 2
     
+    wait_for_apt
     apt-key fingerprint 0EBFCD88 || return 3
     
+    wait_for_apt
     add-apt-repository -y --update "deb [arch=amd64] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" || return 4
     
-    apt-get install -y \
+    wait_for_apt
+    apt-get install -y -qq \
             docker-ce \
             docker-ce-cli \
             containerd.io \
@@ -148,6 +184,41 @@ install_docker() {
 }
 
 
+install_qa_prerequisites() {
+    [ -z "$STORAGE_ROOT" ] \
+        && echo "Error: STORAGE_ROOT not set" 1>&2 \
+        && return 1
+
+    local rc=0
+    
+    # python3-dnspython: is used by the python scripts in 'tests' and is
+    #   not installed by setup
+    wait_for_apt
+    apt-get install -y -qq python3-dnspython
+    
+    # copy in pre-built MiaB-LDAP ssl files
+    #   1. avoid the lengthy generation of DH params
+    mkdir -p $STORAGE_ROOT/ssl \
+        || (echo "Unable to create $STORAGE_ROOT/ssl ($?)" && rc=1)
+    cp tests/assets/ssl/dh2048.pem $STORAGE_ROOT/ssl \
+        || (echo "Copy dhparams failed ($?)" && rc=1)
+
+    # create miab_ldap.conf to specify what the Nextcloud LDAP service
+    # account password will be to avoid a random one created by start.sh
+    if [ ! -z "$LDAP_NEXTCLOUD_PASSWORD" ]; then
+        mkdir -p $STORAGE_ROOT/ldap \
+            || (echo "Could not create $STORAGE_ROOT/ldap" && rc=1)
+        [ -e $STORAGE_ROOT/ldap/miab_ldap.conf ] && \
+            echo "Warning: exists: $STORAGE_ROOT/ldap/miab_ldap.conf" 1>&2
+        touch $STORAGE_ROOT/ldap/miab_ldap.conf || rc=1
+        if ! grep "^LDAP_NEXTCLOUD_PASSWORD=" $STORAGE_ROOT/ldap/miab_ldap.conf >/dev/null; then
+            echo "LDAP_NEXTCLOUD_PASSWORD=\"$LDAP_NEXTCLOUD_PASSWORD\"" >> $STORAGE_ROOT/ldap/miab_ldap.conf
+        fi
+    fi
+    return $rc
+}
+
+
 travis_fix_nsd() {
     if [ "$TRAVIS" != "true" ]; then
         return 0

From 219c3fa020aca3691953e29ca856088285152d00 Mon Sep 17 00:00:00 2001
From: downtownallday <downtownallday@gmail.com>
Date: Wed, 10 Jun 2020 11:41:26 -0400
Subject: [PATCH 31/65] Need a default value or boom

---
 setup/dns.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/setup/dns.sh b/setup/dns.sh
index cfc3879c..eaac56bd 100755
--- a/setup/dns.sh
+++ b/setup/dns.sh
@@ -67,7 +67,7 @@ done
 # default value for control-enable is no, running "nsd-checkconf -o
 # control-enable /etc/nsd/nsd.conf" returns "yes", so we explicitly
 # set it here.
-if [ -z "$PRIVATE_IPV6" -a "$TRAVIS" == "true" ]; then
+if [ -z "$PRIVATE_IPV6" -a "${TRAVIS:-}" == "true" ]; then
     cat >> /etc/nsd/nsd.conf <<EOF
   do-ip4: yes
   do-ip6: no

From f1ba09a0662184ecfb54537591b934dbc47812fb Mon Sep 17 00:00:00 2001
From: downtownallday <downtownallday@gmail.com>
Date: Wed, 10 Jun 2020 12:18:10 -0400
Subject: [PATCH 32/65] Dump nsswitch

---
 tests/system-setup/setup-funcs.sh | 1 +
 1 file changed, 1 insertion(+)

diff --git a/tests/system-setup/setup-funcs.sh b/tests/system-setup/setup-funcs.sh
index 37f78afd..b71591ec 100755
--- a/tests/system-setup/setup-funcs.sh
+++ b/tests/system-setup/setup-funcs.sh
@@ -100,6 +100,7 @@ dump_conf_files() {
     if [ "$skip" == "false" ]; then
         dump_log "/etc/mailinabox.conf"
         dump_log "/etc/hosts"
+        dump_log "/etc/nsswitch.conf"
         dump_log "/etc/nsd/nsd.conf"
         dump_log "/etc/postfix/main.cf"
     fi

From adf7d3ed8d0fa4b25b0c2e9058278e24ae935b33 Mon Sep 17 00:00:00 2001
From: downtownallday <downtownallday@gmail.com>
Date: Wed, 10 Jun 2020 17:05:34 -0400
Subject: [PATCH 33/65] Dump kernel ip6 settings

---
 .travis.yml | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/.travis.yml b/.travis.yml
index b9bed2b8..49948c38 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -21,6 +21,8 @@ before_install:
   - hostname
   - hostname --fqdn
   - ip add
+  #
+  - sysctl -a | grep -i ipv6 | grep disable
   
 install:
   - sudo tests/system-setup/remote-nextcloud-docker.sh

From ae5fbffa408d503332d3c1bddcf97bf0fa28efe0 Mon Sep 17 00:00:00 2001
From: downtownallday <downtownallday@gmail.com>
Date: Wed, 10 Jun 2020 21:56:49 -0400
Subject: [PATCH 34/65] Attempt at dealing with disabled ipv6 on interfaces

---
 setup/dns.sh          | 16 ++++++++++------
 setup/functions.sh    |  6 ++++++
 setup/mail-postfix.sh | 10 ++++++++++
 3 files changed, 26 insertions(+), 6 deletions(-)

diff --git a/setup/dns.sh b/setup/dns.sh
index eaac56bd..546c9c04 100755
--- a/setup/dns.sh
+++ b/setup/dns.sh
@@ -62,12 +62,16 @@ for ip in $PRIVATE_IP $PRIVATE_IPV6; do
 	echo "  ip-address: $ip" >> /etc/nsd/nsd.conf;
 done
 
-# Deal with a failure for nsd to start on Travis-CI by disabling ip6
-# and setting control-enable to "no". Even though the nsd docs say the
-# default value for control-enable is no, running "nsd-checkconf -o
-# control-enable /etc/nsd/nsd.conf" returns "yes", so we explicitly
-# set it here.
-if [ -z "$PRIVATE_IPV6" -a "${TRAVIS:-}" == "true" ]; then
+# nsd fails to start when ipv6 is disabled by the kernel on certain
+# interfaces without "do-ip6" set to "no" and "control-enable" to "no"
+# [confirm]. Even though the nsd docs say the default value for
+# control-enable is no, running "nsd-checkconf -o control-enable
+# /etc/nsd/nsd.conf" returns "yes", so we explicitly set it here.
+#
+# For instance, on Travis-CI, ipv6 is disabled on the lo and docker
+# interfaces, but enabled on the primary interface ens4. nsd fails to
+# start without these additions.
+if kernel_ipv6_lo_disabled; then
     cat >> /etc/nsd/nsd.conf <<EOF
   do-ip4: yes
   do-ip6: no
diff --git a/setup/functions.sh b/setup/functions.sh
index 5d12a6df..c1f7f733 100644
--- a/setup/functions.sh
+++ b/setup/functions.sh
@@ -235,3 +235,9 @@ function generate_password() {
 	dd if=/dev/urandom bs=1 count=$input_len 2>/dev/null | base64 --wrap=0 | awk '{ gsub("/", ",", $0); print $0}'
 }
 
+function kernel_ipv6_lo_disabled() {
+	# Returns 0 if ipv6 is disabled on the loopback adapter
+	local v="$(sysctl -n net.ipv6.conf.lo.disable_ipv6)"
+	[ "$v" == "1" ] && return 0
+	return 0
+}
diff --git a/setup/mail-postfix.sh b/setup/mail-postfix.sh
index 88d884b7..e8fdc263 100755
--- a/setup/mail-postfix.sh
+++ b/setup/mail-postfix.sh
@@ -62,6 +62,16 @@ tools/editconf.py /etc/postfix/main.cf \
 	smtpd_banner="\$myhostname ESMTP Hi, I'm a Mail-in-a-Box (Ubuntu/Postfix; see https://mailinabox.email/)" \
 	mydestination=localhost
 
+# when ipv6 is disabled on the loopback adapter and a mail message
+# delivery attempt fails because the recipient doesn't exist, postfix
+# returns reason "Host or domain name not found. Name service error
+# for name=abc.com type=AAAA: Host not found", instead of the actual
+# reason "User doesn't exist." The sysadmin's intent is probably that
+# ipv6 is not desired at all, so disable ipv6.
+if kernel_ipv6_lo_disabled; then
+	tools/editconf.py /etc/postfix/main.cf inet_protocols=ipv4
+fi
+    
 # Tweak some queue settings:
 # * Inform users when their e-mail delivery is delayed more than 3 hours (default is not to warn).
 # * Stop trying to send an undeliverable e-mail after 2 days (instead of 5), and for bounce messages just try for 1 day.

From a9b5350138b74f733c13d1a79216b7f1e25fdbd0 Mon Sep 17 00:00:00 2001
From: downtownallday <downtownallday@gmail.com>
Date: Wed, 10 Jun 2020 22:00:17 -0400
Subject: [PATCH 35/65] Hide sysctl read errors

---
 .travis.yml | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/.travis.yml b/.travis.yml
index 49948c38..0614e2e1 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -21,8 +21,7 @@ before_install:
   - hostname
   - hostname --fqdn
   - ip add
-  #
-  - sysctl -a | grep -i ipv6 | grep disable
+  - sysctl -a 2>/dev/null | grep -i ipv6 | grep disable
   
 install:
   - sudo tests/system-setup/remote-nextcloud-docker.sh

From 7237f553a4fdce2b337bc25e8ca8fc8e8bb057e9 Mon Sep 17 00:00:00 2001
From: downtownallday <downtownallday@gmail.com>
Date: Wed, 10 Jun 2020 22:37:24 -0400
Subject: [PATCH 36/65] Revert: still get host not found from postfix (type=A
 vs type=AAAA) for PRIMARY_HOSTNAME

---
 setup/mail-postfix.sh | 10 ----------
 1 file changed, 10 deletions(-)

diff --git a/setup/mail-postfix.sh b/setup/mail-postfix.sh
index e8fdc263..91dc30be 100755
--- a/setup/mail-postfix.sh
+++ b/setup/mail-postfix.sh
@@ -61,16 +61,6 @@ tools/editconf.py /etc/postfix/main.cf \
 	myhostname=$PRIMARY_HOSTNAME\
 	smtpd_banner="\$myhostname ESMTP Hi, I'm a Mail-in-a-Box (Ubuntu/Postfix; see https://mailinabox.email/)" \
 	mydestination=localhost
-
-# when ipv6 is disabled on the loopback adapter and a mail message
-# delivery attempt fails because the recipient doesn't exist, postfix
-# returns reason "Host or domain name not found. Name service error
-# for name=abc.com type=AAAA: Host not found", instead of the actual
-# reason "User doesn't exist." The sysadmin's intent is probably that
-# ipv6 is not desired at all, so disable ipv6.
-if kernel_ipv6_lo_disabled; then
-	tools/editconf.py /etc/postfix/main.cf inet_protocols=ipv4
-fi
     
 # Tweak some queue settings:
 # * Inform users when their e-mail delivery is delayed more than 3 hours (default is not to warn).

From 2bdb6f6216454703c5ea738437f254832ec4da6b Mon Sep 17 00:00:00 2001
From: downtownallday <downtownallday@gmail.com>
Date: Wed, 10 Jun 2020 22:37:46 -0400
Subject: [PATCH 37/65] Output more host information post-setup

---
 .travis.yml | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/.travis.yml b/.travis.yml
index 0614e2e1..52661aaa 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -25,6 +25,11 @@ before_install:
   
 install:
   - sudo tests/system-setup/remote-nextcloud-docker.sh
+  - hostname -I || true
+  - hostname -i || true
+  - hostname || true
+  - hostname --fqdn || true
+  - nslookup box.abc.com || true
 
 script:
   #

From 4113ed85011aa082fb38c0be2573d1da86db7ee3 Mon Sep 17 00:00:00 2001
From: downtownallday <downtownallday@gmail.com>
Date: Wed, 10 Jun 2020 23:16:35 -0400
Subject: [PATCH 38/65] Look at resolv.conf this time

---
 .travis.yml                                   | 1 +
 tests/system-setup/remote-nextcloud-docker.sh | 3 +++
 tests/system-setup/setup-funcs.sh             | 1 +
 3 files changed, 5 insertions(+)

diff --git a/.travis.yml b/.travis.yml
index 52661aaa..9d1433f6 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -30,6 +30,7 @@ install:
   - hostname || true
   - hostname --fqdn || true
   - nslookup box.abc.com || true
+  - getent hosts box.abc.com || true
 
 script:
   #
diff --git a/tests/system-setup/remote-nextcloud-docker.sh b/tests/system-setup/remote-nextcloud-docker.sh
index befcd452..03ce27ff 100755
--- a/tests/system-setup/remote-nextcloud-docker.sh
+++ b/tests/system-setup/remote-nextcloud-docker.sh
@@ -197,6 +197,9 @@ after_miab_install() {
            "$PRIMARY_HOSTNAME" \
            "$LDAP_NEXTCLOUD_PASSWORD" \
         || die "docker: error running remote-nextcloud-use-miab.sh"
+
+    # re-add systemd name resolver as a secondary nameserver
+    echo "nameserver 127.0.0.53" >> /etc/resolv.conf
 }
 
 
diff --git a/tests/system-setup/setup-funcs.sh b/tests/system-setup/setup-funcs.sh
index b71591ec..f28c4ef1 100755
--- a/tests/system-setup/setup-funcs.sh
+++ b/tests/system-setup/setup-funcs.sh
@@ -101,6 +101,7 @@ dump_conf_files() {
         dump_log "/etc/mailinabox.conf"
         dump_log "/etc/hosts"
         dump_log "/etc/nsswitch.conf"
+        dump_log "/etc/resolv.conf"
         dump_log "/etc/nsd/nsd.conf"
         dump_log "/etc/postfix/main.cf"
     fi

From 6baf4993dbdcc16d1b582649affc31480c21768d Mon Sep 17 00:00:00 2001
From: downtownallday <downtownallday@gmail.com>
Date: Thu, 11 Jun 2020 06:45:49 -0400
Subject: [PATCH 39/65] Ensure root user has a mailbox for sendmail -bv
 delivery report

---
 .travis.yml                                   | 3 ---
 tests/suites/_mail-functions.sh               | 4 ++--
 tests/suites/mail-access.sh                   | 2 +-
 tests/suites/mail-basic.sh                    | 2 +-
 tests/system-setup/remote-nextcloud-docker.sh | 8 +++-----
 tests/system-setup/setup-funcs.sh             | 2 +-
 6 files changed, 8 insertions(+), 13 deletions(-)

diff --git a/.travis.yml b/.travis.yml
index 9d1433f6..a3cc99a7 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -25,11 +25,8 @@ before_install:
   
 install:
   - sudo tests/system-setup/remote-nextcloud-docker.sh
-  - hostname -I || true
-  - hostname -i || true
   - hostname || true
   - hostname --fqdn || true
-  - nslookup box.abc.com || true
   - getent hosts box.abc.com || true
 
 script:
diff --git a/tests/suites/_mail-functions.sh b/tests/suites/_mail-functions.sh
index a9a64bca..e319a188 100644
--- a/tests/suites/_mail-functions.sh
+++ b/tests/suites/_mail-functions.sh
@@ -10,14 +10,14 @@ ensure_root_user() {
 	# ensure there is a local email account for root.
 	#
 	# on exit, ROOT, ROOT_MAILDROP, and ROOT_DN are set, and if no
-	# account exists, a new root@$(hostname) is created having a
+	# account exists, a new root@$(hostname --fqdn) is created having a
 	# random password
 	#
 	if [ ! -z "$ROOT_MAILDROP" ]; then
 		# already have it
 		return
 	fi
-	ROOT="${USER}@$(hostname)"
+	ROOT="${USER}@$(hostname --fqdn || hostname)"
 	record "[Find user $ROOT]"
 	get_attribute "$LDAP_USERS_BASE" "mail=$ROOT" "maildrop"
 	ROOT_MAILDROP="$ATTR_VALUE"
diff --git a/tests/suites/mail-access.sh b/tests/suites/mail-access.sh
index 31816711..7107e330 100644
--- a/tests/suites/mail-access.sh
+++ b/tests/suites/mail-access.sh
@@ -189,7 +189,7 @@ EOF
 }
 
 
-suite_start "mail-access"
+suite_start "mail-access" ensure_root_user
 
 test_greylisting
 test_relay_prohibited
diff --git a/tests/suites/mail-basic.sh b/tests/suites/mail-basic.sh
index 8994e268..43e551be 100644
--- a/tests/suites/mail-basic.sh
+++ b/tests/suites/mail-basic.sh
@@ -67,7 +67,7 @@ test_self_send_receive() {
 
 
 
-suite_start "mail-basic"
+suite_start "mail-basic" ensure_root_user
 
 test_trial_send_local
 test_trial_send_remote
diff --git a/tests/system-setup/remote-nextcloud-docker.sh b/tests/system-setup/remote-nextcloud-docker.sh
index 03ce27ff..2086eef6 100755
--- a/tests/system-setup/remote-nextcloud-docker.sh
+++ b/tests/system-setup/remote-nextcloud-docker.sh
@@ -79,8 +79,9 @@ before_miab_install() {
     fi
     
     # install prerequisites
-    H2 "QA prerequisites"
-    install_qa_prerequisites || die "Error installing QA prerequisites"
+    H2 "QA pre-setup prerequisites"
+    install_pre_setup_qa_prerequisites \
+        || die "Error installing QA prerequisites"
 
     # enable the remote Nextcloud setup mod, which tells MiaB-LDAP to use
     # the remote Nextcloud for calendar and contacts instead of the
@@ -197,9 +198,6 @@ after_miab_install() {
            "$PRIMARY_HOSTNAME" \
            "$LDAP_NEXTCLOUD_PASSWORD" \
         || die "docker: error running remote-nextcloud-use-miab.sh"
-
-    # re-add systemd name resolver as a secondary nameserver
-    echo "nameserver 127.0.0.53" >> /etc/resolv.conf
 }
 
 
diff --git a/tests/system-setup/setup-funcs.sh b/tests/system-setup/setup-funcs.sh
index f28c4ef1..9f8194d5 100755
--- a/tests/system-setup/setup-funcs.sh
+++ b/tests/system-setup/setup-funcs.sh
@@ -186,7 +186,7 @@ install_docker() {
 }
 
 
-install_qa_prerequisites() {
+install_pre_setup_qa_prerequisites() {
     [ -z "$STORAGE_ROOT" ] \
         && echo "Error: STORAGE_ROOT not set" 1>&2 \
         && return 1

From 0e857916eb500c69811cbba90aa815cbd8464e5c Mon Sep 17 00:00:00 2001
From: downtownallday <downtownallday@gmail.com>
Date: Thu, 11 Jun 2020 08:08:31 -0400
Subject: [PATCH 40/65] Update wording

---
 README.md                        | 50 ++++++++++++++++++++++----------
 tests/suites/remote-nextcloud.sh |  8 ++---
 2 files changed, 38 insertions(+), 20 deletions(-)

diff --git a/README.md b/README.md
index b7730fc7..cfdd4ebf 100644
--- a/README.md
+++ b/README.md
@@ -1,34 +1,50 @@
 [![Build Status](https://travis-ci.com/downtownallday/mailinabox-ldap.svg?branch=master)](https://travis-ci.com/downtownallday/mailinabox-ldap)
 
-Mail-in-a-Box LDAP
-===================
+# Mail-in-a-Box LDAP
 This is a version of [Mail-in-a-Box](https://mailinabox.email) with LDAP used as the user account database instead of sqlite.
 
 All features are supported - you won't find many visible differences. It's only an under-the-hood change.
 
-However it will allow a remote Nextcloud installation to authenticate users against Mail-in-a-Box using [Nextcloud's official LDAP support](https://nextcloud.com/usermanagement/). A single user account database shared with Nextcloud was originally the goal of the project which would simplify deploying a private mail and cloud service for a home or small business. But, there could be many other use cases as well. 
+However, it will allow a remote Nextcloud installation to authenticate users against Mail-in-a-Box using [Nextcloud's official LDAP support](https://nextcloud.com/usermanagement/). A single user account database shared with Nextcloud was originally the goal of the project which would simplify deploying a private mail and cloud service for a home or small business. But, there could be many other use cases as well. 
 
 To add a new account to Nextcloud, you'd simply add a new email account with MiaB-LDAP's admin interface. Quotas and other account settings are made within Nextcloud.
 
-How to connect a remote Nextcloud
+## How to connect to a remote Nextcloud
 ---------------------------------
 
-To fully integrate Mail-in-a-Box w/LDAP (MiaB-LDAP) with Nextcloud, changes must be made on both sides.
+To integrate Mail-in-a-Box w/LDAP (MiaB-LDAP) with Nextcloud, changes must be made on both sides.  These changes are automated.
 
-1. MiaB-LDAP
-  * Remote LDAPS access: the default MiaB-LDAP installation doesn't allow any remote LDAP access, so for Nextcloud to access MiaB-LDAP, firewall rules must be loosened to the LDAPS port (636). This is a one-time change.  Run something like this as root on MiaB-LDAP, where $ip is the ip-address of your Nextcloud server:  `ufw allow proto tcp from $ip to any port ldaps`
-  * Roundcube and Z-Push (ActiveSync) changes: modify the MiaB-LDAP configuration to use the remote Nextcloud for contacts and calendar. A script to do this automatically will be available soon.
-2. Remote Nextcloud
-  * Use MiaB-LDAP for user acccounts: on Nextcloud, enable user-ldap (in Apps, enable "LDAP user and group backend". Then in Settings click on "LDAP / AD integration". There are quite a few settings to make in there and more information on this will be forthcoming, including a script that will use the user-ldap API to configure the LDAP parameters in Nextcloud for you.
+**On MiaB-LDAP**
 
-Details
+Enable the setup mod `remote-nextcloud.sh` by creating a symbolic link to it in setup/mods.d. Do this by running this command from the mailinabox directory: `ln -s ../mods.available/remote-nextcloud.sh setup/mods.d/remote-nextcloud.sh`. *During setup you will be prompted for the hostname and web prefix of your remote Nextcloud box.*
+
+The setup mod will configure Roundcube and Z-Push (ActiveSync) to use the remote Nextcloud for contacts and calendar instead of the local Nextcloud, which will be disabled (browsing to /cloud will fail).
+
+**On the remote Nextcloud**
+
+Copy the file `setup/mods.available/remote-nextcloud-use-miab.sh` to the Nextcloud box and run it. This will configure Nextcloud's "LDAP user and group backend" with the MiaB-LDAP details and ensure the contacts and calendar apps are installed. *This does not replace or alter your ability to log into Nextcloud with any existing local Nextcloud accounts. It only allows MiaB-LDAP users to log into Nextcloud using their MiaB-LDAP credentials.*
+
+**Additional Firewall Rule**
+
+On MiaB-LDAP, a one-time change must be applied manually to allow the remote Nextcloud to query the LDAP server because the default MiaB-LDAP installation doesn't allow any remote LDAP access. As root, run the following: `ufw allow proto tcp from $ip to any port ldaps`, where $ip is the ip-address of your Nextcloud server.
+
+
+## Under-the-Hood Details
 -------
 
-Once installed, you will find all LDAP service account credentials in `/home/user-data/ldap/miab_ldap.conf`, such as those for Nextcloud. Service accounts have limited rights to make changes and should be preferred over the use of the LDAP admin account.
+**Additional directory in user-data**
+
+A new ldap directory is created by setup under STORAGE_ROOT (/home/user-data/ldap) that holds the LDAP database, so that it gets backed up by the normal backup process. In there, you will also find all LDAP service account credentials created by setup in `/home/user-data/ldap/miab_ldap.conf`, such as those for Nextcloud. Service accounts have limited rights to make changes and should be preferred over the use of the LDAP admin account.
+
+**LDAP schema for postfix and dovecot**
 
 See `conf/postfix.schema` for more details on the LDAP schema.
 
-LDAP server access logs are stored in `/var/log/ldap/slapd.log` and rotated daily.
+**LDAP logs**
+
+LDAP server logs are stored in `/var/log/ldap/slapd.log` and rotated daily.
+
+**Command line queries**
 
 To perform general command-line searches against your LDAP database, run `setup/ldap -search "\<query\>"` as root, where _query_ can be a distinguished name to show all attributes of that dn, or an LDAP search enclosed in parenthesis. Some examples:
   * `setup/ldap.sh -search "(mail=alice@mydomain.com)"` (show alice)
@@ -36,12 +52,14 @@ To perform general command-line searches against your LDAP database, run `setup/
   * `setup/ldap.sh -search "(objectClass=mailuser)"` (show all users)
   * etc.
 
-This is a convenient way to run ldapsearch having all the correct command line arguments.
+This is a convenient way to run ldapsearch having all the correct command line arguments, but any LDAP tool will also work.
 
-Caution: do not make direct LDAP database changes, such as adding users or groups using ldapmodify or other LDAP database tool. Instead, use the MiaB admin interface or REST API. Adding or removing a user or group with the admin interface may trigger additional database and system changes by the management daemon, such as updating DNS zones for new email domains, updating group memberships, etc, that would not be performed with a direct change.
+**Caution**
+
+*Do not make direct LDAP database changes, such as adding users or groups using ldapmodify or other LDAP database tools. Instead, use the MiaB admin interface or REST API. Adding or removing a user or group with the admin interface may trigger additional database and system changes by the management daemon, such as updating DNS zones for new email domains, updating group memberships, etc, that would not be performed with a direct change.*
 
 
-Migration
+## Migration
 ---------
 Running any of the setup scripts to install MiaB-LDAP (`miab`, `setup/bootstrap.sh`, `setup/start.sh`, etc) will automatically migrate your current installation from sqlite to LDAP. Make a full MiaB backup before running!
 
diff --git a/tests/suites/remote-nextcloud.sh b/tests/suites/remote-nextcloud.sh
index d2af6c7d..f2dc68a1 100644
--- a/tests/suites/remote-nextcloud.sh
+++ b/tests/suites/remote-nextcloud.sh
@@ -6,7 +6,7 @@
 #    - Nextcloud is already installed and MiaB-LDAP is already
 #      configured to use it.
 #
-#      ie. remote-nextcloud.sh was run on MiaB-LDAP.sh by
+#      ie. remote-nextcloud.sh was run on MiaB-LDAP by
 #          setup/start.sh because there was a symbolic link from
 #          setup/mods.d/remote-nextcloud.sh to the script in
 #          mods.available
@@ -14,7 +14,7 @@
 #    - The remote Nextcloud has been configured to use MiaB-LDAP
 #      for users and groups.
 #
-#      eg. remote-nextcloud-use-miab.sh was copied to the remote Nextcloud
+#      ie. remote-nextcloud-use-miab.sh was copied to the remote Nextcloud
 #      server and was run successfully there
 #
 
@@ -247,9 +247,9 @@ test_mail_from_nextcloud() {
 test_nextcloud_contacts() {
     test_start "nextcloud_contacts"
 
-    assert_is_configured || (test_end && return)
+    assert_is_configured || test_end && return
 
-    local alice="alice3@somedomain.com"
+    local alice="alice.nc@somedomain.com"
     local alice_pw="$(generate_password 16)"
 
 	# create local user alice

From 41642f2f5947f64a267130590afd8d39aee17cb3 Mon Sep 17 00:00:00 2001
From: Faye Duxovni <duxovni@duxovni.org>
Date: Sun, 7 Jun 2020 09:50:04 -0400
Subject: [PATCH 41/65] [backport] Fix roundcube error log file path in setup
 script (#1775)

---
 setup/webmail.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/setup/webmail.sh b/setup/webmail.sh
index 20d43c57..bd31e221 100755
--- a/setup/webmail.sh
+++ b/setup/webmail.sh
@@ -160,7 +160,7 @@ mkdir -p /var/log/roundcubemail /var/tmp/roundcubemail $STORAGE_ROOT/mail/roundc
 chown -R www-data.www-data /var/log/roundcubemail /var/tmp/roundcubemail $STORAGE_ROOT/mail/roundcube
 
 # Ensure the log file monitored by fail2ban exists, or else fail2ban can't start.
-sudo -u www-data touch /var/log/roundcubemail/errors
+sudo -u www-data touch /var/log/roundcubemail/errors.log
 
 # Password changing plugin settings
 # The config comes empty by default, so we need the settings

From e03a6541ced593b6c19a875f3fe59139d193a41c Mon Sep 17 00:00:00 2001
From: Joshua Tauberer <jt@occams.info>
Date: Fri, 5 Jun 2020 13:45:50 -0400
Subject: [PATCH 42/65] Don't make autoconfig/autodiscover subdomains and SRV
 records when the parent domain has no user accounts

These subdomains/records are for automatic configuration of mail clients, but if there are no user accounts on a domain, there is no need to publish a DNS record, provision a TLS certificate, or create an nginx server config block.
---
 CHANGELOG.md             |  4 ++++
 management/dns_update.py | 28 +++++++++++++++-------------
 management/mailconfig.py | 14 ++++++++------
 management/web_update.py |  8 ++++----
 4 files changed, 31 insertions(+), 23 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 01f860fe..04cfb753 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -9,6 +9,10 @@ Mail:
 * An MTA-STS policy for incoming mail is now published (in DNS and over HTTPS) when the primary hostname and email address domain both have a signed TLS certificate installed.
 * MTA-STS reporting is enabled with reports sent to administrator@ the primary hostname.
 
+DNS:
+
+* autoconfig and autodiscover subdomains and CalDAV/CardDAV SRV records are no longer generated for domains that don't have user accounts since they are unnecessary.
+
 v0.45 (May 16, 2020)
 --------------------
 
diff --git a/management/dns_update.py b/management/dns_update.py
index 5fdb3e0f..80273a12 100755
--- a/management/dns_update.py
+++ b/management/dns_update.py
@@ -281,28 +281,30 @@ def build_zone(domain, all_domains, additional_records, www_redirect_domains, en
 		if not has_rec(dmarc_qname, "TXT", prefix="v=DMARC1; "):
 			records.append((dmarc_qname, "TXT", 'v=DMARC1; p=reject', "Recommended. Prevents use of this domain name for outbound mail by specifying that the SPF rule should be honoured for mail from @%s." % (qname + "." + domain)))
 
-	# Add CardDAV/CalDAV SRV records on the non-primary hostname that points to the primary hostname.
+	# Add CardDAV/CalDAV SRV records on the non-primary hostname that points to the primary hostname
+	# for autoconfiguration of mail clients (so only domains hosting user accounts need it).
 	# The SRV record format is priority (0, whatever), weight (0, whatever), port, service provider hostname (w/ trailing dot).
-	if domain != env["PRIMARY_HOSTNAME"]:
+	if domain != env["PRIMARY_HOSTNAME"] and domain in get_mail_domains(env, users_only=True):
 		for dav in ("card", "cal"):
 			qname = "_" + dav + "davs._tcp"
 			if not has_rec(qname, "SRV"):
 				records.append((qname, "SRV", "0 0 443 " + env["PRIMARY_HOSTNAME"] + ".", "Recommended. Specifies the hostname of the server that handles CardDAV/CalDAV services for email addresses on this domain."))
 
-	# Adds autoconfiguration A records for all domains.
+	# Adds autoconfiguration A records for all domains that there are user accounts at.
 	# This allows the following clients to automatically configure email addresses in the respective applications.
 	# autodiscover.* - Z-Push ActiveSync Autodiscover
 	# autoconfig.* - Thunderbird Autoconfig
-	autodiscover_records = [
-		("autodiscover", "A", env["PUBLIC_IP"], "Provides email configuration autodiscovery support for Z-Push ActiveSync Autodiscover."),
-		("autodiscover", "AAAA", env["PUBLIC_IPV6"], "Provides email configuration autodiscovery support for Z-Push ActiveSync Autodiscover."),
-		("autoconfig", "A", env["PUBLIC_IP"], "Provides email configuration autodiscovery support for Thunderbird Autoconfig."),
-		("autoconfig", "AAAA", env["PUBLIC_IPV6"], "Provides email configuration autodiscovery support for Thunderbird Autoconfig.")
-	]
-	for qname, rtype, value, explanation in autodiscover_records:
-		if value is None or value.strip() == "": continue # skip IPV6 if not set
-		if not has_rec(qname, rtype):
-			records.append((qname, rtype, value, explanation))
+	if domain in get_mail_domains(env, users_only=True):
+		autodiscover_records = [
+			("autodiscover", "A", env["PUBLIC_IP"], "Provides email configuration autodiscovery support for Z-Push ActiveSync Autodiscover."),
+			("autodiscover", "AAAA", env["PUBLIC_IPV6"], "Provides email configuration autodiscovery support for Z-Push ActiveSync Autodiscover."),
+			("autoconfig", "A", env["PUBLIC_IP"], "Provides email configuration autodiscovery support for Thunderbird Autoconfig."),
+			("autoconfig", "AAAA", env["PUBLIC_IPV6"], "Provides email configuration autodiscovery support for Thunderbird Autoconfig.")
+		]
+		for qname, rtype, value, explanation in autodiscover_records:
+			if value is None or value.strip() == "": continue # skip IPV6 if not set
+			if not has_rec(qname, rtype):
+				records.append((qname, rtype, value, explanation))
 
 	# If this is a domain name that there are email addresses configured for, i.e. "something@"
 	# this domain name, then the domain name is a MTA-STS (https://tools.ietf.org/html/rfc8461)
diff --git a/management/mailconfig.py b/management/mailconfig.py
index 5f253c14..dd597cd6 100755
--- a/management/mailconfig.py
+++ b/management/mailconfig.py
@@ -258,13 +258,15 @@ def get_domain(emailaddr, as_unicode=True):
 			pass
 	return ret
 
-def get_mail_domains(env, filter_aliases=lambda alias : True):
+def get_mail_domains(env, filter_aliases=lambda alias : True, users_only=False):
 	# Returns the domain names (IDNA-encoded) of all of the email addresses
-	# configured on the system.
-	return set(
-		   [get_domain(login, as_unicode=False) for login in get_mail_users(env)]
-		 + [get_domain(address, as_unicode=False) for address, *_ in get_mail_aliases(env) if filter_aliases(address) ]
-		 )
+	# configured on the system. If users_only is True, only return domains
+	# with email addresses that correspond to user accounts.
+	domains = []
+	domains.extend([get_domain(login, as_unicode=False) for login in get_mail_users(env)])
+	if not users_only:
+		domains.extend([get_domain(address, as_unicode=False) for address, *_ in get_mail_aliases(env) if filter_aliases(address) ])
+	return set(domains)
 
 def add_mail_user(email, pw, privs, env):
 	# validate email
diff --git a/management/web_update.py b/management/web_update.py
index 4a07dc9e..78f86f4c 100644
--- a/management/web_update.py
+++ b/management/web_update.py
@@ -24,13 +24,13 @@ def get_web_domains(env, include_www_redirects=True, exclude_dns_elsewhere=True)
 		# the topmost of each domain we serve.
 		domains |= set('www.' + zone for zone, zonefile in get_dns_zones(env))
 
-	# Add Autoconfiguration domains, allowing us to serve correct SSL certs.
+	# Add Autoconfiguration domains for domains that there are user accounts at:
 	# 'autoconfig.' for Mozilla Thunderbird auto setup.
 	# 'autodiscover.' for Activesync autodiscovery.
-	domains |= set('autoconfig.' + maildomain for maildomain in get_mail_domains(env))
-	domains |= set('autodiscover.' + maildomain for maildomain in get_mail_domains(env))
+	domains |= set('autoconfig.' + maildomain for maildomain in get_mail_domains(env, users_only=True))
+	domains |= set('autodiscover.' + maildomain for maildomain in get_mail_domains(env, users_only=True))
 
-	# 'mta-sts.' for MTA-STS support.
+	# 'mta-sts.' for MTA-STS support for all domains that have email addresses.
 	domains |= set('mta-sts.' + maildomain for maildomain in get_mail_domains(env))
 
 	if exclude_dns_elsewhere:

From 9db2fc7f0551b6ea9b7c73f447495fda722473fb Mon Sep 17 00:00:00 2001
From: Joshua Tauberer <jt@occams.info>
Date: Sun, 7 Jun 2020 09:45:04 -0400
Subject: [PATCH 43/65] In web proxies, add X-{Forwarded-{Host,Proto},Real-IP}
 and 'proxy_set_header Host' when there is a flag

Merges #1432, more or less.
---
 management/web_update.py | 14 ++++++++++++++
 1 file changed, 14 insertions(+)

diff --git a/management/web_update.py b/management/web_update.py
index 78f86f4c..66340619 100644
--- a/management/web_update.py
+++ b/management/web_update.py
@@ -158,9 +158,23 @@ def make_domain_config(domain, templates, ssl_certificates, env):
 
 			# any proxy or redirect here?
 			for path, url in yaml.get("proxies", {}).items():
+				# Parse some flags in the fragment of the URL.
+				pass_http_host_header = False
+				m = re.search("#(.*)$", url)
+				if m:
+					for flag in m.group(1).split(","):
+						if flag == "pass-http-host":
+							pass_http_host_header = True
+					url = re.sub("#(.*)$", "", url)
+
 				nginx_conf_extra += "\tlocation %s {" % path
 				nginx_conf_extra += "\n\t\tproxy_pass %s;" % url
+				if pass_http_host_header:
+					nginx_conf_extra += "\n\t\tproxy_set_header Host $http_host;"
 				nginx_conf_extra += "\n\t\tproxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;"
+				nginx_conf_extra += "\n\t\tproxy_set_header X-Forwarded-Host $http_host;"
+				nginx_conf_extra += "\n\t\tproxy_set_header X-Forwarded-Proto $scheme;"
+				nginx_conf_extra += "\n\t\tproxy_set_header X-Real-IP $remote_addr;"
 				nginx_conf_extra += "\n\t}\n"
 			for path, alias in yaml.get("aliases", {}).items():
 				nginx_conf_extra += "\tlocation %s {" % path

From 12d60d102b0cddf6a09d8b68ba2d0a2531efd0e3 Mon Sep 17 00:00:00 2001
From: Joshua Tauberer <jt@occams.info>
Date: Thu, 11 Jun 2020 12:19:00 -0400
Subject: [PATCH 44/65] Update Roundcube to 1.4.6

Fixes #1776
---
 CHANGELOG.md     | 7 +++++++
 setup/webmail.sh | 4 ++--
 2 files changed, 9 insertions(+), 2 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 3cd9e724..23ddd136 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,6 +1,13 @@
 CHANGELOG
 =========
 
+v0.46 (June 11, 2020)
+---------------------
+
+Security fixes:
+
+* Roundcube is updated to version 1.4.6 (https://roundcube.net/news/2020/06/02/security-updates-1.4.5-and-1.3.12).
+
 v0.45 (May 16, 2020)
 --------------------
 
diff --git a/setup/webmail.sh b/setup/webmail.sh
index bd31e221..7054e38e 100755
--- a/setup/webmail.sh
+++ b/setup/webmail.sh
@@ -28,8 +28,8 @@ apt_install \
 # Install Roundcube from source if it is not already present or if it is out of date.
 # Combine the Roundcube version number with the commit hash of plugins to track
 # whether we have the latest version of everything.
-VERSION=1.4.4
-HASH=4e425263f5bec27d39c07bde524f421bda205c07
+VERSION=1.4.6
+HASH=44961ef62bb9c9875141ca34704bbc7d6f36373d
 PERSISTENT_LOGIN_VERSION=6b3fc450cae23ccb2f393d0ef67aa319e877e435
 HTML5_NOTIFIER_VERSION=4b370e3cd60dabd2f428a26f45b677ad1b7118d5
 CARDDAV_VERSION=3.0.3

From 049bfb6f7f0ce918e5437bcf3a18f66ceef2ea3d Mon Sep 17 00:00:00 2001
From: Joshua Tauberer <jt@occams.info>
Date: Thu, 11 Jun 2020 12:23:18 -0400
Subject: [PATCH 45/65] v0.46

---
 README.md          | 4 ++--
 setup/bootstrap.sh | 2 +-
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/README.md b/README.md
index e787c8d8..1d4452b8 100644
--- a/README.md
+++ b/README.md
@@ -58,7 +58,7 @@ by him:
 	$ curl -s https://keybase.io/joshdata/key.asc | gpg --import
 	gpg: key C10BDD81: public key "Joshua Tauberer <jt@occams.info>" imported
 
-	$ git verify-tag v0.45
+	$ git verify-tag v0.46
 	gpg: Signature made ..... using RSA key ID C10BDD81
 	gpg: Good signature from "Joshua Tauberer <jt@occams.info>"
 	gpg: WARNING: This key is not certified with a trusted signature!
@@ -71,7 +71,7 @@ and on his [personal homepage](https://razor.occams.info/). (Of course, if this
 
 Checkout the tag corresponding to the most recent release:
 
-	$ git checkout v0.45
+	$ git checkout v0.46
 
 Begin the installation.
 
diff --git a/setup/bootstrap.sh b/setup/bootstrap.sh
index 4fcb85cc..6aae9500 100644
--- a/setup/bootstrap.sh
+++ b/setup/bootstrap.sh
@@ -20,7 +20,7 @@ if [ -z "$TAG" ]; then
 	# want to display in status checks.
 	if [ "`lsb_release -d | sed 's/.*:\s*//' | sed 's/18\.04\.[0-9]/18.04/' `" == "Ubuntu 18.04 LTS" ]; then
 		# This machine is running Ubuntu 18.04.
-		TAG=v0.45
+		TAG=v0.46
 
 	elif [ "`lsb_release -d | sed 's/.*:\s*//' | sed 's/14\.04\.[0-9]/14.04/' `" == "Ubuntu 14.04 LTS" ]; then
 		# This machine is running Ubuntu 14.04.

From 99bb128d6091e65455f735be45bf79aeb93b8cfa Mon Sep 17 00:00:00 2001
From: downtownallday <downtownallday@gmail.com>
Date: Thu, 11 Jun 2020 13:47:50 -0400
Subject: [PATCH 46/65] Investigating occasional test failures. See this in the
 log:

[ERROR] Jun 11 17:30:15 lmtp(alice@somedomain.com): Error: open(/etc/dovecot/sieve-spam.svbin.box.abc.com.13806.26a2405f731160b1) failed: Read-only file system
[ERROR] Jun 11 17:30:15 lmtp(alice@somedomain.com): Error: GKVhOyZq4l7uNQAAasegNA: sieve: binary save: failed to create temporary file: open(/etc/dovecot/sieve-spam.svbin.) failed: Read-only file system

systemctl service setting for dovecot includes ProtectSystem=full, so /etc will be read-only.

need to understand why sieve is attempting to create a temp file or maybe
move sieve script to /var
---
 .travis.yml | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/.travis.yml b/.travis.yml
index a3cc99a7..79ad0288 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -33,4 +33,6 @@ script:
   #
   # launch automated tests, but skip tests that require remote
   # smtp support because Travis-CI blocks outgoing port 25
+  - sudo ls -al /etc/dovecot
+  - sudo touch /etc/dovecot/sieve-spam.svbin
   - sudo tests/runner.sh -dumpoutput -no-smtp-remote default remote-nextcloud

From 1f35e9ef918294c32d902f462d2b9f83c184eb26 Mon Sep 17 00:00:00 2001
From: downtownallday <downtownallday@gmail.com>
Date: Thu, 11 Jun 2020 21:18:05 -0400
Subject: [PATCH 47/65] Move directory setup/mods.d to `local`

---
 .travis.yml                                   | 1 -
 README.md                                     | 2 +-
 setup/mods.d/.gitignore                       | 2 --
 setup/start.sh                                | 8 +++++---
 tests/suites/remote-nextcloud.sh              | 6 +++---
 tests/system-setup/remote-nextcloud-docker.sh | 8 ++++----
 6 files changed, 13 insertions(+), 14 deletions(-)
 delete mode 100644 setup/mods.d/.gitignore

diff --git a/.travis.yml b/.travis.yml
index 79ad0288..f3beb134 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -33,6 +33,5 @@ script:
   #
   # launch automated tests, but skip tests that require remote
   # smtp support because Travis-CI blocks outgoing port 25
-  - sudo ls -al /etc/dovecot
   - sudo touch /etc/dovecot/sieve-spam.svbin
   - sudo tests/runner.sh -dumpoutput -no-smtp-remote default remote-nextcloud
diff --git a/README.md b/README.md
index e28849fc..730431c8 100644
--- a/README.md
+++ b/README.md
@@ -16,7 +16,7 @@ To integrate Mail-in-a-Box w/LDAP (MiaB-LDAP) with Nextcloud, changes must be ma
 
 **On MiaB-LDAP**
 
-Enable the setup mod `remote-nextcloud.sh` by creating a symbolic link to it in setup/mods.d. Do this by running this command from the mailinabox directory: `ln -s ../mods.available/remote-nextcloud.sh setup/mods.d/remote-nextcloud.sh`. *During setup you will be prompted for the hostname and web prefix of your remote Nextcloud box.*
+Enable the setup mod `remote-nextcloud.sh` by creating the directory `local` and creating a symbolic link to it. Do this by running this command from the mailinabox directory: `ln -s ../setup/mods.available/remote-nextcloud.sh local/remote-nextcloud.sh`. *During setup you will be prompted for the hostname and web prefix of your remote Nextcloud box.*
 
 The setup mod will configure Roundcube and Z-Push (ActiveSync) to use the remote Nextcloud for contacts and calendar instead of the local Nextcloud, which will be disabled (browsing to /cloud will fail).
 
diff --git a/setup/mods.d/.gitignore b/setup/mods.d/.gitignore
deleted file mode 100644
index d6b7ef32..00000000
--- a/setup/mods.d/.gitignore
+++ /dev/null
@@ -1,2 +0,0 @@
-*
-!.gitignore
diff --git a/setup/start.sh b/setup/start.sh
index cfe239b8..5da27665 100755
--- a/setup/start.sh
+++ b/setup/start.sh
@@ -150,9 +150,11 @@ fi
 #
 # Run setup mods 
 #
-for mod in $(ls setup/mods.d | grep -v '~$'); do
-    setup/mods.d/$mod
-done
+if [ -d local ]; then
+    for mod in $(ls local | grep -v '~$'); do
+        local/$mod
+    done
+fi
 
 # Done.
 echo
diff --git a/tests/suites/remote-nextcloud.sh b/tests/suites/remote-nextcloud.sh
index f2dc68a1..825c3df5 100644
--- a/tests/suites/remote-nextcloud.sh
+++ b/tests/suites/remote-nextcloud.sh
@@ -2,15 +2,15 @@
 #	
 # Test the setup modification script setup/mods.available/remote-nextcloud.sh
 # Prerequisites:
-
+#
 #    - Nextcloud is already installed and MiaB-LDAP is already
 #      configured to use it.
 #
 #      ie. remote-nextcloud.sh was run on MiaB-LDAP by
 #          setup/start.sh because there was a symbolic link from
-#          setup/mods.d/remote-nextcloud.sh to the script in
+#          local/remote-nextcloud.sh to the script in
 #          mods.available
-
+#
 #    - The remote Nextcloud has been configured to use MiaB-LDAP
 #      for users and groups.
 #
diff --git a/tests/system-setup/remote-nextcloud-docker.sh b/tests/system-setup/remote-nextcloud-docker.sh
index 2086eef6..35174bdc 100755
--- a/tests/system-setup/remote-nextcloud-docker.sh
+++ b/tests/system-setup/remote-nextcloud-docker.sh
@@ -56,7 +56,6 @@ before_miab_install() {
     H1 "BEFORE MIAB-LDAP INSTALL"
 
     H2 "Update /etc/hosts"
-    #update_hosts_for_private_ip || die "Could not update /etc/hosts"
     set_system_hostname || die "Could not set hostname"
 
     # update system time
@@ -86,9 +85,10 @@ before_miab_install() {
     # enable the remote Nextcloud setup mod, which tells MiaB-LDAP to use
     # the remote Nextcloud for calendar and contacts instead of the
     # MiaB-installed one
-    H2 "Create setup/mods.d/remote-nextcloud.sh symbolic link"
-    if [ ! -e "setup/mods.d/remote-nextcloud.sh" ]; then
-        ln -s "../mods.available/remote-nextcloud.sh" "setup/mods.d/remote-nextcloud.sh" || die "Could not create remote-nextcloud.sh symlink"
+    H2 "Create local/remote-nextcloud.sh symbolic link"
+    if [ ! -e "local/remote-nextcloud.sh" ]; then
+        mkdir -p local
+        ln -s "../setup/mods.available/remote-nextcloud.sh" "local/remote-nextcloud.sh" || die "Could not create remote-nextcloud.sh symlink"
     fi
     
     # install Docker

From 6fd3195275fdfef3edd748a44b70dc830f320802 Mon Sep 17 00:00:00 2001
From: Joshua Tauberer <jt@occams.info>
Date: Fri, 12 Jun 2020 13:09:11 -0400
Subject: [PATCH 48/65] Fix MTA-STS policy id so it does not have invalid
 characters, fixes #1779

---
 management/dns_update.py | 10 ++++++----
 1 file changed, 6 insertions(+), 4 deletions(-)

diff --git a/management/dns_update.py b/management/dns_update.py
index 80273a12..2fb7b1b8 100755
--- a/management/dns_update.py
+++ b/management/dns_update.py
@@ -340,11 +340,13 @@ def build_zone(domain, all_domains, additional_records, www_redirect_domains, en
 			# 'break' was not encountered above, so both domains are good
 			mta_sts_enabled = True
 	if mta_sts_enabled:
-		# Compute a up-to-32-character hash of the policy file. We'll take a SHA-1 hash of the policy
-		# file (20 bytes) and encode it as base-64 (60 bytes) but then just take its first 20 bytes
-		# which should be sufficient to change whenever the policy file changes.
+		# Compute an up-to-32-character hash of the policy file. We'll take a SHA-1 hash of the policy
+		# file (20 bytes) and encode it as base-64 (28 bytes, using alphanumeric alternate characters
+		# instead of '+' and '/' which are not allowed in an MTA-STS policy id) but then just take its
+		# first 20 characters, which is more than sufficient to change whenever the policy file changes
+		# (and ensures any '=' padding at the end of the base64 encoding is dropped).
 		with open("/var/lib/mailinabox/mta-sts.txt", "rb") as f:
-			mta_sts_policy_id = base64.b64encode(hashlib.sha1(f.read()).digest()).decode("ascii")[0:20]
+			mta_sts_policy_id = base64.b64encode(hashlib.sha1(f.read()).digest(), altchars=b"AA").decode("ascii")[0:20]
 		mta_sts_records.extend([
 			("_mta-sts", "TXT", "v=STSv1; id=" + mta_sts_policy_id, "Optional. Part of the MTA-STS policy for incoming mail. If set, a MTA-STS policy must also be published.")
 		])

From b0090edd52436587ff41588b53e8382a904bd4cf Mon Sep 17 00:00:00 2001
From: downtownallday <downtownallday@gmail.com>
Date: Sun, 14 Jun 2020 13:51:00 -0400
Subject: [PATCH 49/65] Test upgrade to LDAP from upstream Mail-in-a-Box/sqlite

---
 .travis.yml                                   |  67 ++--
 management/mailconfig.py                      |  56 ++--
 setup/ldap.sh                                 |   8 +-
 .../remote-nextcloud-use-miab.sh              |   5 +-
 setup/ssl.sh                                  |  10 +-
 tests/lib/all.sh                              |  16 +
 tests/lib/color-output.sh                     |  32 ++
 .../_locations.sh => lib/locations.sh}        |   0
 tests/lib/misc.sh                             |  65 ++++
 tests/lib/rest.sh                             | 106 +++++++
 tests/lib/system.sh                           | 102 +++++++
 tests/suites/_init.sh                         |  23 +-
 tests/suites/_ldap-functions.sh               |  14 +-
 tests/suites/_mgmt-functions.sh               |  46 +--
 tests/system-setup/remote-nextcloud-docker.sh | 136 +++------
 tests/system-setup/setup-defaults.sh          |   8 +-
 tests/system-setup/setup-funcs.sh             | 228 +++++---------
 tests/system-setup/upgrade-from-upstream.sh   | 289 ++++++++++++++++++
 18 files changed, 831 insertions(+), 380 deletions(-)
 create mode 100644 tests/lib/all.sh
 create mode 100644 tests/lib/color-output.sh
 rename tests/{suites/_locations.sh => lib/locations.sh} (100%)
 create mode 100644 tests/lib/misc.sh
 create mode 100644 tests/lib/rest.sh
 create mode 100644 tests/lib/system.sh
 create mode 100755 tests/system-setup/upgrade-from-upstream.sh

diff --git a/.travis.yml b/.travis.yml
index f3beb134..e4fba7ec 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -1,37 +1,46 @@
 # travisci config
 env:
   global:
-  - PRIMARY_HOSTNAME=box.abc.com
+  - MIAB_LDAP=true
 
 language: shell
 os: linux
 dist: bionic
-  
-before_install:
-  - echo "==== ENVIRONMENT ===="
-  - env | sort
-  - echo "UMASK=$(umask)"
-  #
-  - echo "==== AppArmor Status ===="
-  - (sudo aa-status; true)
-  #
-  - echo "==== NETWORK INFO ===="
-  - hostname -I
-  - hostname -i
-  - hostname
-  - hostname --fqdn
-  - ip add
-  - sysctl -a 2>/dev/null | grep -i ipv6 | grep disable
-  
-install:
-  - sudo tests/system-setup/remote-nextcloud-docker.sh
-  - hostname || true
-  - hostname --fqdn || true
-  - getent hosts box.abc.com || true
 
-script:
-  #
-  # launch automated tests, but skip tests that require remote
-  # smtp support because Travis-CI blocks outgoing port 25
-  - sudo touch /etc/dovecot/sieve-spam.svbin
-  - sudo tests/runner.sh -dumpoutput -no-smtp-remote default remote-nextcloud
+jobs:
+  fast_finish: true
+  include:
+  # JOB: MiaB-LDAP connected to a remote Nextcloud
+  - env: PRIMARY_HOSTNAME=box1.abc.com
+    name: remote-nextcloud-docker
+    before_install:
+      - echo "==== ENVIRONMENT ===="
+      - env | sort
+      - echo "UMASK=$(umask)"
+      - echo "==== AppArmor Status ===="
+      - (sudo aa-status; true)
+      - echo "==== NETWORK INFO ===="
+      - hostname -I
+      - hostname -i
+      - hostname
+      - hostname --fqdn
+      - ip add
+      - sysctl -a 2>/dev/null | grep -i ipv6 | grep disable
+    install:
+      - sudo tests/system-setup/remote-nextcloud-docker.sh
+    script:
+      # launch automated tests, but skip tests that require remote
+      # smtp support because Travis-CI blocks outgoing port 25
+      - sudo touch /etc/dovecot/sieve-spam.svbin
+      - sudo tests/runner.sh -dumpoutput -no-smtp-remote default remote-nextcloud
+
+  # JOB: Upgrade from upstream install
+  - env: PRIMARY_HOSTNAME=box2.abc.com
+    name: upgrade-from-upstream
+    install:
+      - sudo tests/system-setup/upgrade-from-upstream.sh
+    script:
+      # launch automated tests, but skip tests that require remote
+      # smtp support because Travis-CI blocks outgoing port 25
+      - sudo touch /etc/dovecot/sieve-spam.svbin
+      - sudo tests/runner.sh -dumpoutput -no-smtp-remote
diff --git a/management/mailconfig.py b/management/mailconfig.py
index 4847eb8d..4fa63ca7 100755
--- a/management/mailconfig.py
+++ b/management/mailconfig.py
@@ -321,7 +321,7 @@ def get_mail_aliases(env, as_map=False):
 	
 	# make a dict of permitted senders, key=mail(lowercase) value=members
 	permitted_senders = { rec["mail"][0].lower(): rec["member"] for rec in pager }
-	
+
 	# get all alias groups
 	pager = c.paged_search(env.LDAP_ALIASES_BASE, "(objectClass=mailGroup)", attributes=['mail','member','rfc822MailMember'])
 	
@@ -362,7 +362,7 @@ def get_mail_aliases(env, as_map=False):
 			alias = aliases[address]
 			xft = ",".join(alias["forward_tos"])
 			xas = ",".join(alias["permitted_senders"])
-			list.append( (address, xft, xas) )
+			list.append( (address, xft, None if xas is "" else xas) )
 		return list
 	
 	else:
@@ -432,7 +432,7 @@ def get_domain(emailaddr, as_unicode=True):
 			pass
 	return ret
 
-def get_mail_domains(env, as_map=False, filter_aliases=None, category=None, users_only=False):
+def get_mail_domains(env, as_map=False, filter_aliases=lambda alias: True, category=None, users_only=False):
 	# Retrieves all domains, IDNA-encoded, we accept mail for.
 	#
 	# If as_map is False, the function returns the lowercase domain
@@ -457,17 +457,18 @@ def get_mail_domains(env, as_map=False, filter_aliases=None, category=None, user
 	# make it easy for dns_update to get ssl domains]
 	#
 	# If users_only is True, only return domains with email addresses
-	# that correspond to user accounts. [TODO: This currently has no
-	# effect - this function only returns user mail domains]
+	# that correspond to user accounts.
 	#
 	conn = open_database(env)
 	filter = "(&(objectClass=domain)(businessCategory=mail))"
 	if category:
 		filter = "(&(objectClass=domain)(businessCategory=%s))" % category
+
+	domains=None
+
+	# user mail domains
 	id = conn.search(env.LDAP_DOMAINS_BASE, filter, attributes="dc")
 	response = conn.wait(id)
-	filter_candidates=[]
-	domains=None
 	if as_map:
 		domains = {}
 		for rec in response:
@@ -478,36 +479,23 @@ def get_mail_domains(env, as_map=False, filter_aliases=None, category=None, user
 			if filter_aliases: filter_candidates.append(rec['dc'][0].lower())
 	else:
 		domains = set([ rec["dc"][0].lower() for rec in response ])
-		if filter_aliases: filter_candidates += domains
-	
-	for candidate in filter_candidates:
-		# with the filter, there has to be at least one user or
-		# filtered (included) alias in the domain for the domain to be
-		# part of the returned set
 
-		# any users ?
-		response = conn.wait( conn.search(env.LDAP_USERS_BASE, "(&(objectClass=mailUser)(mail=*@%s))" % candidate, size_limit=1) )
-		if response.next():
-			# yes, that domain needs to be in the returned set
-			continue
 
-		# any filtered aliases ?
-		pager = conn.paged_search(
-			env.LDAP_ALIASES_BASE,
-			"(&(objectClass=mailGroup)(mail=*@%s))" % candidate,
-			attributes=['mail'])
+	# alias domains
+	if not users_only:
+		pager = conn.paged_search(env.LDAP_ALIASES_BASE, "(objectClass=mailGroup)", attributes="mail")
+		if as_map:
+			for rec in pager:
+				if filter_aliases(rec["mail"][0].lower()):
+					domain = get_domain(rec["mail"][0].lower(),as_unicode=False)
+					domains[domain] = {
+						"dn": None,
+						"domain": domain
+					}
 
-		remove = True
-		for rec in pager:
-			if filter_aliases(rec['mail'][0]):
-				remove = False
-				pager.abandon()
-				break
-
-		if remove:
-			if as_map: del domains[candidate]
-			else: domains.remove(candidate)
-			
+		else:
+			domains = domains.union(set([ get_domain(rec["mail"][0].lower(), as_unicode=False) for rec in pager if filter_aliases(rec["mail"][0].lower()) ]))
+					
 	return domains
 
 
diff --git a/setup/ldap.sh b/setup/ldap.sh
index 78e2c4cb..252c5cd8 100755
--- a/setup/ldap.sh
+++ b/setup/ldap.sh
@@ -857,7 +857,13 @@ cat > /etc/logrotate.d/slapd <<EOF;
 EOF
 
 # Modify olc server config like TLS
-modify_global_config
+# Skip this step if no ca_certificate.pem exists - this indicates
+# that the system hasn't yet been migrated from sqlite
+if [ -e "$STORAGE_ROOT/ssl/ca_certificate.pem" ]; then
+	modify_global_config
+else
+	say_debug "Not enabling TLS at this time - ca_certificate hasn't been generated yet"
+fi
 
 # Add overlays and ensure mail-related attributes are indexed
 add_overlays
diff --git a/setup/mods.available/remote-nextcloud-use-miab.sh b/setup/mods.available/remote-nextcloud-use-miab.sh
index fd0277bf..e8ce1117 100755
--- a/setup/mods.available/remote-nextcloud-use-miab.sh
+++ b/setup/mods.available/remote-nextcloud-use-miab.sh
@@ -6,8 +6,9 @@
 #
 # The script will:
 #   1. enable the "LDAP user and group backend" in Nextcloud
-#   2. configure Nextcloud to access MiaB-LDAP for users and groups
-#   3. optionally install and configure ssmtp so system mail is
+#   2. install calendar and contacts
+#   3. configure Nextcloud to access MiaB-LDAP for users and groups
+#   4. optionally install and configure ssmtp so system mail is
 #      sent to MiaB-LDAP
 #
 VERBOSE=0
diff --git a/setup/ssl.sh b/setup/ssl.sh
index c8681df4..518450bc 100755
--- a/setup/ssl.sh
+++ b/setup/ssl.sh
@@ -96,11 +96,6 @@ if [ ! -s $STORAGE_ROOT/ssl/ssl_private_key.pem ]; then
 	# Set the umask so the key file is never world-readable.
 	(umask 037; hide_output \
 		openssl genrsa -out $STORAGE_ROOT/ssl/ssl_private_key.pem 2048)
-
-	# Give the group 'ssl-cert' read access so slapd can read it
-	groupadd -fr ssl-cert
-	chgrp ssl-cert $STORAGE_ROOT/ssl/ssl_private_key.pem
-	chmod g+r $STORAGE_ROOT/ssl/ssl_private_key.pem
 	
 	# Remove the ssl_certificate.pem symbolic link to force a
 	# regeneration of the server certificate. It needs to be
@@ -110,6 +105,11 @@ if [ ! -s $STORAGE_ROOT/ssl/ssl_private_key.pem ]; then
 	fi
 fi
 
+# Give the group 'ssl-cert' read access so slapd can read it
+groupadd -fr ssl-cert
+chgrp ssl-cert $STORAGE_ROOT/ssl/ssl_private_key.pem
+chmod g+r $STORAGE_ROOT/ssl/ssl_private_key.pem
+
 #
 # Generate a root CA certificate
 #
diff --git a/tests/lib/all.sh b/tests/lib/all.sh
new file mode 100644
index 00000000..20a0c6de
--- /dev/null
+++ b/tests/lib/all.sh
@@ -0,0 +1,16 @@
+#
+# source all lib scripts
+#
+# from your script, supply the path to this directory as the first argument
+#
+#    eg source "tests/lib/all.sh" "tests/lib"
+#
+# failure to load any script is fatal!
+
+. "$1/color-output.sh" || exit 1
+. "$1/locations.sh"    || exit 2
+. "$1/misc.sh"         || exit 3
+. "$1/rest.sh"         || exit 4
+. "$1/system.sh"       || exit 5
+
+
diff --git a/tests/lib/color-output.sh b/tests/lib/color-output.sh
new file mode 100644
index 00000000..3e1954f8
--- /dev/null
+++ b/tests/lib/color-output.sh
@@ -0,0 +1,32 @@
+# ansi escapes for hilighting text
+F_DANGER=$(echo -e "\033[31m")
+F_WARN=$(echo -e "\033[93m")
+F_RESET=$(echo -e "\033[39m")
+
+
+danger() {
+    local echoarg
+    case "$1" in
+        -n )
+            echoarg="$1"
+            shift
+            ;;
+        * )
+            echoarg=""
+    esac
+    echo $echoarg "${F_DANGER}$1${F_RESET}"
+}
+
+warn() {
+    local echoarg
+    case "$1" in
+        -n )
+            echoarg="$1"
+            shift
+            ;;
+        * )
+            echoarg=""
+    esac
+    echo "${F_WARN}$1${F_RESET}"
+}
+
diff --git a/tests/suites/_locations.sh b/tests/lib/locations.sh
similarity index 100%
rename from tests/suites/_locations.sh
rename to tests/lib/locations.sh
diff --git a/tests/lib/misc.sh b/tests/lib/misc.sh
new file mode 100644
index 00000000..4312d6ae
--- /dev/null
+++ b/tests/lib/misc.sh
@@ -0,0 +1,65 @@
+#
+# misc helpful functions
+#
+# requirements:
+#   system packages: [ python3 ]
+
+
+array_contains() {
+	local searchfor="$1"
+	shift
+	local item
+	for item; do
+		[ "$item" == "$searchfor" ] && return 0
+	done
+	return 1
+}
+
+is_true() {
+    # empty string is not true
+    if [ "$1" == "true" \
+              -o "$1" == "TRUE" \
+              -o "$1" == "True" \
+              -o "$1" == "yes" \
+              -o "$1" == "YES" \
+              -o "$1" == "Yes" \
+              -o "$1" == "1" ]
+    then
+        return 0
+    else
+        return 1
+    fi
+}
+
+is_false() {
+    if is_true $@; then return 1; fi
+    return 0
+}
+
+email_localpart() {
+    local addr="$1"
+    awk -F@ '{print $1}' <<<"$addr"
+}
+
+email_domainpart() {
+    local addr="$1"
+    awk -F@ '{print $2}' <<<"$addr"
+}
+
+
+generate_uuid() {
+	local uuid
+	uuid=$(python3 -c "import uuid; print(uuid.uuid4())")
+	[ $? -ne 0 ] && die "Unable to generate a uuid"
+	echo "$uuid"
+}
+
+generate_qa_password() {
+    echo "Test1234."
+}
+
+sha1() {
+	local txt="$1"
+	python3 -c "import hashlib; m=hashlib.sha1(); m.update(bytearray(r'''$txt''','utf-8')); print(m.hexdigest());" || die "Unable to generate sha1 hash"
+}
+
diff --git a/tests/lib/rest.sh b/tests/lib/rest.sh
new file mode 100644
index 00000000..86951dc1
--- /dev/null
+++ b/tests/lib/rest.sh
@@ -0,0 +1,106 @@
+#
+# REST helper functions
+#
+# requirements:
+#   system packages: [ curl ]
+#   lib scripts: [ color-output.sh ]
+#
+
+rest_urlencoded() {
+	# Issue a REST call having data urlencoded
+    #
+    # eg: rest_urlencoded POST /admin/mail/users/add "email=alice@abc.com" "password=secret"
+    #
+    # When providing a URI (/mail/users/add) and not a URL
+    # (https://host/mail/users/add), PRIMARY_HOSTNAME must be set!
+    #
+    # The function will set the following global variables regardless
+    # of exit c ode:
+    #     REST_HTTP_CODE
+    #     REST_OUTPUT
+    #     REST_ERROR
+    # 
+    # Return values:
+    #   0 indicates success (curl returned 0 or a code deemed to be
+    #     successful and HTTP status is >=200  but <300)
+    #   1 curl returned with non-zero code that indicates and error
+    #   2 the response status was <200 or >= 300
+    #
+    # Messages, errors, and output are all sent to stderr, there is no
+    # stdout output
+    #   
+	local verb="$1" # eg "POST"
+	local uri="$2"  # eg "/mail/users/add"
+	local auth_user="$3"
+	local auth_pass="$4"
+	shift; shift; shift; shift  # remaining arguments are data or curl args
+
+	local url
+    case "$uri" in
+        http:* | https:* )
+            url="$uri"
+            ;;
+        * )
+            url="https://$PRIMARY_HOSTNAME${uri}"
+            ;;
+    esac
+    
+	local data=()
+	local item output onlydata="false"
+	
+	for item; do
+        case "$item" in
+            -- )
+                onlydata="true"
+                ;;
+            --* )
+                # curl argument
+                if $onlydata; then
+                    data+=("--data-urlencode" "$item");
+                else
+                    data+=("$item")
+                fi
+                ;;
+            * )
+                onlydata="true"
+                data+=("--data-urlencode" "$item");
+                ;;
+        esac
+    done
+
+	echo "spawn: curl -w \"%{http_code}\" -X $verb --user \"${auth_user}:xxx\" ${data[@]} $url" 1>&2
+	output=$(curl -s -S -w "%{http_code}" -X $verb --user "${auth_user}:${auth_pass}" "${data[@]}" $url)
+	local code=$?
+
+	# http status is last 3 characters of output, extract it
+	REST_HTTP_CODE=$(awk '{S=substr($0,length($0)-2)} END {print S}' <<<"$output")
+	REST_OUTPUT=$(awk 'BEGIN{L=""}{ if(L!="") print L; L=$0 } END { print substr(L,1,length(L)-3) }' <<<"$output")
+	REST_ERROR=""
+	[ -z "$REST_HTTP_CODE" ] && REST_HTTP_CODE="000"
+
+	if [ $code -ne 0 ]; then
+		if [ $code -eq 56 -a $REST_HTTP_CODE -eq 200 ]; then
+			# this is okay, I guess. happens sometimes during
+			# POST /admin/mail/aliases/remove
+			# 56=Unexpected EOF
+			echo "Ignoring curl return code 56 due to 200 status" 1>&2
+			
+		elif [ $code -ne 16 -o $REST_HTTP_CODE -ne 200 ]; then
+			# any error code will fail the rest call except for a 16
+			# with a 200 HTTP status.
+			# 16="a problem was detected in the HTTP2 framing layer. This is somewhat generic and can be one out of several problems"
+			REST_ERROR="CURL failed with code $code"
+			echo "${F_DANGER}$REST_ERROR${F_RESET}" 1>&2
+			echo "$output" 1>&2
+			return 1
+		fi
+	fi
+	if [ $REST_HTTP_CODE -lt 200 -o $REST_HTTP_CODE -ge 300 ]; then
+		REST_ERROR="REST status $REST_HTTP_CODE: $REST_OUTPUT"
+		echo "${F_DANGER}$REST_ERROR${F_RESET}" 1>&2
+		return 2
+	fi
+	echo "CURL succeded, HTTP status $REST_HTTP_CODE" 1>&2
+	echo "$output" 1>&2
+	return 0	
+}
diff --git a/tests/lib/system.sh b/tests/lib/system.sh
new file mode 100644
index 00000000..39695b10
--- /dev/null
+++ b/tests/lib/system.sh
@@ -0,0 +1,102 @@
+
+wait_for_apt() {
+    # check to see if other package managers have a lock on new
+    # installs, and wait for them to finish
+    #
+    # returns non-zero if waiting times out (currently ~600 seconds)
+    local count=0
+    while fuser /var/lib/dpkg/lock >/dev/null 2>&1 || fuser /var/lib/dpkg/lock-frontend >/dev/null 2>&1; do
+        sleep 6
+        let count+=1
+        if [ $count -eq 1 ]; then
+            echo -n "Waiting for other package manager to finish..."
+        elif [ $count -gt 100 ]; then
+            echo -n "FAILED"
+            return 1
+        else
+            echo -n "${count}.."
+        fi
+    done
+    [ $count -ge 1 ] && echo ""
+}
+
+dump_file() {
+    local log_file="$1"
+    local lines="$2"
+    local title="DUMP OF $log_file"
+    echo ""
+    echo "--------"
+    echo -n "-------- $log_file"
+    if [ ! -z "$lines" ]; then
+        echo " (last $line lines)"
+    else
+        echo ""
+    fi
+    echo "--------"
+
+    if [ !-e "$log_file" ]; then
+        echo "DOES NOT EXIST"
+    elif [ ! -z "$lines" ]; then
+        tail -$lines "$log_file"
+    else
+        cat "$log_file"
+    fi
+}
+
+dump_file_if_exists() {
+    [ ! -e "$1" ] && return
+    dump_file "$@"
+}
+
+update_system_time() {
+    if [ ! -x /usr/sbin/ntpdate ]; then
+        wait_for_apt
+        apt-get install -y -qq ntpdate || return 1
+    fi
+    ntpdate -s ntp.ubuntu.com && echo "System time updated"
+}
+
+set_system_hostname() {
+    # set the system hostname to the FQDN specified or
+    # PRIMARY_HOSTNAME if no FQDN was given
+    local fqdn="${1:-$PRIMARY_HOSTNAME}"
+    local host="$(awk -F. '{print $1}' <<< "$fqdn")"
+    sed -i 's/^127\.0\.1\.1[ \t].*/127.0.1.1 '"$fqdn $host ip4-loopback/" /etc/hosts || return 1
+    #hostname "$host" || return 1
+    #echo "$host" > /etc/hostname
+    return 0
+}
+
+install_docker() {
+    if [ -x /usr/bin/docker ]; then
+        echo "Docker already installed"
+        return 0
+    fi
+    
+    wait_for_apt
+    apt-get install -y -qq \
+            apt-transport-https \
+            ca-certificates \
+            curl \
+            gnupg-agent \
+            software-properties-common \
+        || return 1
+       
+    wait_for_apt
+    curl -fsSL https://download.docker.com/linux/ubuntu/gpg | apt-key add - \
+        || return 2
+    
+    wait_for_apt
+    apt-key fingerprint 0EBFCD88 || return 3
+    
+    wait_for_apt
+    add-apt-repository -y --update "deb [arch=amd64] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" || return 4
+    
+    wait_for_apt
+    apt-get install -y -qq \
+            docker-ce \
+            docker-ce-cli \
+            containerd.io \
+        || return 5
+}
+
diff --git a/tests/suites/_init.sh b/tests/suites/_init.sh
index bee4fb20..829db86d 100644
--- a/tests/suites/_init.sh
+++ b/tests/suites/_init.sh
@@ -6,7 +6,7 @@
 set +eu
 
 # load test suite helper functions
-. suites/_locations.sh      || exit 1
+. lib/all.sh "lib"      || exit 1
 . suites/_ldap-functions.sh || exit 1
 . suites/_mail-functions.sh || exit 1
 . suites/_mgmt-functions.sh || exit 1
@@ -20,10 +20,6 @@ declare -i OVERALL_SKIPPED=0
 declare -i OVERALL_COUNT=0
 declare -i OVERALL_COUNT_SUITES=0
 
-# ansi escapes for hilighting text
-F_DANGER=$(echo -e "\033[31m")
-F_WARN=$(echo -e "\033[93m")
-F_RESET=$(echo -e "\033[39m")
 
 # options
 FAILURE_IS_FATAL=no
@@ -157,7 +153,12 @@ test_skip() {
 }
 
 skip_test() {
-	# return 0 if we should skip the current test
+	# call from within a test to check whether the test will be
+	# skipped
+	#
+	# returns 0 if the current test was skipped in which case your test
+	# function must immediately call 'test_end' and return
+	#
 	if [ "$SKIP_REMOTE_SMTP_TESTS" == "yes" ] &&
 		   array_contains "remote-smtp" "$@";
 	then
@@ -191,16 +192,6 @@ die() {
 	exit 1
 }
 
-array_contains() {
-	local searchfor="$1"
-	shift
-	local item
-	for item; do
-		[ "$item" == "$searchfor" ] && return 0
-	done
-	return 1
-}
-
 python_error() {
 	# finds tracebacks and outputs just the final error message of
 	# each
diff --git a/tests/suites/_ldap-functions.sh b/tests/suites/_ldap-functions.sh
index 77e102fe..6c69200d 100644
--- a/tests/suites/_ldap-functions.sh
+++ b/tests/suites/_ldap-functions.sh
@@ -1,16 +1,10 @@
 # -*- indent-tabs-mode: t; tab-width: 4; -*-
 
-generate_uuid() {
-	local uuid
-	uuid=$(python3 -c "import uuid; print(uuid.uuid4())")
-	[ $? -ne 0 ] && die "Unable to generate a uuid"
-	echo "$uuid"
-}
+# requirements:
+#   system packages: [ ldap-utils ]
+#   setup scripts:   [ functions-ldap.sh ]
+#   setup artifacts: [ miab_ldap.conf ]
 
-sha1() {
-	local txt="$1"
-	python3 -c "import hashlib; m=hashlib.sha1(); m.update(bytearray(r'''$txt''','utf-8')); print(m.hexdigest());" || die "Unable to generate sha1 hash"
-}
 
 delete_user() {
 	local email="$1"
diff --git a/tests/suites/_mgmt-functions.sh b/tests/suites/_mgmt-functions.sh
index 07fbdc5b..5ac50701 100644
--- a/tests/suites/_mgmt-functions.sh
+++ b/tests/suites/_mgmt-functions.sh
@@ -44,49 +44,9 @@ mgmt_rest() {
 	local uri="$2"  # eg "/mail/users/add"
 	shift; shift;   # remaining arguments are data
 
-	local auth_user="${MGMT_ADMIN_EMAIL}"
-	local auth_pass="${MGMT_ADMIN_PW}"
-	local url="https://$PRIMARY_HOSTNAME${uri}"
-	local data=()
-	local item output
-	
-	for item; do data+=("--data-urlencode" "$item"); done
-
-	record "spawn: curl -w \"%{http_code}\" -X $verb --user \"${auth_user}:xxx\" ${data[@]} $url"
-	output=$(curl -s -S -w "%{http_code}" -X $verb --user "${auth_user}:${auth_pass}" "${data[@]}" $url 2>>$TEST_OF)
-	local code=$?
-
-	# http status is last 3 characters of output, extract it
-	REST_HTTP_CODE=$(awk '{S=substr($0,length($0)-2)} END {print S}' <<<"$output")
-	REST_OUTPUT=$(awk 'BEGIN{L=""}{ if(L!="") print L; L=$0 } END { print substr(L,1,length(L)-3) }' <<<"$output")
-	REST_ERROR=""
-	[ -z "$REST_HTTP_CODE" ] && REST_HTTP_CODE="000"
-
-	if [ $code -ne 0 ]; then
-		if [ $code -eq 56 -a $REST_HTTP_CODE -eq 200 ]; then
-			# this is okay, I guess. happens sometimes during
-			# POST /admin/mail/aliases/remove
-			# 56=Unexpected EOF
-			record "Ignoring curl return code 56 due to 200 status"
-			
-		elif [ $code -ne 16 -o $REST_HTTP_CODE -ne 200 ]; then
-			# any error code will fail the rest call except for a 16
-			# with a 200 HTTP status.
-			# 16="a problem was detected in the HTTP2 framing layer. This is somewhat generic and can be one out of several problems"
-			REST_ERROR="CURL failed with code $code"
-			record "${F_DANGER}$REST_ERROR${F_RESET}"
-			record "$output"
-			return 1
-		fi
-	fi
-	if [ $REST_HTTP_CODE -lt 200 -o $REST_HTTP_CODE -ge 300 ]; then
-		REST_ERROR="REST status $REST_HTTP_CODE: $REST_OUTPUT"
-		record "${F_DANGER}$REST_ERROR${F_RESET}"
-		return 2
-	fi
-	record "CURL succeded, HTTP status $REST_HTTP_CODE"
-	record "$output"
-	return 0	
+	# call function from lib/rest.sh
+	rest_urlencoded "$verb" "$uri" "${MGMT_ADMIN_EMAIL}" "${MGMT_ADMIN_PW}" "$@" 2>>$TEST_OF
+	return $?
 }
 
 
diff --git a/tests/system-setup/remote-nextcloud-docker.sh b/tests/system-setup/remote-nextcloud-docker.sh
index 35174bdc..1332c64f 100755
--- a/tests/system-setup/remote-nextcloud-docker.sh
+++ b/tests/system-setup/remote-nextcloud-docker.sh
@@ -27,9 +27,9 @@
 
 
 usage() {
-    echo "Usage: $(basename "$0") [\"before-miab-install\"|\"miab-install\"|\"after-miab-install\"]"
-    echo "Install MiaB-LDAP and a remote Nextcloud running under docker exposed as localhost:8000"
-    echo "With no arguments, all three stages are run."
+    echo "Usage: $(basename "$0")"
+    echo "Install MiaB-LDAP and a remote Nextcloud running under docker"
+    echo "Nextcloud is exposed as http://localhost:8000"
     exit 1
 }
 
@@ -40,10 +40,9 @@ if [ ! -d "tests/system-setup" ]; then
 fi
 
 # load helper scripts
-. "tests/system-setup/setup-defaults.sh" \
-    || die "Could not load setup-defaults"
-. "tests/system-setup/setup-funcs.sh" \
-    || die "Could not load setup-funcs"
+. "tests/lib/all.sh" "tests/lib" || die "Could not load lib scripts"
+. "tests/system-setup/setup-defaults.sh" || die "Could not load setup-defaults"
+. "tests/system-setup/setup-funcs.sh" || die "Could not load setup-funcs"
 
 # ensure running as root
 if [ "$EUID" != "0" ]; then
@@ -54,42 +53,15 @@ fi
 
 before_miab_install() {
     H1 "BEFORE MIAB-LDAP INSTALL"
-
-    H2 "Update /etc/hosts"
-    set_system_hostname || die "Could not set hostname"
-
-    # update system time
-    H2 "Set system time"
-    update_system_time || echo "Ignoring error..."
+    system_init
+    miab_testing_init || die "Initialization failed"
     
-    # update package lists before installing anything
-    H2 "apt-get update"
-    wait_for_apt
-    apt-get update -qq || die "apt-get update failed!"
-
-    # upgrade packages - if we don't do this and something like bind
-    # is upgraded through automatic upgrades (because maybe MiaB was
-    # previously installed), it may cause problems with the rest of
-    # the setup, such as with name resolution failures
-    if is_false "$TRAVIS"; then
-        H2 "apt-get upgrade"
-        wait_for_apt
-        apt-get upgrade -qq || die "apt-get upgrade failed!"
-    fi
-    
-    # install prerequisites
-    H2 "QA pre-setup prerequisites"
-    install_pre_setup_qa_prerequisites \
-        || die "Error installing QA prerequisites"
-
     # enable the remote Nextcloud setup mod, which tells MiaB-LDAP to use
     # the remote Nextcloud for calendar and contacts instead of the
     # MiaB-installed one
-    H2 "Create local/remote-nextcloud.sh symbolic link"
-    if [ ! -e "local/remote-nextcloud.sh" ]; then
-        mkdir -p local
-        ln -s "../setup/mods.available/remote-nextcloud.sh" "local/remote-nextcloud.sh" || die "Could not create remote-nextcloud.sh symlink"
-    fi
+    H2 "Enable local mod remote-nextcloud"
+    enable_miab_mod "remote-nextcloud" \
+        || die "Could not enable remote-nextcloud mod"
     
     # install Docker
     H2 "Install Docker"
@@ -101,7 +73,7 @@ miab_install() {
     H1 "MIAB-LDAP INSTALL"
     if ! setup/start.sh; then
         H1 "OUTPUT OF SELECT FILES"
-        dump_log "/var/log/syslog" 100
+        dump_file "/var/log/syslog" 100
         dump_conf_files "$TRAVIS"
         H2; H2 "End"; H2
         die "setup/start.sh failed!"
@@ -119,22 +91,28 @@ after_miab_install() {
     
     # run Nextcloud docker image
     H2 "Start Nextcloud docker container"
-    docker run -d --name NC -p 8000:80 \
-           --env SQLITE_DATABASE=nextclouddb.sqlite \
-           --env NEXTCLOUD_ADMIN_USER="$NC_ADMIN_USER" \
-           --env NEXTCLOUD_ADMIN_PASSWORD="$NC_ADMIN_PASSWORD" \
-           --env NEXTCLOUD_TRUSTED_DOMAINS="127.0.0.1 ::1" \
-           --env NEXTCLOUD_UPDATE=1 \
-           --env SMTP_HOST="$PRIMARY_HOSTNAME" \
-           --env SMTP_SECURE="tls" \
-           --env SMTP_PORT=587 \
-           --env SMTP_AUTHTYPE="LOGIN" \
-           --env SMTP_NAME="$EMAIL_ADDR" \
-           --env SMTP_PASSWORD="$EMAIL_PW" \
-           --env SMTP_FROM_ADDRESS="$(awk -F@ '{print $1}' <<< "$EMAIL_ADDR")" \
-           --env MAIL_DOMAIN="$(awk -F@ '{print $2}' <<< "$EMAIL_ADDR")" \
-           nextcloud:latest \
-        || die "Docker run failed!"
+    local container_started="true"
+    if [ -z "$(docker ps -f NAME=NC -q)" ]; then
+        docker run -d --name NC -p 8000:80 \
+               --env SQLITE_DATABASE=nextclouddb.sqlite \
+               --env NEXTCLOUD_ADMIN_USER="$NC_ADMIN_USER" \
+               --env NEXTCLOUD_ADMIN_PASSWORD="$NC_ADMIN_PASSWORD" \
+               --env NEXTCLOUD_TRUSTED_DOMAINS="127.0.0.1 ::1" \
+               --env NEXTCLOUD_UPDATE=1 \
+               --env SMTP_HOST="$PRIMARY_HOSTNAME" \
+               --env SMTP_SECURE="tls" \
+               --env SMTP_PORT=587 \
+               --env SMTP_AUTHTYPE="LOGIN" \
+               --env SMTP_NAME="$EMAIL_ADDR" \
+               --env SMTP_PASSWORD="$EMAIL_PW" \
+               --env SMTP_FROM_ADDRESS="$(email_localpart "$EMAIL_ADDR")" \
+               --env MAIL_DOMAIN="$(email_domainpart "$EMAIL_ADDR")" \
+               nextcloud:latest \
+            || die "Docker run failed!"
+    else
+        echo "Container already running"
+        container_started="false"
+    fi
 
     H2 "docker: Update /etc/hosts so it can find MiaB-LDAP by name"
     echo "$PRIVATE_IP $PRIMARY_HOSTNAME" | \
@@ -160,32 +138,20 @@ after_miab_install() {
 
     # wait for Nextcloud installation to complete
     H2 "Wait for Nextcloud installation to complete"
-    echo -n "Waiting ..."
-    local count=0
-    while true; do
-        if [ $count -ge 10 ]; then
-            echo "FAILED"
-            die "Giving up"
-        fi
-        sleep 6
-        let count+=1
-        if [ $(docker exec NC php -n -r "include 'config/config.php'; print \$CONFIG['installed']?'true':'false';") == "true" ]; then
-            echo "ok"
-            break
-        fi
-        echo -n "${count}..."
-    done
+    wait_for_docker_nextcloud NC installed || die "Giving up"
     
-    # install and enable Nextcloud and apps
+    # install and enable Nextcloud apps
     H2 "docker: install Nextcloud calendar app"
     docker exec -u www-data NC ./occ app:install calendar \
-        || die "docker: installing calendar app failed"
+        || $container_started \
+            && die "docker: installing calendar app failed ($?)"
     H2 "docker: install Nextcloud contacts app"
     docker exec -u www-data NC ./occ app:install contacts \
-        || die "docker: installing contacts app failed"
+        || $container_started \
+            && die "docker: installing contacts app failed ($?)"
     H2 "docker: enable user_ldap"
     docker exec -u www-data NC ./occ app:enable user_ldap \
-        || die "docker: enabling user_ldap failed"
+        || die "docker: enabling user_ldap failed ($?)"
 
     # integrate Nextcloud with MiaB-LDAP
     H2 "docker: integrate Nextcloud with MiaB-LDAP"
@@ -201,28 +167,26 @@ after_miab_install() {
 }
 
 
-
 #
-# process command line
+# Main
 #
-
-case "$1" in
-    before-miab-install )
+case "${1:-all}" in
+    before-install )
         before_miab_install
         ;;
-    after-miab-install )
-        after_miab_install
-        ;;
-    miab-install )
+    install )
         miab_install
         ;;
-    "" )
+    after-install )
+        after_miab_install
+        ;;
+    all )
         before_miab_install
         miab_install
         after_miab_install
         ;;
     * )
-        
         usage
         ;;
 esac
+
diff --git a/tests/system-setup/setup-defaults.sh b/tests/system-setup/setup-defaults.sh
index 739a0260..b415e7a0 100755
--- a/tests/system-setup/setup-defaults.sh
+++ b/tests/system-setup/setup-defaults.sh
@@ -1,6 +1,6 @@
 #!/bin/bash
 
-# Used by MiaB-LDAP setup/start.sh
+# Used by setup/start.sh
 export NONINTERACTIVE=${NONINTERACTIVE:-1}
 export SKIP_NETWORK_CHECKS=${SKIP_NETWORK_CHECKS:-1}
 export STORAGE_USER="${STORAGE_USER:-user-data}"
@@ -15,7 +15,6 @@ elif [ -z "$PRIMARY_HOSTNAME" ]; then
     export PRIMARY_HOSTNAME=${PRIMARY_HOSTNAME:-$(hostname --fqdn || hostname)}
 fi
 
-
 # Placing this var in STORAGE_ROOT/ldap/miab_ldap.conf before running
 # setup/start.sh will avoid a random password from being used for the
 # Nextcloud LDAP service account
@@ -28,6 +27,9 @@ export NC_HOST=${NC_HOST:-127.0.0.1}
 export NC_PORT=${NC_PORT:-8000}
 export NC_PREFIX=${NC_PREFIX:-/}
 
-# For setup scripts that are installing a remote Nextcloud
+# For setup scripts that may be installing a remote Nextcloud
 export NC_ADMIN_USER="${NC_ADMIN_USER:-admin}"
 export NC_ADMIN_PASSWORD="${NC_ADMIN_PASSWORD:-Test_1234}"
+
+# For setup scripts that install upstream versions
+export MIAB_UPSTREAM_GIT="https://github.com/mail-in-a-box/mailinabox.git"
diff --git a/tests/system-setup/setup-funcs.sh b/tests/system-setup/setup-funcs.sh
index 9f8194d5..c04a4fa9 100755
--- a/tests/system-setup/setup-funcs.sh
+++ b/tests/system-setup/setup-funcs.sh
@@ -1,4 +1,9 @@
 
+#
+# requires:
+#
+#   test scripts: [ lib/misc.sh, lib/system.sh ]
+#
 
 die() {
     local msg="$1"
@@ -25,64 +30,28 @@ H2() {
     fi
 }
 
-dump_log() {
-    local log_file="$1"
-    local lines="$2"
-    local title="DUMP OF $log_file"
-    echo ""
-    echo "--------"
-    echo -n "-------- $log_file"
-    if [ ! -z "$lines" ]; then
-        echo " (last $line lines)"
-    else
-        echo ""
-    fi
-    echo "--------"
-      
-    if [ ! -z "$lines" ]; then
-        tail -$lines "$log_file"
-    else
-        cat "$log_file"
-    fi
-}
 
-is_true() {
-    # empty string is not true
-    if [ "$1" == "true" \
-              -o "$1" == "TRUE" \
-              -o "$1" == "True" \
-              -o "$1" == "yes" \
-              -o "$1" == "YES" \
-              -o "$1" == "Yes" \
-              -o "$1" == "1" ]
-    then
-        return 0
-    else
-        return 1
-    fi
-}
-
-is_false() {
-    if is_true $@; then return 1; fi
+wait_for_docker_nextcloud() {
+    local container="$1"
+    local config_key="$2"
+    echo -n "Waiting ..."
+    local count=0
+    while true; do
+        if [ $count -ge 10 ]; then
+            echo "FAILED"
+            return 1
+        fi
+        sleep 6
+        let count+=1
+        if [ $(docker exec "$container" php -n -r "include 'config/config.php'; print \$CONFIG['$config_key']?'true':'false';") == "true" ]; then
+            echo "ok"
+            break
+        fi
+        echo -n "${count}..."
+    done
     return 0
 }
 
-wait_for_apt() {
-    local count=0
-    while fuser /var/lib/dpkg/lock >/dev/null 2>&1 || fuser /var/lib/dpkg/lock-frontend >/dev/null 2>&1; do
-        sleep 6
-        let count+=1
-        if [ $count -eq 1 ]; then
-            echo -n "Waiting for other package manager to finish..."
-        elif [ $count -gt 100 ]; then
-            echo -n "FAILED"
-            return 1
-        else
-            echo -n "${count}.."
-        fi
-    done
-    [ $count -ge 1 ] && echo ""
-}
 
 dump_conf_files() {
     local skip
@@ -98,99 +67,58 @@ dump_conf_files() {
         done
     fi
     if [ "$skip" == "false" ]; then
-        dump_log "/etc/mailinabox.conf"
-        dump_log "/etc/hosts"
-        dump_log "/etc/nsswitch.conf"
-        dump_log "/etc/resolv.conf"
-        dump_log "/etc/nsd/nsd.conf"
-        dump_log "/etc/postfix/main.cf"
+        dump_file "/etc/mailinabox.conf"
+        dump_file_if_exists "/etc/mailinabox_mods.conf"
+        dump_file "/etc/hosts"
+        dump_file "/etc/nsswitch.conf"
+        dump_file "/etc/resolv.conf"
+        dump_file "/etc/nsd/nsd.conf"
+        #dump_file "/etc/postfix/main.cf"
     fi
 }
 
 
-update_system_time() {
-    if [ ! -x /usr/sbin/ntpdate ]; then
+
+#
+# Initialize the test system
+#   hostname, time, apt update/upgrade, etc
+#
+system_init() {
+    H2 "Update /etc/hosts"
+    set_system_hostname || die "Could not set hostname"
+
+    # update system time
+    H2 "Set system time"
+    update_system_time || echo "Ignoring error..."
+    
+    # update package lists before installing anything
+    H2 "apt-get update"
+    wait_for_apt
+    apt-get update -qq || die "apt-get update failed!"
+
+    # upgrade packages - if we don't do this and something like bind
+    # is upgraded through automatic upgrades (because maybe MiaB was
+    # previously installed), it may cause problems with the rest of
+    # the setup, such as with name resolution failures
+    if is_false "$TRAVIS"; then
+        H2 "apt-get upgrade"
         wait_for_apt
-        apt-get install -y -qq ntpdate || return 1
+        apt-get upgrade -qq || die "apt-get upgrade failed!"
     fi
-    ntpdate -s ntp.ubuntu.com && echo "System time updated"
-}
-
-update_hosts() {
-    local host="$1"
-    shift
-    local ip
-    for ip; do
-        if [ ! -z "$ip" ]; then
-            local line="$ip $host"
-            if ! grep -F "$line" /etc/hosts 1>/dev/null; then
-                echo "$line" >>/etc/hosts
-            fi
-        fi
-    done
-}
-
-update_hosts_for_private_ip() {
-    # create /etc/hosts entry for PRIVATE_IP and PRIVATE_IPV6
-    # PRIMARY_HOSTNAME must already be set
-    local ip4=$(source setup/functions.sh; get_default_privateip 4)
-    local ip6=$(source setup/functions.sh; get_default_privateip 6)
-    [ -z "$ip4" -a -z "$ip6" ] && return 1
-    [ -z "$ip6" ] && ip6="::1"
-    update_hosts "$PRIMARY_HOSTNAME" "$ip4" "$ip6" || return 1
-}
-
-set_system_hostname() {
-    # set the system hostname to the FQDN specified or
-    # PRIMARY_HOSTNAME if no FQDN was given
-    local fqdn="${1:-$PRIMARY_HOSTNAME}"
-    local host="$(awk -F. '{print $1}' <<< "$fqdn")"
-    sed -i 's/^127\.0\.1\.1[ \t].*/127.0.1.1 '"$fqdn $host ip4-loopback/" /etc/hosts || return 1
-    #hostname "$host" || return 1
-    #echo "$host" > /etc/hostname
-    return 0
 }
 
 
-install_docker() {
-    if [ -x /usr/bin/docker ]; then
-        echo "Docker already installed"
-        return 0
-    fi
-    
-    wait_for_apt
-    apt-get install -y -qq \
-            apt-transport-https \
-            ca-certificates \
-            curl \
-            gnupg-agent \
-            software-properties-common \
-        || return 1
-       
-    wait_for_apt
-    curl -fsSL https://download.docker.com/linux/ubuntu/gpg | apt-key add - \
-        || return 2
-    
-    wait_for_apt
-    apt-key fingerprint 0EBFCD88 || return 3
-    
-    wait_for_apt
-    add-apt-repository -y --update "deb [arch=amd64] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" || return 4
-    
-    wait_for_apt
-    apt-get install -y -qq \
-            docker-ce \
-            docker-ce-cli \
-            containerd.io \
-        || return 5
-}
-
-
-install_pre_setup_qa_prerequisites() {
+#
+# Initialize the test system with QA prerequisites
+# Anything needed to use the test runner, speed up the installation,
+# etc
+#
+miab_testing_init() {
     [ -z "$STORAGE_ROOT" ] \
         && echo "Error: STORAGE_ROOT not set" 1>&2 \
         && return 1
 
+    H2 "QA prerequisites"
     local rc=0
     
     # python3-dnspython: is used by the python scripts in 'tests' and is
@@ -221,22 +149,20 @@ install_pre_setup_qa_prerequisites() {
 }
 
 
-travis_fix_nsd() {
-    if [ "$TRAVIS" != "true" ]; then
-        return 0
+enable_miab_mod() {
+    local name="${1}.sh"
+    if [ ! -e "local/$name" ]; then
+        mkdir -p local
+        ln -s "../setup/mods.available/$name" "local/$name"
     fi
-    
-    # nsd won't start on Travis-CI without the changes below: ip6 off and
-    # control-enable set to no. Even though the nsd docs say the
-    # default value for control-enable is no, running "nsd-checkconf -o
-    # control-enable /etc/nsd/nsd.conf" returns "yes", so we explicitly
-    # set it here.
-    #
-    # we're assuming that the "ip-address" line is the last line in the
-    # "server" section of nsd.conf. if this generated file output
-    # changes, the sed command below may need to be adjusted.
-    sed -i 's/ip-address\(.\)\(.*\)/ip-address\1\2\n  do-ip4\1 yes\n  do-ip6\1 no\n  verbosity\1 3\nremote-control\1\n  control-enable\1 no/' /etc/nsd/nsd.conf || return 1
-    cat /etc/nsd/nsd.conf
-    systemctl reset-failed nsd.service || return 2
-    systemctl restart nsd.service || return 3
 }
+
+tag_from_readme() {
+    # extract the recommended TAG from README.md
+    # sets a global "TAG"
+    local readme="${1:-README.md}"
+    TAG="$(grep -F 'git checkout' "$readme" | sed 's/.*\(v[0123456789]*\.[0123456789]*\).*/\1/')"
+    [ $? -ne 0 -o -z "$TAG" ] && return 1
+    return 0
+}
+
diff --git a/tests/system-setup/upgrade-from-upstream.sh b/tests/system-setup/upgrade-from-upstream.sh
new file mode 100755
index 00000000..45f40fcb
--- /dev/null
+++ b/tests/system-setup/upgrade-from-upstream.sh
@@ -0,0 +1,289 @@
+#!/bin/bash
+
+# setup MiaB-LDAP by:
+#    1. installing upstream MiaB
+#    2. adding some data (users/aliases/etc)
+#    3. upgrading to MiaB-LDAP
+#
+# See setup-defaults.sh for usernames and passwords.
+#
+
+
+usage() {
+    echo "Usage: $(basename "$0")"
+    echo "Install MiaB-LDAP after installing upstream MiaB"
+    exit 1
+}
+
+# ensure working directory
+if [ ! -d "tests/system-setup" ]; then
+    echo "This script must be run from the MiaB root directory"
+    exit 1
+fi
+
+# load helper scripts
+. "tests/lib/all.sh" "tests/lib" || die "Could not load lib scripts"
+. "tests/system-setup/setup-defaults.sh" || die "Could not load setup-defaults"
+. "tests/system-setup/setup-funcs.sh" || die "Could not load setup-funcs"
+
+# ensure running as root
+if [ "$EUID" != "0" ]; then
+    die "This script must be run as root (sudo)"
+fi
+
+
+before_install() {
+    H1 "INIT"
+    system_init
+    miab_testing_init || die "Initialization failed"
+}
+
+upstream_install() {
+    local upstream_dir="$HOME/mailinabox-upstream"
+    H1 "INSTALL UPSTREAM"
+    [ ! -x /usr/bin/git ] && apt-get install -y -qq git
+    
+    if [ ! -d "$upstream_dir" ] || [ -z "$(ls -A "$upstream_dir")" ] ; then
+        H2 "Cloning $MIAB_UPSTREAM_GIT"
+        rm -rf "$upstream_dir"
+        git clone "$MIAB_UPSTREAM_GIT" "$upstream_dir"
+        if [ $? -ne 0 ]; then
+            rm -rf "$upstream_dir"
+            die "git clone upstream failed!"
+        fi
+        if [ -z "$TAG" ]; then
+            tag_from_readme "$upstream_dir/README.md"
+            if [ $? -ne 0 ]; then
+                rm -rf "$upstream_dir"
+                die "Failed to extract TAG from $upstream_dir/README.md"
+            fi
+        fi
+    fi
+
+    pushd "$upstream_dir" >/dev/null
+    if [ ! -z "$TAG" ]; then
+        H2 "Checkout $TAG"
+        git checkout "$TAG" || die "git checkout $TAG failed"
+    fi
+    
+    H2 "Run upstream setup"
+    setup/start.sh || die "Upstream setup failed!"
+    popd >/dev/null
+    
+    H2 "Upstream info"
+    echo "Code version: $(git describe)"
+    echo "Migration version: $(cat "$STORAGE_ROOT/mailinabox.version")"
+}
+
+
+add_data() {
+    H1 "Add some Mail-in-a-Box data"
+    local users=()
+    users+="betsy@$(email_domainpart "$EMAIL_ADDR")"
+
+    local alises=()
+    aliases+="goalias@testdom.com > ${users[0]}"
+    aliases+="nested@testdom.com > goalias@testdom.com"
+
+    local pw="$(generate_qa_password)"
+
+
+    #
+    # get the existing users and aliases
+    #
+    local current_users=() current_aliases=()
+    local user alias
+    if ! rest_urlencoded GET /admin/mail/users "$EMAIL_ADDR" "$EMAIL_PW" --insecure 2>/dev/null; then
+        die "Unable to enumerate users: rc=$? err=$REST_ERROR"
+    fi
+    for user in $REST_OUTPUT; do
+        current_users+=("$user")
+    done
+
+    if ! rest_urlencoded GET /admin/mail/aliases "$EMAIL_ADDR" "$EMAIL_PW" --insecure 2>/dev/null; then
+        die "Unable to enumerate aliases: rc=$? err=$REST_ERROR"
+    fi
+    for alias in $REST_OUTPUT; do
+        current_aliases+=("$alias")
+    done
+
+    
+    #
+    # add users
+    #
+    for user in "${users[@]}"; do
+        if array_contains "$user" "${current_users[@]}"; then
+            echo "Not adding user $user: already exists"
+
+        elif ! rest_urlencoded POST /admin/mail/users/add "$EMAIL_ADDR" "$EMAIL_PW" --insecure -- "email=$user" "password=$pw" 2>/dev/null
+        then
+            die "Unable to add user $user: rc=$? err=$REST_ERROR"
+        fi
+    done
+
+    #
+    # add aliases
+    #
+    local aliasdef
+    for aliasdef in "${aliases[@]}"; do
+        alias="$(awk -F'[> ]' '{print $1}' <<<"$aliasdef")"
+        local forwards_to="$(sed 's/.*> *\(.*\)/\1/' <<<"$aliasdef")"
+        if array_contains "$alias" "${current_aliases[@]}"; then
+            echo "Not adding alias $alias: already exists"
+            
+        elif ! rest_urlencoded POST /admin/mail/aliases/add "$EMAIL_ADDR" "$EMAIL_PW" --insecure -- "address=$alias" "forwards_to=$forwards_to" 2>/dev/null
+        then 
+            die "Unable to add alias $alias: rc=$? err=$REST_ERROR"
+        fi
+    done
+}
+
+capture_state() {
+    # users and aliases lists
+    # dns zone files
+    # tls certificates: expected CN's
+
+    local state_dir="$1"
+    local infojson="$state_dir/info.json"
+
+    H1 "Capture server state to $state_dir"
+    
+    # nuke saved state, if any
+    rm -rf "$state_dir"
+    mkdir -p "$state_dir"
+
+    # create info.json
+    H2 "create info.json"
+    echo "VERSION='$(git describe --abbrev=0)'" >"$infojson"
+    echo "MIGRATION_VERSION=$(cat "$STORAGE_ROOT/mailinabox.version")" >>"$infojson"
+
+    # record users
+    H2 "record users"
+    rest_urlencoded GET "/admin/mail/users?format=json" "$EMAIL_ADDR" "$EMAIL_PW" --insecure 2>/dev/null \
+        || die "Unable to get users: rc=$? err=$REST_ERROR"
+    echo "$REST_OUTPUT" > "$state_dir/users.json"
+
+    # record aliases
+    H2 "record aliases"
+    rest_urlencoded GET "/admin/mail/aliases?format=json" "$EMAIL_ADDR" "$EMAIL_PW" --insecure 2>/dev/null \
+        || die "Unable to get aliases: rc=$? err=$REST_ERROR"
+    echo "$REST_OUTPUT" > "$state_dir/aliases.json"
+
+    # record dns config
+    H2 "record dns details"
+    local file
+    mkdir -p "$state_dir/zones"
+    for file in ls /etc/nsd/zones/*.signed; do
+        cp "$file" "$state_dir/zones"
+    done
+    
+}
+
+miab_ldap_install() {
+    # ensure we're in a MiaB-LDAP working directory
+    if [ ! -e setup/ldap.sh ]; then
+        die "The working directory is not MiaB-LDAP!"
+    fi
+    setup/start.sh -v || die "Upgrade to MiaB-LDAP failed !!!!!!"
+}
+
+compare_state() {
+    local s1="$1"
+    local s2="$2"
+    
+    local output
+    local changed="false"
+
+    H1 "COMPARE STATES $(basename "$s1") TO $(basename "$2")"
+    H2 "Users"
+    # users
+    output="$(diff "$s1/users.json" "$s2/users.json" 2>&1)"
+    if [ $? -ne 0 ]; then
+        changed="true"
+        echo "USERS ARE DIFFERENT!"
+        echo "$output"
+    else
+        echo "OK"
+    fi
+
+    H2 "Aliases"
+    output="$(diff "$s1/aliases.json" "$s2/aliases.json" 2>&1)"
+    if [ $? -ne 0 ]; then
+        change="true"
+        echo "ALIASES ARE DIFFERENT!"
+        echo "$output"
+    else
+        echo "OK"
+    fi
+
+    H2 "DNS - zones missing"
+    local zone
+    for zone in $(cd "$s1/zones"; ls *.signed); do
+        if [ ! -e "$s2/zones/$zone" ]; then
+            echo "MISSING zone: $zone"
+            changed="true"
+        fi
+    done
+
+    H2 "DNS - zones added"
+    for zone in $(cd "$s2/zones"; ls *.signed); do
+        if [ ! -e "$s2/zones/$zone" ]; then
+            echo "ADDED zone: $zone"
+            changed="true"
+        fi
+    done
+
+    H2 "DNS - zones changed"
+    for zone in $(cd "$s1/zones"; ls *.signed); do
+        if [ -e "$s2/zones/$zone" ]; then
+            output="$(diff "$s1/zones/$zone" "$s2/zones/$zone" 2>&1)"
+            if [ $? -ne 0 ]; then
+                echo "CHANGED zone: $zone"
+                echo "$output"
+                changed="true"
+            fi
+        fi
+    done
+
+    if $changed; then
+        return 1
+    else
+        return 0
+    fi
+}
+
+
+if [ "$1" == "c" ]; then
+    capture_state "tests/system-setup/state/miab-ldap"
+    exit $?
+fi
+
+
+# install basic stuff, set the hostname, time, etc
+before_install
+
+# if MiaB-LDAP is already migrated, do not run upstream setup
+if [ -e "$STORAGE_ROOT/mailinabox.version" ] &&
+       [ $(cat "$STORAGE_ROOT/mailinabox.version") -ge 13 ]
+then
+    echo "Warning: MiaB-LDAP is already installed! Skipping installation of upstream"
+else
+    # install upstream
+    upstream_install
+    add_data
+    capture_state "tests/system-setup/state/upstream"
+fi
+
+# install miab-ldap
+miab_ldap_install
+capture_state "tests/system-setup/state/miab-ldap"
+
+# compare states
+if ! compare_state "tests/system-setup/state/upstream" "tests/system-setup/state/miab-ldap"; then
+    die "Upstream and upgraded states are different !"
+fi
+
+#
+# actual verification that mail sends/receives properly is done via
+# the test runner ...
+#

From 6a93af9670030b3648e4754f1dee88ec95578a53 Mon Sep 17 00:00:00 2001
From: downtownallday <downtownallday@gmail.com>
Date: Sun, 14 Jun 2020 16:48:21 -0400
Subject: [PATCH 50/65] Fix upgrade job failure

---
 .travis.yml                                   | 10 +++-
 setup/munin.sh                                |  1 +
 setup/nextcloud.sh                            |  1 +
 tests/lib/rest.sh                             | 10 ++--
 tests/suites/_mgmt-functions.sh               |  2 +-
 tests/system-setup/remote-nextcloud-docker.sh | 16 +++---
 tests/system-setup/upgrade-from-upstream.sh   | 49 +++++++++++++------
 7 files changed, 61 insertions(+), 28 deletions(-)

diff --git a/.travis.yml b/.travis.yml
index e4fba7ec..f435280e 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -11,7 +11,9 @@ jobs:
   fast_finish: true
   include:
   # JOB: MiaB-LDAP connected to a remote Nextcloud
-  - env: PRIMARY_HOSTNAME=box1.abc.com
+  - env:
+      - PRIMARY_HOSTNAME=box1.abc.com
+      - FEATURE_MUNIN=false
     name: remote-nextcloud-docker
     before_install:
       - echo "==== ENVIRONMENT ===="
@@ -35,7 +37,11 @@ jobs:
       - sudo tests/runner.sh -dumpoutput -no-smtp-remote default remote-nextcloud
 
   # JOB: Upgrade from upstream install
-  - env: PRIMARY_HOSTNAME=box2.abc.com
+  - env:
+      - PRIMARY_HOSTNAME=box2.abc.com
+      - UPSTREAM_TAG=master
+      - FEATURE_NEXTCLOUD=false
+      - FEATURE_MUNIN=false
     name: upgrade-from-upstream
     install:
       - sudo tests/system-setup/upgrade-from-upstream.sh
diff --git a/setup/munin.sh b/setup/munin.sh
index 6799cad6..79974dc6 100755
--- a/setup/munin.sh
+++ b/setup/munin.sh
@@ -1,6 +1,7 @@
 #!/bin/bash
 # Munin: resource monitoring tool
 #################################################
+[ "${FEATURE_MUNIN:-true}" == "false" ] && return 0
 
 source setup/functions.sh # load our functions
 source /etc/mailinabox.conf # load global vars
diff --git a/setup/nextcloud.sh b/setup/nextcloud.sh
index 6bf7899c..88a967f5 100755
--- a/setup/nextcloud.sh
+++ b/setup/nextcloud.sh
@@ -1,6 +1,7 @@
 #!/bin/bash
 # Nextcloud
 ##########################
+[ "${FEATURE_NEXTCLOUD:-true}" == "false" ] && return 0
 
 source setup/functions.sh # load our functions
 source /etc/mailinabox.conf # load global vars
diff --git a/tests/lib/rest.sh b/tests/lib/rest.sh
index 86951dc1..dfbccb1b 100644
--- a/tests/lib/rest.sh
+++ b/tests/lib/rest.sh
@@ -3,7 +3,7 @@
 #
 # requirements:
 #   system packages: [ curl ]
-#   lib scripts: [ color-output.sh ]
+#   lib scripts: [ system.sh, color-output.sh ]
 #
 
 rest_urlencoded() {
@@ -26,8 +26,7 @@ rest_urlencoded() {
     #   1 curl returned with non-zero code that indicates and error
     #   2 the response status was <200 or >= 300
     #
-    # Messages, errors, and output are all sent to stderr, there is no
-    # stdout output
+    # Messages and errors are sent to stderr
     #   
 	local verb="$1" # eg "POST"
 	local uri="$2"  # eg "/mail/users/add"
@@ -36,12 +35,14 @@ rest_urlencoded() {
 	shift; shift; shift; shift  # remaining arguments are data or curl args
 
 	local url
+    local is_local="false"
     case "$uri" in
         http:* | https:* )
             url="$uri"
             ;;
         * )
             url="https://$PRIMARY_HOSTNAME${uri}"
+            is_local="true"
             ;;
     esac
     
@@ -98,6 +99,9 @@ rest_urlencoded() {
 	if [ $REST_HTTP_CODE -lt 200 -o $REST_HTTP_CODE -ge 300 ]; then
 		REST_ERROR="REST status $REST_HTTP_CODE: $REST_OUTPUT"
 		echo "${F_DANGER}$REST_ERROR${F_RESET}" 1>&2
+        if $is_local && [ $REST_HTTP_CODE -ge 500 ]; then
+            tail -100 /var/log/syslog
+        fi
 		return 2
 	fi
 	echo "CURL succeded, HTTP status $REST_HTTP_CODE" 1>&2
diff --git a/tests/suites/_mgmt-functions.sh b/tests/suites/_mgmt-functions.sh
index 5ac50701..422e447e 100644
--- a/tests/suites/_mgmt-functions.sh
+++ b/tests/suites/_mgmt-functions.sh
@@ -45,7 +45,7 @@ mgmt_rest() {
 	shift; shift;   # remaining arguments are data
 
 	# call function from lib/rest.sh
-	rest_urlencoded "$verb" "$uri" "${MGMT_ADMIN_EMAIL}" "${MGMT_ADMIN_PW}" "$@" 2>>$TEST_OF
+	rest_urlencoded "$verb" "$uri" "${MGMT_ADMIN_EMAIL}" "${MGMT_ADMIN_PW}" "$@" >>$TEST_OF 2>&1
 	return $?
 }
 
diff --git a/tests/system-setup/remote-nextcloud-docker.sh b/tests/system-setup/remote-nextcloud-docker.sh
index 1332c64f..e24900e4 100755
--- a/tests/system-setup/remote-nextcloud-docker.sh
+++ b/tests/system-setup/remote-nextcloud-docker.sh
@@ -142,13 +142,17 @@ after_miab_install() {
     
     # install and enable Nextcloud apps
     H2 "docker: install Nextcloud calendar app"
-    docker exec -u www-data NC ./occ app:install calendar \
-        || $container_started \
-            && die "docker: installing calendar app failed ($?)"
+    if ! docker exec -u www-data NC ./occ app:install calendar
+    then
+        $container_started || die "docker: installing calendar app failed"
+    fi
+    
     H2 "docker: install Nextcloud contacts app"
-    docker exec -u www-data NC ./occ app:install contacts \
-        || $container_started \
-            && die "docker: installing contacts app failed ($?)"
+    if ! docker exec -u www-data NC ./occ app:install contacts
+    then
+        $container_started || die "docker: installing contacts app failed"
+    fi
+    
     H2 "docker: enable user_ldap"
     docker exec -u www-data NC ./occ app:enable user_ldap \
         || die "docker: enabling user_ldap failed ($?)"
diff --git a/tests/system-setup/upgrade-from-upstream.sh b/tests/system-setup/upgrade-from-upstream.sh
index 45f40fcb..0e321e49 100755
--- a/tests/system-setup/upgrade-from-upstream.sh
+++ b/tests/system-setup/upgrade-from-upstream.sh
@@ -51,7 +51,7 @@ upstream_install() {
             rm -rf "$upstream_dir"
             die "git clone upstream failed!"
         fi
-        if [ -z "$TAG" ]; then
+        if [ -z "$UPSTREAM_TAG" ]; then
             tag_from_readme "$upstream_dir/README.md"
             if [ $? -ne 0 ]; then
                 rm -rf "$upstream_dir"
@@ -61,9 +61,9 @@ upstream_install() {
     fi
 
     pushd "$upstream_dir" >/dev/null
-    if [ ! -z "$TAG" ]; then
-        H2 "Checkout $TAG"
-        git checkout "$TAG" || die "git checkout $TAG failed"
+    if [ ! -z "$UPSTREAM_TAG" ]; then
+        H2 "Checkout $UPSTREAM_TAG"
+        git checkout "$UPSTREAM_TAG" || die "git checkout $UPSTREAM_TAG failed"
     fi
     
     H2 "Run upstream setup"
@@ -93,7 +93,7 @@ add_data() {
     #
     local current_users=() current_aliases=()
     local user alias
-    if ! rest_urlencoded GET /admin/mail/users "$EMAIL_ADDR" "$EMAIL_PW" --insecure 2>/dev/null; then
+    if ! rest_urlencoded GET /admin/mail/users "$EMAIL_ADDR" "$EMAIL_PW" --insecure >/dev/null 2>&1; then
         die "Unable to enumerate users: rc=$? err=$REST_ERROR"
     fi
     for user in $REST_OUTPUT; do
@@ -132,7 +132,7 @@ add_data() {
             echo "Not adding alias $alias: already exists"
             
         elif ! rest_urlencoded POST /admin/mail/aliases/add "$EMAIL_ADDR" "$EMAIL_PW" --insecure -- "address=$alias" "forwards_to=$forwards_to" 2>/dev/null
-        then 
+        then
             die "Unable to add alias $alias: rc=$? err=$REST_ERROR"
         fi
     done
@@ -144,7 +144,7 @@ capture_state() {
     # tls certificates: expected CN's
 
     local state_dir="$1"
-    local infojson="$state_dir/info.json"
+    local info="$state_dir/info.txt"
 
     H1 "Capture server state to $state_dir"
     
@@ -153,9 +153,9 @@ capture_state() {
     mkdir -p "$state_dir"
 
     # create info.json
-    H2 "create info.json"
-    echo "VERSION='$(git describe --abbrev=0)'" >"$infojson"
-    echo "MIGRATION_VERSION=$(cat "$STORAGE_ROOT/mailinabox.version")" >>"$infojson"
+    H2 "create info.txt"
+    echo "VERSION='$(git describe --abbrev=0)'" >"$info"
+    echo "MIGRATION_VERSION=$(cat "$STORAGE_ROOT/mailinabox.version")" >>"$info"
 
     # record users
     H2 "record users"
@@ -173,10 +173,9 @@ capture_state() {
     H2 "record dns details"
     local file
     mkdir -p "$state_dir/zones"
-    for file in ls /etc/nsd/zones/*.signed; do
+    for file in /etc/nsd/zones/*.signed; do
         cp "$file" "$state_dir/zones"
-    done
-    
+    done    
 }
 
 miab_ldap_install() {
@@ -217,33 +216,46 @@ compare_state() {
     fi
 
     H2 "DNS - zones missing"
-    local zone
+    local zone count=0
     for zone in $(cd "$s1/zones"; ls *.signed); do
         if [ ! -e "$s2/zones/$zone" ]; then
             echo "MISSING zone: $zone"
             changed="true"
+            let count+=1
         fi
     done
+    echo "$count missing"
 
     H2 "DNS - zones added"
+    count=0
     for zone in $(cd "$s2/zones"; ls *.signed); do
         if [ ! -e "$s2/zones/$zone" ]; then
             echo "ADDED zone: $zone"
             changed="true"
+            let count+=1
         fi
     done
+    echo "$count added"
 
     H2 "DNS - zones changed"
+    count=0
     for zone in $(cd "$s1/zones"; ls *.signed); do
         if [ -e "$s2/zones/$zone" ]; then
-            output="$(diff "$s1/zones/$zone" "$s2/zones/$zone" 2>&1)"
+            # all the signatures change if we're using self-signed certs
+            local t1="/tmp/s1.$$.txt"
+            local t2="/tmp/s2.$$.txt"
+            awk '$4 == "RRSIG" || $4 == "NSEC3" { next; } $4 == "SOA" { print $1" "$2" "$3" "$4" "$5" "$6" "$8" "$9" "$10" "$11" "$12; next } { print $0 }' "$s1/zones/$zone" > "$t1" 
+            awk '$4 == "RRSIG" || $4 == "NSEC3" { next; } $4 == "SOA" { print $1" "$2" "$3" "$4" "$5" "$6" "$8" "$9" "$10" "$11" "$12; next } { print $0 }' "$s2/zones/$zone" > "$t2" 
+            output="$(diff "$t1" "$t2" 2>&1)"
             if [ $? -ne 0 ]; then
                 echo "CHANGED zone: $zone"
                 echo "$output"
                 changed="true"
+                let count+=1
             fi
         fi
     done
+    echo "$count zone files had differences"
 
     if $changed; then
         return 1
@@ -253,12 +265,17 @@ compare_state() {
 }
 
 
-if [ "$1" == "c" ]; then
+
+if [ "$1" == "cap" ]; then
     capture_state "tests/system-setup/state/miab-ldap"
     exit $?
+elif [ "$1" == "compare" ]; then
+    compare_state "tests/system-setup/state/upstream" "tests/system-setup/state/miab-ldap"
+    exit $?
 fi
 
 
+
 # install basic stuff, set the hostname, time, etc
 before_install
 

From 071c89c420ab07e728300c2fae0f55da19d28345 Mon Sep 17 00:00:00 2001
From: downtownallday <downtownallday@gmail.com>
Date: Sun, 14 Jun 2020 17:19:30 -0400
Subject: [PATCH 51/65] Tail /var/log/messages if upstream setup fails

---
 .travis.yml                                 | 2 +-
 tests/system-setup/upgrade-from-upstream.sh | 7 ++++++-
 2 files changed, 7 insertions(+), 2 deletions(-)

diff --git a/.travis.yml b/.travis.yml
index f435280e..f19da3bd 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -1,7 +1,7 @@
 # travisci config
 env:
   global:
-  - MIAB_LDAP=true
+  - MIAB_LDAP_PROJECT=true
 
 language: shell
 os: linux
diff --git a/tests/system-setup/upgrade-from-upstream.sh b/tests/system-setup/upgrade-from-upstream.sh
index 0e321e49..2db726d0 100755
--- a/tests/system-setup/upgrade-from-upstream.sh
+++ b/tests/system-setup/upgrade-from-upstream.sh
@@ -67,7 +67,12 @@ upstream_install() {
     fi
     
     H2 "Run upstream setup"
-    setup/start.sh || die "Upstream setup failed!"
+    if ! setup/start.sh; then
+        echo "$F_WARN"
+        tail -100 /var/log/syslog
+        echo "$F_RESET"
+        die "Upstream setup failed!"
+    fi
     popd >/dev/null
     
     H2 "Upstream info"

From bce1cd41ff2fb503a5e30c11c27296f200f6c090 Mon Sep 17 00:00:00 2001
From: downtownallday <downtownallday@gmail.com>
Date: Sun, 14 Jun 2020 18:00:54 -0400
Subject: [PATCH 52/65] Apply a patch to setup/dns.sh on Travis so nsd will
 start during upstream install

---
 .travis.yml                                 |  2 --
 tests/lib/rest.sh                           |  2 ++
 tests/system-setup/upgrade-from-upstream.sh | 21 ++++++++++++++++++++-
 3 files changed, 22 insertions(+), 3 deletions(-)

diff --git a/.travis.yml b/.travis.yml
index f19da3bd..3b514ecc 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -40,8 +40,6 @@ jobs:
   - env:
       - PRIMARY_HOSTNAME=box2.abc.com
       - UPSTREAM_TAG=master
-      - FEATURE_NEXTCLOUD=false
-      - FEATURE_MUNIN=false
     name: upgrade-from-upstream
     install:
       - sudo tests/system-setup/upgrade-from-upstream.sh
diff --git a/tests/lib/rest.sh b/tests/lib/rest.sh
index dfbccb1b..e20a142f 100644
--- a/tests/lib/rest.sh
+++ b/tests/lib/rest.sh
@@ -100,7 +100,9 @@ rest_urlencoded() {
 		REST_ERROR="REST status $REST_HTTP_CODE: $REST_OUTPUT"
 		echo "${F_DANGER}$REST_ERROR${F_RESET}" 1>&2
         if $is_local && [ $REST_HTTP_CODE -ge 500 ]; then
+            echo -n "$F_WARN"
             tail -100 /var/log/syslog
+            echo -n "$F_RESET"
         fi
 		return 2
 	fi
diff --git a/tests/system-setup/upgrade-from-upstream.sh b/tests/system-setup/upgrade-from-upstream.sh
index 2db726d0..26f7b6a2 100755
--- a/tests/system-setup/upgrade-from-upstream.sh
+++ b/tests/system-setup/upgrade-from-upstream.sh
@@ -65,11 +65,24 @@ upstream_install() {
         H2 "Checkout $UPSTREAM_TAG"
         git checkout "$UPSTREAM_TAG" || die "git checkout $UPSTREAM_TAG failed"
     fi
+
+    if [ "$TRAVIS" == "true" ]; then
+        # Apply a patch to setup/dns.sh so nsd will start. We must do
+        # it in the script and not after setup.sh runs because part of
+        # setup includes adding a new user via the management
+        # interface and that's where the management daemon crashes:
+        #
+        # "subprocess.CalledProcessError: Command '['/usr/sbin/service', 'nsd', 'restart']' returned non-zero exit status 1"
+        #
+        H2 "Patching upstream setup/dns.sh for Travis-CI"
+        sed -i 's|\(.*include:.*zones\.conf.*\)|cat >> /etc/nsd/nsd.conf <<EOF\n  do-ip4: yes\n  do-ip6: no\nremote-control:\n  control-enable: no\nEOF\n\n\1|' setup/dns.sh \
+            || die "Couldn't patch setup/dns.sh !!"
+    fi
     
     H2 "Run upstream setup"
     if ! setup/start.sh; then
         echo "$F_WARN"
-        tail -100 /var/log/syslog
+        dump_file /var/log/syslog 100
         echo "$F_RESET"
         die "Upstream setup failed!"
     fi
@@ -116,6 +129,7 @@ add_data() {
     #
     # add users
     #
+    H2 "Add users"
     for user in "${users[@]}"; do
         if array_contains "$user" "${current_users[@]}"; then
             echo "Not adding user $user: already exists"
@@ -123,12 +137,15 @@ add_data() {
         elif ! rest_urlencoded POST /admin/mail/users/add "$EMAIL_ADDR" "$EMAIL_PW" --insecure -- "email=$user" "password=$pw" 2>/dev/null
         then
             die "Unable to add user $user: rc=$? err=$REST_ERROR"
+        else
+            echo "Add: $user"
         fi
     done
 
     #
     # add aliases
     #
+    H2 "Add aliases"
     local aliasdef
     for aliasdef in "${aliases[@]}"; do
         alias="$(awk -F'[> ]' '{print $1}' <<<"$aliasdef")"
@@ -139,6 +156,8 @@ add_data() {
         elif ! rest_urlencoded POST /admin/mail/aliases/add "$EMAIL_ADDR" "$EMAIL_PW" --insecure -- "address=$alias" "forwards_to=$forwards_to" 2>/dev/null
         then
             die "Unable to add alias $alias: rc=$? err=$REST_ERROR"
+        else
+            echo "Add: $aliasdef"
         fi
     done
 }

From 41188ad42c9043bc7e681133a3284c0fb8fb586e Mon Sep 17 00:00:00 2001
From: downtownallday <downtownallday@gmail.com>
Date: Sun, 14 Jun 2020 18:02:27 -0400
Subject: [PATCH 53/65] This fixes the warning about setting LC_ALL on Ubuntu
 server

---
 setup/start.sh | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/setup/start.sh b/setup/start.sh
index 5da27665..b56616bf 100755
--- a/setup/start.sh
+++ b/setup/start.sh
@@ -16,7 +16,8 @@ source setup/preflight.sh
 
 if ! locale -a | grep en_US.utf8 > /dev/null; then
     # Generate locale if not exists
-    hide_output locale-gen en_US.UTF-8
+    hide_output localedef -f UTF-8 -i en_US en_US.UTF-8
+    #hide_output locale-gen en_US.UTF-8
 fi
 
 export LANGUAGE=en_US.UTF-8

From a6f54a9bd37c0b264858701f04b29afd03c66d10 Mon Sep 17 00:00:00 2001
From: downtownallday <downtownallday@gmail.com>
Date: Sun, 14 Jun 2020 18:23:11 -0400
Subject: [PATCH 54/65] Minor code issues

---
 tests/lib/system.sh                         | 2 +-
 tests/system-setup/upgrade-from-upstream.sh | 6 +++---
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/tests/lib/system.sh b/tests/lib/system.sh
index 39695b10..1b8cb147 100644
--- a/tests/lib/system.sh
+++ b/tests/lib/system.sh
@@ -34,7 +34,7 @@ dump_file() {
     fi
     echo "--------"
 
-    if [ !-e "$log_file" ]; then
+    if [ ! -e "$log_file" ]; then
         echo "DOES NOT EXIST"
     elif [ ! -z "$lines" ]; then
         tail -$lines "$log_file"
diff --git a/tests/system-setup/upgrade-from-upstream.sh b/tests/system-setup/upgrade-from-upstream.sh
index 26f7b6a2..ed3530f8 100755
--- a/tests/system-setup/upgrade-from-upstream.sh
+++ b/tests/system-setup/upgrade-from-upstream.sh
@@ -97,11 +97,11 @@ upstream_install() {
 add_data() {
     H1 "Add some Mail-in-a-Box data"
     local users=()
-    users+="betsy@$(email_domainpart "$EMAIL_ADDR")"
+    users+=("betsy@$(email_domainpart "$EMAIL_ADDR")")
 
     local alises=()
-    aliases+="goalias@testdom.com > ${users[0]}"
-    aliases+="nested@testdom.com > goalias@testdom.com"
+    aliases+=("goalias@testdom.com > ${users[0]}")
+    aliases+=("nested@testdom.com > goalias@testdom.com")
 
     local pw="$(generate_qa_password)"
 

From 7de362a168886ff2f77a60398e29e00ad058010d Mon Sep 17 00:00:00 2001
From: downtownallday <downtownallday@gmail.com>
Date: Sun, 14 Jun 2020 20:22:14 -0400
Subject: [PATCH 55/65] Wording changes

---
 README.md                                   | 2 +-
 tests/system-setup/upgrade-from-upstream.sh | 7 ++++---
 2 files changed, 5 insertions(+), 4 deletions(-)

diff --git a/README.md b/README.md
index 730431c8..66557fac 100644
--- a/README.md
+++ b/README.md
@@ -16,7 +16,7 @@ To integrate Mail-in-a-Box w/LDAP (MiaB-LDAP) with Nextcloud, changes must be ma
 
 **On MiaB-LDAP**
 
-Enable the setup mod `remote-nextcloud.sh` by creating the directory `local` and creating a symbolic link to it. Do this by running this command from the mailinabox directory: `ln -s ../setup/mods.available/remote-nextcloud.sh local/remote-nextcloud.sh`. *During setup you will be prompted for the hostname and web prefix of your remote Nextcloud box.*
+Enable the setup mod `remote-nextcloud.sh` by creating the directory `local` in the directory where mailinabox is installed (usually $HOME/mailinabox), then creat a symbolic link to remote-nextcloud.sh. e.g. run this command from the mailinabox directory: `mkdir -p local; ln -s ../setup/mods.available/remote-nextcloud.sh local/remote-nextcloud.sh`. *During setup you will be prompted for the hostname and web prefix of your remote Nextcloud box.*
 
 The setup mod will configure Roundcube and Z-Push (ActiveSync) to use the remote Nextcloud for contacts and calendar instead of the local Nextcloud, which will be disabled (browsing to /cloud will fail).
 
diff --git a/tests/system-setup/upgrade-from-upstream.sh b/tests/system-setup/upgrade-from-upstream.sh
index ed3530f8..aba6c6d2 100755
--- a/tests/system-setup/upgrade-from-upstream.sh
+++ b/tests/system-setup/upgrade-from-upstream.sh
@@ -203,6 +203,7 @@ capture_state() {
 }
 
 miab_ldap_install() {
+    H1 "INSTALL MIAB-LDAP"
     # ensure we're in a MiaB-LDAP working directory
     if [ ! -e setup/ldap.sh ]; then
         die "The working directory is not MiaB-LDAP!"
@@ -217,7 +218,7 @@ compare_state() {
     local output
     local changed="false"
 
-    H1 "COMPARE STATES $(basename "$s1") TO $(basename "$2")"
+    H1 "COMPARE STATES: $(basename "$s1") VS $(basename "$2")"
     H2 "Users"
     # users
     output="$(diff "$s1/users.json" "$s2/users.json" 2>&1)"
@@ -226,7 +227,7 @@ compare_state() {
         echo "USERS ARE DIFFERENT!"
         echo "$output"
     else
-        echo "OK"
+        echo "No change"
     fi
 
     H2 "Aliases"
@@ -236,7 +237,7 @@ compare_state() {
         echo "ALIASES ARE DIFFERENT!"
         echo "$output"
     else
-        echo "OK"
+        echo "No change"
     fi
 
     H2 "DNS - zones missing"

From 94fd0fd067f34b0bd8c8470798ec8bb0d5f49527 Mon Sep 17 00:00:00 2001
From: downtownallday <downtownallday@gmail.com>
Date: Mon, 15 Jun 2020 12:22:55 -0400
Subject: [PATCH 56/65] Title shortened

---
 README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/README.md b/README.md
index 66557fac..469bcc72 100644
--- a/README.md
+++ b/README.md
@@ -29,7 +29,7 @@ Copy the file `setup/mods.available/remote-nextcloud-use-miab.sh` to the Nextclo
 On MiaB-LDAP, a one-time change must be applied manually to allow the remote Nextcloud to query the LDAP server because the default MiaB-LDAP installation doesn't allow any remote LDAP access. As root, run the following: `ufw allow proto tcp from $ip to any port ldaps`, where $ip is the ip-address of your Nextcloud server.
 
 
-## Under-the-Hood Details
+## Under-the-Hood
 
 **Additional directory in user-data**
 

From 77145e2f3899c3903262cc7530dc6d1e912bef4b Mon Sep 17 00:00:00 2001
From: downtownallday <downtownallday@gmail.com>
Date: Mon, 15 Jun 2020 12:24:03 -0400
Subject: [PATCH 57/65] Fix server error on non-change:
 "ldap3.core.exceptions.LDAPChangeError: no changes in modify request"

---
 management/backend.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/management/backend.py b/management/backend.py
index bce126d0..363fe491 100644
--- a/management/backend.py
+++ b/management/backend.py
@@ -203,10 +203,11 @@ class LdapConnection(ldap3.Connection):
 					existing_record,
 					values[attr])
 				if modify_op: changes[attr] = modify_op
-			self.wait ( self.modify(dn, changes) )
+			if len(changes)>0:
+				self.wait ( self.modify(dn, changes) )
 			return 'modify'
 		else:
-			# add new alias
+			# add new entry
 			self.wait ( self.add(dn, objectClasses, values) )
 			return 'add'
 

From 5f008d91b84fc4fab09de0b72ac720bc06d66636 Mon Sep 17 00:00:00 2001
From: downtownallday <downtownallday@gmail.com>
Date: Mon, 15 Jun 2020 12:26:12 -0400
Subject: [PATCH 58/65] Ignore alias domains that have no forward-to because we
 don't accept mail locally for the alias

---
 management/mailconfig.py | 29 +++++++++++++++++++++++++----
 1 file changed, 25 insertions(+), 4 deletions(-)

diff --git a/management/mailconfig.py b/management/mailconfig.py
index 4fa63ca7..216c548c 100755
--- a/management/mailconfig.py
+++ b/management/mailconfig.py
@@ -362,7 +362,7 @@ def get_mail_aliases(env, as_map=False):
 			alias = aliases[address]
 			xft = ",".join(alias["forward_tos"])
 			xas = ",".join(alias["permitted_senders"])
-			list.append( (address, xft, None if xas is "" else xas) )
+			list.append( (address, xft, None if xas == "" else xas) )
 		return list
 	
 	else:
@@ -482,11 +482,27 @@ def get_mail_domains(env, as_map=False, filter_aliases=lambda alias: True, categ
 
 
 	# alias domains
+	#
+	# Ignore aliases that have no forward-to. We don't need DNS
+	# handling in that case becuase the alias is there only for the
+	# permitted-senders. We don't accept mail locally for the alias.
+	#
+	# Aliases with only permitted-senders are useful when a server has
+	# a configured smarthost (eg. sendmail with a smarthost, or using
+	# ssmtp on Ubuntu, etc). The server drops mail off for delivery to
+	# the smarthost (MiaB) using its MiaB login but needs to MAIL FROM
+	# a host login (user@host.tld). Replies should bounce.
+	#
+	# A smarthost configuration should be a catch-all, one for each server:
+	#   Alias=@host.tld
+	#   Forward-to=<blank>
+	#   Permitted-senders:<the email that the smarthost used to authenticate with MiaB>
+	#
 	if not users_only:
-		pager = conn.paged_search(env.LDAP_ALIASES_BASE, "(objectClass=mailGroup)", attributes="mail")
+		pager = conn.paged_search(env.LDAP_ALIASES_BASE, "(objectClass=mailGroup)", attributes=["mail","member","rfc822MailMember"])
 		if as_map:
 			for rec in pager:
-				if filter_aliases(rec["mail"][0].lower()):
+				if filter_aliases(rec["mail"][0].lower()) and ( len(rec["member"]) >0 or len(rec["rfc822MailMember"]) >0 ):
 					domain = get_domain(rec["mail"][0].lower(),as_unicode=False)
 					domains[domain] = {
 						"dn": None,
@@ -494,7 +510,12 @@ def get_mail_domains(env, as_map=False, filter_aliases=lambda alias: True, categ
 					}
 
 		else:
-			domains = domains.union(set([ get_domain(rec["mail"][0].lower(), as_unicode=False) for rec in pager if filter_aliases(rec["mail"][0].lower()) ]))
+			alias_domains = set([
+				get_domain(rec["mail"][0].lower(), as_unicode=False)
+				for rec in pager if filter_aliases(rec["mail"][0].lower()) and
+				( len(rec["member"]) >0 or len(rec["rfc822MailMember"]) >0 )
+			])			
+			domains = domains.union( alias_domains )
 					
 	return domains
 

From 582b12b33a0fc615623266801e783ab4edf48090 Mon Sep 17 00:00:00 2001
From: downtownallday <downtownallday@gmail.com>
Date: Tue, 16 Jun 2020 06:44:14 -0400
Subject: [PATCH 59/65] Revert: does not fix the error message "warning: cannot
 change locale" during 'export LC_ALL'. This appears to only be fixable by
 restarting bash after the locale-gen call.

---
 setup/start.sh | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/setup/start.sh b/setup/start.sh
index b56616bf..5da27665 100755
--- a/setup/start.sh
+++ b/setup/start.sh
@@ -16,8 +16,7 @@ source setup/preflight.sh
 
 if ! locale -a | grep en_US.utf8 > /dev/null; then
     # Generate locale if not exists
-    hide_output localedef -f UTF-8 -i en_US en_US.UTF-8
-    #hide_output locale-gen en_US.UTF-8
+    hide_output locale-gen en_US.UTF-8
 fi
 
 export LANGUAGE=en_US.UTF-8

From 2d7cb869c556d095ab03143869339ae2002684a3 Mon Sep 17 00:00:00 2001
From: downtownallday <downtownallday@gmail.com>
Date: Fri, 19 Jun 2020 12:02:15 -0400
Subject: [PATCH 60/65] Continue to make old ownCloud contacts available in
 Roundcube

---
 setup/mods.available/remote-nextcloud.sh | 27 +++++++++++++++++++-----
 1 file changed, 22 insertions(+), 5 deletions(-)

diff --git a/setup/mods.available/remote-nextcloud.sh b/setup/mods.available/remote-nextcloud.sh
index c09e2130..b50add94 100755
--- a/setup/mods.available/remote-nextcloud.sh
+++ b/setup/mods.available/remote-nextcloud.sh
@@ -35,12 +35,29 @@ configure_roundcube() {
     local name="${1:-$NC_HOST}"
     local baseurl="$NC_PROTO://$NC_HOST:$NC_PORT$NC_PREFIX"
     
-    # Configure CardDav
-    cat > ${RCM_PLUGIN_DIR}/carddav/config.inc.php <<EOF
+    # Configure CardDav plugin
+    #
+    # 1. make MiaB ownCloud contacts read-only so users can still
+    #    access them, but not change them, and no sync occurs
+    #
+    # a. set 'active' to 'false'
+    #    regular expression before "bashing" it:
+    #       (['"]active['"][ \t]*=>[ \t]*)true
+    #
+    sed -i 's/\(['"'"'"]active['"'"'"][ \t]*=>[ \t]*\)true/\1false/' ${RCM_PLUGIN_DIR}/carddav/config.inc.php
+
+    # b. set 'readonly' to 'true'
+    #    regular expressions is like above
+    sed -i 's/\(['"'"'"]readonly['"'"'"][ \t]*=>[ \t]*\)false/\1true/' ${RCM_PLUGIN_DIR}/carddav/config.inc.php
+
+    #
+    # 2. add the remote Nextcloud
+    #
+    cat >> ${RCM_PLUGIN_DIR}/carddav/config.inc.php <<EOF
 <?php
-/* Do not edit. Written by Mail-in-a-Box-LDAP mods. Regenerated on updates. */
-\$prefs['_GLOBAL']['hide_preferences'] = true;
-\$prefs['_GLOBAL']['suppress_version_warning'] = true;
+/* Do not edit. Written by Mail-in-a-Box-LDAP. Regenerated on updates. */
+//\$prefs['_GLOBAL']['hide_preferences'] = true;
+//\$prefs['_GLOBAL']['suppress_version_warning'] = true;
 \$prefs['cloud'] = array(
 	 'name'         =>  '$name',
 	 'username'     =>  '%u', // login username

From 144aa6e5d6d87b0ba69a884f6d0ebc1141299f1d Mon Sep 17 00:00:00 2001
From: downtownallday <downtownallday@gmail.com>
Date: Fri, 19 Jun 2020 12:03:29 -0400
Subject: [PATCH 61/65] 1. Catch connection errors and report them 2. Limit
 address book updates to just the user given

---
 .../assets/mail/roundcube/carddav_refresh.sh  | 48 +++++++++++++------
 1 file changed, 34 insertions(+), 14 deletions(-)

diff --git a/tests/assets/mail/roundcube/carddav_refresh.sh b/tests/assets/mail/roundcube/carddav_refresh.sh
index e399c9bc..335a4eaa 100755
--- a/tests/assets/mail/roundcube/carddav_refresh.sh
+++ b/tests/assets/mail/roundcube/carddav_refresh.sh
@@ -69,7 +69,21 @@ if ($auth['valid'] && !$auth['abort']
      _die("login failed");
    }
 
- 
+
+// ----------------------------------------------------
+// Get the user id (see deluser.sh)
+// ----------------------------------------------------
+$host = $auth['host']; # can be a url (eg: ssl://localhost)
+$host_url = parse_url($host);
+if ($host_url['host']) {
+       $host = $host_url['host'];
+}
+$user = rcube_user::query($auth['user'], $host);
+if (!$user) {
+       _die("User not found auth[host]=" . $auth['host'] . " host=" . $host . "\n");
+}
+   
+
 // ----------------------------------------------------
 // ensure the carddav tables are created and populated
 // ----------------------------------------------------
@@ -77,16 +91,22 @@ if ($auth['valid'] && !$auth['abort']
 require_once('plugins/carddav/carddav_backend.php');
 require_once('plugins/carddav/carddav.php');
 
-$c = new carddav(rcube_plugin_api::get_instance());
-$c->task .= "|cli";
-$c->init();
-print "done: init\n";
-// this ensures the carddav tables are created
-$c->checkMigrations();
-print "done: init tables\n";
-// this populates carddav_addressbooks from config
-$c->init_presets();
-print "done: init addressbooks\n";
+try {
+   $c = new carddav(rcube_plugin_api::get_instance());
+   $c->task .= "|cli";
+   $c->init();
+   print "done: init\n";
+   // this ensures the carddav tables are created
+   $c->checkMigrations();
+   print "done: init tables\n";
+   // this populates carddav_addressbooks from config
+   $c->init_presets();
+   print "done: init addressbooks\n";
+} catch(exception $e) {
+    print $e . "\n";
+    _die("failed");
+}
+
 
 // -------------------------------------------------------------
 // Set the last_updated field for addressbooks to an old date.
@@ -99,7 +119,7 @@ if (!$db->is_connected() || $db->is_error()) {
 }
 print "db connected\n";
 
-$db->query("update " . $db->table_name('carddav_addressbooks') . " set last_updated=? WHERE active=1", '2000-01-01 00:00:00');
+$db->query("update " . $db->table_name('carddav_addressbooks') . " set last_updated=? WHERE active=1 and user_id=" . $user->ID, '2000-01-01 00:00:00');
 print "update made\n";
 if ($db->is_error()) {
   _die("DB error occurred: " . $db->is_error());
@@ -107,7 +127,7 @@ if ($db->is_error()) {
 
 
 // ------------------------------------------------------
-// Update/sync all active address books
+// Update/sync all out-of-date address books
 // ------------------------------------------------------
 
 // first get all row ids
@@ -120,7 +140,7 @@ if ($db->is_error()) {
 }
    
 while ($row = $db->fetch_assoc($sql_result)) {
-  $dbid += array(intval($row['id']));
+  array_push($dbid, intval($row['id']));
   print "carddav_addressbooks id: " . $row['id'] . "\n";
 }
 

From 1bd7b2c4c77df2168b3fc53bb74528b623e6304c Mon Sep 17 00:00:00 2001
From: downtownallday <downtownallday@gmail.com>
Date: Fri, 19 Jun 2020 12:12:49 -0400
Subject: [PATCH 62/65] 1. Better code organization & simplify 2. Add
 "populate" data for upgrades - enabled in both system-setup scripts 3. Add
 "upgrade" test runner suite

---
 .travis.yml                                   |  11 +-
 tests/lib/all.sh                              |   3 +
 tests/lib/carddav.sh                          | 280 ++++++++++++++++++
 tests/lib/color-output.sh                     |  20 +-
 tests/lib/installed-state.sh                  | 141 +++++++++
 tests/lib/misc.sh                             |   6 +-
 tests/lib/populate.sh                         |  99 +++++++
 tests/lib/rest.sh                             |   4 +-
 tests/runner.sh                               |  43 +--
 tests/suites/_init.sh                         |   2 +
 tests/suites/remote-nextcloud.sh              | 229 ++------------
 tests/suites/upgrade.sh                       |  48 +++
 tests/system-setup/populate/README.txt        |  31 ++
 tests/system-setup/populate/basic-data.sh     |  10 +
 tests/system-setup/populate/basic-populate.sh |  48 +++
 tests/system-setup/populate/basic-verify.sh   |  53 ++++
 tests/system-setup/remote-nextcloud-docker.sh | 155 ++++++----
 tests/system-setup/setup-defaults.sh          |   2 +-
 tests/system-setup/setup-funcs.sh             |  65 ++--
 tests/system-setup/upgrade-from-upstream.sh   | 246 +++------------
 20 files changed, 987 insertions(+), 509 deletions(-)
 create mode 100644 tests/lib/carddav.sh
 create mode 100644 tests/lib/installed-state.sh
 create mode 100644 tests/lib/populate.sh
 create mode 100644 tests/suites/upgrade.sh
 create mode 100644 tests/system-setup/populate/README.txt
 create mode 100755 tests/system-setup/populate/basic-data.sh
 create mode 100755 tests/system-setup/populate/basic-populate.sh
 create mode 100755 tests/system-setup/populate/basic-verify.sh

diff --git a/.travis.yml b/.travis.yml
index 3b514ecc..71f87913 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -29,12 +29,15 @@ jobs:
       - ip add
       - sysctl -a 2>/dev/null | grep -i ipv6 | grep disable
     install:
-      - sudo tests/system-setup/remote-nextcloud-docker.sh
+      # setup with 'basic' data before setting up again using
+      # a remote nextcloud to verify ownCloud contacts are still
+      # available
+      - sudo tests/system-setup/remote-nextcloud-docker.sh basic
     script:
       # launch automated tests, but skip tests that require remote
       # smtp support because Travis-CI blocks outgoing port 25
       - sudo touch /etc/dovecot/sieve-spam.svbin
-      - sudo tests/runner.sh -dumpoutput -no-smtp-remote default remote-nextcloud
+      - sudo tests/runner.sh -dumpoutput -no-smtp-remote default remote-nextcloud upgrade-basic
 
   # JOB: Upgrade from upstream install
   - env:
@@ -42,9 +45,9 @@ jobs:
       - UPSTREAM_TAG=master
     name: upgrade-from-upstream
     install:
-      - sudo tests/system-setup/upgrade-from-upstream.sh
+      - sudo tests/system-setup/upgrade-from-upstream.sh basic
     script:
       # launch automated tests, but skip tests that require remote
       # smtp support because Travis-CI blocks outgoing port 25
       - sudo touch /etc/dovecot/sieve-spam.svbin
-      - sudo tests/runner.sh -dumpoutput -no-smtp-remote
+      - sudo tests/runner.sh -dumpoutput -no-smtp-remote default upgrade-basic
diff --git a/tests/lib/all.sh b/tests/lib/all.sh
index 20a0c6de..0fa22f6c 100644
--- a/tests/lib/all.sh
+++ b/tests/lib/all.sh
@@ -12,5 +12,8 @@
 . "$1/misc.sh"         || exit 3
 . "$1/rest.sh"         || exit 4
 . "$1/system.sh"       || exit 5
+. "$1/carddav.sh"      || exit 6
 
+. "$1/populate.sh"     || exit 7
+. "$1/installed-state.sh" || exit 8
 
diff --git a/tests/lib/carddav.sh b/tests/lib/carddav.sh
new file mode 100644
index 00000000..6ff58011
--- /dev/null
+++ b/tests/lib/carddav.sh
@@ -0,0 +1,280 @@
+#
+# requires:
+#    system packages: [ curl, python3, sqlite3 ]
+#    scripts: [ color-output.sh, misc.sh, locations.sh ]
+#
+# ASSETS_DIR: where the assets directory is located (defaults to
+# tests/assets)
+#
+
+nextcloud_url() {
+    # eg: http://localhost/cloud/
+    carddav_url | sed 's|\(.*\)/remote.php/.*|\1/|'
+}
+
+carddav_url() {
+    # get the carddav url as configured in z-push for the user specified
+    # eg: http://localhost/cloud/remote.php/dav/addressbooks/users/admin/contacts/
+    local user="${1:-%u}"
+    local path="${2:-CARDDAV_DEFAULT_PATH}"
+    local php="include \"$ZPUSH_DIR/backend/carddav/config.php\"; print CARDDAV_PROTOCOL . \"://\" . CARDDAV_SERVER . \":\" . CARDDAV_PORT . "
+    php="$php$path;"
+    local url
+    url="$(php -n -r "$php")"
+    [ $? -ne 0 ] && die "Unable to run php to extract carddav url from z-push"
+    sed "s/%u/$user/" <<< "$url"
+}
+
+carddav_rest() {
+    # issue a CardDAV rest call to Nextcloud
+    # SEE: https://tools.ietf.org/html/rfc6352
+    #
+    # The function will set the following global variables regardless
+    # of exit code:
+    #     REST_HTTP_CODE
+    #     REST_OUTPUT
+    #     REST_ERROR
+    #     REST_ERROR_BRIEF
+    # 
+    # Return values:
+    #   0 indicates success (curl returned 0 or a code deemed to be
+    #     successful and HTTP status is >=200  but <300)
+    #   1 curl returned with non-zero code that indicates and error
+    #   2 the response status was <200 or >= 300
+    #
+    # Debug messages are sent to stderr
+    #   
+    local verb="$1"
+    local uri="$2"
+    local auth_user="$3"
+    local auth_pass="$4"
+	shift; shift; shift; shift  # remaining arguments are data
+
+    local url
+    case "$uri" in
+        /* )
+            url="$(nextcloud_url)${uri#/}"
+            ;;
+        http* )
+            url="$uri"
+            ;;
+        * )
+            url="$(carddav_url "$auth_user")${uri#/}"
+            ;;
+    esac
+    
+    local data=()
+	local item output onlydata="false"
+	
+	for item; do
+        case "$item" in
+            -- )
+                onlydata="true"
+                ;;
+            --* )
+                # curl argument
+                if $onlydata; then
+                    data+=("--data" "$item");
+                else
+                    data+=("$item")
+                fi
+                ;;
+            * )
+                onlydata="true"
+                data+=("--data" "$item");
+                ;;
+        esac
+    done
+    
+    local ct
+    case "${data[1]}" in
+        BEGIN:VCARD* )
+            ct="text/vcard"
+            ;;
+        * )
+            ct='text/xml; charset="utf-8"'
+    esac
+
+    local tmp1="/tmp/curl.$$.tmp"
+    
+    echo "spawn: curl -w \"%{http_code}\" -X $verb -H 'Content-Type: $ct' --user \"${auth_user}:xxx\" ${data[@]} \"$url\"" 1>&2
+	output=$(curl -s -S -w "%{http_code}" -X $verb -H "Content-Type: $ct" --user "${auth_user}:${auth_pass}" "${data[@]}" "$url" 2>$tmp1)
+	local code=$?
+
+	# http status is last 3 characters of output, extract it
+	REST_HTTP_CODE=$(awk '{S=substr($0,length($0)-2)} END {print S}' <<<"$output")
+	REST_OUTPUT=$(awk 'BEGIN{L=""}{ if(L!="") print L; L=$0 } END { print substr(L,1,length(L)-3) }' <<<"$output")
+	REST_ERROR=""
+    REST_ERROR_BRIEF=""
+	[ -z "$REST_HTTP_CODE" ] && REST_HTTP_CODE="000"
+
+    if [ $code -ne 0 -o \
+               $REST_HTTP_CODE -lt 200 -o \
+               $REST_HTTP_CODE -ge 300 ]
+    then
+        if [ $code -ne 0 -a "$REST_HTTP_CODE" == "000" ]; then
+            REST_ERROR="exit code $code"
+            REST_ERROR_BRIEF="$REST_ERROR"
+        else
+		    REST_ERROR="REST status $REST_HTTP_CODE: $REST_OUTPUT"
+            REST_ERROR_BRIEF=$(python3 -c "import xml.etree.ElementTree as ET; print(ET.fromstring(r'''$REST_OUTPUT''').find('s:message',{'s':'http://sabredav.org/ns'}).text)" 2>/dev/null)
+            if [ -z "$REST_ERROR_BRIEF" ]; then
+                REST_ERROR_BRIEF="$REST_ERROR"
+            else
+                REST_ERROR_BRIEF="$REST_HTTP_CODE: $REST_ERROR_BRIEF"
+            fi
+            if [ $code -ne 0 ]; then
+                REST_ERROR_BRIEF="exit code $code: $REST_ERROR_BRIEF"
+                REST_ERROR="exit code $code: $REST_ERROR"
+            fi
+        fi
+
+        if [ -s $tmp1 ]; then
+            REST_ERROR="$REST_ERROR: $(cat $tmp1)"
+            REST_ERROR_BRIEF="$REST_ERROR_BRIEF: $(cat $tmp1)"
+        fi
+        rm -f $tmp1
+        
+		echo "${F_DANGER}$REST_ERROR${F_RESET}" 1>&2
+		[ $code -ne 0 ] && return 1
+        return 2
+	fi
+    
+	echo "CURL succeded, HTTP status $REST_HTTP_CODE" 1>&2
+	echo "$output" 1>&2
+    rm -f $tmp1
+	return 0    
+}
+
+
+carddav_ls() {
+    # place all .vcf files into global FILES
+    # debug messages are sent to stderr
+    local user="$1"
+    local pass="$2"
+    shift; shift
+    FILES=()
+    if ! carddav_rest PROPFIND "" "$user" "$pass" $@
+    then
+        return 1
+    fi
+    
+    FILES=( $(python3 -c "import xml.etree.ElementTree as ET; [print(el.find('d:href',{'d':'DAV:'}).text) for el in ET.fromstring(r'''$REST_OUTPUT''').findall('d:response',{'d':'DAV:'}) if el.find('d:href',{'d':'DAV:'}) is not None]") )
+
+    local idx=${#FILES[*]}
+    let idx-=1
+    while [ $idx -ge 0 ]; do
+        # remove non .vcf entries, take basename contact href
+        case "${FILES[$idx]}" in
+            *.vcf )
+                FILES[$idx]=$(basename "${FILES[$idx]}")
+                ;;
+            * )
+                unset "FILES[$idx]"
+                ;;
+        esac
+        let idx-=1
+    done
+}
+
+
+carddav_make_addressbook() {
+    local user="$1"
+    local pass="$2"
+    local name="$3"
+    local desc="${4:-$name}"
+    local xml="<?xml version=\"1.0\" encoding=\"utf-8\" ?>
+<D:mkcol xmlns:D=\"DAV:\"
+         xmlns:C=\"urn:ietf:params:xml:ns:carddav\">
+  <D:set>
+    <D:prop>
+      <D:resourcetype>
+        <D:collection/>
+        <C:addressbook/>
+      </D:resourcetype>
+      <D:displayname>$name</D:displayname>
+      <C:addressbook-description xml:lang=\"en\">$desc</C:addressbook-description>
+    </D:prop>
+  </D:set>
+</D:mkcol>"
+    local url="$(carddav_url "$user" CARDDAV_PATH)"
+    carddav_rest MKCOL "$url" "$user" "$pass" "$xml"
+}
+
+
+carddav_add_contact() {
+    # debug messages are sent to stderr
+    local user="$1"
+    local pass="$2"
+    local c_name="$3"
+    local c_phone="$4"
+    local c_email="$5"
+    local c_uid="${6:-$(generate_uuid)}"
+    shift; shift; shift; shift; shift; shift
+    
+    local vcard="BEGIN:VCARD
+VERSION:3.0
+UID:$c_uid
+REV;VALUE=DATE-AND-OR-TIME:$(date -u +%Y%m%dT%H%M%SZ)
+FN:$c_name
+EMAIL;TYPE=INTERNET,PREF:$c_email
+NOTE:Miab-LDAP QA
+ORG:Miab-LDAP
+TEL;TYPE=WORK,VOICE:$c_phone
+END:VCARD"
+    carddav_rest PUT "$c_uid.vcf" "$user" "$pass" $@ -- "$vcard"
+}
+
+
+carddav_delete_contact() {
+    local user="$1"
+    local pass="$2"
+    local c_uid="$3"
+    shift; shift; shift
+    carddav_rest DELETE "$c_uid.vcf" "$user" "$pass" $@
+}
+
+
+roundcube_force_carddav_refresh() {
+    local user="$1"
+    local pass="$2"
+    local assets_dir="${ASSETS_DIR:-tests/assets}"
+    local code
+    if ! cp "$assets_dir/mail/roundcube/carddav_refresh.sh" $RCM_DIR/bin
+    then
+        return 1
+    fi
+    pushd "$RCM_DIR" >/dev/null
+    bin/carddav_refresh.sh "$user" "$pass"
+    code=$?
+    popd >/dev/null
+    return $code
+}
+
+
+roundcube_carddav_contact_exists() {
+    # returns 0 if contact exists
+    #         1 if contact does not exist
+    #         2 if an error occurred
+    # stderr receives error messages
+    local user="$1"
+    local pass="$2"
+    local c_uid="$3"
+    local db="${4:-$STORAGE_ROOT/mail/roundcube/roundcube.sqlite}"
+    local output
+    output="$(sqlite3 "$db" "select name from carddav_contacts where cuid='$c_uid'")"
+    [ $? -ne 0 ] && return 2
+    if [ -z "$output" ]; then
+        return 1
+    else
+        return 0
+    fi
+}
+
+
+roundcube_dump_contacts() {
+    local db="${1:-$STORAGE_ROOT/mail/roundcube/roundcube.sqlite}"
+    local cols="${2:-name,cuid}"
+    sqlite3 "$db" "select $cols FROM carddav_contacts"
+}
+
diff --git a/tests/lib/color-output.sh b/tests/lib/color-output.sh
index 3e1954f8..3652e98b 100644
--- a/tests/lib/color-output.sh
+++ b/tests/lib/color-output.sh
@@ -27,6 +27,24 @@ warn() {
         * )
             echoarg=""
     esac
-    echo "${F_WARN}$1${F_RESET}"
+    echo $echoarg "${F_WARN}$1${F_RESET}"
+}
+
+H1() {
+    local msg="$1"
+    echo "----------------------------------------------"
+    if [ ! -z "$msg" ]; then
+        echo "           $msg"
+        echo "----------------------------------------------"
+    fi
+}
+
+H2() {
+    local msg="$1"
+    if [ -z "$msg" ]; then
+        echo "***"
+    else
+        echo "*** $msg ***"
+    fi
 }
 
diff --git a/tests/lib/installed-state.sh b/tests/lib/installed-state.sh
new file mode 100644
index 00000000..3efc0411
--- /dev/null
+++ b/tests/lib/installed-state.sh
@@ -0,0 +1,141 @@
+#
+# requires:
+#    scripts: [ colored-output.sh, rest.sh ]
+#
+# these functions are meant for comparing upstream (non-LDAP)
+# installations to a subsequent MiaB-LDAP upgrade
+#
+
+
+installed_state_capture() {
+    # users and aliases
+    # dns zone files
+    # TOOD: tls certificates: expected CN's
+
+    local state_dir="$1"
+    local info="$state_dir/info.txt"
+
+    H1 "Capture installed estate to $state_dir"
+
+    # nuke saved state, if any
+    rm -rf "$state_dir"
+    mkdir -p "$state_dir"
+
+    # create info.json
+    H2 "create info.txt"
+    echo "STATE_VERSION=1" > "$info"
+    echo "GIT_VERSION='$(git describe --abbrev=0)'" >>"$info"
+    echo "MIGRATION_VERSION=$(cat "$STORAGE_ROOT/mailinabox.version")" >>"$info"
+
+    # record users
+    H2 "record users"
+    if ! rest_urlencoded GET "/admin/mail/users?format=json" "$EMAIL_ADDR" "$EMAIL_PW" --insecure 2>/dev/null
+    then
+        echo "Unable to get users: rc=$? err=$REST_ERROR" 1>&2
+        return 1
+    fi
+    echo "$REST_OUTPUT" > "$state_dir/users.json"
+
+    # record aliases
+    H2 "record aliases"
+    if ! rest_urlencoded GET "/admin/mail/aliases?format=json" "$EMAIL_ADDR" "$EMAIL_PW" --insecure 2>/dev/null
+    then
+        echo "Unable to get aliases: rc=$? err=$REST_ERROR" 1>&2
+        return 2
+    fi
+    echo "$REST_OUTPUT" > "$state_dir/aliases.json"
+
+    # record dns config
+    H2 "record dns details"
+    local file
+    mkdir -p "$state_dir/zones"
+    for file in /etc/nsd/zones/*.signed; do
+        if ! cp "$file" "$state_dir/zones"
+        then
+            echo "Copy $file -> $state_dir/zones failed" 1>&2
+            return 3
+        fi
+    done
+    
+    return 0
+}
+
+
+
+installed_state_compare() {
+    local s1="$1"
+    local s2="$2"
+    
+    local output
+    local changed="false"
+
+    H1 "COMPARE STATES: $(basename "$s1") VS $(basename "$2")"
+    H2 "Users"
+    # users
+    output="$(diff "$s1/users.json" "$s2/users.json" 2>&1)"
+    if [ $? -ne 0 ]; then
+        changed="true"
+        echo "USERS ARE DIFFERENT!"
+        echo "$output"
+    else
+        echo "No change"
+    fi
+
+    H2 "Aliases"
+    output="$(diff "$s1/aliases.json" "$s2/aliases.json" 2>&1)"
+    if [ $? -ne 0 ]; then
+        change="true"
+        echo "ALIASES ARE DIFFERENT!"
+        echo "$output"
+    else
+        echo "No change"
+    fi
+
+    H2 "DNS - zones missing"
+    local zone count=0
+    for zone in $(cd "$s1/zones"; ls *.signed); do
+        if [ ! -e "$s2/zones/$zone" ]; then
+            echo "MISSING zone: $zone"
+            changed="true"
+            let count+=1
+        fi
+    done
+    echo "$count missing"
+
+    H2 "DNS - zones added"
+    count=0
+    for zone in $(cd "$s2/zones"; ls *.signed); do
+        if [ ! -e "$s2/zones/$zone" ]; then
+            echo "ADDED zone: $zone"
+            changed="true"
+            let count+=1
+        fi
+    done
+    echo "$count added"
+
+    H2 "DNS - zones changed"
+    count=0
+    for zone in $(cd "$s1/zones"; ls *.signed); do
+        if [ -e "$s2/zones/$zone" ]; then
+            # all the signatures change if we're using self-signed certs
+            local t1="/tmp/s1.$$.txt"
+            local t2="/tmp/s2.$$.txt"
+            awk '$4 == "RRSIG" || $4 == "NSEC3" { next; } $4 == "SOA" { print $1" "$2" "$3" "$4" "$5" "$6" "$8" "$9" "$10" "$11" "$12; next } { print $0 }' "$s1/zones/$zone" > "$t1" 
+            awk '$4 == "RRSIG" || $4 == "NSEC3" { next; } $4 == "SOA" { print $1" "$2" "$3" "$4" "$5" "$6" "$8" "$9" "$10" "$11" "$12; next } { print $0 }' "$s2/zones/$zone" > "$t2" 
+            output="$(diff "$t1" "$t2" 2>&1)"
+            if [ $? -ne 0 ]; then
+                echo "CHANGED zone: $zone"
+                echo "$output"
+                changed="true"
+                let count+=1
+            fi
+        fi
+    done
+    echo "$count zone files had differences"
+
+    if $changed; then
+        return 1
+    else
+        return 0
+    fi
+}
diff --git a/tests/lib/misc.sh b/tests/lib/misc.sh
index 4312d6ae..52a2b8a6 100644
--- a/tests/lib/misc.sh
+++ b/tests/lib/misc.sh
@@ -55,7 +55,11 @@ generate_uuid() {
 }
 
 generate_qa_password() {
-    echo "Test1234."
+    echo "Test$(date +%s)"
+}
+
+static_qa_password() {
+    echo "Test_1234"
 }
 
 sha1() {
diff --git a/tests/lib/populate.sh b/tests/lib/populate.sh
new file mode 100644
index 00000000..8a8b0c5f
--- /dev/null
+++ b/tests/lib/populate.sh
@@ -0,0 +1,99 @@
+
+#
+# requires:
+#   scripts: [ rest.sh, misc.sh ]
+#
+
+populate_miab_users() {
+    local url="$1"
+    local admin_email="${2:-$EMAIL_ADDR}"
+    local admin_pass="${3:-$EMAIL_PW}"
+    shift; shift; shift  # remaining arguments are users to add
+
+    # each "user" argument is in the format "email:password"
+    # if no password is given a "qa" password will be generated
+
+    [ $# -eq 0 ] && return 0
+    
+    #
+    # get the existing users
+    #
+    local current_users=() user
+    if ! rest_urlencoded GET ${url%/}/admin/mail/users "$admin_email" "$admin_pass" --insecure 2>/dev/null; then
+        echo "Unable to enumerate users: rc=$? err=$REST_ERROR" 1>&2
+        return 1
+    fi
+    for user in $REST_OUTPUT; do
+        current_users+=("$user")
+    done
+
+    #
+    # add the new users
+    #
+    local pw="$(generate_qa_password)"
+    
+    for user; do
+        local user_email="$(awk -F: '{print $1}' <<< "$user")"
+        local user_pass="$(awk -F: '{print $2}' <<< "$user")"
+        if array_contains "$user_email" "${current_users[@]}"; then
+            echo "Not adding user $user_email: already exists"
+
+        elif ! rest_urlencoded POST ${url%/}/admin/mail/users/add "$admin_email" "$admin_pass" --insecure -- "email=$user_email" "password=${user_pass:-$pw}" 2>/dev/null
+        then
+            echo "Unable to add user $user_email: rc=$? err=$REST_ERROR" 1>&2
+            return 2
+        else
+            echo "Add: $user"
+        fi
+    done
+
+    return 0
+}
+
+
+
+populate_miab_aliases() {
+    local url="$1"
+    local admin_email="${2:-$EMAIL_ADDR}"
+    local admin_pass="${3:-$EMAIL_PW}"
+    shift; shift; shift  # remaining arguments are aliases to add
+
+    # each "alias" argument is in the format "email-alias > forward-to"
+
+    [ $# -eq 0 ] && return 0
+    
+    #
+    # get the existing aliases
+    #
+    local current_aliases=() alias
+    if ! rest_urlencoded GET ${url%/}/admin/mail/aliases "$admin_email" "$admin_pass" --insecure 2>/dev/null; then
+        echo "Unable to enumerate aliases: rc=$? err=$REST_ERROR" 1>&2
+        return 1
+    fi
+    for alias in $REST_OUTPUT; do
+        current_aliases+=("$alias")
+    done
+
+    #
+    # add the new aliases
+    #
+    local aliasdef
+    for aliasdef; do
+        alias="$(awk -F'[> ]' '{print $1}' <<<"$aliasdef")"
+        local forwards_to="$(sed 's/.*> *\(.*\)/\1/' <<<"$aliasdef")"
+        if array_contains "$alias" "${current_aliases[@]}"; then
+            echo "Not adding alias $aliasdef: already exists"
+            
+        elif ! rest_urlencoded POST ${url%/}/admin/mail/aliases/add "$admin_email" "$admin_pass" --insecure -- "address=$alias" "forwards_to=$forwards_to" 2>/dev/null
+        then
+            echo "Unable to add alias $alias: rc=$? err=$REST_ERROR" 1>&2
+            return 2
+        else
+            echo "Add: $aliasdef"
+        fi
+    done
+
+    return 0
+}
+
+
diff --git a/tests/lib/rest.sh b/tests/lib/rest.sh
index e20a142f..2c8fd39d 100644
--- a/tests/lib/rest.sh
+++ b/tests/lib/rest.sh
@@ -15,7 +15,7 @@ rest_urlencoded() {
     # (https://host/mail/users/add), PRIMARY_HOSTNAME must be set!
     #
     # The function will set the following global variables regardless
-    # of exit c ode:
+    # of exit code:
     #     REST_HTTP_CODE
     #     REST_OUTPUT
     #     REST_ERROR
@@ -26,7 +26,7 @@ rest_urlencoded() {
     #   1 curl returned with non-zero code that indicates and error
     #   2 the response status was <200 or >= 300
     #
-    # Messages and errors are sent to stderr
+    # Debug messages are sent to stderr
     #   
 	local verb="$1" # eg "POST"
 	local uri="$2"  # eg "/mail/users/add"
diff --git a/tests/runner.sh b/tests/runner.sh
index 8119d462..a6fe566c 100755
--- a/tests/runner.sh
+++ b/tests/runner.sh
@@ -23,6 +23,7 @@ default_suites=(
 
 extra_suites=(
 	remote-nextcloud
+	"upgrade-<name>"
 )
 
 usage() {
@@ -41,6 +42,7 @@ usage() {
 	echo "Extra test suites:"
 	echo "------------------"
 	echo "  remote-nextcloud   : test the setup mod for remote Nextcloud"
+	echo "  upgrade-<name>     : verify an upgrade using named populate data"
 	echo ""
 
 	echo "If no suite-name(s) are given, all default suites are run"
@@ -78,23 +80,30 @@ while [ $# -gt 0 ]; do
 			if [ $OVERALL_COUNT_SUITES -eq 0 ]; then
 				rm -rf "${BASE_OUTPUTDIR}"
 			fi
-			
-			if [ "$1" == "default" ]
-			then
-				# run all default suites
-				for suite in ${default_suites[@]}; do
-					. suites/$suite.sh
-				done
-			elif array_contains "$1" ${default_suites[@]} || \
-					array_contains "$1" ${extra_suites[@]}
-			then
-				# run specified suite
-				. "suites/$1.sh"
-			else
-				echo "Unknown suite '$1'" 1>&2
-				usage
-			fi
-			;;
+
+			case "$1" in
+				default )
+					# run all default suites
+					for suite in ${default_suites[@]}; do
+						. suites/$suite.sh
+					done
+					;;
+				upgrade-* )
+					# run upgrade suite with named populate data
+					. "suites/upgrade.sh" "$(awk -F- '{print $2}' <<< "$1")"
+					;;
+				* )
+					if array_contains "$1" "${default_suites[@]}" || \
+							array_contains "$1" "${extra_suites[@]}"
+					then
+						# run specified suite
+						. "suites/$1.sh"
+					else
+						echo "Unknown suite '$1'" 1>&2
+						usage
+					fi
+					;;
+			esac
 	esac
 	shift
 done
diff --git a/tests/suites/_init.sh b/tests/suites/_init.sh
index 829db86d..2a116001 100644
--- a/tests/suites/_init.sh
+++ b/tests/suites/_init.sh
@@ -12,6 +12,8 @@ set +eu
 . suites/_mgmt-functions.sh || exit 1
 
 # globals - all global variables are UPPERCASE
+ASSETS_DIR="assets"
+MIAB_DIR=".."
 BASE_OUTPUTDIR="$(realpath out)"
 PYMAIL="./test_mail.py"
 declare -i OVERALL_SUCCESSES=0
diff --git a/tests/suites/remote-nextcloud.sh b/tests/suites/remote-nextcloud.sh
index 825c3df5..5a4533ba 100644
--- a/tests/suites/remote-nextcloud.sh
+++ b/tests/suites/remote-nextcloud.sh
@@ -35,207 +35,26 @@ assert_is_configured() {
 }
 
 
-nextcloud_url() {
-    # eg: http://localhost/cloud/
-    carddav_url | sed 's|\(.*\)/remote.php/.*|\1/|'
-}
-
-carddav_url() {
-    # get the carddav url as configured in z-push for the user specified
-    # eg: http://localhost/cloud/remote.php/dav/addressbooks/users/admin/contacts/
-    local user="${1:-%u}"
-    local path="${2:-CARDDAV_DEFAULT_PATH}"
-    local php='include "/usr/local/lib/z-push/backend/carddav/config.php"; print CARDDAV_PROTOCOL . "://" . CARDDAV_SERVER . ":" . CARDDAV_PORT . '
-    php="$php$path;"
-    local url
-    url="$(php -n -r "$php")"
-    [ $? -ne 0 ] && die "Unable to run php to extract carddav url from z-push"
-    sed "s/%u/$user/" <<< "$url"
-}
-
-
-carddav_rest() {
-    # issue a CardDAV rest call to Nextcloud
-    # SEE: https://tools.ietf.org/html/rfc6352
-    local verb="$1"
-    local uri="$2"
-    local auth_user="$3"
-    local auth_pass="$4"
-	shift; shift; shift; shift  # remaining arguments are data
-
-    local url
-    case "$uri" in
-        /* )
-            url="$(nextcloud_url)${uri#/}"
-            ;;
-        http*)
-            url="$uri"
-            ;;
-        * )
-            url="$(carddav_url "$auth_user")${uri#/}"
-            ;;
-    esac
-    
-    local data=()
-	local item output
-
-    for item; do data+=("--data" "$item"); done
-
-    local ct
-    case "${data[1]}" in
-        BEGIN:VCARD* )
-            ct="text/vcard"
-            ;;
-        * )
-            ct='text/xml; charset="utf-8"'
-    esac
-    
-    record "spawn: curl -w \"%{http_code}\" -X $verb -H 'Content-Type: $ct' --user \"${auth_user}:xxx\" ${data[@]} \"$url\""
-	output=$(curl -s -S -w "%{http_code}" -X $verb -H "Content-Type: $ct" --user "${auth_user}:${auth_pass}" "${data[@]}" "$url" 2>>$TEST_OF)
-	local code=$?
-
-	# http status is last 3 characters of output, extract it
-	REST_HTTP_CODE=$(awk '{S=substr($0,length($0)-2)} END {print S}' <<<"$output")
-	REST_OUTPUT=$(awk 'BEGIN{L=""}{ if(L!="") print L; L=$0 } END { print substr(L,1,length(L)-3) }' <<<"$output")
-	REST_ERROR=""
-	[ -z "$REST_HTTP_CODE" ] && REST_HTTP_CODE="000"
-
-    if [ $code -ne 0 -o \
-               $REST_HTTP_CODE -lt 200 -o \
-               $REST_HTTP_CODE -ge 300 ]
-    then
-		REST_ERROR="REST status $REST_HTTP_CODE: $REST_OUTPUT"
-        REST_ERROR_BRIEF=$(python3 -c "import xml.etree.ElementTree as ET; print(ET.fromstring(r'''$REST_OUTPUT''').find('s:message',{'s':'http://sabredav.org/ns'}).text)" 2>/dev/null)
-        if [ -z "$REST_ERROR_BRIEF" ]; then
-            REST_ERROR_BRIEF="$REST_ERROR"
-        else
-            REST_ERROR_BRIEF="$REST_HTTP_CODE: $REST_ERROR_BRIEF"
-        fi
-        if [ $code -ne 0 ]; then
-            REST_ERROR_BRIEF="curl exit code $code: $REST_ERROR_BRIEF"
-            REST_ERROR="curl exit code $code: $REST_ERROR"
-        fi
-		record "${F_DANGER}$REST_ERROR${F_RESET}"
-		return 2
-	fi
-	record "CURL succeded, HTTP status $REST_HTTP_CODE"
-	record "$output"
-	return 0    
-}
-
-
-carddav_ls() {
-    # return all .vcf files in array 'FILES'
-    local user="$1"
-    local pass="$2"
-    carddav_rest PROPFIND "" "$user" "$pass" || return $?
-    local file FILES=()
-    python3 -c "import xml.etree.ElementTree as ET; [print(el.find('d:href',{'d':'DAV:'}).text) for el in ET.fromstring(r'''$REST_OUTPUT''').findall('d:response',{'d':'DAV:'}) if el.find('d:href',{'d':'DAV:'}) is not None]" |
-        while read file; do
-            # skip non .vcf entries
-            case "$file" in
-                *.vcf )
-                    FILES+=( "$(basename "$file")" )
-                    ;;
-                * )
-                    ;;
-            esac
-        done
-}
-
-
-make_collection() {
-    local user="$1"
-    local pass="$2"
-    local name="$3"
-    local desc="${4:-$name}"
-    local xml="<?xml version=\"1.0\" encoding=\"utf-8\" ?>
-<D:mkcol xmlns:D=\"DAV:\"
-         xmlns:C=\"urn:ietf:params:xml:ns:carddav\">
-  <D:set>
-    <D:prop>
-      <D:resourcetype>
-        <D:collection/>
-        <C:addressbook/>
-      </D:resourcetype>
-      <D:displayname>$name</D:displayname>
-      <C:addressbook-description xml:lang=\"en\">$desc</C:addressbook-description>
-    </D:prop>
-  </D:set>
-</D:mkcol>"
-    record "[create address book '$name' for $user]"
-    local url="$(carddav_url "$user" CARDDAV_PATH)"
-    carddav_rest MKCOL "$url" "$user" "$pass" "$xml"
-}
-
-
-
-add_contact() {
-    local user="$1"
-    local pass="$2"
-    local c_name="$3"
-    local c_phone="$4"
-    local c_email="$5"
-    local c_uid="${6:-$(generate_uuid)}"
-    local file_name="$c_uid.vcf"
-    
-    local vcard="BEGIN:VCARD
-VERSION:3.0
-UID:$c_uid
-REV;VALUE=DATE-AND-OR-TIME:$(date -u +%Y%m%dT%H%M%SZ)
-FN:$c_name
-EMAIL;TYPE=INTERNET,PREF:$c_email
-NOTE:Miab-LDAP QA
-ORG:Miab-LDAP
-TEL;TYPE=WORK,VOICE:$c_phone
-END:VCARD"
-    record "[add contact '$c_name' to $user]"
-    carddav_rest PUT "$file_name" "$user" "$pass" "$vcard"
-}
-
-delete_contact() {
-    local user="$1"
-    local pass="$2"
-    local c_uid="$3"
-    local file_name="$c_uid.vcf"
-    record "[delete contact with vcard uid '$c_uid' from $user]"
-    carddav_rest DELETE "$file_name" "$user" "$pass"
-}
-
-force_roundcube_carddav_refresh() {
-    local user="$1"
-    local pass="$2"
-    local code
-    record "[forcing refresh of roundcube contact for $user]"
-    copy_or_die assets/mail/roundcube/carddav_refresh.sh $RCM_DIR/bin
-    pushd "$RCM_DIR" >/dev/null
-    bin/carddav_refresh.sh "$user" "$pass" >>$TEST_OF 2>&1
-    code=$?
-    popd >/dev/null
-    return $code
-}
-
 assert_roundcube_carddav_contact_exists() {
     local user="$1"
     local pass="$2"
     local c_uid="$3"
     local output
     record "[checking that roundcube contact with vcard UID=$c_uid exists]"
-    output="$(sqlite3 $STORAGE_ROOT/mail/roundcube/roundcube.sqlite "select name from carddav_contacts where cuid='$c_uid'" 2>>$TEST_OF)"
-    if [ $? -ne 0 ]; then
-        test_failure "Error querying roundcube sqlite database"
-        return 1
-    fi
-    if [ -z "$output" ]; then
+	roundcube_carddav_contact_exists "$user" "$pass" "$c_uid" 2>>$TEST_OF
+	local rc=$?
+	
+    if [ $rc -eq 0 ]; then
+		return
+	elif [ $rc -eq 1 ]; then
         test_failure "Contact not found in Roundcube"
         record "Not found"
-        record "Existing entries (name,vcard-uid):"
-        output="$(sqlite3 $STORAGE_ROOT/mail/roundcube/roundcube.sqlite "select name,cuid FROM carddav_contacts" 2>>$TEST_OF)"
-        return 1
-    else
-        record "$output"
+        record "Existing entries:"
+		roundcube_dump_contacts >>$TEST_OF 2>&1
+	else
+		test_failure "Error querying roundcube contacts"
+		return
     fi
-    return 0
 }
 
 
@@ -245,9 +64,12 @@ test_mail_from_nextcloud() {
 }
 
 test_nextcloud_contacts() {
-    test_start "nextcloud_contacts"
+    test_start "nextcloud-contacts"
 
-    assert_is_configured || test_end && return
+    if ! assert_is_configured; then
+		test_end
+		return
+	fi
 
     local alice="alice.nc@somedomain.com"
     local alice_pw="$(generate_password 16)"
@@ -263,28 +85,37 @@ test_nextcloud_contacts() {
     # LDAP and that Roundcube is able to reach Nextcloud for contacts
     #
 
-    #make_collection "$alice" "$alice_pw" "contacts"
+	#record "[create address book 'contacts' for $alice]"
+	#carddav_make_addressbook "$alice" "$alice_pw" "contacts" 2>>$TEST_OF
 
     # add new contact to alice's Nextcloud account using CardDAV API
     local c_uid="$(generate_uuid)"
-    add_contact \
+    record "[add contact 'JimIno' to $alice]"
+    if ! carddav_add_contact \
         "$alice" \
         "$alice_pw" \
         "JimIno" \
         "555-1212" \
         "jim@ino.com" \
         "$c_uid" \
-        || test_failure "Could not add contact for $alice in Nextcloud: $REST_ERROR_BRIEF"
+		2>>$TEST_OF
+	then
+		test_failure "Could not add contact for $alice in Nextcloud: $REST_ERROR_BRIEF"
+		test_end
+		return
+	fi
     
     # force a refresh/sync of the contacts in Roundcube
-    force_roundcube_carddav_refresh "$alice" "$alice_pw" || \
+    record "[forcing refresh of roundcube contact for $alice]"
+    roundcube_force_carddav_refresh "$alice" "$alice_pw" >>$TEST_OF 2>&1 || \
         test_failure "Could not refresh roundcube contacts for $alice"
 
     # query the roundcube sqlite database for the new contact
     assert_roundcube_carddav_contact_exists "$alice" "$alice_pw" "$c_uid"
     
     # delete the contact
-    delete_contact "$alice" "$alice_pw" "$c_uid" || \
+    record "[delete contact with vcard uid '$c_uid' from $alice]"
+    carddav_delete_contact "$alice" "$alice_pw" "$c_uid" 2>>$TEST_OF || \
         test_failure "Unable to delete contact for $alice in Nextcloud"
     
 
diff --git a/tests/suites/upgrade.sh b/tests/suites/upgrade.sh
new file mode 100644
index 00000000..a8586f70
--- /dev/null
+++ b/tests/suites/upgrade.sh
@@ -0,0 +1,48 @@
+
+#
+# the system must have been populated proir to any upgrade with one of
+# the tests/system-setup/populate scripts to use this suite
+#
+# supply the name of the populate script that was used as an argument
+# eg. if basic-populate.sh was used to populate, supply "basic" to the
+# script as an argument
+#
+
+
+verify_populate() {    
+    local populate_name="$1"
+    local verify_script="system-setup/populate/${populate_name}-verify.sh"
+
+    test_start "verify '$populate_name' population set"
+
+    if [ ! -e "$verify_script" ]; then
+        test_failure "Verify script $(basename "$verify_script") does not exist"
+
+    else
+        record "[run verify-upgrade script $verify_script]"
+        local output rc
+        output=$("$verify_script" 2>>$TEST_OF)
+        rc=$?
+        if [ $rc -ne 0 ]
+        then
+           if [ $rc -eq 127 ]; then
+               test_failure "verify script would not run (wd=$(pwd))"
+           else
+               test_failure "verify script exited with $rc: $output"
+           fi
+        fi
+    fi
+
+    test_end
+}
+
+
+
+suite_start "upgrade"
+
+export ASSETS_DIR
+export MIAB_DIR
+
+verify_populate "$1"
+
+suite_end
diff --git a/tests/system-setup/populate/README.txt b/tests/system-setup/populate/README.txt
new file mode 100644
index 00000000..095f65dc
--- /dev/null
+++ b/tests/system-setup/populate/README.txt
@@ -0,0 +1,31 @@
+This directory contains scripts used to populate a MiaB installation
+with known values, and then subsequently verify that MiaB continues to
+operate poperly after an upgrade or setup mod change.
+
+Each "named" populate set of scripts should contain at least two
+shell scripts:
+
+   1. <name>-populate.sh  : populates the installation
+   2. <name>-verify.sh    : verifies operation after upgrade
+
+The system-setup/* scripts run the populate script, and the test
+runner's 'upgrade' test suite runs the verify script.
+
+These scripts are run, not sourced.
+
+
+Expected script output and return value:
+
+   1. All debug output must go to stderr
+   2. Result messages must be sent to stdout (a single line, preferrably)
+   3. Return 0 if successfully passed verification
+   4. Return non-zero if failed verification
+
+The working directory for <name>-populate.sh is the Mail-in-a-Box root
+directory.
+
+The working directory for <name>-verify.sh is 'tests' (because the
+test runner always changes the working directory there to limit
+contamination of the source tree). Use MIAB_DIR and ASSETS_DIR, if
+needed.
+
diff --git a/tests/system-setup/populate/basic-data.sh b/tests/system-setup/populate/basic-data.sh
new file mode 100755
index 00000000..bf41b0a7
--- /dev/null
+++ b/tests/system-setup/populate/basic-data.sh
@@ -0,0 +1,10 @@
+#
+# requires:
+#    lib scripts: [ misc.sh ]
+#    system-setup scripts: [ setup-defaults.sh ]
+#
+
+TEST_USER="anna@$(email_domainpart "$EMAIL_ADDR")"
+TEST_USER_PASS="$(static_qa_password)"
+TEST_USER_CONTACT_UUID="e0642b47-9104-4adb-adfd-5f907d04216a"
+TEST_USER_CONTACT_EMAIL="sam@bff.org"
diff --git a/tests/system-setup/populate/basic-populate.sh b/tests/system-setup/populate/basic-populate.sh
new file mode 100755
index 00000000..d4f3d9dc
--- /dev/null
+++ b/tests/system-setup/populate/basic-populate.sh
@@ -0,0 +1,48 @@
+#!/bin/bash
+
+. "$(dirname "$0")/../setup-defaults.sh" || exit 1
+. "$(dirname "$0")/../../lib/all.sh" "$(dirname "$0")/../../lib" || exit 1
+. "$(dirname "$0")/basic-data.sh" || exit 1
+
+
+#
+# Add user
+#
+if ! populate_miab_users "" "" "" "${TEST_USER}:${TEST_USER_PASS}"
+then
+    echo "Unable to add user"
+    exit 1
+fi
+
+#
+# Add Nextcloud contact and force Roundcube contact sync to ensure the
+# roundcube carddav addressbooks and contacts tables are populated in
+# case a remote nextcloud is subsequently configured and the
+# syncronization disabled.
+#
+if ! carddav_ls "$TEST_USER" "$TEST_USER_PASS" --insecure 2>/dev/null
+then
+    echo "Could not enumerate contacts: $REST_ERROR"
+    exit 1
+fi
+echo "Current contacts count: ${#FILES[@]}"
+    
+if array_contains "$TEST_USER_CONTACT_UUID.vcf" "${FILES[@]}"; then
+    echo "Contact $TEST_USER_CONTACT_UUID already present"
+else
+    if ! carddav_add_contact "$TEST_USER" "$TEST_USER_PASS" "Anna" "666-1111" "$TEST_USER_CONTACT_EMAIL" "$TEST_USER_CONTACT_UUID" --insecure 2>/dev/null
+    then
+        echo "Could not add contact: $REST_ERROR"
+        exit 1
+    fi
+    
+    echo "Force Roundcube contact sync"
+    if ! roundcube_force_carddav_refresh "$TEST_USER" "$TEST_USER_PASS"
+    then
+        echo "Roundcube <-> Nextcloud contact sync failed"
+        exit 1
+    fi
+fi
+
+exit 0
+
diff --git a/tests/system-setup/populate/basic-verify.sh b/tests/system-setup/populate/basic-verify.sh
new file mode 100755
index 00000000..1165e7f6
--- /dev/null
+++ b/tests/system-setup/populate/basic-verify.sh
@@ -0,0 +1,53 @@
+#!/bin/bash
+
+. "$(dirname "$0")/../setup-defaults.sh" || exit 1
+. "$(dirname "$0")/../../lib/all.sh" "$(dirname "$0")/../../lib" || exit 1
+. "$(dirname "$0")/basic-data.sh" || exit 1
+. /etc/mailinabox.conf || exit 1
+
+
+# 1. the test user can still log in and send mail
+
+echo "[User can still log in with their old passwords and send mail]" 1>&2
+echo "python3 test_mail.py $PRIVATE_IP $TEST_USER $TEST_USER_PASS" 1>&2
+python3 test_mail.py "$PRIVATE_IP" "$TEST_USER" "$TEST_USER_PASS" 1>&2
+if [ $? -ne 0 ]; then
+    echo "Basic mail functionality test failed"
+    exit 1
+fi
+
+
+# 2. the test user's contact is still accessible in Roundcube
+
+echo "[Force Roundcube contact sync]" 1>&2
+# if MiaB's Nextcloud carddav configuration was removed all the
+# contacts for it will be removed in the Roundcube database after the
+# sync
+
+if ! roundcube_force_carddav_refresh "$TEST_USER" "$TEST_USER_PASS" 1>&2
+then
+    echo "Roundcube <-> Nextcloud contact sync failed ($?)"
+    exit 1
+fi
+
+echo "[Ensure old Nextcloud contacts are still present]" 1>&2
+echo "sqlite3 $STORAGE_ROOT/mail/roundcube/roundcube.sqlite \"select email from carddav_contacts where cuid='$TEST_USER_CONTACT_UUID'\"" 1>&2
+output=$(sqlite3 "$STORAGE_ROOT/mail/roundcube/roundcube.sqlite" "select email from carddav_contacts where cuid='$TEST_USER_CONTACT_UUID'")
+rc=$?
+if [ $rc -ne 0 ]
+then
+    echo "Querying Roundcube's sqlite database failed ($rc)"
+    exit 1
+else
+    echo "Success, found $output" 1>&2
+fi
+
+if [ "$output" != "$TEST_USER_CONTACT_EMAIL" ]
+then
+    echo "Unexpected email for contact uuid: got '$output', expected '$TEST_USER_CONTACT_EMAIL'"
+    exit 1
+fi
+
+echo "OK basic-verify passed"
+exit 0
+
diff --git a/tests/system-setup/remote-nextcloud-docker.sh b/tests/system-setup/remote-nextcloud-docker.sh
index e24900e4..c68470e3 100755
--- a/tests/system-setup/remote-nextcloud-docker.sh
+++ b/tests/system-setup/remote-nextcloud-docker.sh
@@ -26,13 +26,6 @@
 #
 
 
-usage() {
-    echo "Usage: $(basename "$0")"
-    echo "Install MiaB-LDAP and a remote Nextcloud running under docker"
-    echo "Nextcloud is exposed as http://localhost:8000"
-    exit 1
-}
-
 # ensure working directory
 if [ ! -d "tests/system-setup" ]; then
     echo "This script must be run from the MiaB root directory"
@@ -51,49 +44,26 @@ fi
 
 
 
-before_miab_install() {
-    H1 "BEFORE MIAB-LDAP INSTALL"
-    system_init
-    miab_testing_init || die "Initialization failed"
-    
-    # enable the remote Nextcloud setup mod, which tells MiaB-LDAP to use
-    # the remote Nextcloud for calendar and contacts instead of the
-    # MiaB-installed one
-    H2 "Enable local mod remote-nextcloud"
-    enable_miab_mod "remote-nextcloud" \
-        || die "Could not enable remote-nextcloud mod"
-    
+init() {
+    H1 "INIT"
+    init_test_system
+    init_miab_testing || die "Initialization failed"    
+}
+
+
+install_nextcloud_docker() {
+    H1 "INSTALL NEXTCLOUD ON DOCKER"
+
     # install Docker
     H2 "Install Docker"
     install_docker || die "Could not install Docker! ($?)"
-}
 
-
-miab_install() {
-    H1 "MIAB-LDAP INSTALL"
-    if ! setup/start.sh; then
-        H1 "OUTPUT OF SELECT FILES"
-        dump_file "/var/log/syslog" 100
-        dump_conf_files "$TRAVIS"
-        H2; H2 "End"; H2
-        die "setup/start.sh failed!"
-    fi
-    H1 "OUTPUT OF SELECT FILES"
-    dump_conf_files "$TRAVIS"
-    H2; H2 "End"; H2
-}
-
-
-after_miab_install() {
-    H1 "AFTER MIAB-LDAP INSTALL"
-    
-    . /etc/mailinabox.conf || die "Could not load /etc/mailinabox.conf"
-    
     # run Nextcloud docker image
     H2 "Start Nextcloud docker container"
     local container_started="true"
     if [ -z "$(docker ps -f NAME=NC -q)" ]; then
         docker run -d --name NC -p 8000:80 \
+               --add-host "$PRIMARY_HOSTNAME:$PRIVATE_IP" \
                --env SQLITE_DATABASE=nextclouddb.sqlite \
                --env NEXTCLOUD_ADMIN_USER="$NC_ADMIN_USER" \
                --env NEXTCLOUD_ADMIN_PASSWORD="$NC_ADMIN_PASSWORD" \
@@ -114,10 +84,10 @@ after_miab_install() {
         container_started="false"
     fi
 
-    H2 "docker: Update /etc/hosts so it can find MiaB-LDAP by name"
-    echo "$PRIVATE_IP $PRIMARY_HOSTNAME" | \
-        docker exec -i NC bash -c 'cat >>/etc/hosts' \
-        || die "docker: could not update /etc/hosts"
+    # H2 "docker: Update /etc/hosts so it can find MiaB-LDAP by name"
+    # echo "$PRIVATE_IP $PRIMARY_HOSTNAME" | \
+    #     docker exec -i NC bash -c 'cat >>/etc/hosts' \
+    #     || die "docker: could not update /etc/hosts"
     
     # apt-get update
     H2 "docker: apt-get update"
@@ -171,26 +141,87 @@ after_miab_install() {
 }
 
 
-#
-# Main
-#
-case "${1:-all}" in
-    before-install )
-        before_miab_install
+
+
+do_upgrade() {
+    local populate_name="$1"
+
+    if [ -e "local/remote-nextcloud.sh" ]; then
+        # we install w/o remote nextcloud first so we can add
+        # a user w/contacts and ensure the contact exists in the
+        # new system
+        if [ ! -L "local/remote-nextcloud.sh" ]; then
+            echo "Warning: local/remote-nextcloud.sh is a regular file - should be a symlink"
+        fi
+        die "Error: local/remote-nextcloud.sh exists - delete it and try again"
+    fi
+
+    # initialize test system
+    init
+
+    # install w/o remote Nextcloud
+    miab_ldap_install
+    
+    # populate some data
+    [ ! -z "$populate_name" ] && populate_by_name "$populate_name"
+    
+    # install Nextcloud in a Docker container (MiaB must be available)
+    install_nextcloud_docker
+    
+    H1 "Enable remote-nextcloud mod"
+    enable_miab_mod "remote-nextcloud" \
+        || die "Could not enable remote-nextcloud mod"
+
+    # re-run setup to use the remote Nextcloud
+    miab_ldap_install
+}
+
+
+do_default() {
+    # initialize test system
+    init
+
+    H1 "Enable remote-nextcloud mod"
+    enable_miab_mod "remote-nextcloud" \
+        || die "Could not enable remote-nextcloud mod"
+    
+    # run setup to use the remote Nextcloud (doesn't need to be available)
+    miab_ldap_install
+
+    # install Nextcloud in a Docker container (MiaB must be available)
+    install_nextcloud_docker    
+}
+
+
+
+
+
+case "$1" in
+    upgrade )
+        # Runs this sequence:
+        #   1. setup w/o remote nextcloud
+        #   2. if an additional argument is given, populate the MiaB
+        #      installation
+        #   3. install a remote nextcloud
+        #   4. enable remote-nextcloud mod
+        #   5. re-run setup
+        #
+
+        shift
+        do_upgrade "$@"
         ;;
-    install )
-        miab_install
-        ;;
-    after-install )
-        after_miab_install
-        ;;
-    all )
-        before_miab_install
-        miab_install
-        after_miab_install
+
+    "" | default )
+        # Runs this sequence:
+        #   1. setup w/remote nextcloud
+        #   2. install and connect the remote nextcloud
+        do_default
         ;;
+    
     * )
-        usage
+        echo "Unknown option $1"
+        exit 1
         ;;
 esac
 
+
diff --git a/tests/system-setup/setup-defaults.sh b/tests/system-setup/setup-defaults.sh
index b415e7a0..7d44f93a 100755
--- a/tests/system-setup/setup-defaults.sh
+++ b/tests/system-setup/setup-defaults.sh
@@ -7,7 +7,7 @@ export STORAGE_USER="${STORAGE_USER:-user-data}"
 export STORAGE_ROOT="${STORAGE_ROOT:-/home/$STORAGE_USER}"
 export EMAIL_ADDR="${EMAIL_ADDR:-qa@abc.com}"
 export EMAIL_PW="${EMAIL_PW:-Test_1234}"
-export PUBLIC_IP="${PUBLIC_IP:-$(source setup/functions.sh; get_default_privateip 4)}"
+export PUBLIC_IP="${PUBLIC_IP:-$(source ${MIAB_DIR:-.}/setup/functions.sh; get_default_privateip 4)}"
 
 if [ "$TRAVIS" == "true" ]; then
     export PRIMARY_HOSTNAME=${PRIMARY_HOSTNAME:-box.abc.com}
diff --git a/tests/system-setup/setup-funcs.sh b/tests/system-setup/setup-funcs.sh
index c04a4fa9..3f6db83e 100755
--- a/tests/system-setup/setup-funcs.sh
+++ b/tests/system-setup/setup-funcs.sh
@@ -5,6 +5,7 @@
 #   test scripts: [ lib/misc.sh, lib/system.sh ]
 #
 
+
 die() {
     local msg="$1"
     echo "$msg" 1>&2
@@ -12,25 +13,6 @@ die() {
 }
 
 
-H1() {
-    local msg="$1"
-    echo "----------------------------------------------"
-    if [ ! -z "$msg" ]; then
-        echo "           $msg"
-        echo "----------------------------------------------"
-    fi
-}
-
-H2() {
-    local msg="$1"
-    if [ -z "$msg" ]; then
-        echo "***"
-    else
-        echo "*** $msg ***"
-    fi
-}
-
-
 wait_for_docker_nextcloud() {
     local container="$1"
     local config_key="$2"
@@ -83,7 +65,9 @@ dump_conf_files() {
 # Initialize the test system
 #   hostname, time, apt update/upgrade, etc
 #
-system_init() {
+# Errors are fatal
+#
+init_test_system() {
     H2 "Update /etc/hosts"
     set_system_hostname || die "Could not set hostname"
 
@@ -113,7 +97,7 @@ system_init() {
 # Anything needed to use the test runner, speed up the installation,
 # etc
 #
-miab_testing_init() {
+init_miab_testing() {
     [ -z "$STORAGE_ROOT" ] \
         && echo "Error: STORAGE_ROOT not set" 1>&2 \
         && return 1
@@ -152,8 +136,12 @@ miab_testing_init() {
 enable_miab_mod() {
     local name="${1}.sh"
     if [ ! -e "local/$name" ]; then
-        mkdir -p local
-        ln -s "../setup/mods.available/$name" "local/$name"
+        mkdir -p "local"
+        if ! ln -s "../setup/mods.available/$name" "local/$name"
+        then
+            echo "Warning: copying instead of symlinking local/$name"
+            cp "setup/mods.available/$name" "local/$name"
+        fi
     fi
 }
 
@@ -166,3 +154,34 @@ tag_from_readme() {
     return 0
 }
 
+
+miab_ldap_install() {
+    H1 "MIAB-LDAP INSTALL"
+    # ensure we're in a MiaB-LDAP working directory
+    if [ ! -e setup/ldap.sh ]; then
+        die "Cannot install: the working directory is not MiaB-LDAP!"
+    fi
+    
+    if ! setup/start.sh; then
+        H1 "OUTPUT OF SELECT FILES"
+        dump_file "/var/log/syslog" 100
+        dump_conf_files "$TRAVIS"
+        H2; H2 "End"; H2
+        die "MiaB-LDAP setup/start.sh failed!"
+    fi
+
+    # set actual STORAGE_ROOT, STORAGE_USER, PRIVATE_IP, etc
+    . /etc/mailinabox.conf || die "Could not source /etc/mailinabox.conf"
+}
+
+
+populate_by_name() {
+    local populate_name="$1"
+
+    H1 "Populate Mail-in-a-Box ($populate_name)"
+    local populate_script="tests/system-setup/populate/${populate_name}-populate.sh"
+    if [ ! -e "$populate_script" ]; then
+        die "Does not exist: $populate_script"
+    fi
+    "$populate_script" || die "Failed: $populate_script"
+}
diff --git a/tests/system-setup/upgrade-from-upstream.sh b/tests/system-setup/upgrade-from-upstream.sh
index aba6c6d2..721d7c11 100755
--- a/tests/system-setup/upgrade-from-upstream.sh
+++ b/tests/system-setup/upgrade-from-upstream.sh
@@ -32,10 +32,10 @@ if [ "$EUID" != "0" ]; then
 fi
 
 
-before_install() {
+init() {
     H1 "INIT"
-    system_init
-    miab_testing_init || die "Initialization failed"
+    init_test_system
+    init_miab_testing || die "Initialization failed"
 }
 
 upstream_install() {
@@ -94,217 +94,60 @@ upstream_install() {
 }
 
 
-add_data() {
+populate() {
+    local pw="$(static_qa_password)"
+    
     H1 "Add some Mail-in-a-Box data"
     local users=()
-    users+=("betsy@$(email_domainpart "$EMAIL_ADDR")")
+    users+=("betsy@$(email_domainpart "$EMAIL_ADDR"):$pw")
 
     local alises=()
-    aliases+=("goalias@testdom.com > ${users[0]}")
+    aliases+=("goalias@testdom.com > $(awk -F: {'print $1'} <<<"${users[0]}")")
     aliases+=("nested@testdom.com > goalias@testdom.com")
 
-    local pw="$(generate_qa_password)"
-
-
-    #
-    # get the existing users and aliases
-    #
-    local current_users=() current_aliases=()
-    local user alias
-    if ! rest_urlencoded GET /admin/mail/users "$EMAIL_ADDR" "$EMAIL_PW" --insecure >/dev/null 2>&1; then
-        die "Unable to enumerate users: rc=$? err=$REST_ERROR"
-    fi
-    for user in $REST_OUTPUT; do
-        current_users+=("$user")
-    done
-
-    if ! rest_urlencoded GET /admin/mail/aliases "$EMAIL_ADDR" "$EMAIL_PW" --insecure 2>/dev/null; then
-        die "Unable to enumerate aliases: rc=$? err=$REST_ERROR"
-    fi
-    for alias in $REST_OUTPUT; do
-        current_aliases+=("$alias")
-    done
-
-    
-    #
-    # add users
-    #
     H2 "Add users"
-    for user in "${users[@]}"; do
-        if array_contains "$user" "${current_users[@]}"; then
-            echo "Not adding user $user: already exists"
+    if ! populate_miab_users "" "" "" "${users[@]}"
+    then
+        die "Unable to add users"
+    fi
 
-        elif ! rest_urlencoded POST /admin/mail/users/add "$EMAIL_ADDR" "$EMAIL_PW" --insecure -- "email=$user" "password=$pw" 2>/dev/null
-        then
-            die "Unable to add user $user: rc=$? err=$REST_ERROR"
-        else
-            echo "Add: $user"
-        fi
-    done
-
-    #
-    # add aliases
-    #
     H2 "Add aliases"
-    local aliasdef
-    for aliasdef in "${aliases[@]}"; do
-        alias="$(awk -F'[> ]' '{print $1}' <<<"$aliasdef")"
-        local forwards_to="$(sed 's/.*> *\(.*\)/\1/' <<<"$aliasdef")"
-        if array_contains "$alias" "${current_aliases[@]}"; then
-            echo "Not adding alias $alias: already exists"
-            
-        elif ! rest_urlencoded POST /admin/mail/aliases/add "$EMAIL_ADDR" "$EMAIL_PW" --insecure -- "address=$alias" "forwards_to=$forwards_to" 2>/dev/null
-        then
-            die "Unable to add alias $alias: rc=$? err=$REST_ERROR"
-        else
-            echo "Add: $aliasdef"
-        fi
-    done
-}
-
-capture_state() {
-    # users and aliases lists
-    # dns zone files
-    # tls certificates: expected CN's
-
-    local state_dir="$1"
-    local info="$state_dir/info.txt"
-
-    H1 "Capture server state to $state_dir"
-    
-    # nuke saved state, if any
-    rm -rf "$state_dir"
-    mkdir -p "$state_dir"
-
-    # create info.json
-    H2 "create info.txt"
-    echo "VERSION='$(git describe --abbrev=0)'" >"$info"
-    echo "MIGRATION_VERSION=$(cat "$STORAGE_ROOT/mailinabox.version")" >>"$info"
-
-    # record users
-    H2 "record users"
-    rest_urlencoded GET "/admin/mail/users?format=json" "$EMAIL_ADDR" "$EMAIL_PW" --insecure 2>/dev/null \
-        || die "Unable to get users: rc=$? err=$REST_ERROR"
-    echo "$REST_OUTPUT" > "$state_dir/users.json"
-
-    # record aliases
-    H2 "record aliases"
-    rest_urlencoded GET "/admin/mail/aliases?format=json" "$EMAIL_ADDR" "$EMAIL_PW" --insecure 2>/dev/null \
-        || die "Unable to get aliases: rc=$? err=$REST_ERROR"
-    echo "$REST_OUTPUT" > "$state_dir/aliases.json"
-
-    # record dns config
-    H2 "record dns details"
-    local file
-    mkdir -p "$state_dir/zones"
-    for file in /etc/nsd/zones/*.signed; do
-        cp "$file" "$state_dir/zones"
-    done    
-}
-
-miab_ldap_install() {
-    H1 "INSTALL MIAB-LDAP"
-    # ensure we're in a MiaB-LDAP working directory
-    if [ ! -e setup/ldap.sh ]; then
-        die "The working directory is not MiaB-LDAP!"
-    fi
-    setup/start.sh -v || die "Upgrade to MiaB-LDAP failed !!!!!!"
-}
-
-compare_state() {
-    local s1="$1"
-    local s2="$2"
-    
-    local output
-    local changed="false"
-
-    H1 "COMPARE STATES: $(basename "$s1") VS $(basename "$2")"
-    H2 "Users"
-    # users
-    output="$(diff "$s1/users.json" "$s2/users.json" 2>&1)"
-    if [ $? -ne 0 ]; then
-        changed="true"
-        echo "USERS ARE DIFFERENT!"
-        echo "$output"
-    else
-        echo "No change"
-    fi
-
-    H2 "Aliases"
-    output="$(diff "$s1/aliases.json" "$s2/aliases.json" 2>&1)"
-    if [ $? -ne 0 ]; then
-        change="true"
-        echo "ALIASES ARE DIFFERENT!"
-        echo "$output"
-    else
-        echo "No change"
-    fi
-
-    H2 "DNS - zones missing"
-    local zone count=0
-    for zone in $(cd "$s1/zones"; ls *.signed); do
-        if [ ! -e "$s2/zones/$zone" ]; then
-            echo "MISSING zone: $zone"
-            changed="true"
-            let count+=1
-        fi
-    done
-    echo "$count missing"
-
-    H2 "DNS - zones added"
-    count=0
-    for zone in $(cd "$s2/zones"; ls *.signed); do
-        if [ ! -e "$s2/zones/$zone" ]; then
-            echo "ADDED zone: $zone"
-            changed="true"
-            let count+=1
-        fi
-    done
-    echo "$count added"
-
-    H2 "DNS - zones changed"
-    count=0
-    for zone in $(cd "$s1/zones"; ls *.signed); do
-        if [ -e "$s2/zones/$zone" ]; then
-            # all the signatures change if we're using self-signed certs
-            local t1="/tmp/s1.$$.txt"
-            local t2="/tmp/s2.$$.txt"
-            awk '$4 == "RRSIG" || $4 == "NSEC3" { next; } $4 == "SOA" { print $1" "$2" "$3" "$4" "$5" "$6" "$8" "$9" "$10" "$11" "$12; next } { print $0 }' "$s1/zones/$zone" > "$t1" 
-            awk '$4 == "RRSIG" || $4 == "NSEC3" { next; } $4 == "SOA" { print $1" "$2" "$3" "$4" "$5" "$6" "$8" "$9" "$10" "$11" "$12; next } { print $0 }' "$s2/zones/$zone" > "$t2" 
-            output="$(diff "$t1" "$t2" 2>&1)"
-            if [ $? -ne 0 ]; then
-                echo "CHANGED zone: $zone"
-                echo "$output"
-                changed="true"
-                let count+=1
-            fi
-        fi
-    done
-    echo "$count zone files had differences"
-
-    if $changed; then
-        return 1
-    else
-        return 0
+    if ! populate_miab_aliases "" "" "" "${aliases[@]}"
+    then
+        die "Unable to add aliases"
     fi
 }
 
 
 
-if [ "$1" == "cap" ]; then
-    capture_state "tests/system-setup/state/miab-ldap"
-    exit $?
-elif [ "$1" == "compare" ]; then
-    compare_state "tests/system-setup/state/upstream" "tests/system-setup/state/miab-ldap"
-    exit $?
-fi
+
+# these are for debugging/testing
+case "$1" in
+    capture )
+        . /etc/mailinabox.conf
+        installed_state_capture "tests/system-setup/state/miab-ldap"
+        exit $?
+        ;;
+    compare )
+        . /etc/mailinabox.conf
+        installed_state_compare "tests/system-setup/state/upstream" "tests/system-setup/state/miab-ldap"
+        exit $?
+        ;;
+    populate )
+        . /etc/mailinabox.conf
+        populate_by_name "${1:-basic}"
+        exit $?
+        ;;
+esac
+
 
 
 
 # install basic stuff, set the hostname, time, etc
-before_install
+init
 
 # if MiaB-LDAP is already migrated, do not run upstream setup
+[ -e /etc/mailinabox.conf ] && . /etc/mailinabox.conf
 if [ -e "$STORAGE_ROOT/mailinabox.version" ] &&
        [ $(cat "$STORAGE_ROOT/mailinabox.version") -ge 13 ]
 then
@@ -312,16 +155,21 @@ then
 else
     # install upstream
     upstream_install
-    add_data
-    capture_state "tests/system-setup/state/upstream"
+    . /etc/mailinabox.conf
+    
+    # populate some data
+    populate_by_name "${1:-basic}"
+
+    # capture upstream state
+    installed_state_capture "tests/system-setup/state/upstream"
 fi
 
-# install miab-ldap
+# install miab-ldap and capture state
 miab_ldap_install
-capture_state "tests/system-setup/state/miab-ldap"
+installed_state_capture "tests/system-setup/state/miab-ldap"
 
 # compare states
-if ! compare_state "tests/system-setup/state/upstream" "tests/system-setup/state/miab-ldap"; then
+if ! installed_state_compare "tests/system-setup/state/upstream" "tests/system-setup/state/miab-ldap"; then
     die "Upstream and upgraded states are different !"
 fi
 

From 5aa3a6fd5d35a1a15a250a304e13ed932a3fc312 Mon Sep 17 00:00:00 2001
From: downtownallday <downtownallday@gmail.com>
Date: Fri, 19 Jun 2020 12:26:55 -0400
Subject: [PATCH 63/65] Add missing argument

---
 .travis.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.travis.yml b/.travis.yml
index 71f87913..9ca03542 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -32,7 +32,7 @@ jobs:
       # setup with 'basic' data before setting up again using
       # a remote nextcloud to verify ownCloud contacts are still
       # available
-      - sudo tests/system-setup/remote-nextcloud-docker.sh basic
+      - sudo tests/system-setup/remote-nextcloud-docker.sh upgrade basic
     script:
       # launch automated tests, but skip tests that require remote
       # smtp support because Travis-CI blocks outgoing port 25

From 25f5690655ea08bef18f68d35763bc9fae66ecac Mon Sep 17 00:00:00 2001
From: downtownallday <downtownallday@gmail.com>
Date: Fri, 19 Jun 2020 18:05:57 -0400
Subject: [PATCH 64/65] Fix wording

---
 README.md                    | 2 +-
 tests/lib/installed-state.sh | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/README.md b/README.md
index 469bcc72..5ad5a7b1 100644
--- a/README.md
+++ b/README.md
@@ -18,7 +18,7 @@ To integrate Mail-in-a-Box w/LDAP (MiaB-LDAP) with Nextcloud, changes must be ma
 
 Enable the setup mod `remote-nextcloud.sh` by creating the directory `local` in the directory where mailinabox is installed (usually $HOME/mailinabox), then creat a symbolic link to remote-nextcloud.sh. e.g. run this command from the mailinabox directory: `mkdir -p local; ln -s ../setup/mods.available/remote-nextcloud.sh local/remote-nextcloud.sh`. *During setup you will be prompted for the hostname and web prefix of your remote Nextcloud box.*
 
-The setup mod will configure Roundcube and Z-Push (ActiveSync) to use the remote Nextcloud for contacts and calendar instead of the local Nextcloud, which will be disabled (browsing to /cloud will fail).
+The setup mod will configure Roundcube and Z-Push (ActiveSync) to use the remote Nextcloud for contacts and calendar instead of the local Nextcloud, which will be disabled (browsing to /cloud will fail). Old contacts will still be available in Roundcube, but read-only. Users can drag them into the remote Nextcloud.
 
 **On the remote Nextcloud**
 
diff --git a/tests/lib/installed-state.sh b/tests/lib/installed-state.sh
index 3efc0411..3e3eacf0 100644
--- a/tests/lib/installed-state.sh
+++ b/tests/lib/installed-state.sh
@@ -15,7 +15,7 @@ installed_state_capture() {
     local state_dir="$1"
     local info="$state_dir/info.txt"
 
-    H1 "Capture installed estate to $state_dir"
+    H1 "Capture installed state to $state_dir"
 
     # nuke saved state, if any
     rm -rf "$state_dir"

From a5ab29c83f3c00684de2cda5036e2675a44a274f Mon Sep 17 00:00:00 2001
From: downtownallday <downtownallday@gmail.com>
Date: Sun, 21 Jun 2020 09:13:54 -0400
Subject: [PATCH 65/65] Add Vagrant support for running automated tests

---
 setup/start.sh                                |  6 +-
 tests/lib/color-output.sh                     | 14 ++++
 tests/lib/misc.sh                             | 14 ++++
 tests/suites/_init.sh                         |  2 +-
 tests/system-setup/remote-nextcloud-docker.sh | 14 ++--
 tests/system-setup/setup-defaults.sh          |  1 +
 tests/system-setup/setup-funcs.sh             | 10 +--
 tests/system-setup/upgrade-from-upstream.sh   | 10 +--
 tests/vagrant/.gitignore                      |  3 +
 tests/vagrant/Vagrantfile                     | 42 +++++++++++
 tests/vagrant/globals.sh                      |  2 +
 tests/vagrant/parallel.sh                     | 71 +++++++++++++++++++
 12 files changed, 166 insertions(+), 23 deletions(-)
 create mode 100644 tests/vagrant/.gitignore
 create mode 100644 tests/vagrant/Vagrantfile
 create mode 100644 tests/vagrant/globals.sh
 create mode 100755 tests/vagrant/parallel.sh

diff --git a/setup/start.sh b/setup/start.sh
index 5da27665..91848873 100755
--- a/setup/start.sh
+++ b/setup/start.sh
@@ -150,9 +150,9 @@ fi
 #
 # Run setup mods 
 #
-if [ -d local ]; then
-    for mod in $(ls local | grep -v '~$'); do
-        local/$mod
+if [ -d "${LOCAL_MODS_DIR:-local}" ]; then
+    for mod in $(ls "${LOCAL_MODS_DIR:-local}" | grep -v '~$'); do
+        ${LOCAL_MODS_DIR:-local}/$mod
     done
 fi
 
diff --git a/tests/lib/color-output.sh b/tests/lib/color-output.sh
index 3652e98b..29ee80f6 100644
--- a/tests/lib/color-output.sh
+++ b/tests/lib/color-output.sh
@@ -1,9 +1,23 @@
 # ansi escapes for hilighting text
 F_DANGER=$(echo -e "\033[31m")
 F_WARN=$(echo -e "\033[93m")
+F_SUCCESS=$(echo -e "\033[32m")
 F_RESET=$(echo -e "\033[39m")
 
 
+success() {
+    local echoarg
+    case "$1" in
+        -n )
+            echoarg="$1"
+            shift
+            ;;
+        * )
+            echoarg=""
+    esac
+    echo $echoarg "${F_SUCCESS}$1${F_RESET}"
+}
+
 danger() {
     local echoarg
     case "$1" in
diff --git a/tests/lib/misc.sh b/tests/lib/misc.sh
index 52a2b8a6..dc49809a 100644
--- a/tests/lib/misc.sh
+++ b/tests/lib/misc.sh
@@ -67,3 +67,17 @@ sha1() {
 	python3 -c "import hashlib; m=hashlib.sha1(); m.update(bytearray(r'''$txt''','utf-8')); print(m.hexdigest());" || die "Unable to generate sha1 hash"
 }
 
+elapsed_pretty() {
+    local start_s="$1"
+    local end_s="$2"
+    local elapsed elapsed_m elapsed_s
+    if [ -z "$end_s" ]; then
+        elapsed="$start_s"
+    else
+        let elapsed="$end_s - $start_s"
+    fi
+    
+    let elapsed_m="$elapsed / 60"
+    let elapsed_s="$elapsed % 60"
+    echo "${elapsed_m}m ${elapsed_s}s"
+}
diff --git a/tests/suites/_init.sh b/tests/suites/_init.sh
index 2a116001..5466ad3b 100644
--- a/tests/suites/_init.sh
+++ b/tests/suites/_init.sh
@@ -14,7 +14,7 @@ set +eu
 # globals - all global variables are UPPERCASE
 ASSETS_DIR="assets"
 MIAB_DIR=".."
-BASE_OUTPUTDIR="$(realpath out)"
+BASE_OUTPUTDIR="$(realpath out)/$(hostname | awk -F. '{print $1}')"
 PYMAIL="./test_mail.py"
 declare -i OVERALL_SUCCESSES=0
 declare -i OVERALL_FAILURES=0
diff --git a/tests/system-setup/remote-nextcloud-docker.sh b/tests/system-setup/remote-nextcloud-docker.sh
index c68470e3..c1a72826 100755
--- a/tests/system-setup/remote-nextcloud-docker.sh
+++ b/tests/system-setup/remote-nextcloud-docker.sh
@@ -84,11 +84,6 @@ install_nextcloud_docker() {
         container_started="false"
     fi
 
-    # H2 "docker: Update /etc/hosts so it can find MiaB-LDAP by name"
-    # echo "$PRIVATE_IP $PRIMARY_HOSTNAME" | \
-    #     docker exec -i NC bash -c 'cat >>/etc/hosts' \
-    #     || die "docker: could not update /etc/hosts"
-    
     # apt-get update
     H2 "docker: apt-get update"
     docker exec NC apt-get update || die "docker: apt-get update failed"
@@ -98,6 +93,7 @@ install_nextcloud_docker() {
     ufw allow ldaps || die "Unable to modify firewall to permit ldaps"
 
     # add MiaB-LDAP's ca_certificate.pem to docker's trusted cert list
+    # (because setup/ssl.sh created its own self-signed ca)
     H2 "docker: update trusted CA list"
     docker cp \
            $STORAGE_ROOT/ssl/ca_certificate.pem \
@@ -146,14 +142,14 @@ install_nextcloud_docker() {
 do_upgrade() {
     local populate_name="$1"
 
-    if [ -e "local/remote-nextcloud.sh" ]; then
+    if [ -e "$LOCAL_MODS_DIR/remote-nextcloud.sh" ]; then
         # we install w/o remote nextcloud first so we can add
         # a user w/contacts and ensure the contact exists in the
         # new system
-        if [ ! -L "local/remote-nextcloud.sh" ]; then
-            echo "Warning: local/remote-nextcloud.sh is a regular file - should be a symlink"
+        if [ ! -L "$LOCAL_MODS_DIR/remote-nextcloud.sh" ]; then
+            echo "Warning: $LOCAL_MODS_DIR/remote-nextcloud.sh is a regular file - should be a symlink"
         fi
-        die "Error: local/remote-nextcloud.sh exists - delete it and try again"
+        die "Error: $LOCAL_MODS_DIR/remote-nextcloud.sh exists - delete it and try again"
     fi
 
     # initialize test system
diff --git a/tests/system-setup/setup-defaults.sh b/tests/system-setup/setup-defaults.sh
index 7d44f93a..2f2bfe06 100755
--- a/tests/system-setup/setup-defaults.sh
+++ b/tests/system-setup/setup-defaults.sh
@@ -8,6 +8,7 @@ export STORAGE_ROOT="${STORAGE_ROOT:-/home/$STORAGE_USER}"
 export EMAIL_ADDR="${EMAIL_ADDR:-qa@abc.com}"
 export EMAIL_PW="${EMAIL_PW:-Test_1234}"
 export PUBLIC_IP="${PUBLIC_IP:-$(source ${MIAB_DIR:-.}/setup/functions.sh; get_default_privateip 4)}"
+export LOCAL_MODS_DIR="${LOCAL_MODS_DIR:-local}"
 
 if [ "$TRAVIS" == "true" ]; then
     export PRIMARY_HOSTNAME=${PRIMARY_HOSTNAME:-box.abc.com}
diff --git a/tests/system-setup/setup-funcs.sh b/tests/system-setup/setup-funcs.sh
index 3f6db83e..2416bd1e 100755
--- a/tests/system-setup/setup-funcs.sh
+++ b/tests/system-setup/setup-funcs.sh
@@ -135,12 +135,12 @@ init_miab_testing() {
 
 enable_miab_mod() {
     local name="${1}.sh"
-    if [ ! -e "local/$name" ]; then
-        mkdir -p "local"
-        if ! ln -s "../setup/mods.available/$name" "local/$name"
+    if [ ! -e "$LOCAL_MODS_DIR/$name" ]; then
+        mkdir -p "$LOCAL_MODS_DIR"
+        if ! ln -s "$(pwd)/setup/mods.available/$name" "$LOCAL_MODS_DIR/$name"
         then
-            echo "Warning: copying instead of symlinking local/$name"
-            cp "setup/mods.available/$name" "local/$name"
+            echo "Warning: copying instead of symlinking $LOCAL_MODS_DIR/$name"
+            cp "setup/mods.available/$name" "$LOCAL_MODS_DIR/$name"
         fi
     fi
 }
diff --git a/tests/system-setup/upgrade-from-upstream.sh b/tests/system-setup/upgrade-from-upstream.sh
index 721d7c11..7384f188 100755
--- a/tests/system-setup/upgrade-from-upstream.sh
+++ b/tests/system-setup/upgrade-from-upstream.sh
@@ -125,12 +125,12 @@ populate() {
 case "$1" in
     capture )
         . /etc/mailinabox.conf
-        installed_state_capture "tests/system-setup/state/miab-ldap"
+        installed_state_capture "/tmp/state/miab-ldap"
         exit $?
         ;;
     compare )
         . /etc/mailinabox.conf
-        installed_state_compare "tests/system-setup/state/upstream" "tests/system-setup/state/miab-ldap"
+        installed_state_compare "/tmp/state/upstream" "/tmp/state/miab-ldap"
         exit $?
         ;;
     populate )
@@ -161,15 +161,15 @@ else
     populate_by_name "${1:-basic}"
 
     # capture upstream state
-    installed_state_capture "tests/system-setup/state/upstream"
+    installed_state_capture "/tmp/state/upstream"
 fi
 
 # install miab-ldap and capture state
 miab_ldap_install
-installed_state_capture "tests/system-setup/state/miab-ldap"
+installed_state_capture "/tmp/state/miab-ldap"
 
 # compare states
-if ! installed_state_compare "tests/system-setup/state/upstream" "tests/system-setup/state/miab-ldap"; then
+if ! installed_state_compare "/tmp/state/upstream" "/tmp/state/miab-ldap"; then
     die "Upstream and upgraded states are different !"
 fi
 
diff --git a/tests/vagrant/.gitignore b/tests/vagrant/.gitignore
new file mode 100644
index 00000000..6cfd7a63
--- /dev/null
+++ b/tests/vagrant/.gitignore
@@ -0,0 +1,3 @@
+.vagrant
+out
+*-console.log
diff --git a/tests/vagrant/Vagrantfile b/tests/vagrant/Vagrantfile
new file mode 100644
index 00000000..aecd35bd
--- /dev/null
+++ b/tests/vagrant/Vagrantfile
@@ -0,0 +1,42 @@
+
+Vagrant.configure("2") do |config|
+
+  config.vm.synced_folder "../..", "/mailinabox", id: "mailinabox", automount: false
+  config.vm.provision "file", source:"globals.sh", destination:"globals.sh"
+
+  # remote-nextcloud-docker
+
+  config.vm.define "remote-nextcloud-docker" do |m1|
+    m1.vm.box = "ubuntu/bionic64"
+    m1.vm.provision :shell, :inline => <<-SH
+source globals.sh || exit 1
+export PRIMARY_HOSTNAME=qa1.abc.com
+export FEATURE_MUNIN=false
+cd /mailinabox
+if tests/system-setup/remote-nextcloud-docker.sh upgrade basic
+then
+   tests/runner.sh default remote-nextcloud upgrade-basic
+fi
+echo "EXITCODE: $?"
+SH
+  end
+  
+
+  # upgrade-from-upstream
+  
+  config.vm.define "upgrade-from-upstream" do |m2|
+    m2.vm.box = "ubuntu/bionic64"
+    m2.vm.provision :shell, :inline => <<-SH
+source globals.sh || exit 1
+export PRIMARY_HOSTNAME=qa2.abc.com
+export UPSTREAM_TAG=master
+cd /mailinabox
+if tests/system-setup/upgrade-from-upstream.sh basic
+then
+  tests/runner.sh default upgrade-basic
+fi
+echo "EXITCODE: $?"
+SH
+  end
+
+end
diff --git a/tests/vagrant/globals.sh b/tests/vagrant/globals.sh
new file mode 100644
index 00000000..2a2c6b7b
--- /dev/null
+++ b/tests/vagrant/globals.sh
@@ -0,0 +1,2 @@
+export MIAB_LDAP_PROJECT=true
+export LOCAL_MODS_DIR=/local
diff --git a/tests/vagrant/parallel.sh b/tests/vagrant/parallel.sh
new file mode 100755
index 00000000..27d1e203
--- /dev/null
+++ b/tests/vagrant/parallel.sh
@@ -0,0 +1,71 @@
+#!/bin/bash
+
+# Parallel provisioning for virtualbox because "The Vagrant VirtualBox
+# provider does not support parallel execution at this time"
+# (https://www.vagrantup.com/docs/providers/virtualbox/usage.html)
+#
+# Credit to:
+#    https://dzone.com/articles/parallel-provisioning-speeding
+#
+
+. "$(dirname "$0")/../lib/color-output.sh"
+. "$(dirname "$0")/../lib/misc.sh"
+
+
+OUTPUT_DIR=out
+rm -rf "$OUTPUT_DIR"
+mkdir -p "$OUTPUT_DIR"
+
+# set total parallel vms to (#cores minus 1)
+MAX_PROCS=$(cat /proc/cpuinfo | grep processor | wc -l)
+let MAX_PROCS-=1
+
+ 
+parallel_provision() {
+    while read box; do
+        outfile="$OUTPUT_DIR/$box.out.txt"
+        echo "Provisioning '$box'. Output will be in: $outfile" 1>&2
+        echo $box
+    done | xargs -P $MAX_PROCS -I"BOXNAME" \
+        sh -c 'vagrant provision BOXNAME >'"$OUTPUT_DIR/"'BOXNAME.out.txt 2>&1 || echo "Error Occurred: BOXNAME"'
+}
+ 
+## -- main -- ##
+
+start_time="$(date +%s)"
+
+# start boxes sequentially to avoid vbox explosions
+vagrant up --no-provision
+ 
+# but run provision tasks in parallel
+vagrant status | grep running | awk '{print $1}' | parallel_provision
+
+
+# output overall result - Vagrantfile script must output "EXITCODE: <num>"
+H1 "Results"
+
+rc=0
+for file in "$OUTPUT_DIR"/*.out.txt; do
+    box=$(basename $file | awk -F. '{print $1}')
+    exitcode="$(tail "$file" | grep EXITCODE: | awk '{print $NF}')"
+    echo -n "$box: "
+    if [ -z "$exitcode" ]; then
+        danger "NO EXITCODE!"
+        [ $rc -eq 0 ] && rc=2
+    elif [ "$exitcode" == "0" ]; then
+        success "SUCCESS"
+    else
+        danger "FAILURE ($exitcode)"
+        rc=1
+    fi
+done
+
+# output elapsed time
+end_time="$(date +%s)"
+echo ""
+echo "Elapsed time: $(elapsed_pretty $start_time $end_time)"
+
+# exit
+echo ""
+echo "Guest VMs are running! Destroy them with 'vagrant destroy -f'"
+exit $rc