1. Better code organization & simplify

2. Add "populate" data for upgrades - enabled in both system-setup scripts
3. Add "upgrade" test runner suite

tests/system-setup/populate/README.txt

@@ -0,0 +1,31 @@
+This directory contains scripts used to populate a MiaB installation
+with known values, and then subsequently verify that MiaB continues to
+operate poperly after an upgrade or setup mod change.
+
+Each "named" populate set of scripts should contain at least two
+shell scripts:
+
+   1. <name>-populate.sh  : populates the installation
+   2. <name>-verify.sh    : verifies operation after upgrade
+
+The system-setup/* scripts run the populate script, and the test
+runner's 'upgrade' test suite runs the verify script.
+
+These scripts are run, not sourced.
+
+
+Expected script output and return value:
+
+   1. All debug output must go to stderr
+   2. Result messages must be sent to stdout (a single line, preferrably)
+   3. Return 0 if successfully passed verification
+   4. Return non-zero if failed verification
+
+The working directory for <name>-populate.sh is the Mail-in-a-Box root
+directory.
+
+The working directory for <name>-verify.sh is 'tests' (because the
+test runner always changes the working directory there to limit
+contamination of the source tree). Use MIAB_DIR and ASSETS_DIR, if
+needed.
+

tests/system-setup/populate/basic-data.sh

@@ -0,0 +1,10 @@
+#
+# requires:
+#    lib scripts: [ misc.sh ]
+#    system-setup scripts: [ setup-defaults.sh ]
+#
+
+TEST_USER="anna@$(email_domainpart "$EMAIL_ADDR")"
+TEST_USER_PASS="$(static_qa_password)"
+TEST_USER_CONTACT_UUID="e0642b47-9104-4adb-adfd-5f907d04216a"
+TEST_USER_CONTACT_EMAIL="sam@bff.org"

tests/system-setup/populate/basic-populate.sh

@@ -0,0 +1,48 @@
+#!/bin/bash
+
+. "$(dirname "$0")/../setup-defaults.sh" || exit 1
+. "$(dirname "$0")/../../lib/all.sh" "$(dirname "$0")/../../lib" || exit 1
+. "$(dirname "$0")/basic-data.sh" || exit 1
+
+
+#
+# Add user
+#
+if ! populate_miab_users "" "" "" "${TEST_USER}:${TEST_USER_PASS}"
+then
+    echo "Unable to add user"
+    exit 1
+fi
+
+#
+# Add Nextcloud contact and force Roundcube contact sync to ensure the
+# roundcube carddav addressbooks and contacts tables are populated in
+# case a remote nextcloud is subsequently configured and the
+# syncronization disabled.
+#
+if ! carddav_ls "$TEST_USER" "$TEST_USER_PASS" --insecure 2>/dev/null
+then
+    echo "Could not enumerate contacts: $REST_ERROR"
+    exit 1
+fi
+echo "Current contacts count: ${#FILES[@]}"
+    
+if array_contains "$TEST_USER_CONTACT_UUID.vcf" "${FILES[@]}"; then
+    echo "Contact $TEST_USER_CONTACT_UUID already present"
+else
+    if ! carddav_add_contact "$TEST_USER" "$TEST_USER_PASS" "Anna" "666-1111" "$TEST_USER_CONTACT_EMAIL" "$TEST_USER_CONTACT_UUID" --insecure 2>/dev/null
+    then
+        echo "Could not add contact: $REST_ERROR"
+        exit 1
+    fi
+    
+    echo "Force Roundcube contact sync"
+    if ! roundcube_force_carddav_refresh "$TEST_USER" "$TEST_USER_PASS"
+    then
+        echo "Roundcube <-> Nextcloud contact sync failed"
+        exit 1
+    fi
+fi
+
+exit 0
+

tests/system-setup/populate/basic-verify.sh

@@ -0,0 +1,53 @@
+#!/bin/bash
+
+. "$(dirname "$0")/../setup-defaults.sh" || exit 1
+. "$(dirname "$0")/../../lib/all.sh" "$(dirname "$0")/../../lib" || exit 1
+. "$(dirname "$0")/basic-data.sh" || exit 1
+. /etc/mailinabox.conf || exit 1
+
+
+# 1. the test user can still log in and send mail
+
+echo "[User can still log in with their old passwords and send mail]" 1>&2
+echo "python3 test_mail.py $PRIVATE_IP $TEST_USER $TEST_USER_PASS" 1>&2
+python3 test_mail.py "$PRIVATE_IP" "$TEST_USER" "$TEST_USER_PASS" 1>&2
+if [ $? -ne 0 ]; then
+    echo "Basic mail functionality test failed"
+    exit 1
+fi
+
+
+# 2. the test user's contact is still accessible in Roundcube
+
+echo "[Force Roundcube contact sync]" 1>&2
+# if MiaB's Nextcloud carddav configuration was removed all the
+# contacts for it will be removed in the Roundcube database after the
+# sync
+
+if ! roundcube_force_carddav_refresh "$TEST_USER" "$TEST_USER_PASS" 1>&2
+then
+    echo "Roundcube <-> Nextcloud contact sync failed ($?)"
+    exit 1
+fi
+
+echo "[Ensure old Nextcloud contacts are still present]" 1>&2
+echo "sqlite3 $STORAGE_ROOT/mail/roundcube/roundcube.sqlite \"select email from carddav_contacts where cuid='$TEST_USER_CONTACT_UUID'\"" 1>&2
+output=$(sqlite3 "$STORAGE_ROOT/mail/roundcube/roundcube.sqlite" "select email from carddav_contacts where cuid='$TEST_USER_CONTACT_UUID'")
+rc=$?
+if [ $rc -ne 0 ]
+then
+    echo "Querying Roundcube's sqlite database failed ($rc)"
+    exit 1
+else
+    echo "Success, found $output" 1>&2
+fi
+
+if [ "$output" != "$TEST_USER_CONTACT_EMAIL" ]
+then
+    echo "Unexpected email for contact uuid: got '$output', expected '$TEST_USER_CONTACT_EMAIL'"
+    exit 1
+fi
+
+echo "OK basic-verify passed"
+exit 0
+