From b0090edd52436587ff41588b53e8382a904bd4cf Mon Sep 17 00:00:00 2001
From: downtownallday <downtownallday@gmail.com>
Date: Sun, 14 Jun 2020 13:51:00 -0400
Subject: [PATCH] Test upgrade to LDAP from upstream Mail-in-a-Box/sqlite
---
.travis.yml | 67 ++--
management/mailconfig.py | 56 ++--
setup/ldap.sh | 8 +-
.../remote-nextcloud-use-miab.sh | 5 +-
setup/ssl.sh | 10 +-
tests/lib/all.sh | 16 +
tests/lib/color-output.sh | 32 ++
.../_locations.sh => lib/locations.sh} | 0
tests/lib/misc.sh | 65 ++++
tests/lib/rest.sh | 106 +++++++
tests/lib/system.sh | 102 +++++++
tests/suites/_init.sh | 23 +-
tests/suites/_ldap-functions.sh | 14 +-
tests/suites/_mgmt-functions.sh | 46 +--
tests/system-setup/remote-nextcloud-docker.sh | 136 +++------
tests/system-setup/setup-defaults.sh | 8 +-
tests/system-setup/setup-funcs.sh | 228 +++++---------
tests/system-setup/upgrade-from-upstream.sh | 289 ++++++++++++++++++
18 files changed, 831 insertions(+), 380 deletions(-)
create mode 100644 tests/lib/all.sh
create mode 100644 tests/lib/color-output.sh
rename tests/{suites/_locations.sh => lib/locations.sh} (100%)
create mode 100644 tests/lib/misc.sh
create mode 100644 tests/lib/rest.sh
create mode 100644 tests/lib/system.sh
create mode 100755 tests/system-setup/upgrade-from-upstream.sh
diff --git a/.travis.yml b/.travis.yml
index f3beb134..e4fba7ec 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -1,37 +1,46 @@
# travisci config
env:
global:
- - PRIMARY_HOSTNAME=box.abc.com
+ - MIAB_LDAP=true
language: shell
os: linux
dist: bionic
-
-before_install:
- - echo "==== ENVIRONMENT ===="
- - env | sort
- - echo "UMASK=$(umask)"
- #
- - echo "==== AppArmor Status ===="
- - (sudo aa-status; true)
- #
- - echo "==== NETWORK INFO ===="
- - hostname -I
- - hostname -i
- - hostname
- - hostname --fqdn
- - ip add
- - sysctl -a 2>/dev/null | grep -i ipv6 | grep disable
-
-install:
- - sudo tests/system-setup/remote-nextcloud-docker.sh
- - hostname || true
- - hostname --fqdn || true
- - getent hosts box.abc.com || true
-script:
- #
- # launch automated tests, but skip tests that require remote
- # smtp support because Travis-CI blocks outgoing port 25
- - sudo touch /etc/dovecot/sieve-spam.svbin
- - sudo tests/runner.sh -dumpoutput -no-smtp-remote default remote-nextcloud
+jobs:
+ fast_finish: true
+ include:
+ # JOB: MiaB-LDAP connected to a remote Nextcloud
+ - env: PRIMARY_HOSTNAME=box1.abc.com
+ name: remote-nextcloud-docker
+ before_install:
+ - echo "==== ENVIRONMENT ===="
+ - env | sort
+ - echo "UMASK=$(umask)"
+ - echo "==== AppArmor Status ===="
+ - (sudo aa-status; true)
+ - echo "==== NETWORK INFO ===="
+ - hostname -I
+ - hostname -i
+ - hostname
+ - hostname --fqdn
+ - ip add
+ - sysctl -a 2>/dev/null | grep -i ipv6 | grep disable
+ install:
+ - sudo tests/system-setup/remote-nextcloud-docker.sh
+ script:
+ # launch automated tests, but skip tests that require remote
+ # smtp support because Travis-CI blocks outgoing port 25
+ - sudo touch /etc/dovecot/sieve-spam.svbin
+ - sudo tests/runner.sh -dumpoutput -no-smtp-remote default remote-nextcloud
+
+ # JOB: Upgrade from upstream install
+ - env: PRIMARY_HOSTNAME=box2.abc.com
+ name: upgrade-from-upstream
+ install:
+ - sudo tests/system-setup/upgrade-from-upstream.sh
+ script:
+ # launch automated tests, but skip tests that require remote
+ # smtp support because Travis-CI blocks outgoing port 25
+ - sudo touch /etc/dovecot/sieve-spam.svbin
+ - sudo tests/runner.sh -dumpoutput -no-smtp-remote
diff --git a/management/mailconfig.py b/management/mailconfig.py
index 4847eb8d..4fa63ca7 100755
--- a/management/mailconfig.py
+++ b/management/mailconfig.py
@@ -321,7 +321,7 @@ def get_mail_aliases(env, as_map=False):
# make a dict of permitted senders, key=mail(lowercase) value=members
permitted_senders = { rec["mail"][0].lower(): rec["member"] for rec in pager }
-
+
# get all alias groups
pager = c.paged_search(env.LDAP_ALIASES_BASE, "(objectClass=mailGroup)", attributes=['mail','member','rfc822MailMember'])
@@ -362,7 +362,7 @@ def get_mail_aliases(env, as_map=False):
alias = aliases[address]
xft = ",".join(alias["forward_tos"])
xas = ",".join(alias["permitted_senders"])
- list.append( (address, xft, xas) )
+ list.append( (address, xft, None if xas is "" else xas) )
return list
else:
@@ -432,7 +432,7 @@ def get_domain(emailaddr, as_unicode=True):
pass
return ret
-def get_mail_domains(env, as_map=False, filter_aliases=None, category=None, users_only=False):
+def get_mail_domains(env, as_map=False, filter_aliases=lambda alias: True, category=None, users_only=False):
# Retrieves all domains, IDNA-encoded, we accept mail for.
#
# If as_map is False, the function returns the lowercase domain
@@ -457,17 +457,18 @@ def get_mail_domains(env, as_map=False, filter_aliases=None, category=None, user
# make it easy for dns_update to get ssl domains]
#
# If users_only is True, only return domains with email addresses
- # that correspond to user accounts. [TODO: This currently has no
- # effect - this function only returns user mail domains]
+ # that correspond to user accounts.
#
conn = open_database(env)
filter = "(&(objectClass=domain)(businessCategory=mail))"
if category:
filter = "(&(objectClass=domain)(businessCategory=%s))" % category
+
+ domains=None
+
+ # user mail domains
id = conn.search(env.LDAP_DOMAINS_BASE, filter, attributes="dc")
response = conn.wait(id)
- filter_candidates=[]
- domains=None
if as_map:
domains = {}
for rec in response:
@@ -478,36 +479,23 @@ def get_mail_domains(env, as_map=False, filter_aliases=None, category=None, user
if filter_aliases: filter_candidates.append(rec['dc'][0].lower())
else:
domains = set([ rec["dc"][0].lower() for rec in response ])
- if filter_aliases: filter_candidates += domains
-
- for candidate in filter_candidates:
- # with the filter, there has to be at least one user or
- # filtered (included) alias in the domain for the domain to be
- # part of the returned set
- # any users ?
- response = conn.wait( conn.search(env.LDAP_USERS_BASE, "(&(objectClass=mailUser)(mail=*@%s))" % candidate, size_limit=1) )
- if response.next():
- # yes, that domain needs to be in the returned set
- continue
- # any filtered aliases ?
- pager = conn.paged_search(
- env.LDAP_ALIASES_BASE,
- "(&(objectClass=mailGroup)(mail=*@%s))" % candidate,
- attributes=['mail'])
+ # alias domains
+ if not users_only:
+ pager = conn.paged_search(env.LDAP_ALIASES_BASE, "(objectClass=mailGroup)", attributes="mail")
+ if as_map:
+ for rec in pager:
+ if filter_aliases(rec["mail"][0].lower()):
+ domain = get_domain(rec["mail"][0].lower(),as_unicode=False)
+ domains[domain] = {
+ "dn": None,
+ "domain": domain
+ }
- remove = True
- for rec in pager:
- if filter_aliases(rec['mail'][0]):
- remove = False
- pager.abandon()
- break
-
- if remove:
- if as_map: del domains[candidate]
- else: domains.remove(candidate)
-
+ else:
+ domains = domains.union(set([ get_domain(rec["mail"][0].lower(), as_unicode=False) for rec in pager if filter_aliases(rec["mail"][0].lower()) ]))
+
return domains
diff --git a/setup/ldap.sh b/setup/ldap.sh
index 78e2c4cb..252c5cd8 100755
--- a/setup/ldap.sh
+++ b/setup/ldap.sh
@@ -857,7 +857,13 @@ cat > /etc/logrotate.d/slapd <<EOF;
EOF
# Modify olc server config like TLS
-modify_global_config
+# Skip this step if no ca_certificate.pem exists - this indicates
+# that the system hasn't yet been migrated from sqlite
+if [ -e "$STORAGE_ROOT/ssl/ca_certificate.pem" ]; then
+ modify_global_config
+else
+ say_debug "Not enabling TLS at this time - ca_certificate hasn't been generated yet"
+fi
# Add overlays and ensure mail-related attributes are indexed
add_overlays
diff --git a/setup/mods.available/remote-nextcloud-use-miab.sh b/setup/mods.available/remote-nextcloud-use-miab.sh
index fd0277bf..e8ce1117 100755
--- a/setup/mods.available/remote-nextcloud-use-miab.sh
+++ b/setup/mods.available/remote-nextcloud-use-miab.sh
@@ -6,8 +6,9 @@
#
# The script will:
# 1. enable the "LDAP user and group backend" in Nextcloud
-# 2. configure Nextcloud to access MiaB-LDAP for users and groups
-# 3. optionally install and configure ssmtp so system mail is
+# 2. install calendar and contacts
+# 3. configure Nextcloud to access MiaB-LDAP for users and groups
+# 4. optionally install and configure ssmtp so system mail is
# sent to MiaB-LDAP
#
VERBOSE=0
diff --git a/setup/ssl.sh b/setup/ssl.sh
index c8681df4..518450bc 100755
--- a/setup/ssl.sh
+++ b/setup/ssl.sh
@@ -96,11 +96,6 @@ if [ ! -s $STORAGE_ROOT/ssl/ssl_private_key.pem ]; then
# Set the umask so the key file is never world-readable.
(umask 037; hide_output \
openssl genrsa -out $STORAGE_ROOT/ssl/ssl_private_key.pem 2048)
-
- # Give the group 'ssl-cert' read access so slapd can read it
- groupadd -fr ssl-cert
- chgrp ssl-cert $STORAGE_ROOT/ssl/ssl_private_key.pem
- chmod g+r $STORAGE_ROOT/ssl/ssl_private_key.pem
# Remove the ssl_certificate.pem symbolic link to force a
# regeneration of the server certificate. It needs to be
@@ -110,6 +105,11 @@ if [ ! -s $STORAGE_ROOT/ssl/ssl_private_key.pem ]; then
fi
fi
+# Give the group 'ssl-cert' read access so slapd can read it
+groupadd -fr ssl-cert
+chgrp ssl-cert $STORAGE_ROOT/ssl/ssl_private_key.pem
+chmod g+r $STORAGE_ROOT/ssl/ssl_private_key.pem
+
#
# Generate a root CA certificate
#
diff --git a/tests/lib/all.sh b/tests/lib/all.sh
new file mode 100644
index 00000000..20a0c6de
--- /dev/null
+++ b/tests/lib/all.sh
@@ -0,0 +1,16 @@
+#
+# source all lib scripts
+#
+# from your script, supply the path to this directory as the first argument
+#
+# eg source "tests/lib/all.sh" "tests/lib"
+#
+# failure to load any script is fatal!
+
+. "$1/color-output.sh" || exit 1
+. "$1/locations.sh" || exit 2
+. "$1/misc.sh" || exit 3
+. "$1/rest.sh" || exit 4
+. "$1/system.sh" || exit 5
+
+
diff --git a/tests/lib/color-output.sh b/tests/lib/color-output.sh
new file mode 100644
index 00000000..3e1954f8
--- /dev/null
+++ b/tests/lib/color-output.sh
@@ -0,0 +1,32 @@
+# ansi escapes for hilighting text
+F_DANGER=$(echo -e "\033[31m")
+F_WARN=$(echo -e "\033[93m")
+F_RESET=$(echo -e "\033[39m")
+
+
+danger() {
+ local echoarg
+ case "$1" in
+ -n )
+ echoarg="$1"
+ shift
+ ;;
+ * )
+ echoarg=""
+ esac
+ echo $echoarg "${F_DANGER}$1${F_RESET}"
+}
+
+warn() {
+ local echoarg
+ case "$1" in
+ -n )
+ echoarg="$1"
+ shift
+ ;;
+ * )
+ echoarg=""
+ esac
+ echo "${F_WARN}$1${F_RESET}"
+}
+
diff --git a/tests/suites/_locations.sh b/tests/lib/locations.sh
similarity index 100%
rename from tests/suites/_locations.sh
rename to tests/lib/locations.sh
diff --git a/tests/lib/misc.sh b/tests/lib/misc.sh
new file mode 100644
index 00000000..4312d6ae
--- /dev/null
+++ b/tests/lib/misc.sh
@@ -0,0 +1,65 @@
+#
+# misc helpful functions
+#
+# requirements:
+# system packages: [ python3 ]
+
+
+array_contains() {
+ local searchfor="$1"
+ shift
+ local item
+ for item; do
+ [ "$item" == "$searchfor" ] && return 0
+ done
+ return 1
+}
+
+is_true() {
+ # empty string is not true
+ if [ "$1" == "true" \
+ -o "$1" == "TRUE" \
+ -o "$1" == "True" \
+ -o "$1" == "yes" \
+ -o "$1" == "YES" \
+ -o "$1" == "Yes" \
+ -o "$1" == "1" ]
+ then
+ return 0
+ else
+ return 1
+ fi
+}
+
+is_false() {
+ if is_true $@; then return 1; fi
+ return 0
+}
+
+email_localpart() {
+ local addr="$1"
+ awk -F@ '{print $1}' <<<"$addr"
+}
+
+email_domainpart() {
+ local addr="$1"
+ awk -F@ '{print $2}' <<<"$addr"
+}
+
+
+generate_uuid() {
+ local uuid
+ uuid=$(python3 -c "import uuid; print(uuid.uuid4())")
+ [ $? -ne 0 ] && die "Unable to generate a uuid"
+ echo "$uuid"
+}
+
+generate_qa_password() {
+ echo "Test1234."
+}
+
+sha1() {
+ local txt="$1"
+ python3 -c "import hashlib; m=hashlib.sha1(); m.update(bytearray(r'''$txt''','utf-8')); print(m.hexdigest());" || die "Unable to generate sha1 hash"
+}
+
diff --git a/tests/lib/rest.sh b/tests/lib/rest.sh
new file mode 100644
index 00000000..86951dc1
--- /dev/null
+++ b/tests/lib/rest.sh
@@ -0,0 +1,106 @@
+#
+# REST helper functions
+#
+# requirements:
+# system packages: [ curl ]
+# lib scripts: [ color-output.sh ]
+#
+
+rest_urlencoded() {
+ # Issue a REST call having data urlencoded
+ #
+ # eg: rest_urlencoded POST /admin/mail/users/add "email=alice@abc.com" "password=secret"
+ #
+ # When providing a URI (/mail/users/add) and not a URL
+ # (https://host/mail/users/add), PRIMARY_HOSTNAME must be set!
+ #
+ # The function will set the following global variables regardless
+ # of exit c ode:
+ # REST_HTTP_CODE
+ # REST_OUTPUT
+ # REST_ERROR
+ #
+ # Return values:
+ # 0 indicates success (curl returned 0 or a code deemed to be
+ # successful and HTTP status is >=200 but <300)
+ # 1 curl returned with non-zero code that indicates and error
+ # 2 the response status was <200 or >= 300
+ #
+ # Messages, errors, and output are all sent to stderr, there is no
+ # stdout output
+ #
+ local verb="$1" # eg "POST"
+ local uri="$2" # eg "/mail/users/add"
+ local auth_user="$3"
+ local auth_pass="$4"
+ shift; shift; shift; shift # remaining arguments are data or curl args
+
+ local url
+ case "$uri" in
+ http:* | https:* )
+ url="$uri"
+ ;;
+ * )
+ url="https://$PRIMARY_HOSTNAME${uri}"
+ ;;
+ esac
+
+ local data=()
+ local item output onlydata="false"
+
+ for item; do
+ case "$item" in
+ -- )
+ onlydata="true"
+ ;;
+ --* )
+ # curl argument
+ if $onlydata; then
+ data+=("--data-urlencode" "$item");
+ else
+ data+=("$item")
+ fi
+ ;;
+ * )
+ onlydata="true"
+ data+=("--data-urlencode" "$item");
+ ;;
+ esac
+ done
+
+ echo "spawn: curl -w \"%{http_code}\" -X $verb --user \"${auth_user}:xxx\" ${data[@]} $url" 1>&2
+ output=$(curl -s -S -w "%{http_code}" -X $verb --user "${auth_user}:${auth_pass}" "${data[@]}" $url)
+ local code=$?
+
+ # http status is last 3 characters of output, extract it
+ REST_HTTP_CODE=$(awk '{S=substr($0,length($0)-2)} END {print S}' <<<"$output")
+ REST_OUTPUT=$(awk 'BEGIN{L=""}{ if(L!="") print L; L=$0 } END { print substr(L,1,length(L)-3) }' <<<"$output")
+ REST_ERROR=""
+ [ -z "$REST_HTTP_CODE" ] && REST_HTTP_CODE="000"
+
+ if [ $code -ne 0 ]; then
+ if [ $code -eq 56 -a $REST_HTTP_CODE -eq 200 ]; then
+ # this is okay, I guess. happens sometimes during
+ # POST /admin/mail/aliases/remove
+ # 56=Unexpected EOF
+ echo "Ignoring curl return code 56 due to 200 status" 1>&2
+
+ elif [ $code -ne 16 -o $REST_HTTP_CODE -ne 200 ]; then
+ # any error code will fail the rest call except for a 16
+ # with a 200 HTTP status.
+ # 16="a problem was detected in the HTTP2 framing layer. This is somewhat generic and can be one out of several problems"
+ REST_ERROR="CURL failed with code $code"
+ echo "${F_DANGER}$REST_ERROR${F_RESET}" 1>&2
+ echo "$output" 1>&2
+ return 1
+ fi
+ fi
+ if [ $REST_HTTP_CODE -lt 200 -o $REST_HTTP_CODE -ge 300 ]; then
+ REST_ERROR="REST status $REST_HTTP_CODE: $REST_OUTPUT"
+ echo "${F_DANGER}$REST_ERROR${F_RESET}" 1>&2
+ return 2
+ fi
+ echo "CURL succeded, HTTP status $REST_HTTP_CODE" 1>&2
+ echo "$output" 1>&2
+ return 0
+}
diff --git a/tests/lib/system.sh b/tests/lib/system.sh
new file mode 100644
index 00000000..39695b10
--- /dev/null
+++ b/tests/lib/system.sh
@@ -0,0 +1,102 @@
+
+wait_for_apt() {
+ # check to see if other package managers have a lock on new
+ # installs, and wait for them to finish
+ #
+ # returns non-zero if waiting times out (currently ~600 seconds)
+ local count=0
+ while fuser /var/lib/dpkg/lock >/dev/null 2>&1 || fuser /var/lib/dpkg/lock-frontend >/dev/null 2>&1; do
+ sleep 6
+ let count+=1
+ if [ $count -eq 1 ]; then
+ echo -n "Waiting for other package manager to finish..."
+ elif [ $count -gt 100 ]; then
+ echo -n "FAILED"
+ return 1
+ else
+ echo -n "${count}.."
+ fi
+ done
+ [ $count -ge 1 ] && echo ""
+}
+
+dump_file() {
+ local log_file="$1"
+ local lines="$2"
+ local title="DUMP OF $log_file"
+ echo ""
+ echo "--------"
+ echo -n "-------- $log_file"
+ if [ ! -z "$lines" ]; then
+ echo " (last $line lines)"
+ else
+ echo ""
+ fi
+ echo "--------"
+
+ if [ !-e "$log_file" ]; then
+ echo "DOES NOT EXIST"
+ elif [ ! -z "$lines" ]; then
+ tail -$lines "$log_file"
+ else
+ cat "$log_file"
+ fi
+}
+
+dump_file_if_exists() {
+ [ ! -e "$1" ] && return
+ dump_file "$@"
+}
+
+update_system_time() {
+ if [ ! -x /usr/sbin/ntpdate ]; then
+ wait_for_apt
+ apt-get install -y -qq ntpdate || return 1
+ fi
+ ntpdate -s ntp.ubuntu.com && echo "System time updated"
+}
+
+set_system_hostname() {
+ # set the system hostname to the FQDN specified or
+ # PRIMARY_HOSTNAME if no FQDN was given
+ local fqdn="${1:-$PRIMARY_HOSTNAME}"
+ local host="$(awk -F. '{print $1}' <<< "$fqdn")"
+ sed -i 's/^127\.0\.1\.1[ \t].*/127.0.1.1 '"$fqdn $host ip4-loopback/" /etc/hosts || return 1
+ #hostname "$host" || return 1
+ #echo "$host" > /etc/hostname
+ return 0
+}
+
+install_docker() {
+ if [ -x /usr/bin/docker ]; then
+ echo "Docker already installed"
+ return 0
+ fi
+
+ wait_for_apt
+ apt-get install -y -qq \
+ apt-transport-https \
+ ca-certificates \
+ curl \
+ gnupg-agent \
+ software-properties-common \
+ || return 1
+
+ wait_for_apt
+ curl -fsSL https://download.docker.com/linux/ubuntu/gpg | apt-key add - \
+ || return 2
+
+ wait_for_apt
+ apt-key fingerprint 0EBFCD88 || return 3
+
+ wait_for_apt
+ add-apt-repository -y --update "deb [arch=amd64] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" || return 4
+
+ wait_for_apt
+ apt-get install -y -qq \
+ docker-ce \
+ docker-ce-cli \
+ containerd.io \
+ || return 5
+}
+
diff --git a/tests/suites/_init.sh b/tests/suites/_init.sh
index bee4fb20..829db86d 100644
--- a/tests/suites/_init.sh
+++ b/tests/suites/_init.sh
@@ -6,7 +6,7 @@
set +eu
# load test suite helper functions
-. suites/_locations.sh || exit 1
+. lib/all.sh "lib" || exit 1
. suites/_ldap-functions.sh || exit 1
. suites/_mail-functions.sh || exit 1
. suites/_mgmt-functions.sh || exit 1
@@ -20,10 +20,6 @@ declare -i OVERALL_SKIPPED=0
declare -i OVERALL_COUNT=0
declare -i OVERALL_COUNT_SUITES=0
-# ansi escapes for hilighting text
-F_DANGER=$(echo -e "\033[31m")
-F_WARN=$(echo -e "\033[93m")
-F_RESET=$(echo -e "\033[39m")
# options
FAILURE_IS_FATAL=no
@@ -157,7 +153,12 @@ test_skip() {
}
skip_test() {
- # return 0 if we should skip the current test
+ # call from within a test to check whether the test will be
+ # skipped
+ #
+ # returns 0 if the current test was skipped in which case your test
+ # function must immediately call 'test_end' and return
+ #
if [ "$SKIP_REMOTE_SMTP_TESTS" == "yes" ] &&
array_contains "remote-smtp" "$@";
then
@@ -191,16 +192,6 @@ die() {
exit 1
}
-array_contains() {
- local searchfor="$1"
- shift
- local item
- for item; do
- [ "$item" == "$searchfor" ] && return 0
- done
- return 1
-}
-
python_error() {
# finds tracebacks and outputs just the final error message of
# each
diff --git a/tests/suites/_ldap-functions.sh b/tests/suites/_ldap-functions.sh
index 77e102fe..6c69200d 100644
--- a/tests/suites/_ldap-functions.sh
+++ b/tests/suites/_ldap-functions.sh
@@ -1,16 +1,10 @@
# -*- indent-tabs-mode: t; tab-width: 4; -*-
-generate_uuid() {
- local uuid
- uuid=$(python3 -c "import uuid; print(uuid.uuid4())")
- [ $? -ne 0 ] && die "Unable to generate a uuid"
- echo "$uuid"
-}
+# requirements:
+# system packages: [ ldap-utils ]
+# setup scripts: [ functions-ldap.sh ]
+# setup artifacts: [ miab_ldap.conf ]
-sha1() {
- local txt="$1"
- python3 -c "import hashlib; m=hashlib.sha1(); m.update(bytearray(r'''$txt''','utf-8')); print(m.hexdigest());" || die "Unable to generate sha1 hash"
-}
delete_user() {
local email="$1"
diff --git a/tests/suites/_mgmt-functions.sh b/tests/suites/_mgmt-functions.sh
index 07fbdc5b..5ac50701 100644
--- a/tests/suites/_mgmt-functions.sh
+++ b/tests/suites/_mgmt-functions.sh
@@ -44,49 +44,9 @@ mgmt_rest() {
local uri="$2" # eg "/mail/users/add"
shift; shift; # remaining arguments are data
- local auth_user="${MGMT_ADMIN_EMAIL}"
- local auth_pass="${MGMT_ADMIN_PW}"
- local url="https://$PRIMARY_HOSTNAME${uri}"
- local data=()
- local item output
-
- for item; do data+=("--data-urlencode" "$item"); done
-
- record "spawn: curl -w \"%{http_code}\" -X $verb --user \"${auth_user}:xxx\" ${data[@]} $url"
- output=$(curl -s -S -w "%{http_code}" -X $verb --user "${auth_user}:${auth_pass}" "${data[@]}" $url 2>>$TEST_OF)
- local code=$?
-
- # http status is last 3 characters of output, extract it
- REST_HTTP_CODE=$(awk '{S=substr($0,length($0)-2)} END {print S}' <<<"$output")
- REST_OUTPUT=$(awk 'BEGIN{L=""}{ if(L!="") print L; L=$0 } END { print substr(L,1,length(L)-3) }' <<<"$output")
- REST_ERROR=""
- [ -z "$REST_HTTP_CODE" ] && REST_HTTP_CODE="000"
-
- if [ $code -ne 0 ]; then
- if [ $code -eq 56 -a $REST_HTTP_CODE -eq 200 ]; then
- # this is okay, I guess. happens sometimes during
- # POST /admin/mail/aliases/remove
- # 56=Unexpected EOF
- record "Ignoring curl return code 56 due to 200 status"
-
- elif [ $code -ne 16 -o $REST_HTTP_CODE -ne 200 ]; then
- # any error code will fail the rest call except for a 16
- # with a 200 HTTP status.
- # 16="a problem was detected in the HTTP2 framing layer. This is somewhat generic and can be one out of several problems"
- REST_ERROR="CURL failed with code $code"
- record "${F_DANGER}$REST_ERROR${F_RESET}"
- record "$output"
- return 1
- fi
- fi
- if [ $REST_HTTP_CODE -lt 200 -o $REST_HTTP_CODE -ge 300 ]; then
- REST_ERROR="REST status $REST_HTTP_CODE: $REST_OUTPUT"
- record "${F_DANGER}$REST_ERROR${F_RESET}"
- return 2
- fi
- record "CURL succeded, HTTP status $REST_HTTP_CODE"
- record "$output"
- return 0
+ # call function from lib/rest.sh
+ rest_urlencoded "$verb" "$uri" "${MGMT_ADMIN_EMAIL}" "${MGMT_ADMIN_PW}" "$@" 2>>$TEST_OF
+ return $?
}
diff --git a/tests/system-setup/remote-nextcloud-docker.sh b/tests/system-setup/remote-nextcloud-docker.sh
index 35174bdc..1332c64f 100755
--- a/tests/system-setup/remote-nextcloud-docker.sh
+++ b/tests/system-setup/remote-nextcloud-docker.sh
@@ -27,9 +27,9 @@
usage() {
- echo "Usage: $(basename "$0") [\"before-miab-install\"|\"miab-install\"|\"after-miab-install\"]"
- echo "Install MiaB-LDAP and a remote Nextcloud running under docker exposed as localhost:8000"
- echo "With no arguments, all three stages are run."
+ echo "Usage: $(basename "$0")"
+ echo "Install MiaB-LDAP and a remote Nextcloud running under docker"
+ echo "Nextcloud is exposed as http://localhost:8000"
exit 1
}
@@ -40,10 +40,9 @@ if [ ! -d "tests/system-setup" ]; then
fi
# load helper scripts
-. "tests/system-setup/setup-defaults.sh" \
- || die "Could not load setup-defaults"
-. "tests/system-setup/setup-funcs.sh" \
- || die "Could not load setup-funcs"
+. "tests/lib/all.sh" "tests/lib" || die "Could not load lib scripts"
+. "tests/system-setup/setup-defaults.sh" || die "Could not load setup-defaults"
+. "tests/system-setup/setup-funcs.sh" || die "Could not load setup-funcs"
# ensure running as root
if [ "$EUID" != "0" ]; then
@@ -54,42 +53,15 @@ fi
before_miab_install() {
H1 "BEFORE MIAB-LDAP INSTALL"
-
- H2 "Update /etc/hosts"
- set_system_hostname || die "Could not set hostname"
-
- # update system time
- H2 "Set system time"
- update_system_time || echo "Ignoring error..."
+ system_init
+ miab_testing_init || die "Initialization failed"
- # update package lists before installing anything
- H2 "apt-get update"
- wait_for_apt
- apt-get update -qq || die "apt-get update failed!"
-
- # upgrade packages - if we don't do this and something like bind
- # is upgraded through automatic upgrades (because maybe MiaB was
- # previously installed), it may cause problems with the rest of
- # the setup, such as with name resolution failures
- if is_false "$TRAVIS"; then
- H2 "apt-get upgrade"
- wait_for_apt
- apt-get upgrade -qq || die "apt-get upgrade failed!"
- fi
-
- # install prerequisites
- H2 "QA pre-setup prerequisites"
- install_pre_setup_qa_prerequisites \
- || die "Error installing QA prerequisites"
-
# enable the remote Nextcloud setup mod, which tells MiaB-LDAP to use
# the remote Nextcloud for calendar and contacts instead of the
# MiaB-installed one
- H2 "Create local/remote-nextcloud.sh symbolic link"
- if [ ! -e "local/remote-nextcloud.sh" ]; then
- mkdir -p local
- ln -s "../setup/mods.available/remote-nextcloud.sh" "local/remote-nextcloud.sh" || die "Could not create remote-nextcloud.sh symlink"
- fi
+ H2 "Enable local mod remote-nextcloud"
+ enable_miab_mod "remote-nextcloud" \
+ || die "Could not enable remote-nextcloud mod"
# install Docker
H2 "Install Docker"
@@ -101,7 +73,7 @@ miab_install() {
H1 "MIAB-LDAP INSTALL"
if ! setup/start.sh; then
H1 "OUTPUT OF SELECT FILES"
- dump_log "/var/log/syslog" 100
+ dump_file "/var/log/syslog" 100
dump_conf_files "$TRAVIS"
H2; H2 "End"; H2
die "setup/start.sh failed!"
@@ -119,22 +91,28 @@ after_miab_install() {
# run Nextcloud docker image
H2 "Start Nextcloud docker container"
- docker run -d --name NC -p 8000:80 \
- --env SQLITE_DATABASE=nextclouddb.sqlite \
- --env NEXTCLOUD_ADMIN_USER="$NC_ADMIN_USER" \
- --env NEXTCLOUD_ADMIN_PASSWORD="$NC_ADMIN_PASSWORD" \
- --env NEXTCLOUD_TRUSTED_DOMAINS="127.0.0.1 ::1" \
- --env NEXTCLOUD_UPDATE=1 \
- --env SMTP_HOST="$PRIMARY_HOSTNAME" \
- --env SMTP_SECURE="tls" \
- --env SMTP_PORT=587 \
- --env SMTP_AUTHTYPE="LOGIN" \
- --env SMTP_NAME="$EMAIL_ADDR" \
- --env SMTP_PASSWORD="$EMAIL_PW" \
- --env SMTP_FROM_ADDRESS="$(awk -F@ '{print $1}' <<< "$EMAIL_ADDR")" \
- --env MAIL_DOMAIN="$(awk -F@ '{print $2}' <<< "$EMAIL_ADDR")" \
- nextcloud:latest \
- || die "Docker run failed!"
+ local container_started="true"
+ if [ -z "$(docker ps -f NAME=NC -q)" ]; then
+ docker run -d --name NC -p 8000:80 \
+ --env SQLITE_DATABASE=nextclouddb.sqlite \
+ --env NEXTCLOUD_ADMIN_USER="$NC_ADMIN_USER" \
+ --env NEXTCLOUD_ADMIN_PASSWORD="$NC_ADMIN_PASSWORD" \
+ --env NEXTCLOUD_TRUSTED_DOMAINS="127.0.0.1 ::1" \
+ --env NEXTCLOUD_UPDATE=1 \
+ --env SMTP_HOST="$PRIMARY_HOSTNAME" \
+ --env SMTP_SECURE="tls" \
+ --env SMTP_PORT=587 \
+ --env SMTP_AUTHTYPE="LOGIN" \
+ --env SMTP_NAME="$EMAIL_ADDR" \
+ --env SMTP_PASSWORD="$EMAIL_PW" \
+ --env SMTP_FROM_ADDRESS="$(email_localpart "$EMAIL_ADDR")" \
+ --env MAIL_DOMAIN="$(email_domainpart "$EMAIL_ADDR")" \
+ nextcloud:latest \
+ || die "Docker run failed!"
+ else
+ echo "Container already running"
+ container_started="false"
+ fi
H2 "docker: Update /etc/hosts so it can find MiaB-LDAP by name"
echo "$PRIVATE_IP $PRIMARY_HOSTNAME" | \
@@ -160,32 +138,20 @@ after_miab_install() {
# wait for Nextcloud installation to complete
H2 "Wait for Nextcloud installation to complete"
- echo -n "Waiting ..."
- local count=0
- while true; do
- if [ $count -ge 10 ]; then
- echo "FAILED"
- die "Giving up"
- fi
- sleep 6
- let count+=1
- if [ $(docker exec NC php -n -r "include 'config/config.php'; print \$CONFIG['installed']?'true':'false';") == "true" ]; then
- echo "ok"
- break
- fi
- echo -n "${count}..."
- done
+ wait_for_docker_nextcloud NC installed || die "Giving up"
- # install and enable Nextcloud and apps
+ # install and enable Nextcloud apps
H2 "docker: install Nextcloud calendar app"
docker exec -u www-data NC ./occ app:install calendar \
- || die "docker: installing calendar app failed"
+ || $container_started \
+ && die "docker: installing calendar app failed ($?)"
H2 "docker: install Nextcloud contacts app"
docker exec -u www-data NC ./occ app:install contacts \
- || die "docker: installing contacts app failed"
+ || $container_started \
+ && die "docker: installing contacts app failed ($?)"
H2 "docker: enable user_ldap"
docker exec -u www-data NC ./occ app:enable user_ldap \
- || die "docker: enabling user_ldap failed"
+ || die "docker: enabling user_ldap failed ($?)"
# integrate Nextcloud with MiaB-LDAP
H2 "docker: integrate Nextcloud with MiaB-LDAP"
@@ -201,28 +167,26 @@ after_miab_install() {
}
-
#
-# process command line
+# Main
#
-
-case "$1" in
- before-miab-install )
+case "${1:-all}" in
+ before-install )
before_miab_install
;;
- after-miab-install )
- after_miab_install
- ;;
- miab-install )
+ install )
miab_install
;;
- "" )
+ after-install )
+ after_miab_install
+ ;;
+ all )
before_miab_install
miab_install
after_miab_install
;;
* )
-
usage
;;
esac
+
diff --git a/tests/system-setup/setup-defaults.sh b/tests/system-setup/setup-defaults.sh
index 739a0260..b415e7a0 100755
--- a/tests/system-setup/setup-defaults.sh
+++ b/tests/system-setup/setup-defaults.sh
@@ -1,6 +1,6 @@
#!/bin/bash
-# Used by MiaB-LDAP setup/start.sh
+# Used by setup/start.sh
export NONINTERACTIVE=${NONINTERACTIVE:-1}
export SKIP_NETWORK_CHECKS=${SKIP_NETWORK_CHECKS:-1}
export STORAGE_USER="${STORAGE_USER:-user-data}"
@@ -15,7 +15,6 @@ elif [ -z "$PRIMARY_HOSTNAME" ]; then
export PRIMARY_HOSTNAME=${PRIMARY_HOSTNAME:-$(hostname --fqdn || hostname)}
fi
-
# Placing this var in STORAGE_ROOT/ldap/miab_ldap.conf before running
# setup/start.sh will avoid a random password from being used for the
# Nextcloud LDAP service account
@@ -28,6 +27,9 @@ export NC_HOST=${NC_HOST:-127.0.0.1}
export NC_PORT=${NC_PORT:-8000}
export NC_PREFIX=${NC_PREFIX:-/}
-# For setup scripts that are installing a remote Nextcloud
+# For setup scripts that may be installing a remote Nextcloud
export NC_ADMIN_USER="${NC_ADMIN_USER:-admin}"
export NC_ADMIN_PASSWORD="${NC_ADMIN_PASSWORD:-Test_1234}"
+
+# For setup scripts that install upstream versions
+export MIAB_UPSTREAM_GIT="https://github.com/mail-in-a-box/mailinabox.git"
diff --git a/tests/system-setup/setup-funcs.sh b/tests/system-setup/setup-funcs.sh
index 9f8194d5..c04a4fa9 100755
--- a/tests/system-setup/setup-funcs.sh
+++ b/tests/system-setup/setup-funcs.sh
@@ -1,4 +1,9 @@
+#
+# requires:
+#
+# test scripts: [ lib/misc.sh, lib/system.sh ]
+#
die() {
local msg="$1"
@@ -25,64 +30,28 @@ H2() {
fi
}
-dump_log() {
- local log_file="$1"
- local lines="$2"
- local title="DUMP OF $log_file"
- echo ""
- echo "--------"
- echo -n "-------- $log_file"
- if [ ! -z "$lines" ]; then
- echo " (last $line lines)"
- else
- echo ""
- fi
- echo "--------"
-
- if [ ! -z "$lines" ]; then
- tail -$lines "$log_file"
- else
- cat "$log_file"
- fi
-}
-is_true() {
- # empty string is not true
- if [ "$1" == "true" \
- -o "$1" == "TRUE" \
- -o "$1" == "True" \
- -o "$1" == "yes" \
- -o "$1" == "YES" \
- -o "$1" == "Yes" \
- -o "$1" == "1" ]
- then
- return 0
- else
- return 1
- fi
-}
-
-is_false() {
- if is_true $@; then return 1; fi
+wait_for_docker_nextcloud() {
+ local container="$1"
+ local config_key="$2"
+ echo -n "Waiting ..."
+ local count=0
+ while true; do
+ if [ $count -ge 10 ]; then
+ echo "FAILED"
+ return 1
+ fi
+ sleep 6
+ let count+=1
+ if [ $(docker exec "$container" php -n -r "include 'config/config.php'; print \$CONFIG['$config_key']?'true':'false';") == "true" ]; then
+ echo "ok"
+ break
+ fi
+ echo -n "${count}..."
+ done
return 0
}
-wait_for_apt() {
- local count=0
- while fuser /var/lib/dpkg/lock >/dev/null 2>&1 || fuser /var/lib/dpkg/lock-frontend >/dev/null 2>&1; do
- sleep 6
- let count+=1
- if [ $count -eq 1 ]; then
- echo -n "Waiting for other package manager to finish..."
- elif [ $count -gt 100 ]; then
- echo -n "FAILED"
- return 1
- else
- echo -n "${count}.."
- fi
- done
- [ $count -ge 1 ] && echo ""
-}
dump_conf_files() {
local skip
@@ -98,99 +67,58 @@ dump_conf_files() {
done
fi
if [ "$skip" == "false" ]; then
- dump_log "/etc/mailinabox.conf"
- dump_log "/etc/hosts"
- dump_log "/etc/nsswitch.conf"
- dump_log "/etc/resolv.conf"
- dump_log "/etc/nsd/nsd.conf"
- dump_log "/etc/postfix/main.cf"
+ dump_file "/etc/mailinabox.conf"
+ dump_file_if_exists "/etc/mailinabox_mods.conf"
+ dump_file "/etc/hosts"
+ dump_file "/etc/nsswitch.conf"
+ dump_file "/etc/resolv.conf"
+ dump_file "/etc/nsd/nsd.conf"
+ #dump_file "/etc/postfix/main.cf"
fi
}
-update_system_time() {
- if [ ! -x /usr/sbin/ntpdate ]; then
+
+#
+# Initialize the test system
+# hostname, time, apt update/upgrade, etc
+#
+system_init() {
+ H2 "Update /etc/hosts"
+ set_system_hostname || die "Could not set hostname"
+
+ # update system time
+ H2 "Set system time"
+ update_system_time || echo "Ignoring error..."
+
+ # update package lists before installing anything
+ H2 "apt-get update"
+ wait_for_apt
+ apt-get update -qq || die "apt-get update failed!"
+
+ # upgrade packages - if we don't do this and something like bind
+ # is upgraded through automatic upgrades (because maybe MiaB was
+ # previously installed), it may cause problems with the rest of
+ # the setup, such as with name resolution failures
+ if is_false "$TRAVIS"; then
+ H2 "apt-get upgrade"
wait_for_apt
- apt-get install -y -qq ntpdate || return 1
+ apt-get upgrade -qq || die "apt-get upgrade failed!"
fi
- ntpdate -s ntp.ubuntu.com && echo "System time updated"
-}
-
-update_hosts() {
- local host="$1"
- shift
- local ip
- for ip; do
- if [ ! -z "$ip" ]; then
- local line="$ip $host"
- if ! grep -F "$line" /etc/hosts 1>/dev/null; then
- echo "$line" >>/etc/hosts
- fi
- fi
- done
-}
-
-update_hosts_for_private_ip() {
- # create /etc/hosts entry for PRIVATE_IP and PRIVATE_IPV6
- # PRIMARY_HOSTNAME must already be set
- local ip4=$(source setup/functions.sh; get_default_privateip 4)
- local ip6=$(source setup/functions.sh; get_default_privateip 6)
- [ -z "$ip4" -a -z "$ip6" ] && return 1
- [ -z "$ip6" ] && ip6="::1"
- update_hosts "$PRIMARY_HOSTNAME" "$ip4" "$ip6" || return 1
-}
-
-set_system_hostname() {
- # set the system hostname to the FQDN specified or
- # PRIMARY_HOSTNAME if no FQDN was given
- local fqdn="${1:-$PRIMARY_HOSTNAME}"
- local host="$(awk -F. '{print $1}' <<< "$fqdn")"
- sed -i 's/^127\.0\.1\.1[ \t].*/127.0.1.1 '"$fqdn $host ip4-loopback/" /etc/hosts || return 1
- #hostname "$host" || return 1
- #echo "$host" > /etc/hostname
- return 0
}
-install_docker() {
- if [ -x /usr/bin/docker ]; then
- echo "Docker already installed"
- return 0
- fi
-
- wait_for_apt
- apt-get install -y -qq \
- apt-transport-https \
- ca-certificates \
- curl \
- gnupg-agent \
- software-properties-common \
- || return 1
-
- wait_for_apt
- curl -fsSL https://download.docker.com/linux/ubuntu/gpg | apt-key add - \
- || return 2
-
- wait_for_apt
- apt-key fingerprint 0EBFCD88 || return 3
-
- wait_for_apt
- add-apt-repository -y --update "deb [arch=amd64] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" || return 4
-
- wait_for_apt
- apt-get install -y -qq \
- docker-ce \
- docker-ce-cli \
- containerd.io \
- || return 5
-}
-
-
-install_pre_setup_qa_prerequisites() {
+#
+# Initialize the test system with QA prerequisites
+# Anything needed to use the test runner, speed up the installation,
+# etc
+#
+miab_testing_init() {
[ -z "$STORAGE_ROOT" ] \
&& echo "Error: STORAGE_ROOT not set" 1>&2 \
&& return 1
+ H2 "QA prerequisites"
local rc=0
# python3-dnspython: is used by the python scripts in 'tests' and is
@@ -221,22 +149,20 @@ install_pre_setup_qa_prerequisites() {
}
-travis_fix_nsd() {
- if [ "$TRAVIS" != "true" ]; then
- return 0
+enable_miab_mod() {
+ local name="${1}.sh"
+ if [ ! -e "local/$name" ]; then
+ mkdir -p local
+ ln -s "../setup/mods.available/$name" "local/$name"
fi
-
- # nsd won't start on Travis-CI without the changes below: ip6 off and
- # control-enable set to no. Even though the nsd docs say the
- # default value for control-enable is no, running "nsd-checkconf -o
- # control-enable /etc/nsd/nsd.conf" returns "yes", so we explicitly
- # set it here.
- #
- # we're assuming that the "ip-address" line is the last line in the
- # "server" section of nsd.conf. if this generated file output
- # changes, the sed command below may need to be adjusted.
- sed -i 's/ip-address\(.\)\(.*\)/ip-address\1\2\n do-ip4\1 yes\n do-ip6\1 no\n verbosity\1 3\nremote-control\1\n control-enable\1 no/' /etc/nsd/nsd.conf || return 1
- cat /etc/nsd/nsd.conf
- systemctl reset-failed nsd.service || return 2
- systemctl restart nsd.service || return 3
}
+
+tag_from_readme() {
+ # extract the recommended TAG from README.md
+ # sets a global "TAG"
+ local readme="${1:-README.md}"
+ TAG="$(grep -F 'git checkout' "$readme" | sed 's/.*\(v[0123456789]*\.[0123456789]*\).*/\1/')"
+ [ $? -ne 0 -o -z "$TAG" ] && return 1
+ return 0
+}
+
diff --git a/tests/system-setup/upgrade-from-upstream.sh b/tests/system-setup/upgrade-from-upstream.sh
new file mode 100755
index 00000000..45f40fcb
--- /dev/null
+++ b/tests/system-setup/upgrade-from-upstream.sh
@@ -0,0 +1,289 @@
+#!/bin/bash
+
+# setup MiaB-LDAP by:
+# 1. installing upstream MiaB
+# 2. adding some data (users/aliases/etc)
+# 3. upgrading to MiaB-LDAP
+#
+# See setup-defaults.sh for usernames and passwords.
+#
+
+
+usage() {
+ echo "Usage: $(basename "$0")"
+ echo "Install MiaB-LDAP after installing upstream MiaB"
+ exit 1
+}
+
+# ensure working directory
+if [ ! -d "tests/system-setup" ]; then
+ echo "This script must be run from the MiaB root directory"
+ exit 1
+fi
+
+# load helper scripts
+. "tests/lib/all.sh" "tests/lib" || die "Could not load lib scripts"
+. "tests/system-setup/setup-defaults.sh" || die "Could not load setup-defaults"
+. "tests/system-setup/setup-funcs.sh" || die "Could not load setup-funcs"
+
+# ensure running as root
+if [ "$EUID" != "0" ]; then
+ die "This script must be run as root (sudo)"
+fi
+
+
+before_install() {
+ H1 "INIT"
+ system_init
+ miab_testing_init || die "Initialization failed"
+}
+
+upstream_install() {
+ local upstream_dir="$HOME/mailinabox-upstream"
+ H1 "INSTALL UPSTREAM"
+ [ ! -x /usr/bin/git ] && apt-get install -y -qq git
+
+ if [ ! -d "$upstream_dir" ] || [ -z "$(ls -A "$upstream_dir")" ] ; then
+ H2 "Cloning $MIAB_UPSTREAM_GIT"
+ rm -rf "$upstream_dir"
+ git clone "$MIAB_UPSTREAM_GIT" "$upstream_dir"
+ if [ $? -ne 0 ]; then
+ rm -rf "$upstream_dir"
+ die "git clone upstream failed!"
+ fi
+ if [ -z "$TAG" ]; then
+ tag_from_readme "$upstream_dir/README.md"
+ if [ $? -ne 0 ]; then
+ rm -rf "$upstream_dir"
+ die "Failed to extract TAG from $upstream_dir/README.md"
+ fi
+ fi
+ fi
+
+ pushd "$upstream_dir" >/dev/null
+ if [ ! -z "$TAG" ]; then
+ H2 "Checkout $TAG"
+ git checkout "$TAG" || die "git checkout $TAG failed"
+ fi
+
+ H2 "Run upstream setup"
+ setup/start.sh || die "Upstream setup failed!"
+ popd >/dev/null
+
+ H2 "Upstream info"
+ echo "Code version: $(git describe)"
+ echo "Migration version: $(cat "$STORAGE_ROOT/mailinabox.version")"
+}
+
+
+add_data() {
+ H1 "Add some Mail-in-a-Box data"
+ local users=()
+ users+="betsy@$(email_domainpart "$EMAIL_ADDR")"
+
+ local alises=()
+ aliases+="goalias@testdom.com > ${users[0]}"
+ aliases+="nested@testdom.com > goalias@testdom.com"
+
+ local pw="$(generate_qa_password)"
+
+
+ #
+ # get the existing users and aliases
+ #
+ local current_users=() current_aliases=()
+ local user alias
+ if ! rest_urlencoded GET /admin/mail/users "$EMAIL_ADDR" "$EMAIL_PW" --insecure 2>/dev/null; then
+ die "Unable to enumerate users: rc=$? err=$REST_ERROR"
+ fi
+ for user in $REST_OUTPUT; do
+ current_users+=("$user")
+ done
+
+ if ! rest_urlencoded GET /admin/mail/aliases "$EMAIL_ADDR" "$EMAIL_PW" --insecure 2>/dev/null; then
+ die "Unable to enumerate aliases: rc=$? err=$REST_ERROR"
+ fi
+ for alias in $REST_OUTPUT; do
+ current_aliases+=("$alias")
+ done
+
+
+ #
+ # add users
+ #
+ for user in "${users[@]}"; do
+ if array_contains "$user" "${current_users[@]}"; then
+ echo "Not adding user $user: already exists"
+
+ elif ! rest_urlencoded POST /admin/mail/users/add "$EMAIL_ADDR" "$EMAIL_PW" --insecure -- "email=$user" "password=$pw" 2>/dev/null
+ then
+ die "Unable to add user $user: rc=$? err=$REST_ERROR"
+ fi
+ done
+
+ #
+ # add aliases
+ #
+ local aliasdef
+ for aliasdef in "${aliases[@]}"; do
+ alias="$(awk -F'[> ]' '{print $1}' <<<"$aliasdef")"
+ local forwards_to="$(sed 's/.*> *\(.*\)/\1/' <<<"$aliasdef")"
+ if array_contains "$alias" "${current_aliases[@]}"; then
+ echo "Not adding alias $alias: already exists"
+
+ elif ! rest_urlencoded POST /admin/mail/aliases/add "$EMAIL_ADDR" "$EMAIL_PW" --insecure -- "address=$alias" "forwards_to=$forwards_to" 2>/dev/null
+ then
+ die "Unable to add alias $alias: rc=$? err=$REST_ERROR"
+ fi
+ done
+}
+
+capture_state() {
+ # users and aliases lists
+ # dns zone files
+ # tls certificates: expected CN's
+
+ local state_dir="$1"
+ local infojson="$state_dir/info.json"
+
+ H1 "Capture server state to $state_dir"
+
+ # nuke saved state, if any
+ rm -rf "$state_dir"
+ mkdir -p "$state_dir"
+
+ # create info.json
+ H2 "create info.json"
+ echo "VERSION='$(git describe --abbrev=0)'" >"$infojson"
+ echo "MIGRATION_VERSION=$(cat "$STORAGE_ROOT/mailinabox.version")" >>"$infojson"
+
+ # record users
+ H2 "record users"
+ rest_urlencoded GET "/admin/mail/users?format=json" "$EMAIL_ADDR" "$EMAIL_PW" --insecure 2>/dev/null \
+ || die "Unable to get users: rc=$? err=$REST_ERROR"
+ echo "$REST_OUTPUT" > "$state_dir/users.json"
+
+ # record aliases
+ H2 "record aliases"
+ rest_urlencoded GET "/admin/mail/aliases?format=json" "$EMAIL_ADDR" "$EMAIL_PW" --insecure 2>/dev/null \
+ || die "Unable to get aliases: rc=$? err=$REST_ERROR"
+ echo "$REST_OUTPUT" > "$state_dir/aliases.json"
+
+ # record dns config
+ H2 "record dns details"
+ local file
+ mkdir -p "$state_dir/zones"
+ for file in ls /etc/nsd/zones/*.signed; do
+ cp "$file" "$state_dir/zones"
+ done
+
+}
+
+miab_ldap_install() {
+ # ensure we're in a MiaB-LDAP working directory
+ if [ ! -e setup/ldap.sh ]; then
+ die "The working directory is not MiaB-LDAP!"
+ fi
+ setup/start.sh -v || die "Upgrade to MiaB-LDAP failed !!!!!!"
+}
+
+compare_state() {
+ local s1="$1"
+ local s2="$2"
+
+ local output
+ local changed="false"
+
+ H1 "COMPARE STATES $(basename "$s1") TO $(basename "$2")"
+ H2 "Users"
+ # users
+ output="$(diff "$s1/users.json" "$s2/users.json" 2>&1)"
+ if [ $? -ne 0 ]; then
+ changed="true"
+ echo "USERS ARE DIFFERENT!"
+ echo "$output"
+ else
+ echo "OK"
+ fi
+
+ H2 "Aliases"
+ output="$(diff "$s1/aliases.json" "$s2/aliases.json" 2>&1)"
+ if [ $? -ne 0 ]; then
+ change="true"
+ echo "ALIASES ARE DIFFERENT!"
+ echo "$output"
+ else
+ echo "OK"
+ fi
+
+ H2 "DNS - zones missing"
+ local zone
+ for zone in $(cd "$s1/zones"; ls *.signed); do
+ if [ ! -e "$s2/zones/$zone" ]; then
+ echo "MISSING zone: $zone"
+ changed="true"
+ fi
+ done
+
+ H2 "DNS - zones added"
+ for zone in $(cd "$s2/zones"; ls *.signed); do
+ if [ ! -e "$s2/zones/$zone" ]; then
+ echo "ADDED zone: $zone"
+ changed="true"
+ fi
+ done
+
+ H2 "DNS - zones changed"
+ for zone in $(cd "$s1/zones"; ls *.signed); do
+ if [ -e "$s2/zones/$zone" ]; then
+ output="$(diff "$s1/zones/$zone" "$s2/zones/$zone" 2>&1)"
+ if [ $? -ne 0 ]; then
+ echo "CHANGED zone: $zone"
+ echo "$output"
+ changed="true"
+ fi
+ fi
+ done
+
+ if $changed; then
+ return 1
+ else
+ return 0
+ fi
+}
+
+
+if [ "$1" == "c" ]; then
+ capture_state "tests/system-setup/state/miab-ldap"
+ exit $?
+fi
+
+
+# install basic stuff, set the hostname, time, etc
+before_install
+
+# if MiaB-LDAP is already migrated, do not run upstream setup
+if [ -e "$STORAGE_ROOT/mailinabox.version" ] &&
+ [ $(cat "$STORAGE_ROOT/mailinabox.version") -ge 13 ]
+then
+ echo "Warning: MiaB-LDAP is already installed! Skipping installation of upstream"
+else
+ # install upstream
+ upstream_install
+ add_data
+ capture_state "tests/system-setup/state/upstream"
+fi
+
+# install miab-ldap
+miab_ldap_install
+capture_state "tests/system-setup/state/miab-ldap"
+
+# compare states
+if ! compare_state "tests/system-setup/state/upstream" "tests/system-setup/state/miab-ldap"; then
+ die "Upstream and upgraded states are different !"
+fi
+
+#
+# actual verification that mail sends/receives properly is done via
+# the test runner ...
+#