external/mailinabox
1
0
Fork 0
mirror of https://github.com/mail-in-a-box/mailinabox.git synced 2026-03-04 15:54:48 +01:00
Code Issues Releases Wiki Activity

1. Better code organization & simplify

Browse Source 
2. Add "populate" data for upgrades - enabled in both system-setup scripts
3. Add "upgrade" test runner suite
This commit is contained in:
downtownallday
2020-06-19 12:12:49 -04:00
parent 144aa6e5d6
commit 1bd7b2c4c7
20 changed files with 987 additions and 509 deletions
Download Patch File Download Diff File Expand all files Collapse all files

31
tests/system-setup/populate/README.txt Normal file
View File

@@ -0,0 +1,31 @@
This directory contains scripts used to populate a MiaB installation
with known values, and then subsequently verify that MiaB continues to
operate poperly after an upgrade or setup mod change.
Each "named" populate set of scripts should contain at least two
shell scripts:
1. <name>-populate.sh : populates the installation
2. <name>-verify.sh : verifies operation after upgrade
The system-setup/* scripts run the populate script, and the test
runner's 'upgrade' test suite runs the verify script.
These scripts are run, not sourced.
Expected script output and return value:
1. All debug output must go to stderr
2. Result messages must be sent to stdout (a single line, preferrably)
3. Return 0 if successfully passed verification
4. Return non-zero if failed verification
The working directory for <name>-populate.sh is the Mail-in-a-Box root
directory.
The working directory for <name>-verify.sh is 'tests' (because the
test runner always changes the working directory there to limit
contamination of the source tree). Use MIAB_DIR and ASSETS_DIR, if
needed.

10
tests/system-setup/populate/basic-data.sh Executable file
View File

@@ -0,0 +1,10 @@
#
# requires:
# lib scripts: [ misc.sh ]
# system-setup scripts: [ setup-defaults.sh ]
#
TEST_USER="anna@$(email_domainpart "$EMAIL_ADDR")"
TEST_USER_PASS="$(static_qa_password)"
TEST_USER_CONTACT_UUID="e0642b47-9104-4adb-adfd-5f907d04216a"
TEST_USER_CONTACT_EMAIL="sam@bff.org"

48
tests/system-setup/populate/basic-populate.sh Executable file
View File

@@ -0,0 +1,48 @@
#!/bin/bash
. "$(dirname "$0")/../setup-defaults.sh" || exit 1
. "$(dirname "$0")/../../lib/all.sh" "$(dirname "$0")/../../lib" || exit 1
. "$(dirname "$0")/basic-data.sh" || exit 1
#
# Add user
#
if ! populate_miab_users "" "" "" "${TEST_USER}:${TEST_USER_PASS}"
then
echo "Unable to add user"
exit 1
fi
#
# Add Nextcloud contact and force Roundcube contact sync to ensure the
# roundcube carddav addressbooks and contacts tables are populated in
# case a remote nextcloud is subsequently configured and the
# syncronization disabled.
#
if ! carddav_ls "$TEST_USER" "$TEST_USER_PASS" --insecure 2>/dev/null
then
echo "Could not enumerate contacts: $REST_ERROR"
exit 1
fi
echo "Current contacts count: ${#FILES[@]}"
if array_contains "$TEST_USER_CONTACT_UUID.vcf" "${FILES[@]}"; then
echo "Contact $TEST_USER_CONTACT_UUID already present"
else
if ! carddav_add_contact "$TEST_USER" "$TEST_USER_PASS" "Anna" "666-1111" "$TEST_USER_CONTACT_EMAIL" "$TEST_USER_CONTACT_UUID" --insecure 2>/dev/null
then
echo "Could not add contact: $REST_ERROR"
exit 1
fi
echo "Force Roundcube contact sync"
if ! roundcube_force_carddav_refresh "$TEST_USER" "$TEST_USER_PASS"
then
echo "Roundcube <-> Nextcloud contact sync failed"
exit 1
fi
fi
exit 0

53
tests/system-setup/populate/basic-verify.sh Executable file
View File

@@ -0,0 +1,53 @@
#!/bin/bash
. "$(dirname "$0")/../setup-defaults.sh" || exit 1
. "$(dirname "$0")/../../lib/all.sh" "$(dirname "$0")/../../lib" || exit 1
. "$(dirname "$0")/basic-data.sh" || exit 1
. /etc/mailinabox.conf || exit 1
# 1. the test user can still log in and send mail
echo "[User can still log in with their old passwords and send mail]" 1>&2
echo "python3 test_mail.py $PRIVATE_IP $TEST_USER $TEST_USER_PASS" 1>&2
python3 test_mail.py "$PRIVATE_IP" "$TEST_USER" "$TEST_USER_PASS" 1>&2
if [ $? -ne 0 ]; then
echo "Basic mail functionality test failed"
exit 1
fi
# 2. the test user's contact is still accessible in Roundcube
echo "[Force Roundcube contact sync]" 1>&2
# if MiaB's Nextcloud carddav configuration was removed all the
# contacts for it will be removed in the Roundcube database after the
# sync
if ! roundcube_force_carddav_refresh "$TEST_USER" "$TEST_USER_PASS" 1>&2
then
echo "Roundcube <-> Nextcloud contact sync failed ($?)"
exit 1
fi
echo "[Ensure old Nextcloud contacts are still present]" 1>&2
echo "sqlite3 $STORAGE_ROOT/mail/roundcube/roundcube.sqlite \"select email from carddav_contacts where cuid='$TEST_USER_CONTACT_UUID'\"" 1>&2
output=$(sqlite3 "$STORAGE_ROOT/mail/roundcube/roundcube.sqlite" "select email from carddav_contacts where cuid='$TEST_USER_CONTACT_UUID'")
rc=$?
if [ $rc -ne 0 ]
then
echo "Querying Roundcube's sqlite database failed ($rc)"
exit 1
else
echo "Success, found $output" 1>&2
fi
if [ "$output" != "$TEST_USER_CONTACT_EMAIL" ]
then
echo "Unexpected email for contact uuid: got '$output', expected '$TEST_USER_CONTACT_EMAIL'"
exit 1
fi
echo "OK basic-verify passed"
exit 0

155
tests/system-setup/remote-nextcloud-docker.sh
View File

@@ -26,13 +26,6 @@
#
usage() {
echo "Usage: $(basename "$0")"
echo "Install MiaB-LDAP and a remote Nextcloud running under docker"
echo "Nextcloud is exposed as http://localhost:8000"
exit 1
}
# ensure working directory
if [ ! -d "tests/system-setup" ]; then
echo "This script must be run from the MiaB root directory"
@@ -51,49 +44,26 @@ fi
before_miab_install() {
H1 "BEFORE MIAB-LDAP INSTALL"
system_init
miab_testing_init || die "Initialization failed"
# enable the remote Nextcloud setup mod, which tells MiaB-LDAP to use
# the remote Nextcloud for calendar and contacts instead of the
# MiaB-installed one
H2 "Enable local mod remote-nextcloud"
enable_miab_mod "remote-nextcloud" \
|| die "Could not enable remote-nextcloud mod"
init() {
H1 "INIT"
init_test_system
init_miab_testing || die "Initialization failed"
}
install_nextcloud_docker() {
H1 "INSTALL NEXTCLOUD ON DOCKER"
# install Docker
H2 "Install Docker"
install_docker || die "Could not install Docker! ($?)"
}
miab_install() {
H1 "MIAB-LDAP INSTALL"
if ! setup/start.sh; then
H1 "OUTPUT OF SELECT FILES"
dump_file "/var/log/syslog" 100
dump_conf_files "$TRAVIS"
H2; H2 "End"; H2
die "setup/start.sh failed!"
fi
H1 "OUTPUT OF SELECT FILES"
dump_conf_files "$TRAVIS"
H2; H2 "End"; H2
}
after_miab_install() {
H1 "AFTER MIAB-LDAP INSTALL"
. /etc/mailinabox.conf || die "Could not load /etc/mailinabox.conf"
# run Nextcloud docker image
H2 "Start Nextcloud docker container"
local container_started="true"
if [ -z "$(docker ps -f NAME=NC -q)" ]; then
docker run -d --name NC -p 8000:80 \
--add-host "$PRIMARY_HOSTNAME:$PRIVATE_IP" \
--env SQLITE_DATABASE=nextclouddb.sqlite \
--env NEXTCLOUD_ADMIN_USER="$NC_ADMIN_USER" \
--env NEXTCLOUD_ADMIN_PASSWORD="$NC_ADMIN_PASSWORD" \
@@ -114,10 +84,10 @@ after_miab_install() {
container_started="false"
fi
H2 "docker: Update /etc/hosts so it can find MiaB-LDAP by name"
echo "$PRIVATE_IP $PRIMARY_HOSTNAME" | \
docker exec -i NC bash -c 'cat >>/etc/hosts' \
|| die "docker: could not update /etc/hosts"
# H2 "docker: Update /etc/hosts so it can find MiaB-LDAP by name"
# echo "$PRIVATE_IP $PRIMARY_HOSTNAME" | \
# docker exec -i NC bash -c 'cat >>/etc/hosts' \
# || die "docker: could not update /etc/hosts"
# apt-get update
H2 "docker: apt-get update"
@@ -171,26 +141,87 @@ after_miab_install() {
}
#
# Main
#
case "${1:-all}" in
before-install )
before_miab_install
do_upgrade() {
local populate_name="$1"
if [ -e "local/remote-nextcloud.sh" ]; then
# we install w/o remote nextcloud first so we can add
# a user w/contacts and ensure the contact exists in the
# new system
if [ ! -L "local/remote-nextcloud.sh" ]; then
echo "Warning: local/remote-nextcloud.sh is a regular file - should be a symlink"
fi
die "Error: local/remote-nextcloud.sh exists - delete it and try again"
fi
# initialize test system
init
# install w/o remote Nextcloud
miab_ldap_install
# populate some data
[ ! -z "$populate_name" ] && populate_by_name "$populate_name"
# install Nextcloud in a Docker container (MiaB must be available)
install_nextcloud_docker
H1 "Enable remote-nextcloud mod"
enable_miab_mod "remote-nextcloud" \
|| die "Could not enable remote-nextcloud mod"
# re-run setup to use the remote Nextcloud
miab_ldap_install
}
do_default() {
# initialize test system
init
H1 "Enable remote-nextcloud mod"
enable_miab_mod "remote-nextcloud" \
|| die "Could not enable remote-nextcloud mod"
# run setup to use the remote Nextcloud (doesn't need to be available)
miab_ldap_install
# install Nextcloud in a Docker container (MiaB must be available)
install_nextcloud_docker
}
case "$1" in
upgrade )
# Runs this sequence:
# 1. setup w/o remote nextcloud
# 2. if an additional argument is given, populate the MiaB
# installation
# 3. install a remote nextcloud
# 4. enable remote-nextcloud mod
# 5. re-run setup
#
shift
do_upgrade "$@"
;;
install )
miab_install
;;
after-install )
after_miab_install
;;
all )
before_miab_install
miab_install
after_miab_install
"" | default )
# Runs this sequence:
# 1. setup w/remote nextcloud
# 2. install and connect the remote nextcloud
do_default
;;
* )
usage
echo "Unknown option $1"
exit 1
;;
esac

2
tests/system-setup/setup-defaults.sh
View File

@@ -7,7 +7,7 @@ export STORAGE_USER="${STORAGE_USER:-user-data}"
export STORAGE_ROOT="${STORAGE_ROOT:-/home/$STORAGE_USER}"
export EMAIL_ADDR="${EMAIL_ADDR:-qa@abc.com}"
export EMAIL_PW="${EMAIL_PW:-Test_1234}"
export PUBLIC_IP="${PUBLIC_IP:-$(source setup/functions.sh; get_default_privateip 4)}"
export PUBLIC_IP="${PUBLIC_IP:-$(source ${MIAB_DIR:-.}/setup/functions.sh; get_default_privateip 4)}"
if [ "$TRAVIS" == "true" ]; then
export PRIMARY_HOSTNAME=${PRIMARY_HOSTNAME:-box.abc.com}

65
tests/system-setup/setup-funcs.sh
View File

@@ -5,6 +5,7 @@
# test scripts: [ lib/misc.sh, lib/system.sh ]
#
die() {
local msg="$1"
echo "$msg" 1>&2
@@ -12,25 +13,6 @@ die() {
}
H1() {
local msg="$1"
echo "----------------------------------------------"
if [ ! -z "$msg" ]; then
echo " $msg"
echo "----------------------------------------------"
fi
}
H2() {
local msg="$1"
if [ -z "$msg" ]; then
echo "***"
else
echo "*** $msg ***"
fi
}
wait_for_docker_nextcloud() {
local container="$1"
local config_key="$2"
@@ -83,7 +65,9 @@ dump_conf_files() {
# Initialize the test system
# hostname, time, apt update/upgrade, etc
#
system_init() {
# Errors are fatal
#
init_test_system() {
H2 "Update /etc/hosts"
set_system_hostname || die "Could not set hostname"
@@ -113,7 +97,7 @@ system_init() {
# Anything needed to use the test runner, speed up the installation,
# etc
#
miab_testing_init() {
init_miab_testing() {
[ -z "$STORAGE_ROOT" ] \
&& echo "Error: STORAGE_ROOT not set" 1>&2 \
&& return 1
@@ -152,8 +136,12 @@ miab_testing_init() {
enable_miab_mod() {
local name="${1}.sh"
if [ ! -e "local/$name" ]; then
mkdir -p local
ln -s "../setup/mods.available/$name" "local/$name"
mkdir -p "local"
if ! ln -s "../setup/mods.available/$name" "local/$name"
then
echo "Warning: copying instead of symlinking local/$name"
cp "setup/mods.available/$name" "local/$name"
fi
fi
}
@@ -166,3 +154,34 @@ tag_from_readme() {
return 0
}
miab_ldap_install() {
H1 "MIAB-LDAP INSTALL"
# ensure we're in a MiaB-LDAP working directory
if [ ! -e setup/ldap.sh ]; then
die "Cannot install: the working directory is not MiaB-LDAP!"
fi
if ! setup/start.sh; then
H1 "OUTPUT OF SELECT FILES"
dump_file "/var/log/syslog" 100
dump_conf_files "$TRAVIS"
H2; H2 "End"; H2
die "MiaB-LDAP setup/start.sh failed!"
fi
# set actual STORAGE_ROOT, STORAGE_USER, PRIVATE_IP, etc
. /etc/mailinabox.conf || die "Could not source /etc/mailinabox.conf"
}
populate_by_name() {
local populate_name="$1"
H1 "Populate Mail-in-a-Box ($populate_name)"
local populate_script="tests/system-setup/populate/${populate_name}-populate.sh"
if [ ! -e "$populate_script" ]; then
die "Does not exist: $populate_script"
fi
"$populate_script" || die "Failed: $populate_script"
}

246
tests/system-setup/upgrade-from-upstream.sh
View File

@@ -32,10 +32,10 @@ if [ "$EUID" != "0" ]; then
fi
before_install() {
init() {
H1 "INIT"
system_init
miab_testing_init || die "Initialization failed"
init_test_system
init_miab_testing || die "Initialization failed"
}
upstream_install() {
@@ -94,217 +94,60 @@ upstream_install() {
}
add_data() {
populate() {
local pw="$(static_qa_password)"
H1 "Add some Mail-in-a-Box data"
local users=()
users+=("betsy@$(email_domainpart "$EMAIL_ADDR")")
users+=("betsy@$(email_domainpart "$EMAIL_ADDR"):$pw")
local alises=()
aliases+=("goalias@testdom.com > ${users[0]}")
aliases+=("goalias@testdom.com > $(awk -F: {'print $1'} <<<"${users[0]}")")
aliases+=("nested@testdom.com > goalias@testdom.com")
local pw="$(generate_qa_password)"
#
# get the existing users and aliases
#
local current_users=() current_aliases=()
local user alias
if ! rest_urlencoded GET /admin/mail/users "$EMAIL_ADDR" "$EMAIL_PW" --insecure >/dev/null 2>&1; then
die "Unable to enumerate users: rc=$? err=$REST_ERROR"
fi
for user in $REST_OUTPUT; do
current_users+=("$user")
done
if ! rest_urlencoded GET /admin/mail/aliases "$EMAIL_ADDR" "$EMAIL_PW" --insecure 2>/dev/null; then
die "Unable to enumerate aliases: rc=$? err=$REST_ERROR"
fi
for alias in $REST_OUTPUT; do
current_aliases+=("$alias")
done
#
# add users
#
H2 "Add users"
for user in "${users[@]}"; do
if array_contains "$user" "${current_users[@]}"; then
echo "Not adding user $user: already exists"
if ! populate_miab_users "" "" "" "${users[@]}"
then
die "Unable to add users"
fi
elif ! rest_urlencoded POST /admin/mail/users/add "$EMAIL_ADDR" "$EMAIL_PW" --insecure -- "email=$user" "password=$pw" 2>/dev/null
then
die "Unable to add user $user: rc=$? err=$REST_ERROR"
else
echo "Add: $user"
fi
done
#
# add aliases
#
H2 "Add aliases"
local aliasdef
for aliasdef in "${aliases[@]}"; do
alias="$(awk -F'[> ]' '{print $1}' <<<"$aliasdef")"
local forwards_to="$(sed 's/.*> *\(.*\)/\1/' <<<"$aliasdef")"
if array_contains "$alias" "${current_aliases[@]}"; then
echo "Not adding alias $alias: already exists"
elif ! rest_urlencoded POST /admin/mail/aliases/add "$EMAIL_ADDR" "$EMAIL_PW" --insecure -- "address=$alias" "forwards_to=$forwards_to" 2>/dev/null
then
die "Unable to add alias $alias: rc=$? err=$REST_ERROR"
else
echo "Add: $aliasdef"
fi
done
}
capture_state() {
# users and aliases lists
# dns zone files
# tls certificates: expected CN's
local state_dir="$1"
local info="$state_dir/info.txt"
H1 "Capture server state to $state_dir"
# nuke saved state, if any
rm -rf "$state_dir"
mkdir -p "$state_dir"
# create info.json
H2 "create info.txt"
echo "VERSION='$(git describe --abbrev=0)'" >"$info"
echo "MIGRATION_VERSION=$(cat "$STORAGE_ROOT/mailinabox.version")" >>"$info"
# record users
H2 "record users"
rest_urlencoded GET "/admin/mail/users?format=json" "$EMAIL_ADDR" "$EMAIL_PW" --insecure 2>/dev/null \
|| die "Unable to get users: rc=$? err=$REST_ERROR"
echo "$REST_OUTPUT" > "$state_dir/users.json"
# record aliases
H2 "record aliases"
rest_urlencoded GET "/admin/mail/aliases?format=json" "$EMAIL_ADDR" "$EMAIL_PW" --insecure 2>/dev/null \
|| die "Unable to get aliases: rc=$? err=$REST_ERROR"
echo "$REST_OUTPUT" > "$state_dir/aliases.json"
# record dns config
H2 "record dns details"
local file
mkdir -p "$state_dir/zones"
for file in /etc/nsd/zones/*.signed; do
cp "$file" "$state_dir/zones"
done
}
miab_ldap_install() {
H1 "INSTALL MIAB-LDAP"
# ensure we're in a MiaB-LDAP working directory
if [ ! -e setup/ldap.sh ]; then
die "The working directory is not MiaB-LDAP!"
fi
setup/start.sh -v || die "Upgrade to MiaB-LDAP failed !!!!!!"
}
compare_state() {
local s1="$1"
local s2="$2"
local output
local changed="false"
H1 "COMPARE STATES: $(basename "$s1") VS $(basename "$2")"
H2 "Users"
# users
output="$(diff "$s1/users.json" "$s2/users.json" 2>&1)"
if [ $? -ne 0 ]; then
changed="true"
echo "USERS ARE DIFFERENT!"
echo "$output"
else
echo "No change"
fi
H2 "Aliases"
output="$(diff "$s1/aliases.json" "$s2/aliases.json" 2>&1)"
if [ $? -ne 0 ]; then
change="true"
echo "ALIASES ARE DIFFERENT!"
echo "$output"
else
echo "No change"
fi
H2 "DNS - zones missing"
local zone count=0
for zone in $(cd "$s1/zones"; ls *.signed); do
if [ ! -e "$s2/zones/$zone" ]; then
echo "MISSING zone: $zone"
changed="true"
let count+=1
fi
done
echo "$count missing"
H2 "DNS - zones added"
count=0
for zone in $(cd "$s2/zones"; ls *.signed); do
if [ ! -e "$s2/zones/$zone" ]; then
echo "ADDED zone: $zone"
changed="true"
let count+=1
fi
done
echo "$count added"
H2 "DNS - zones changed"
count=0
for zone in $(cd "$s1/zones"; ls *.signed); do
if [ -e "$s2/zones/$zone" ]; then
# all the signatures change if we're using self-signed certs
local t1="/tmp/s1.$$.txt"
local t2="/tmp/s2.$$.txt"
awk '$4 == "RRSIG" || $4 == "NSEC3" { next; } $4 == "SOA" { print $1" "$2" "$3" "$4" "$5" "$6" "$8" "$9" "$10" "$11" "$12; next } { print $0 }' "$s1/zones/$zone" > "$t1"
awk '$4 == "RRSIG" || $4 == "NSEC3" { next; } $4 == "SOA" { print $1" "$2" "$3" "$4" "$5" "$6" "$8" "$9" "$10" "$11" "$12; next } { print $0 }' "$s2/zones/$zone" > "$t2"
output="$(diff "$t1" "$t2" 2>&1)"
if [ $? -ne 0 ]; then
echo "CHANGED zone: $zone"
echo "$output"
changed="true"
let count+=1
fi
fi
done
echo "$count zone files had differences"
if $changed; then
return 1
else
return 0
if ! populate_miab_aliases "" "" "" "${aliases[@]}"
then
die "Unable to add aliases"
fi
}
if [ "$1" == "cap" ]; then
capture_state "tests/system-setup/state/miab-ldap"
exit $?
elif [ "$1" == "compare" ]; then
compare_state "tests/system-setup/state/upstream" "tests/system-setup/state/miab-ldap"
exit $?
fi
# these are for debugging/testing
case "$1" in
capture )
. /etc/mailinabox.conf
installed_state_capture "tests/system-setup/state/miab-ldap"
exit $?
;;
compare )
. /etc/mailinabox.conf
installed_state_compare "tests/system-setup/state/upstream" "tests/system-setup/state/miab-ldap"
exit $?
;;
populate )
. /etc/mailinabox.conf
populate_by_name "${1:-basic}"
exit $?
;;
esac
# install basic stuff, set the hostname, time, etc
before_install
init
# if MiaB-LDAP is already migrated, do not run upstream setup
[ -e /etc/mailinabox.conf ] && . /etc/mailinabox.conf
if [ -e "$STORAGE_ROOT/mailinabox.version" ] &&
[ $(cat "$STORAGE_ROOT/mailinabox.version") -ge 13 ]
then
@@ -312,16 +155,21 @@ then
else
# install upstream
upstream_install
add_data
capture_state "tests/system-setup/state/upstream"
. /etc/mailinabox.conf
# populate some data
populate_by_name "${1:-basic}"
# capture upstream state
installed_state_capture "tests/system-setup/state/upstream"
fi
# install miab-ldap
# install miab-ldap and capture state
miab_ldap_install
capture_state "tests/system-setup/state/miab-ldap"
installed_state_capture "tests/system-setup/state/miab-ldap"
# compare states
if ! compare_state "tests/system-setup/state/upstream" "tests/system-setup/state/miab-ldap"; then
if ! installed_state_compare "tests/system-setup/state/upstream" "tests/system-setup/state/miab-ldap"; then
die "Upstream and upgraded states are different !"
fi