1
0
mirror of https://github.com/mail-in-a-box/mailinabox.git synced 2026-03-04 15:54:48 +01:00

1. Better code organization & simplify

2. Add "populate" data for upgrades - enabled in both system-setup scripts
3. Add "upgrade" test runner suite
This commit is contained in:
downtownallday
2020-06-19 12:12:49 -04:00
parent 144aa6e5d6
commit 1bd7b2c4c7
20 changed files with 987 additions and 509 deletions

View File

@@ -12,5 +12,8 @@
. "$1/misc.sh" || exit 3
. "$1/rest.sh" || exit 4
. "$1/system.sh" || exit 5
. "$1/carddav.sh" || exit 6
. "$1/populate.sh" || exit 7
. "$1/installed-state.sh" || exit 8

280
tests/lib/carddav.sh Normal file
View File

@@ -0,0 +1,280 @@
#
# requires:
# system packages: [ curl, python3, sqlite3 ]
# scripts: [ color-output.sh, misc.sh, locations.sh ]
#
# ASSETS_DIR: where the assets directory is located (defaults to
# tests/assets)
#
nextcloud_url() {
# eg: http://localhost/cloud/
carddav_url | sed 's|\(.*\)/remote.php/.*|\1/|'
}
carddav_url() {
# get the carddav url as configured in z-push for the user specified
# eg: http://localhost/cloud/remote.php/dav/addressbooks/users/admin/contacts/
local user="${1:-%u}"
local path="${2:-CARDDAV_DEFAULT_PATH}"
local php="include \"$ZPUSH_DIR/backend/carddav/config.php\"; print CARDDAV_PROTOCOL . \"://\" . CARDDAV_SERVER . \":\" . CARDDAV_PORT . "
php="$php$path;"
local url
url="$(php -n -r "$php")"
[ $? -ne 0 ] && die "Unable to run php to extract carddav url from z-push"
sed "s/%u/$user/" <<< "$url"
}
carddav_rest() {
# issue a CardDAV rest call to Nextcloud
# SEE: https://tools.ietf.org/html/rfc6352
#
# The function will set the following global variables regardless
# of exit code:
# REST_HTTP_CODE
# REST_OUTPUT
# REST_ERROR
# REST_ERROR_BRIEF
#
# Return values:
# 0 indicates success (curl returned 0 or a code deemed to be
# successful and HTTP status is >=200 but <300)
# 1 curl returned with non-zero code that indicates and error
# 2 the response status was <200 or >= 300
#
# Debug messages are sent to stderr
#
local verb="$1"
local uri="$2"
local auth_user="$3"
local auth_pass="$4"
shift; shift; shift; shift # remaining arguments are data
local url
case "$uri" in
/* )
url="$(nextcloud_url)${uri#/}"
;;
http* )
url="$uri"
;;
* )
url="$(carddav_url "$auth_user")${uri#/}"
;;
esac
local data=()
local item output onlydata="false"
for item; do
case "$item" in
-- )
onlydata="true"
;;
--* )
# curl argument
if $onlydata; then
data+=("--data" "$item");
else
data+=("$item")
fi
;;
* )
onlydata="true"
data+=("--data" "$item");
;;
esac
done
local ct
case "${data[1]}" in
BEGIN:VCARD* )
ct="text/vcard"
;;
* )
ct='text/xml; charset="utf-8"'
esac
local tmp1="/tmp/curl.$$.tmp"
echo "spawn: curl -w \"%{http_code}\" -X $verb -H 'Content-Type: $ct' --user \"${auth_user}:xxx\" ${data[@]} \"$url\"" 1>&2
output=$(curl -s -S -w "%{http_code}" -X $verb -H "Content-Type: $ct" --user "${auth_user}:${auth_pass}" "${data[@]}" "$url" 2>$tmp1)
local code=$?
# http status is last 3 characters of output, extract it
REST_HTTP_CODE=$(awk '{S=substr($0,length($0)-2)} END {print S}' <<<"$output")
REST_OUTPUT=$(awk 'BEGIN{L=""}{ if(L!="") print L; L=$0 } END { print substr(L,1,length(L)-3) }' <<<"$output")
REST_ERROR=""
REST_ERROR_BRIEF=""
[ -z "$REST_HTTP_CODE" ] && REST_HTTP_CODE="000"
if [ $code -ne 0 -o \
$REST_HTTP_CODE -lt 200 -o \
$REST_HTTP_CODE -ge 300 ]
then
if [ $code -ne 0 -a "$REST_HTTP_CODE" == "000" ]; then
REST_ERROR="exit code $code"
REST_ERROR_BRIEF="$REST_ERROR"
else
REST_ERROR="REST status $REST_HTTP_CODE: $REST_OUTPUT"
REST_ERROR_BRIEF=$(python3 -c "import xml.etree.ElementTree as ET; print(ET.fromstring(r'''$REST_OUTPUT''').find('s:message',{'s':'http://sabredav.org/ns'}).text)" 2>/dev/null)
if [ -z "$REST_ERROR_BRIEF" ]; then
REST_ERROR_BRIEF="$REST_ERROR"
else
REST_ERROR_BRIEF="$REST_HTTP_CODE: $REST_ERROR_BRIEF"
fi
if [ $code -ne 0 ]; then
REST_ERROR_BRIEF="exit code $code: $REST_ERROR_BRIEF"
REST_ERROR="exit code $code: $REST_ERROR"
fi
fi
if [ -s $tmp1 ]; then
REST_ERROR="$REST_ERROR: $(cat $tmp1)"
REST_ERROR_BRIEF="$REST_ERROR_BRIEF: $(cat $tmp1)"
fi
rm -f $tmp1
echo "${F_DANGER}$REST_ERROR${F_RESET}" 1>&2
[ $code -ne 0 ] && return 1
return 2
fi
echo "CURL succeded, HTTP status $REST_HTTP_CODE" 1>&2
echo "$output" 1>&2
rm -f $tmp1
return 0
}
carddav_ls() {
# place all .vcf files into global FILES
# debug messages are sent to stderr
local user="$1"
local pass="$2"
shift; shift
FILES=()
if ! carddav_rest PROPFIND "" "$user" "$pass" $@
then
return 1
fi
FILES=( $(python3 -c "import xml.etree.ElementTree as ET; [print(el.find('d:href',{'d':'DAV:'}).text) for el in ET.fromstring(r'''$REST_OUTPUT''').findall('d:response',{'d':'DAV:'}) if el.find('d:href',{'d':'DAV:'}) is not None]") )
local idx=${#FILES[*]}
let idx-=1
while [ $idx -ge 0 ]; do
# remove non .vcf entries, take basename contact href
case "${FILES[$idx]}" in
*.vcf )
FILES[$idx]=$(basename "${FILES[$idx]}")
;;
* )
unset "FILES[$idx]"
;;
esac
let idx-=1
done
}
carddav_make_addressbook() {
local user="$1"
local pass="$2"
local name="$3"
local desc="${4:-$name}"
local xml="<?xml version=\"1.0\" encoding=\"utf-8\" ?>
<D:mkcol xmlns:D=\"DAV:\"
xmlns:C=\"urn:ietf:params:xml:ns:carddav\">
<D:set>
<D:prop>
<D:resourcetype>
<D:collection/>
<C:addressbook/>
</D:resourcetype>
<D:displayname>$name</D:displayname>
<C:addressbook-description xml:lang=\"en\">$desc</C:addressbook-description>
</D:prop>
</D:set>
</D:mkcol>"
local url="$(carddav_url "$user" CARDDAV_PATH)"
carddav_rest MKCOL "$url" "$user" "$pass" "$xml"
}
carddav_add_contact() {
# debug messages are sent to stderr
local user="$1"
local pass="$2"
local c_name="$3"
local c_phone="$4"
local c_email="$5"
local c_uid="${6:-$(generate_uuid)}"
shift; shift; shift; shift; shift; shift
local vcard="BEGIN:VCARD
VERSION:3.0
UID:$c_uid
REV;VALUE=DATE-AND-OR-TIME:$(date -u +%Y%m%dT%H%M%SZ)
FN:$c_name
EMAIL;TYPE=INTERNET,PREF:$c_email
NOTE:Miab-LDAP QA
ORG:Miab-LDAP
TEL;TYPE=WORK,VOICE:$c_phone
END:VCARD"
carddav_rest PUT "$c_uid.vcf" "$user" "$pass" $@ -- "$vcard"
}
carddav_delete_contact() {
local user="$1"
local pass="$2"
local c_uid="$3"
shift; shift; shift
carddav_rest DELETE "$c_uid.vcf" "$user" "$pass" $@
}
roundcube_force_carddav_refresh() {
local user="$1"
local pass="$2"
local assets_dir="${ASSETS_DIR:-tests/assets}"
local code
if ! cp "$assets_dir/mail/roundcube/carddav_refresh.sh" $RCM_DIR/bin
then
return 1
fi
pushd "$RCM_DIR" >/dev/null
bin/carddav_refresh.sh "$user" "$pass"
code=$?
popd >/dev/null
return $code
}
roundcube_carddav_contact_exists() {
# returns 0 if contact exists
# 1 if contact does not exist
# 2 if an error occurred
# stderr receives error messages
local user="$1"
local pass="$2"
local c_uid="$3"
local db="${4:-$STORAGE_ROOT/mail/roundcube/roundcube.sqlite}"
local output
output="$(sqlite3 "$db" "select name from carddav_contacts where cuid='$c_uid'")"
[ $? -ne 0 ] && return 2
if [ -z "$output" ]; then
return 1
else
return 0
fi
}
roundcube_dump_contacts() {
local db="${1:-$STORAGE_ROOT/mail/roundcube/roundcube.sqlite}"
local cols="${2:-name,cuid}"
sqlite3 "$db" "select $cols FROM carddav_contacts"
}

View File

@@ -27,6 +27,24 @@ warn() {
* )
echoarg=""
esac
echo "${F_WARN}$1${F_RESET}"
echo $echoarg "${F_WARN}$1${F_RESET}"
}
H1() {
local msg="$1"
echo "----------------------------------------------"
if [ ! -z "$msg" ]; then
echo " $msg"
echo "----------------------------------------------"
fi
}
H2() {
local msg="$1"
if [ -z "$msg" ]; then
echo "***"
else
echo "*** $msg ***"
fi
}

View File

@@ -0,0 +1,141 @@
#
# requires:
# scripts: [ colored-output.sh, rest.sh ]
#
# these functions are meant for comparing upstream (non-LDAP)
# installations to a subsequent MiaB-LDAP upgrade
#
installed_state_capture() {
# users and aliases
# dns zone files
# TOOD: tls certificates: expected CN's
local state_dir="$1"
local info="$state_dir/info.txt"
H1 "Capture installed estate to $state_dir"
# nuke saved state, if any
rm -rf "$state_dir"
mkdir -p "$state_dir"
# create info.json
H2 "create info.txt"
echo "STATE_VERSION=1" > "$info"
echo "GIT_VERSION='$(git describe --abbrev=0)'" >>"$info"
echo "MIGRATION_VERSION=$(cat "$STORAGE_ROOT/mailinabox.version")" >>"$info"
# record users
H2 "record users"
if ! rest_urlencoded GET "/admin/mail/users?format=json" "$EMAIL_ADDR" "$EMAIL_PW" --insecure 2>/dev/null
then
echo "Unable to get users: rc=$? err=$REST_ERROR" 1>&2
return 1
fi
echo "$REST_OUTPUT" > "$state_dir/users.json"
# record aliases
H2 "record aliases"
if ! rest_urlencoded GET "/admin/mail/aliases?format=json" "$EMAIL_ADDR" "$EMAIL_PW" --insecure 2>/dev/null
then
echo "Unable to get aliases: rc=$? err=$REST_ERROR" 1>&2
return 2
fi
echo "$REST_OUTPUT" > "$state_dir/aliases.json"
# record dns config
H2 "record dns details"
local file
mkdir -p "$state_dir/zones"
for file in /etc/nsd/zones/*.signed; do
if ! cp "$file" "$state_dir/zones"
then
echo "Copy $file -> $state_dir/zones failed" 1>&2
return 3
fi
done
return 0
}
installed_state_compare() {
local s1="$1"
local s2="$2"
local output
local changed="false"
H1 "COMPARE STATES: $(basename "$s1") VS $(basename "$2")"
H2 "Users"
# users
output="$(diff "$s1/users.json" "$s2/users.json" 2>&1)"
if [ $? -ne 0 ]; then
changed="true"
echo "USERS ARE DIFFERENT!"
echo "$output"
else
echo "No change"
fi
H2 "Aliases"
output="$(diff "$s1/aliases.json" "$s2/aliases.json" 2>&1)"
if [ $? -ne 0 ]; then
change="true"
echo "ALIASES ARE DIFFERENT!"
echo "$output"
else
echo "No change"
fi
H2 "DNS - zones missing"
local zone count=0
for zone in $(cd "$s1/zones"; ls *.signed); do
if [ ! -e "$s2/zones/$zone" ]; then
echo "MISSING zone: $zone"
changed="true"
let count+=1
fi
done
echo "$count missing"
H2 "DNS - zones added"
count=0
for zone in $(cd "$s2/zones"; ls *.signed); do
if [ ! -e "$s2/zones/$zone" ]; then
echo "ADDED zone: $zone"
changed="true"
let count+=1
fi
done
echo "$count added"
H2 "DNS - zones changed"
count=0
for zone in $(cd "$s1/zones"; ls *.signed); do
if [ -e "$s2/zones/$zone" ]; then
# all the signatures change if we're using self-signed certs
local t1="/tmp/s1.$$.txt"
local t2="/tmp/s2.$$.txt"
awk '$4 == "RRSIG" || $4 == "NSEC3" { next; } $4 == "SOA" { print $1" "$2" "$3" "$4" "$5" "$6" "$8" "$9" "$10" "$11" "$12; next } { print $0 }' "$s1/zones/$zone" > "$t1"
awk '$4 == "RRSIG" || $4 == "NSEC3" { next; } $4 == "SOA" { print $1" "$2" "$3" "$4" "$5" "$6" "$8" "$9" "$10" "$11" "$12; next } { print $0 }' "$s2/zones/$zone" > "$t2"
output="$(diff "$t1" "$t2" 2>&1)"
if [ $? -ne 0 ]; then
echo "CHANGED zone: $zone"
echo "$output"
changed="true"
let count+=1
fi
fi
done
echo "$count zone files had differences"
if $changed; then
return 1
else
return 0
fi
}

View File

@@ -55,7 +55,11 @@ generate_uuid() {
}
generate_qa_password() {
echo "Test1234."
echo "Test$(date +%s)"
}
static_qa_password() {
echo "Test_1234"
}
sha1() {

99
tests/lib/populate.sh Normal file
View File

@@ -0,0 +1,99 @@
#
# requires:
# scripts: [ rest.sh, misc.sh ]
#
populate_miab_users() {
local url="$1"
local admin_email="${2:-$EMAIL_ADDR}"
local admin_pass="${3:-$EMAIL_PW}"
shift; shift; shift # remaining arguments are users to add
# each "user" argument is in the format "email:password"
# if no password is given a "qa" password will be generated
[ $# -eq 0 ] && return 0
#
# get the existing users
#
local current_users=() user
if ! rest_urlencoded GET ${url%/}/admin/mail/users "$admin_email" "$admin_pass" --insecure 2>/dev/null; then
echo "Unable to enumerate users: rc=$? err=$REST_ERROR" 1>&2
return 1
fi
for user in $REST_OUTPUT; do
current_users+=("$user")
done
#
# add the new users
#
local pw="$(generate_qa_password)"
for user; do
local user_email="$(awk -F: '{print $1}' <<< "$user")"
local user_pass="$(awk -F: '{print $2}' <<< "$user")"
if array_contains "$user_email" "${current_users[@]}"; then
echo "Not adding user $user_email: already exists"
elif ! rest_urlencoded POST ${url%/}/admin/mail/users/add "$admin_email" "$admin_pass" --insecure -- "email=$user_email" "password=${user_pass:-$pw}" 2>/dev/null
then
echo "Unable to add user $user_email: rc=$? err=$REST_ERROR" 1>&2
return 2
else
echo "Add: $user"
fi
done
return 0
}
populate_miab_aliases() {
local url="$1"
local admin_email="${2:-$EMAIL_ADDR}"
local admin_pass="${3:-$EMAIL_PW}"
shift; shift; shift # remaining arguments are aliases to add
# each "alias" argument is in the format "email-alias > forward-to"
[ $# -eq 0 ] && return 0
#
# get the existing aliases
#
local current_aliases=() alias
if ! rest_urlencoded GET ${url%/}/admin/mail/aliases "$admin_email" "$admin_pass" --insecure 2>/dev/null; then
echo "Unable to enumerate aliases: rc=$? err=$REST_ERROR" 1>&2
return 1
fi
for alias in $REST_OUTPUT; do
current_aliases+=("$alias")
done
#
# add the new aliases
#
local aliasdef
for aliasdef; do
alias="$(awk -F'[> ]' '{print $1}' <<<"$aliasdef")"
local forwards_to="$(sed 's/.*> *\(.*\)/\1/' <<<"$aliasdef")"
if array_contains "$alias" "${current_aliases[@]}"; then
echo "Not adding alias $aliasdef: already exists"
elif ! rest_urlencoded POST ${url%/}/admin/mail/aliases/add "$admin_email" "$admin_pass" --insecure -- "address=$alias" "forwards_to=$forwards_to" 2>/dev/null
then
echo "Unable to add alias $alias: rc=$? err=$REST_ERROR" 1>&2
return 2
else
echo "Add: $aliasdef"
fi
done
return 0
}

View File

@@ -15,7 +15,7 @@ rest_urlencoded() {
# (https://host/mail/users/add), PRIMARY_HOSTNAME must be set!
#
# The function will set the following global variables regardless
# of exit c ode:
# of exit code:
# REST_HTTP_CODE
# REST_OUTPUT
# REST_ERROR
@@ -26,7 +26,7 @@ rest_urlencoded() {
# 1 curl returned with non-zero code that indicates and error
# 2 the response status was <200 or >= 300
#
# Messages and errors are sent to stderr
# Debug messages are sent to stderr
#
local verb="$1" # eg "POST"
local uri="$2" # eg "/mail/users/add"