external/mailinabox
1
0
Fork 0
mirror of https://github.com/mail-in-a-box/mailinabox.git synced 2025-04-04 00:17:06 +00:00
Code Issues Releases Wiki Activity

Test upgrade to LDAP from upstream Mail-in-a-Box/sqlite

Browse Source
This commit is contained in:
downtownallday 2020-06-14 13:51:00 -04:00
parent 1f35e9ef91
commit b0090edd52
18 changed files with 831 additions and 380 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

33
.travis.yml
View File

@ -1,20 +1,24 @@
# travisci config
env:
global:
- PRIMARY_HOSTNAME=box.abc.com
- MIAB_LDAP=true
language: shell
os: linux
dist: bionic
before_install:
jobs:
fast_finish: true
include:
# JOB: MiaB-LDAP connected to a remote Nextcloud
- env: PRIMARY_HOSTNAME=box1.abc.com
name: remote-nextcloud-docker
before_install:
- echo "==== ENVIRONMENT ===="
- env | sort
- echo "UMASK=$(umask)"
#
- echo "==== AppArmor Status ===="
- (sudo aa-status; true)
#
- echo "==== NETWORK INFO ===="
- hostname -I
- hostname -i
@ -22,16 +26,21 @@ before_install:
- hostname --fqdn
- ip add
- sysctl -a 2>/dev/null | grep -i ipv6 | grep disable
install:
install:
- sudo tests/system-setup/remote-nextcloud-docker.sh
- hostname || true
- hostname --fqdn || true
- getent hosts box.abc.com || true
script:
#
script:
# launch automated tests, but skip tests that require remote
# smtp support because Travis-CI blocks outgoing port 25
- sudo touch /etc/dovecot/sieve-spam.svbin
- sudo tests/runner.sh -dumpoutput -no-smtp-remote default remote-nextcloud
# JOB: Upgrade from upstream install
- env: PRIMARY_HOSTNAME=box2.abc.com
name: upgrade-from-upstream
install:
- sudo tests/system-setup/upgrade-from-upstream.sh
script:
# launch automated tests, but skip tests that require remote
# smtp support because Travis-CI blocks outgoing port 25
- sudo touch /etc/dovecot/sieve-spam.svbin
- sudo tests/runner.sh -dumpoutput -no-smtp-remote

50
management/mailconfig.py
View File

@ -362,7 +362,7 @@ def get_mail_aliases(env, as_map=False):
alias = aliases[address]
xft = ",".join(alias["forward_tos"])
xas = ",".join(alias["permitted_senders"])
list.append( (address, xft, xas) )
list.append( (address, xft, None if xas is "" else xas) )
return list
else:
@ -432,7 +432,7 @@ def get_domain(emailaddr, as_unicode=True):
pass
return ret
def get_mail_domains(env, as_map=False, filter_aliases=None, category=None, users_only=False):
def get_mail_domains(env, as_map=False, filter_aliases=lambda alias: True, category=None, users_only=False):
# Retrieves all domains, IDNA-encoded, we accept mail for.
#
# If as_map is False, the function returns the lowercase domain
@ -457,17 +457,18 @@ def get_mail_domains(env, as_map=False, filter_aliases=None, category=None, user
# make it easy for dns_update to get ssl domains]
#
# If users_only is True, only return domains with email addresses
# that correspond to user accounts. [TODO: This currently has no
# effect - this function only returns user mail domains]
# that correspond to user accounts.
#
conn = open_database(env)
filter = "(&(objectClass=domain)(businessCategory=mail))"
if category:
filter = "(&(objectClass=domain)(businessCategory=%s))" % category
domains=None
# user mail domains
id = conn.search(env.LDAP_DOMAINS_BASE, filter, attributes="dc")
response = conn.wait(id)
filter_candidates=[]
domains=None
if as_map:
domains = {}
for rec in response:
@ -478,35 +479,22 @@ def get_mail_domains(env, as_map=False, filter_aliases=None, category=None, user
if filter_aliases: filter_candidates.append(rec['dc'][0].lower())
else:
domains = set([ rec["dc"][0].lower() for rec in response ])
if filter_aliases: filter_candidates += domains
for candidate in filter_candidates:
# with the filter, there has to be at least one user or
# filtered (included) alias in the domain for the domain to be
# part of the returned set
# any users ?
response = conn.wait( conn.search(env.LDAP_USERS_BASE, "(&(objectClass=mailUser)(mail=*@%s))" % candidate, size_limit=1) )
if response.next():
# yes, that domain needs to be in the returned set
continue
# any filtered aliases ?
pager = conn.paged_search(
env.LDAP_ALIASES_BASE,
"(&(objectClass=mailGroup)(mail=*@%s))" % candidate,
attributes=['mail'])
remove = True
# alias domains
if not users_only:
pager = conn.paged_search(env.LDAP_ALIASES_BASE, "(objectClass=mailGroup)", attributes="mail")
if as_map:
for rec in pager:
if filter_aliases(rec['mail'][0]):
remove = False
pager.abandon()
break
if filter_aliases(rec["mail"][0].lower()):
domain = get_domain(rec["mail"][0].lower(),as_unicode=False)
domains[domain] = {
"dn": None,
"domain": domain
}
if remove:
if as_map: del domains[candidate]
else: domains.remove(candidate)
else:
domains = domains.union(set([ get_domain(rec["mail"][0].lower(), as_unicode=False) for rec in pager if filter_aliases(rec["mail"][0].lower()) ]))
return domains

8
setup/ldap.sh
View File

@ -857,7 +857,13 @@ cat > /etc/logrotate.d/slapd <<EOF;
EOF
# Modify olc server config like TLS
modify_global_config
# Skip this step if no ca_certificate.pem exists - this indicates
# that the system hasn't yet been migrated from sqlite
if [ -e "$STORAGE_ROOT/ssl/ca_certificate.pem" ]; then
modify_global_config
else
say_debug "Not enabling TLS at this time - ca_certificate hasn't been generated yet"
fi
# Add overlays and ensure mail-related attributes are indexed
add_overlays

5
setup/mods.available/remote-nextcloud-use-miab.sh
View File

@ -6,8 +6,9 @@
#
# The script will:
# 1. enable the "LDAP user and group backend" in Nextcloud
# 2. configure Nextcloud to access MiaB-LDAP for users and groups
# 3. optionally install and configure ssmtp so system mail is
# 2. install calendar and contacts
# 3. configure Nextcloud to access MiaB-LDAP for users and groups
# 4. optionally install and configure ssmtp so system mail is
# sent to MiaB-LDAP
#
VERBOSE=0

10
setup/ssl.sh
View File

@ -97,11 +97,6 @@ if [ ! -s $STORAGE_ROOT/ssl/ssl_private_key.pem ]; then
(umask 037; hide_output \
openssl genrsa -out $STORAGE_ROOT/ssl/ssl_private_key.pem 2048)
# Give the group 'ssl-cert' read access so slapd can read it
groupadd -fr ssl-cert
chgrp ssl-cert $STORAGE_ROOT/ssl/ssl_private_key.pem
chmod g+r $STORAGE_ROOT/ssl/ssl_private_key.pem
# Remove the ssl_certificate.pem symbolic link to force a
# regeneration of the server certificate. It needs to be
# signed by the new ca.
@ -110,6 +105,11 @@ if [ ! -s $STORAGE_ROOT/ssl/ssl_private_key.pem ]; then
fi
fi
# Give the group 'ssl-cert' read access so slapd can read it
groupadd -fr ssl-cert
chgrp ssl-cert $STORAGE_ROOT/ssl/ssl_private_key.pem
chmod g+r $STORAGE_ROOT/ssl/ssl_private_key.pem
#
# Generate a root CA certificate
#

16
tests/lib/all.sh Normal file
View File

@ -0,0 +1,16 @@
#
# source all lib scripts
#
# from your script, supply the path to this directory as the first argument
#
# eg source "tests/lib/all.sh" "tests/lib"
#
# failure to load any script is fatal!
. "$1/color-output.sh" || exit 1
. "$1/locations.sh" || exit 2
. "$1/misc.sh" || exit 3
. "$1/rest.sh" || exit 4
. "$1/system.sh" || exit 5

32
tests/lib/color-output.sh Normal file
View File

@ -0,0 +1,32 @@
# ansi escapes for hilighting text
F_DANGER=$(echo -e "\033[31m")
F_WARN=$(echo -e "\033[93m")
F_RESET=$(echo -e "\033[39m")
danger() {
local echoarg
case "$1" in
-n )
echoarg="$1"
shift
;;
* )
echoarg=""
esac
echo $echoarg "${F_DANGER}$1${F_RESET}"
}
warn() {
local echoarg
case "$1" in
-n )
echoarg="$1"
shift
;;
* )
echoarg=""
esac
echo "${F_WARN}$1${F_RESET}"
}

0
tests/suites/_locations.sh → tests/lib/locations.sh
View File

65
tests/lib/misc.sh Normal file
View File

@ -0,0 +1,65 @@
#
# misc helpful functions
#
# requirements:
# system packages: [ python3 ]
array_contains() {
local searchfor="$1"
shift
local item
for item; do
[ "$item" == "$searchfor" ] && return 0
done
return 1
}
is_true() {
# empty string is not true
if [ "$1" == "true" \
-o "$1" == "TRUE" \
-o "$1" == "True" \
-o "$1" == "yes" \
-o "$1" == "YES" \
-o "$1" == "Yes" \
-o "$1" == "1" ]
then
return 0
else
return 1
fi
}
is_false() {
if is_true $@; then return 1; fi
return 0
}
email_localpart() {
local addr="$1"
awk -F@ '{print $1}' <<<"$addr"
}
email_domainpart() {
local addr="$1"
awk -F@ '{print $2}' <<<"$addr"
}
generate_uuid() {
local uuid
uuid=$(python3 -c "import uuid; print(uuid.uuid4())")
[ $? -ne 0 ] && die "Unable to generate a uuid"
echo "$uuid"
}
generate_qa_password() {
echo "Test1234."
}
sha1() {
local txt="$1"
python3 -c "import hashlib; m=hashlib.sha1(); m.update(bytearray(r'''$txt''','utf-8')); print(m.hexdigest());" || die "Unable to generate sha1 hash"
}

106
tests/lib/rest.sh Normal file
View File

@ -0,0 +1,106 @@
#
# REST helper functions
#
# requirements:
# system packages: [ curl ]
# lib scripts: [ color-output.sh ]
#
rest_urlencoded() {
# Issue a REST call having data urlencoded
#
# eg: rest_urlencoded POST /admin/mail/users/add "email=alice@abc.com" "password=secret"
#
# When providing a URI (/mail/users/add) and not a URL
# (https://host/mail/users/add), PRIMARY_HOSTNAME must be set!
#
# The function will set the following global variables regardless
# of exit c ode:
# REST_HTTP_CODE
# REST_OUTPUT
# REST_ERROR
#
# Return values:
# 0 indicates success (curl returned 0 or a code deemed to be
# successful and HTTP status is >=200 but <300)
# 1 curl returned with non-zero code that indicates and error
# 2 the response status was <200 or >= 300
#
# Messages, errors, and output are all sent to stderr, there is no
# stdout output
#
local verb="$1" # eg "POST"
local uri="$2" # eg "/mail/users/add"
local auth_user="$3"
local auth_pass="$4"
shift; shift; shift; shift # remaining arguments are data or curl args
local url
case "$uri" in
http:* | https:* )
url="$uri"
;;
* )
url="https://$PRIMARY_HOSTNAME${uri}"
;;
esac
local data=()
local item output onlydata="false"
for item; do
case "$item" in
-- )
onlydata="true"
;;
--* )
# curl argument
if $onlydata; then
data+=("--data-urlencode" "$item");
else
data+=("$item")
fi
;;
* )
onlydata="true"
data+=("--data-urlencode" "$item");
;;
esac
done
echo "spawn: curl -w \"%{http_code}\" -X $verb --user \"${auth_user}:xxx\" ${data[@]} $url" 1>&2
output=$(curl -s -S -w "%{http_code}" -X $verb --user "${auth_user}:${auth_pass}" "${data[@]}" $url)
local code=$?
# http status is last 3 characters of output, extract it
REST_HTTP_CODE=$(awk '{S=substr($0,length($0)-2)} END {print S}' <<<"$output")
REST_OUTPUT=$(awk 'BEGIN{L=""}{ if(L!="") print L; L=$0 } END { print substr(L,1,length(L)-3) }' <<<"$output")
REST_ERROR=""
[ -z "$REST_HTTP_CODE" ] && REST_HTTP_CODE="000"
if [ $code -ne 0 ]; then
if [ $code -eq 56 -a $REST_HTTP_CODE -eq 200 ]; then
# this is okay, I guess. happens sometimes during
# POST /admin/mail/aliases/remove
# 56=Unexpected EOF
echo "Ignoring curl return code 56 due to 200 status" 1>&2
elif [ $code -ne 16 -o $REST_HTTP_CODE -ne 200 ]; then
# any error code will fail the rest call except for a 16
# with a 200 HTTP status.
# 16="a problem was detected in the HTTP2 framing layer. This is somewhat generic and can be one out of several problems"
REST_ERROR="CURL failed with code $code"
echo "${F_DANGER}$REST_ERROR${F_RESET}" 1>&2
echo "$output" 1>&2
return 1
fi
fi
if [ $REST_HTTP_CODE -lt 200 -o $REST_HTTP_CODE -ge 300 ]; then
REST_ERROR="REST status $REST_HTTP_CODE: $REST_OUTPUT"
echo "${F_DANGER}$REST_ERROR${F_RESET}" 1>&2
return 2
fi
echo "CURL succeded, HTTP status $REST_HTTP_CODE" 1>&2
echo "$output" 1>&2
return 0
}

102
tests/lib/system.sh Normal file
View File

@ -0,0 +1,102 @@
wait_for_apt() {
# check to see if other package managers have a lock on new
# installs, and wait for them to finish
#
# returns non-zero if waiting times out (currently ~600 seconds)
local count=0
while fuser /var/lib/dpkg/lock >/dev/null 2>&1 || fuser /var/lib/dpkg/lock-frontend >/dev/null 2>&1; do
sleep 6
let count+=1
if [ $count -eq 1 ]; then
echo -n "Waiting for other package manager to finish..."
elif [ $count -gt 100 ]; then
echo -n "FAILED"
return 1
else
echo -n "${count}.."
fi
done
[ $count -ge 1 ] && echo ""
}
dump_file() {
local log_file="$1"
local lines="$2"
local title="DUMP OF $log_file"
echo ""
echo "--------"
echo -n "-------- $log_file"
if [ ! -z "$lines" ]; then
echo " (last $line lines)"
else
echo ""
fi
echo "--------"
if [ !-e "$log_file" ]; then
echo "DOES NOT EXIST"
elif [ ! -z "$lines" ]; then
tail -$lines "$log_file"
else
cat "$log_file"
fi
}
dump_file_if_exists() {
[ ! -e "$1" ] && return
dump_file "$@"
}
update_system_time() {
if [ ! -x /usr/sbin/ntpdate ]; then
wait_for_apt
apt-get install -y -qq ntpdate || return 1
fi
ntpdate -s ntp.ubuntu.com && echo "System time updated"
}
set_system_hostname() {
# set the system hostname to the FQDN specified or
# PRIMARY_HOSTNAME if no FQDN was given
local fqdn="${1:-$PRIMARY_HOSTNAME}"
local host="$(awk -F. '{print $1}' <<< "$fqdn")"
sed -i 's/^127\.0\.1\.1[ \t].*/127.0.1.1 '"$fqdn $host ip4-loopback/" /etc/hosts || return 1
#hostname "$host" || return 1
#echo "$host" > /etc/hostname
return 0
}
install_docker() {
if [ -x /usr/bin/docker ]; then
echo "Docker already installed"
return 0
fi
wait_for_apt
apt-get install -y -qq \
apt-transport-https \
ca-certificates \
curl \
gnupg-agent \
software-properties-common \
|| return 1
wait_for_apt
curl -fsSL https://download.docker.com/linux/ubuntu/gpg | apt-key add - \
|| return 2
wait_for_apt
apt-key fingerprint 0EBFCD88 || return 3
wait_for_apt
add-apt-repository -y --update "deb [arch=amd64] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" || return 4
wait_for_apt
apt-get install -y -qq \
docker-ce \
docker-ce-cli \
containerd.io \
|| return 5
}

23
tests/suites/_init.sh
View File

@ -6,7 +6,7 @@
set +eu
# load test suite helper functions
. suites/_locations.sh || exit 1
. lib/all.sh "lib" || exit 1
. suites/_ldap-functions.sh || exit 1
. suites/_mail-functions.sh || exit 1
. suites/_mgmt-functions.sh || exit 1
@ -20,10 +20,6 @@ declare -i OVERALL_SKIPPED=0
declare -i OVERALL_COUNT=0
declare -i OVERALL_COUNT_SUITES=0
# ansi escapes for hilighting text
F_DANGER=$(echo -e "\033[31m")
F_WARN=$(echo -e "\033[93m")
F_RESET=$(echo -e "\033[39m")
# options
FAILURE_IS_FATAL=no
@ -157,7 +153,12 @@ test_skip() {
}
skip_test() {
# return 0 if we should skip the current test
# call from within a test to check whether the test will be
# skipped
#
# returns 0 if the current test was skipped in which case your test
# function must immediately call 'test_end' and return
#
if [ "$SKIP_REMOTE_SMTP_TESTS" == "yes" ] &&
array_contains "remote-smtp" "$@";
then
@ -191,16 +192,6 @@ die() {
exit 1
}
array_contains() {
local searchfor="$1"
shift
local item
for item; do
[ "$item" == "$searchfor" ] && return 0
done
return 1
}
python_error() {
# finds tracebacks and outputs just the final error message of
# each

14
tests/suites/_ldap-functions.sh
View File

@ -1,16 +1,10 @@
# -*- indent-tabs-mode: t; tab-width: 4; -*-
generate_uuid() {
local uuid
uuid=$(python3 -c "import uuid; print(uuid.uuid4())")
[ $? -ne 0 ] && die "Unable to generate a uuid"
echo "$uuid"
}
# requirements:
# system packages: [ ldap-utils ]
# setup scripts: [ functions-ldap.sh ]
# setup artifacts: [ miab_ldap.conf ]
sha1() {
local txt="$1"
python3 -c "import hashlib; m=hashlib.sha1(); m.update(bytearray(r'''$txt''','utf-8')); print(m.hexdigest());" || die "Unable to generate sha1 hash"
}
delete_user() {
local email="$1"

46
tests/suites/_mgmt-functions.sh
View File

@ -44,49 +44,9 @@ mgmt_rest() {
local uri="$2" # eg "/mail/users/add"
shift; shift; # remaining arguments are data
local auth_user="${MGMT_ADMIN_EMAIL}"
local auth_pass="${MGMT_ADMIN_PW}"
local url="https://$PRIMARY_HOSTNAME${uri}"
local data=()
local item output
for item; do data+=("--data-urlencode" "$item"); done
record "spawn: curl -w \"%{http_code}\" -X $verb --user \"${auth_user}:xxx\" ${data[@]} $url"
output=$(curl -s -S -w "%{http_code}" -X $verb --user "${auth_user}:${auth_pass}" "${data[@]}" $url 2>>$TEST_OF)
local code=$?
# http status is last 3 characters of output, extract it
REST_HTTP_CODE=$(awk '{S=substr($0,length($0)-2)} END {print S}' <<<"$output")
REST_OUTPUT=$(awk 'BEGIN{L=""}{ if(L!="") print L; L=$0 } END { print substr(L,1,length(L)-3) }' <<<"$output")
REST_ERROR=""
[ -z "$REST_HTTP_CODE" ] && REST_HTTP_CODE="000"
if [ $code -ne 0 ]; then
if [ $code -eq 56 -a $REST_HTTP_CODE -eq 200 ]; then
# this is okay, I guess. happens sometimes during
# POST /admin/mail/aliases/remove
# 56=Unexpected EOF
record "Ignoring curl return code 56 due to 200 status"
elif [ $code -ne 16 -o $REST_HTTP_CODE -ne 200 ]; then
# any error code will fail the rest call except for a 16
# with a 200 HTTP status.
# 16="a problem was detected in the HTTP2 framing layer. This is somewhat generic and can be one out of several problems"
REST_ERROR="CURL failed with code $code"
record "${F_DANGER}$REST_ERROR${F_RESET}"
record "$output"
return 1
fi
fi
if [ $REST_HTTP_CODE -lt 200 -o $REST_HTTP_CODE -ge 300 ]; then
REST_ERROR="REST status $REST_HTTP_CODE: $REST_OUTPUT"
record "${F_DANGER}$REST_ERROR${F_RESET}"
return 2
fi
record "CURL succeded, HTTP status $REST_HTTP_CODE"
record "$output"
return 0
# call function from lib/rest.sh
rest_urlencoded "$verb" "$uri" "${MGMT_ADMIN_EMAIL}" "${MGMT_ADMIN_PW}" "$@" 2>>$TEST_OF
return $?
}

108
tests/system-setup/remote-nextcloud-docker.sh
View File

@ -27,9 +27,9 @@
usage() {
echo "Usage: $(basename "$0") [\"before-miab-install\"|\"miab-install\"|\"after-miab-install\"]"
echo "Install MiaB-LDAP and a remote Nextcloud running under docker exposed as localhost:8000"
echo "With no arguments, all three stages are run."
echo "Usage: $(basename "$0")"
echo "Install MiaB-LDAP and a remote Nextcloud running under docker"
echo "Nextcloud is exposed as http://localhost:8000"
exit 1
}
@ -40,10 +40,9 @@ if [ ! -d "tests/system-setup" ]; then
fi
# load helper scripts
. "tests/system-setup/setup-defaults.sh" \
|| die "Could not load setup-defaults"
. "tests/system-setup/setup-funcs.sh" \
|| die "Could not load setup-funcs"
. "tests/lib/all.sh" "tests/lib" || die "Could not load lib scripts"
. "tests/system-setup/setup-defaults.sh" || die "Could not load setup-defaults"
. "tests/system-setup/setup-funcs.sh" || die "Could not load setup-funcs"
# ensure running as root
if [ "$EUID" != "0" ]; then
@ -54,42 +53,15 @@ fi
before_miab_install() {
H1 "BEFORE MIAB-LDAP INSTALL"
H2 "Update /etc/hosts"
set_system_hostname || die "Could not set hostname"
# update system time
H2 "Set system time"
update_system_time || echo "Ignoring error..."
# update package lists before installing anything
H2 "apt-get update"
wait_for_apt
apt-get update -qq || die "apt-get update failed!"
# upgrade packages - if we don't do this and something like bind
# is upgraded through automatic upgrades (because maybe MiaB was
# previously installed), it may cause problems with the rest of
# the setup, such as with name resolution failures
if is_false "$TRAVIS"; then
H2 "apt-get upgrade"
wait_for_apt
apt-get upgrade -qq || die "apt-get upgrade failed!"
fi
# install prerequisites
H2 "QA pre-setup prerequisites"
install_pre_setup_qa_prerequisites \
|| die "Error installing QA prerequisites"
system_init
miab_testing_init || die "Initialization failed"
# enable the remote Nextcloud setup mod, which tells MiaB-LDAP to use
# the remote Nextcloud for calendar and contacts instead of the
# MiaB-installed one
H2 "Create local/remote-nextcloud.sh symbolic link"
if [ ! -e "local/remote-nextcloud.sh" ]; then
mkdir -p local
ln -s "../setup/mods.available/remote-nextcloud.sh" "local/remote-nextcloud.sh" || die "Could not create remote-nextcloud.sh symlink"
fi
H2 "Enable local mod remote-nextcloud"
enable_miab_mod "remote-nextcloud" \
|| die "Could not enable remote-nextcloud mod"
# install Docker
H2 "Install Docker"
@ -101,7 +73,7 @@ miab_install() {
H1 "MIAB-LDAP INSTALL"
if ! setup/start.sh; then
H1 "OUTPUT OF SELECT FILES"
dump_log "/var/log/syslog" 100
dump_file "/var/log/syslog" 100
dump_conf_files "$TRAVIS"
H2; H2 "End"; H2
die "setup/start.sh failed!"
@ -119,6 +91,8 @@ after_miab_install() {
# run Nextcloud docker image
H2 "Start Nextcloud docker container"
local container_started="true"
if [ -z "$(docker ps -f NAME=NC -q)" ]; then
docker run -d --name NC -p 8000:80 \
--env SQLITE_DATABASE=nextclouddb.sqlite \
--env NEXTCLOUD_ADMIN_USER="$NC_ADMIN_USER" \
@ -131,10 +105,14 @@ after_miab_install() {
--env SMTP_AUTHTYPE="LOGIN" \
--env SMTP_NAME="$EMAIL_ADDR" \
--env SMTP_PASSWORD="$EMAIL_PW" \
--env SMTP_FROM_ADDRESS="$(awk -F@ '{print $1}' <<< "$EMAIL_ADDR")" \
--env MAIL_DOMAIN="$(awk -F@ '{print $2}' <<< "$EMAIL_ADDR")" \
--env SMTP_FROM_ADDRESS="$(email_localpart "$EMAIL_ADDR")" \
--env MAIL_DOMAIN="$(email_domainpart "$EMAIL_ADDR")" \
nextcloud:latest \
|| die "Docker run failed!"
else
echo "Container already running"
container_started="false"
fi
H2 "docker: Update /etc/hosts so it can find MiaB-LDAP by name"
echo "$PRIVATE_IP $PRIMARY_HOSTNAME" | \
@ -160,32 +138,20 @@ after_miab_install() {
# wait for Nextcloud installation to complete
H2 "Wait for Nextcloud installation to complete"
echo -n "Waiting ..."
local count=0
while true; do
if [ $count -ge 10 ]; then
echo "FAILED"
die "Giving up"
fi
sleep 6
let count+=1
if [ $(docker exec NC php -n -r "include 'config/config.php'; print \$CONFIG['installed']?'true':'false';") == "true" ]; then
echo "ok"
break
fi
echo -n "${count}..."
done
wait_for_docker_nextcloud NC installed || die "Giving up"
# install and enable Nextcloud and apps
# install and enable Nextcloud apps
H2 "docker: install Nextcloud calendar app"
docker exec -u www-data NC ./occ app:install calendar \
|| die "docker: installing calendar app failed"
|| $container_started \
&& die "docker: installing calendar app failed ($?)"
H2 "docker: install Nextcloud contacts app"
docker exec -u www-data NC ./occ app:install contacts \
|| die "docker: installing contacts app failed"
|| $container_started \
&& die "docker: installing contacts app failed ($?)"
H2 "docker: enable user_ldap"
docker exec -u www-data NC ./occ app:enable user_ldap \
|| die "docker: enabling user_ldap failed"
|| die "docker: enabling user_ldap failed ($?)"
# integrate Nextcloud with MiaB-LDAP
H2 "docker: integrate Nextcloud with MiaB-LDAP"
@ -201,28 +167,26 @@ after_miab_install() {
}
#
# process command line
# Main
#
case "$1" in
before-miab-install )
case "${1:-all}" in
before-install )
before_miab_install
;;
after-miab-install )
after_miab_install
;;
miab-install )
install )
miab_install
;;
"" )
after-install )
after_miab_install
;;
all )
before_miab_install
miab_install
after_miab_install
;;
* )
usage
;;
esac

8
tests/system-setup/setup-defaults.sh
View File

@ -1,6 +1,6 @@
#!/bin/bash
# Used by MiaB-LDAP setup/start.sh
# Used by setup/start.sh
export NONINTERACTIVE=${NONINTERACTIVE:-1}
export SKIP_NETWORK_CHECKS=${SKIP_NETWORK_CHECKS:-1}
export STORAGE_USER="${STORAGE_USER:-user-data}"
@ -15,7 +15,6 @@ elif [ -z "$PRIMARY_HOSTNAME" ]; then
export PRIMARY_HOSTNAME=${PRIMARY_HOSTNAME:-$(hostname --fqdn || hostname)}
fi
# Placing this var in STORAGE_ROOT/ldap/miab_ldap.conf before running
# setup/start.sh will avoid a random password from being used for the
# Nextcloud LDAP service account
@ -28,6 +27,9 @@ export NC_HOST=${NC_HOST:-127.0.0.1}
export NC_PORT=${NC_PORT:-8000}
export NC_PREFIX=${NC_PREFIX:-/}
# For setup scripts that are installing a remote Nextcloud
# For setup scripts that may be installing a remote Nextcloud
export NC_ADMIN_USER="${NC_ADMIN_USER:-admin}"
export NC_ADMIN_PASSWORD="${NC_ADMIN_PASSWORD:-Test_1234}"
# For setup scripts that install upstream versions
export MIAB_UPSTREAM_GIT="https://github.com/mail-in-a-box/mailinabox.git"

220
tests/system-setup/setup-funcs.sh
View File

@ -1,4 +1,9 @@
#
# requires:
#
# test scripts: [ lib/misc.sh, lib/system.sh ]
#
die() {
local msg="$1"
@ -25,65 +30,29 @@ H2() {
fi
}
dump_log() {
local log_file="$1"
local lines="$2"
local title="DUMP OF $log_file"
echo ""
echo "--------"
echo -n "-------- $log_file"
if [ ! -z "$lines" ]; then
echo " (last $line lines)"
else
echo ""
fi
echo "--------"
if [ ! -z "$lines" ]; then
tail -$lines "$log_file"
else
cat "$log_file"
fi
}
is_true() {
# empty string is not true
if [ "$1" == "true" \
-o "$1" == "TRUE" \
-o "$1" == "True" \
-o "$1" == "yes" \
-o "$1" == "YES" \
-o "$1" == "Yes" \
-o "$1" == "1" ]
then
return 0
else
wait_for_docker_nextcloud() {
local container="$1"
local config_key="$2"
echo -n "Waiting ..."
local count=0
while true; do
if [ $count -ge 10 ]; then
echo "FAILED"
return 1
fi
}
is_false() {
if is_true $@; then return 1; fi
return 0
}
wait_for_apt() {
local count=0
while fuser /var/lib/dpkg/lock >/dev/null 2>&1 || fuser /var/lib/dpkg/lock-frontend >/dev/null 2>&1; do
sleep 6
let count+=1
if [ $count -eq 1 ]; then
echo -n "Waiting for other package manager to finish..."
elif [ $count -gt 100 ]; then
echo -n "FAILED"
return 1
else
echo -n "${count}.."
if [ $(docker exec "$container" php -n -r "include 'config/config.php'; print \$CONFIG['$config_key']?'true':'false';") == "true" ]; then
echo "ok"
break
fi
echo -n "${count}..."
done
[ $count -ge 1 ] && echo ""
return 0
}
dump_conf_files() {
local skip
if [ $# -eq 0 ]; then
@ -98,99 +67,58 @@ dump_conf_files() {
done
fi
if [ "$skip" == "false" ]; then
dump_log "/etc/mailinabox.conf"
dump_log "/etc/hosts"
dump_log "/etc/nsswitch.conf"
dump_log "/etc/resolv.conf"
dump_log "/etc/nsd/nsd.conf"
dump_log "/etc/postfix/main.cf"
dump_file "/etc/mailinabox.conf"
dump_file_if_exists "/etc/mailinabox_mods.conf"
dump_file "/etc/hosts"
dump_file "/etc/nsswitch.conf"
dump_file "/etc/resolv.conf"
dump_file "/etc/nsd/nsd.conf"
#dump_file "/etc/postfix/main.cf"
fi
}
update_system_time() {
if [ ! -x /usr/sbin/ntpdate ]; then
#
# Initialize the test system
# hostname, time, apt update/upgrade, etc
#
system_init() {
H2 "Update /etc/hosts"
set_system_hostname || die "Could not set hostname"
# update system time
H2 "Set system time"
update_system_time || echo "Ignoring error..."
# update package lists before installing anything
H2 "apt-get update"
wait_for_apt
apt-get install -y -qq ntpdate || return 1
apt-get update -qq || die "apt-get update failed!"
# upgrade packages - if we don't do this and something like bind
# is upgraded through automatic upgrades (because maybe MiaB was
# previously installed), it may cause problems with the rest of
# the setup, such as with name resolution failures
if is_false "$TRAVIS"; then
H2 "apt-get upgrade"
wait_for_apt
apt-get upgrade -qq || die "apt-get upgrade failed!"
fi
ntpdate -s ntp.ubuntu.com && echo "System time updated"
}
update_hosts() {
local host="$1"
shift
local ip
for ip; do
if [ ! -z "$ip" ]; then
local line="$ip $host"
if ! grep -F "$line" /etc/hosts 1>/dev/null; then
echo "$line" >>/etc/hosts
fi
fi
done
}
update_hosts_for_private_ip() {
# create /etc/hosts entry for PRIVATE_IP and PRIVATE_IPV6
# PRIMARY_HOSTNAME must already be set
local ip4=$(source setup/functions.sh; get_default_privateip 4)
local ip6=$(source setup/functions.sh; get_default_privateip 6)
[ -z "$ip4" -a -z "$ip6" ] && return 1
[ -z "$ip6" ] && ip6="::1"
update_hosts "$PRIMARY_HOSTNAME" "$ip4" "$ip6" || return 1
}
set_system_hostname() {
# set the system hostname to the FQDN specified or
# PRIMARY_HOSTNAME if no FQDN was given
local fqdn="${1:-$PRIMARY_HOSTNAME}"
local host="$(awk -F. '{print $1}' <<< "$fqdn")"
sed -i 's/^127\.0\.1\.1[ \t].*/127.0.1.1 '"$fqdn $host ip4-loopback/" /etc/hosts || return 1
#hostname "$host" || return 1
#echo "$host" > /etc/hostname
return 0
}
install_docker() {
if [ -x /usr/bin/docker ]; then
echo "Docker already installed"
return 0
fi
wait_for_apt
apt-get install -y -qq \
apt-transport-https \
ca-certificates \
curl \
gnupg-agent \
software-properties-common \
|| return 1
wait_for_apt
curl -fsSL https://download.docker.com/linux/ubuntu/gpg | apt-key add - \
|| return 2
wait_for_apt
apt-key fingerprint 0EBFCD88 || return 3
wait_for_apt
add-apt-repository -y --update "deb [arch=amd64] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" || return 4
wait_for_apt
apt-get install -y -qq \
docker-ce \
docker-ce-cli \
containerd.io \
|| return 5
}
install_pre_setup_qa_prerequisites() {
#
# Initialize the test system with QA prerequisites
# Anything needed to use the test runner, speed up the installation,
# etc
#
miab_testing_init() {
[ -z "$STORAGE_ROOT" ] \
&& echo "Error: STORAGE_ROOT not set" 1>&2 \
&& return 1
H2 "QA prerequisites"
local rc=0
# python3-dnspython: is used by the python scripts in 'tests' and is
@ -221,22 +149,20 @@ install_pre_setup_qa_prerequisites() {
}
travis_fix_nsd() {
if [ "$TRAVIS" != "true" ]; then
return 0
enable_miab_mod() {
local name="${1}.sh"
if [ ! -e "local/$name" ]; then
mkdir -p local
ln -s "../setup/mods.available/$name" "local/$name"
fi
# nsd won't start on Travis-CI without the changes below: ip6 off and
# control-enable set to no. Even though the nsd docs say the
# default value for control-enable is no, running "nsd-checkconf -o
# control-enable /etc/nsd/nsd.conf" returns "yes", so we explicitly
# set it here.
#
# we're assuming that the "ip-address" line is the last line in the
# "server" section of nsd.conf. if this generated file output
# changes, the sed command below may need to be adjusted.
sed -i 's/ip-address\(.\)\(.*\)/ip-address\1\2\n do-ip4\1 yes\n do-ip6\1 no\n verbosity\1 3\nremote-control\1\n control-enable\1 no/' /etc/nsd/nsd.conf || return 1
cat /etc/nsd/nsd.conf
systemctl reset-failed nsd.service || return 2
systemctl restart nsd.service || return 3
}
tag_from_readme() {
# extract the recommended TAG from README.md
# sets a global "TAG"
local readme="${1:-README.md}"
TAG="$(grep -F 'git checkout' "$readme" | sed 's/.*\(v[0123456789]*\.[0123456789]*\).*/\1/')"
[ $? -ne 0 -o -z "$TAG" ] && return 1
return 0
}

289
tests/system-setup/upgrade-from-upstream.sh Executable file
View File

@ -0,0 +1,289 @@
#!/bin/bash
# setup MiaB-LDAP by:
# 1. installing upstream MiaB
# 2. adding some data (users/aliases/etc)
# 3. upgrading to MiaB-LDAP
#
# See setup-defaults.sh for usernames and passwords.
#
usage() {
echo "Usage: $(basename "$0")"
echo "Install MiaB-LDAP after installing upstream MiaB"
exit 1
}
# ensure working directory
if [ ! -d "tests/system-setup" ]; then
echo "This script must be run from the MiaB root directory"
exit 1
fi
# load helper scripts
. "tests/lib/all.sh" "tests/lib" || die "Could not load lib scripts"
. "tests/system-setup/setup-defaults.sh" || die "Could not load setup-defaults"
. "tests/system-setup/setup-funcs.sh" || die "Could not load setup-funcs"
# ensure running as root
if [ "$EUID" != "0" ]; then
die "This script must be run as root (sudo)"
fi
before_install() {
H1 "INIT"
system_init
miab_testing_init || die "Initialization failed"
}
upstream_install() {
local upstream_dir="$HOME/mailinabox-upstream"
H1 "INSTALL UPSTREAM"
[ ! -x /usr/bin/git ] && apt-get install -y -qq git
if [ ! -d "$upstream_dir" ] || [ -z "$(ls -A "$upstream_dir")" ] ; then
H2 "Cloning $MIAB_UPSTREAM_GIT"
rm -rf "$upstream_dir"
git clone "$MIAB_UPSTREAM_GIT" "$upstream_dir"
if [ $? -ne 0 ]; then
rm -rf "$upstream_dir"
die "git clone upstream failed!"
fi
if [ -z "$TAG" ]; then
tag_from_readme "$upstream_dir/README.md"
if [ $? -ne 0 ]; then
rm -rf "$upstream_dir"
die "Failed to extract TAG from $upstream_dir/README.md"
fi
fi
fi
pushd "$upstream_dir" >/dev/null
if [ ! -z "$TAG" ]; then
H2 "Checkout $TAG"
git checkout "$TAG" || die "git checkout $TAG failed"
fi
H2 "Run upstream setup"
setup/start.sh || die "Upstream setup failed!"
popd >/dev/null
H2 "Upstream info"
echo "Code version: $(git describe)"
echo "Migration version: $(cat "$STORAGE_ROOT/mailinabox.version")"
}
add_data() {
H1 "Add some Mail-in-a-Box data"
local users=()
users+="betsy@$(email_domainpart "$EMAIL_ADDR")"
local alises=()
aliases+="goalias@testdom.com > ${users[0]}"
aliases+="nested@testdom.com > goalias@testdom.com"
local pw="$(generate_qa_password)"
#
# get the existing users and aliases
#
local current_users=() current_aliases=()
local user alias
if ! rest_urlencoded GET /admin/mail/users "$EMAIL_ADDR" "$EMAIL_PW" --insecure 2>/dev/null; then
die "Unable to enumerate users: rc=$? err=$REST_ERROR"
fi
for user in $REST_OUTPUT; do
current_users+=("$user")
done
if ! rest_urlencoded GET /admin/mail/aliases "$EMAIL_ADDR" "$EMAIL_PW" --insecure 2>/dev/null; then
die "Unable to enumerate aliases: rc=$? err=$REST_ERROR"
fi
for alias in $REST_OUTPUT; do
current_aliases+=("$alias")
done
#
# add users
#
for user in "${users[@]}"; do
if array_contains "$user" "${current_users[@]}"; then
echo "Not adding user $user: already exists"
elif ! rest_urlencoded POST /admin/mail/users/add "$EMAIL_ADDR" "$EMAIL_PW" --insecure -- "email=$user" "password=$pw" 2>/dev/null
then
die "Unable to add user $user: rc=$? err=$REST_ERROR"
fi
done
#
# add aliases
#
local aliasdef
for aliasdef in "${aliases[@]}"; do
alias="$(awk -F'[> ]' '{print $1}' <<<"$aliasdef")"
local forwards_to="$(sed 's/.*> *\(.*\)/\1/' <<<"$aliasdef")"
if array_contains "$alias" "${current_aliases[@]}"; then
echo "Not adding alias $alias: already exists"
elif ! rest_urlencoded POST /admin/mail/aliases/add "$EMAIL_ADDR" "$EMAIL_PW" --insecure -- "address=$alias" "forwards_to=$forwards_to" 2>/dev/null
then
die "Unable to add alias $alias: rc=$? err=$REST_ERROR"
fi
done
}
capture_state() {
# users and aliases lists
# dns zone files
# tls certificates: expected CN's
local state_dir="$1"
local infojson="$state_dir/info.json"
H1 "Capture server state to $state_dir"
# nuke saved state, if any
rm -rf "$state_dir"
mkdir -p "$state_dir"
# create info.json
H2 "create info.json"
echo "VERSION='$(git describe --abbrev=0)'" >"$infojson"
echo "MIGRATION_VERSION=$(cat "$STORAGE_ROOT/mailinabox.version")" >>"$infojson"
# record users
H2 "record users"
rest_urlencoded GET "/admin/mail/users?format=json" "$EMAIL_ADDR" "$EMAIL_PW" --insecure 2>/dev/null \
|| die "Unable to get users: rc=$? err=$REST_ERROR"
echo "$REST_OUTPUT" > "$state_dir/users.json"
# record aliases
H2 "record aliases"
rest_urlencoded GET "/admin/mail/aliases?format=json" "$EMAIL_ADDR" "$EMAIL_PW" --insecure 2>/dev/null \
|| die "Unable to get aliases: rc=$? err=$REST_ERROR"
echo "$REST_OUTPUT" > "$state_dir/aliases.json"
# record dns config
H2 "record dns details"
local file
mkdir -p "$state_dir/zones"
for file in ls /etc/nsd/zones/*.signed; do
cp "$file" "$state_dir/zones"
done
}
miab_ldap_install() {
# ensure we're in a MiaB-LDAP working directory
if [ ! -e setup/ldap.sh ]; then
die "The working directory is not MiaB-LDAP!"
fi
setup/start.sh -v || die "Upgrade to MiaB-LDAP failed !!!!!!"
}
compare_state() {
local s1="$1"
local s2="$2"
local output
local changed="false"
H1 "COMPARE STATES $(basename "$s1") TO $(basename "$2")"
H2 "Users"
# users
output="$(diff "$s1/users.json" "$s2/users.json" 2>&1)"
if [ $? -ne 0 ]; then
changed="true"
echo "USERS ARE DIFFERENT!"
echo "$output"
else
echo "OK"
fi
H2 "Aliases"
output="$(diff "$s1/aliases.json" "$s2/aliases.json" 2>&1)"
if [ $? -ne 0 ]; then
change="true"
echo "ALIASES ARE DIFFERENT!"
echo "$output"
else
echo "OK"
fi
H2 "DNS - zones missing"
local zone
for zone in $(cd "$s1/zones"; ls *.signed); do
if [ ! -e "$s2/zones/$zone" ]; then
echo "MISSING zone: $zone"
changed="true"
fi
done
H2 "DNS - zones added"
for zone in $(cd "$s2/zones"; ls *.signed); do
if [ ! -e "$s2/zones/$zone" ]; then
echo "ADDED zone: $zone"
changed="true"
fi
done
H2 "DNS - zones changed"
for zone in $(cd "$s1/zones"; ls *.signed); do
if [ -e "$s2/zones/$zone" ]; then
output="$(diff "$s1/zones/$zone" "$s2/zones/$zone" 2>&1)"
if [ $? -ne 0 ]; then
echo "CHANGED zone: $zone"
echo "$output"
changed="true"
fi
fi
done
if $changed; then
return 1
else
return 0
fi
}
if [ "$1" == "c" ]; then
capture_state "tests/system-setup/state/miab-ldap"
exit $?
fi
# install basic stuff, set the hostname, time, etc
before_install
# if MiaB-LDAP is already migrated, do not run upstream setup
if [ -e "$STORAGE_ROOT/mailinabox.version" ] &&
[ $(cat "$STORAGE_ROOT/mailinabox.version") -ge 13 ]
then
echo "Warning: MiaB-LDAP is already installed! Skipping installation of upstream"
else
# install upstream
upstream_install
add_data
capture_state "tests/system-setup/state/upstream"
fi
# install miab-ldap
miab_ldap_install
capture_state "tests/system-setup/state/miab-ldap"
# compare states
if ! compare_state "tests/system-setup/state/upstream" "tests/system-setup/state/miab-ldap"; then
die "Upstream and upgraded states are different !"
fi
#
# actual verification that mail sends/receives properly is done via
# the test runner ...
#