external/mailinabox
1
0
Fork 0
mirror of https://github.com/mail-in-a-box/mailinabox.git synced 2026-03-04 15:54:48 +01:00
Code Issues Releases Wiki Activity

Merge branch 'main' of https://github.com/mail-in-a-box/mailinabox

Browse Source 
Upstream is adding handling for utf8 domains by creating a domain alias @utf8 -> @idna. I'm deviating from this approach by setting multiple email address (idna and utf8) per user and alias where a domain contains non-ascii characters. The maildrop (mailbox) remains the same - all mail goes to the user's mailbox regardless of which email address was used. This is more in line with how other systems (eg. active directory), handle multiple email addresses for a single user.

# Conflicts:
#	README.md
#	management/mailconfig.py
#	management/templates/index.html
#	setup/dns.sh
#	setup/mail-users.sh
This commit is contained in:
downtownallday
2021-10-01 17:43:48 -04:00
parent 4403e65053 113b7bd827
commit 66ac35871e
30 changed files with 1326 additions and 458 deletions
Download Patch File Download Diff File Expand all files Collapse all files

107
tests/lib/installed-state.sh
View File

@@ -13,6 +13,7 @@ installed_state_capture() {
# TOOD: tls certificates: expected CN's
local state_dir="$1"
local install_dir="${2:-.}"
local info="$state_dir/info.txt"
H1 "Capture installed state to $state_dir"
@@ -22,11 +23,18 @@ installed_state_capture() {
mkdir -p "$state_dir"
# create info.json
if ! pushd "$install_dir" >/dev/null; then
echo "Directory '$install_dir' no accessible"
return 1
fi
H2 "create info.txt"
echo "STATE_VERSION=1" > "$info"
echo "GIT_VERSION='$(git describe --abbrev=0)'" >>"$info"
echo "GIT_VERSION='$(git describe)'" >>"$info"
git describe | awk -F- '{ split($1,a,"."); print "MAJOR="substr(a[1],2); print "MINOR="a[2]; print "RELEASE="$2 }' >>"$info"
echo "GIT_ORIGIN='$(git remote -v | grep ^origin | grep 'fetch)$' | awk '{print $2}')'" >>"$info"
echo "MIGRATION_VERSION=$(cat "$STORAGE_ROOT/mailinabox.version")" >>"$info"
echo "MIGRATION_VERSION=$([ -e "$STORAGE_ROOT/mailinabox.version" ] && cat "$STORAGE_ROOT/mailinabox.version")" >>"$info"
echo "MIGRATION_ML_VERSION=$([ -e "$STORAGE_ROOT/mailinabox-ldap.version" ] && cat "$STORAGE_ROOT/mailinabox-ldap.version")" >>"$info"
popd >/dev/null
# record users
H2 "record users"
@@ -75,17 +83,21 @@ installed_state_compare() {
#
# determine compare type id (incorporating repo, branch, version, etc)
#
local compare_type="all"
source "$s1/info.txt"
MAJOR_A="$MAJOR"
MINOR_A="$MINOR"
RELEASE_A="${RELEASE:-0}"
PROD_A="miab"
grep "mailinabox-ldap" <<<"$GIT_ORIGIN" >/dev/null && PROD_A="miabldap"
source "$s2/info.txt"
if grep "mailinabox-ldap" <<<"$GIT_ORIGIN" >/dev/null; then
GIT_ORIGIN=""
source "$s1/info.txt"
if ! grep "mailinabox-ldap.git" <<<"$GIT_ORIGIN" >/dev/null; then
compare_type="miab2miab-ldap"
fi
fi
echo "Compare type: $compare_type"
MAJOR_B="$MAJOR"
MINOR_B="$MINOR"
RELEASE_B="${RELEASE:-0}"
PROD_B="miab"
grep "mailinabox-ldap" <<<"$GIT_ORIGIN" >/dev/null && PROD_B="miabldap"
cmptype="${PROD_A}2${PROD_B}"
#
# filter data for compare type
@@ -95,7 +107,7 @@ installed_state_compare() {
cp "$s2/users.json" "$s2/users-cmp.json" || changed="true"
cp "$s2/aliases.json" "$s2/aliases-cmp.json" || changed="true"
if [ "$compare_type" == "miab2miab-ldap" ]
if [ "$cmptype" = "miab2miabldap" ]
then
# user display names is a feature added to MiaB-LDAP that is
# not in MiaB
@@ -104,7 +116,19 @@ installed_state_compare() {
# alias descriptions is a feature added to MiaB-LDAP that is
# not in MiaB
grep -v '"description":' "$s2/aliases.json" > "$s2/aliases-cmp.json" || changed="true"
fi
fi
# cmp: v0.54 to current
if [ "$cmptype" = "miabldap2miabldap" -a $MAJOR_A -eq 0 -a $MINOR_A -le 54 -a $RELEASE_A -eq 0 ]
then
# s1: convert aliases 'required' to 'auto' and resort
jq -c ".[] | .aliases | sort_by(.address) | .[] | {address:.address, forwards_to:.forwards_to, permitted_senders:.permitted_senders, auto:.required, description:.description}" "$s1/aliases.json" > "$s1/aliases-cmp.json"
sed -i 's/\("address":"administrator@.*"auto":\)true/\1false/' "$s1/aliases-cmp.json"
# s2: re-sort aliases
jq -c ".[] | .aliases | sort_by(.address) | .[] | {address:.address, forwards_to:.forwards_to, permitted_senders:.permitted_senders, auto:.auto, description:.description}" "$s2/aliases.json" > "$s2/aliases-cmp.json"
fi
#
# users
@@ -122,6 +146,7 @@ installed_state_compare() {
#
# aliases
#
H2 "Aliases"
output="$(diff "$s1/aliases-cmp.json" "$s2/aliases-cmp.json" 2>&1)"
if [ $? -ne 0 ]; then
@@ -154,36 +179,36 @@ installed_state_compare() {
done
echo "$count added"
H2 "DNS - zones changed"
count=0
for zone in $(cd "$s1/zones"; ls *.signed); do
if [ -e "$s2/zones/$zone" ]; then
# all the signatures change if we're using self-signed certs
# ignore ttl changes
local t1="/tmp/s1.$$.txt"
local t2="/tmp/s2.$$.txt"
awk '\
$4 == "RRSIG" || $4 == "NSEC3" { next; } \
$4 == "SOA" { print $1" "$3" "$4" "$5" "$6" "$8" "$10" "$12; next } \
{ for(i=1;i<=NF;i++) if (i!=2) printf("%s ",$i); print ""; }' \
"$s1/zones/$zone" > "$t1"
# H2 "DNS - zones changed"
# count=0
# for zone in $(cd "$s1/zones"; ls *.signed); do
# if [ -e "$s2/zones/$zone" ]; then
# # all the signatures change if we're using self-signed certs
# # ignore ttl changes
# local t1="/tmp/s1.$$.txt"
# local t2="/tmp/s2.$$.txt"
# awk '\
# $4 == "RRSIG" || $4 == "NSEC3" { next; } \
# $4 == "SOA" { print $1" "$3" "$4" "$5" "$6" "$8" "$10" "$12; next } \
# { for(i=1;i<=NF;i++) if (i!=2) printf("%s ",$i); print ""; }' \
# "$s1/zones/$zone" > "$t1"
awk '\
$4 == "RRSIG" || $4 == "NSEC3" { next; } \
$4 == "SOA" { print $1" "$3" "$4" "$5" "$6" "$8" "$10" "$12; next } \
{ for(i=1;i<=NF;i++) if (i!=2) printf("%s ",$i); print ""; }' \
"$s2/zones/$zone" > "$t2"
# awk '\
# $4 == "RRSIG" || $4 == "NSEC3" { next; } \
# $4 == "SOA" { print $1" "$3" "$4" "$5" "$6" "$8" "$10" "$12; next } \
# { for(i=1;i<=NF;i++) if (i!=2) printf("%s ",$i); print ""; }' \
# "$s2/zones/$zone" > "$t2"
output="$(diff "$t1" "$t2" 2>&1)"
if [ $? -ne 0 ]; then
echo "CHANGED zone: $zone"
echo "$output"
changed="true"
let count+=1
fi
fi
done
echo "$count zone files had differences"
# output="$(diff "$t1" "$t2" 2>&1)"
# if [ $? -ne 0 ]; then
# echo "CHANGED zone: $zone"
# echo "$output"
# changed="true"
# let count+=1
# fi
# fi
# done
# echo "$count zone files had differences"
if $changed; then
return 1

1
tests/suites/_init.sh
View File

@@ -133,6 +133,7 @@ test_success() {
test_failure() {
local why="$1"
[ -z "$TEST_OF" ] && return
record "** TEST_FAILURE: $why **"
TEST_STATE="FAILURE"
TEST_STATE_MSG+=( "$why" )
}

4
tests/suites/_ldap-functions.sh
View File

@@ -147,8 +147,8 @@ EOF
for member; do
case $member in
*@* )
echo "rfc822MailMember: $member" >>$TEST_OF
echo "rfc822MailMember: $member" >>$of 2>>$TEST_OF
echo "mailMember: $member" >>$TEST_OF
echo "mailMember: $member" >>$of 2>>$TEST_OF
;;
* )
echo "member: $member" >>$TEST_OF

2
tests/suites/mail-aliases.sh
View File

@@ -134,7 +134,7 @@ test_shared_alias_delivery() {
test_trial_nonlocal_alias_delivery() {
# verify that mail sent to an alias with a non-local address
# (rfc822MailMember) can be delivered
# (mailMember) can be delivered
test_start "trial-nonlocal-alias-delivery"
if skip_test remote-smtp; then
test_end

64
tests/suites/management-users.sh
View File

@@ -136,6 +136,8 @@ test_intl_domains() {
# remote intl user / forward-to
local intl_person="hans@bücher.example"
local intl_person_idna="hans@xn--bcher-kva.example"
local intl_person_domain=$(email_domainpart "$intl_person")
local intl_person_idna_domain=$(email_domainpart "$intl_person_idna")
# local users
local bob="bob@somedomain.com"
@@ -149,10 +151,50 @@ test_intl_domains() {
if mgmt_create_user "$intl_person" "$bob_pw"; then
test_failure "A user account is not permitted to have an international domain"
# ensure user is removed as is expected by the remaining tests
mgmt_delele_user "$intl_person"
mgmt_delete_user "$intl_person"
delete_user "$intl_person"
delete_user "$intl_person_idna"
fi
# given an idna encoded user - the user should have 2 mail addresses
if ! mgmt_create_user "$intl_person_idna" "$bob_pw"; then
test_failure "Could not create idna-encoded user account $intl_person_idna"
else
get_attribute "$LDAP_USERS_BASE" "(mail=$intl_person_idna)" "mail"
if [ -z "$ATTR_DN" ] || \
! array_contains "$intl_person" "${ATTR_VALUE[@]}" || \
! array_contains "$intl_person_idna" "${ATTR_VALUE[@]}"
then
test_failure "Alias's ($intl_person) mail attribute expected to have both the idna and utf8 names, got ${#ATTR_VALUE[@]}: ${ATTR_VALUE[*]}, expected: $intl_person,$intl_person_idna"
[ ! -z "$ATTR_DN" ] && record_search "$ATTR_DN"
else
record_search "$ATTR_DN"
# required aliases are automatically created and should
# have both mail addresses (idna and utf8)
get_attribute "$LDAP_ALIASES_BASE" "(mail=abuse@$intl_person_idna_domain)" "mail"
if [ -z "$ATTR_DN" ]; then
test_failure "Required alias not created!"
debug_search "(objectClass=mailGroup)" >>$TEST_OF
elif ! array_contains "abuse@$intl_person_domain" "${ATTR_VALUE[@]}" || \
! array_contains "abuse@$intl_person_idna_domain" "${ATTR_VALUE[@]}"
then
test_failure "Require alias abuse@$intl_person_idna_domain expected to contain both idna and utf8 mail addresses"
record_search "$ATTR_DN"
fi
# ensure user is removed as is expected by the remaining tests
mgmt_delete_user "$intl_person_idna"
fi
fi
# at this point intl_person does not exist, so all required aliases
# should also not be present
get_attribute "$LDAP_ALIASES_BASE" "(mail=*@$intl_person_idna_domain)"
if [ ! -z "$ATTR_DN" ]; then
test_failure "No required alias should not exist for the $intl_person_domain domain"
record_search "$ATTR_DN"
fi
# create local users bob and mary
mgmt_assert_create_user "$bob" "$bob_pw"
@@ -161,11 +203,27 @@ test_intl_domains() {
# create intl alias with local user bob and intl_person in it
if mgmt_assert_create_alias_group "$alias" "$bob" "$intl_person"; then
# examine LDAP server to verify IDNA-encodings
get_attribute "$LDAP_ALIASES_BASE" "(mail=$alias_idna)" "rfc822MailMember"
# 1. the mail attribute for the alias should have both the
# idna and utf8 addresses
get_attribute "$LDAP_ALIASES_BASE" "(mail=$alias)" "mail"
if [ -z "$ATTR_DN" ] || \
! array_contains "$alias" "${ATTR_VALUE[@]}" || \
! array_contains "$alias_idna" "${ATTR_VALUE[@]}"
then
test_failure "Alias's ($alias) mail attribute expected to have both the idna and utf8 names, got: ${ATTR_VALUE[*]}, expected: $alias,$alias_idna"
[ ! -z "$ATTR_DN" ] && record_search "$ATTR_DN"
fi
record_search "$ATTR_DN"
# 2. the mailMember attribute for the alias should contain the
# idna encoded intl_person (who is external - not a system user)
get_attribute "$LDAP_ALIASES_BASE" "(mail=$alias_idna)" "mailMember"
if [ -z "$ATTR_DN" ]; then
test_failure "IDNA-encoded alias group not found! created as:$alias expected:$alias_idna"
elif [ "$ATTR_VALUE" != "$intl_person_idna" ]; then
test_failure "Alias group with user having an international domain was not ecoded properly. added as:$intl_person expected:$intl_person_idna"
test_failure "Alias group with user having an international domain was not encoded properly. added as:$intl_person expected:$intl_person_idna"
fi
fi

4
tests/system-setup/setup-defaults.sh
View File

@@ -39,3 +39,7 @@ export NC_ADMIN_PASSWORD="${NC_ADMIN_PASSWORD:-Test_1234}"
# For setup scripts that install upstream versions
export MIAB_UPSTREAM_GIT="${MIAB_UPSTREAM_GIT:-https://github.com/mail-in-a-box/mailinabox.git}"
export UPSTREAM_TAG="${UPSTREAM_TAG:-}"
# For setup scripts that install miabldap releases
export MIABLDAP_GIT="${MIABLDAP_GIT:-https://github.com/downtownallday/mailinabox-ldap.git}"
export MIABLDAP_RELEASE_TAG="${MIABLDAP_RELEASE_TAG:-v0.54}"

8
tests/system-setup/setup-funcs.sh
View File

@@ -224,6 +224,14 @@ miab_ldap_install() {
die "Cannot install: the working directory is not MiaB-LDAP!"
fi
# setup/questions.sh installs the email_validator python3 module
# but only when in interactive mode. make sure it's also installed
# in non-interactive mode
if [ ! -z "${NONINTERACTIVE:-}" ]; then
H2 "Install email_validator python3 module"
pip3 install -q "email_validator>=1.0.0" || die "Unable to install email_validator python3 module!"
fi
# if EHDD_KEYFILE is set, use encryption-at-rest support
if [ ! -z "$EHDD_KEYFILE" ]; then
ehdd/start-encrypted.sh

6
tests/system-setup/upgrade-from-upstream.sh
View File

@@ -96,13 +96,13 @@ upstream_install() {
echo "$F_RESET"
die "Upstream setup failed!"
fi
popd >/dev/null
workaround_dovecot_sieve_bug
H2 "Upstream info"
echo "Code version: $(git describe)"
echo "Migration version: $(cat "$STORAGE_ROOT/mailinabox.version")"
popd >/dev/null
}
@@ -154,9 +154,7 @@ else
fi
# capture upstream state
pushd "$upstream_dir" >/dev/null
installed_state_capture "/tmp/state/upstream"
popd >/dev/null
installed_state_capture "/tmp/state/upstream" "$upstream_dir"
fi
# install miab-ldap and capture state

111
tests/system-setup/upgrade.sh Executable file
View File

@@ -0,0 +1,111 @@
#!/bin/bash
# setup MiaB-LDAP by:
# 1. installing upstream MiaB
# 2. adding some data (users/aliases/etc)
# 3. upgrading to MiaB-LDAP
#
# See setup-defaults.sh for usernames and passwords.
#
usage() {
echo "Usage: $(basename "$0")"
echo "Install MiaB-LDAP after installing upstream MiaB"
exit 1
}
# ensure working directory
if [ ! -d "tests/system-setup" ]; then
echo "This script must be run from the MiaB root directory"
exit 1
fi
# load helper scripts
. "tests/lib/all.sh" "tests/lib" || die "Could not load lib scripts"
. "tests/system-setup/setup-defaults.sh" || die "Could not load setup-defaults"
. "tests/system-setup/setup-funcs.sh" || die "Could not load setup-funcs"
# ensure running as root
if [ "$EUID" != "0" ]; then
die "This script must be run as root (sudo)"
fi
init() {
H1 "INIT"
init_test_system
init_miab_testing || die "Initialization failed"
}
install_release() {
install_dir="$1"
H1 "INSTALL RELEASE $MIABLDAP_RELEASE_TAG"
[ ! -x /usr/bin/git ] && apt-get install -y -qq git
if [ ! -d "$install_dir" ] || [ -z "$(ls -A "$install_dir")" ] ; then
H2 "Cloning $MIABLDAP_GIT"
rm -rf "$install_dir"
git clone "$MIABLDAP_GIT" "$install_dir"
if [ $? -ne 0 ]; then
rm -rf "$install_dir"
die "git clone failed!"
fi
fi
pushd "$install_dir" >/dev/null
H2 "Checkout $MIABLDAP_RELEASE_TAG"
git checkout "$MIABLDAP_RELEASE_TAG" || die "git checkout $MIABLDAP_RELEASE_TAG failed"
H2 "Run setup"
if ! setup/start.sh; then
echo "$F_WARN"
dump_file /var/log/syslog 100
echo "$F_RESET"
die "Release $RELEASE_TAG setup failed!"
fi
workaround_dovecot_sieve_bug
H2 "Release info"
echo "Code version: $(git describe)"
echo "Migration version (miabldap): $(cat "$STORAGE_ROOT/mailinabox-ldap.version")"
popd >/dev/null
}
# install basic stuff, set the hostname, time, etc
init
# install release
release_dir="$HOME/miabldap_$MIABLDAP_RELEASE_TAG"
install_release "$release_dir"
. /etc/mailinabox.conf
# populate some data
if [ $# -gt 0 ]; then
populate_by_name "$@"
else
populate_by_name "basic" "totpuser"
fi
# capture release state
installed_state_capture "/tmp/state/release" "$release_dir"
# install master miab-ldap and capture state
H2 "New miabldap"
echo "git branch: $(git branch | grep '*')"
miab_ldap_install
installed_state_capture "/tmp/state/master"
# compare states
if ! installed_state_compare "/tmp/state/release" "/tmp/state/master"; then
dump_file "/tmp/state/release/info.txt"
dump_file "/tmp/state/master/info.txt"
die "Release $RELEASE_TAG and master states are different !"
fi
#
# actual verification that mail sends/receives properly is done via
# the test runner ...
#

17
tests/vagrant/Vagrantfile vendored
View File

@@ -49,6 +49,23 @@ tests/runner.sh upgrade-basic upgrade-totpuser default || exit 2
SH
end
# upgrade
# this test is only needed when testing migrations from miabldap
# to a newer miabldap with a migration step
#
# upgrade will handle testing upgrades of
# miabldap with or without a new migration step
config.vm.define "upgrade" do |m1|
m1.vm.provision :shell, :inline => <<-SH
cd /mailinabox
source tests/vagrant/globals.sh || exit 1
export PRIMARY_HOSTNAME=upgrade.abc.com
tests/system-setup/upgrade.sh basic totpuser || exit 1
tests/runner.sh upgrade-basic upgrade-totpuser default || exit 2
SH
end
# unsetvars: because miab sets bash '-e' to fail any setup script
# when a script command returns a non-zero exit code, and more
# importantly '-u' which fails scripts when any unset variable is