1
0
mirror of https://github.com/mail-in-a-box/mailinabox.git synced 2026-03-14 17:27:23 +01:00

change from vagrant to lxd as the virtualization system

This commit is contained in:
downtownallday
2024-09-02 17:09:59 -04:00
parent a79a6c00eb
commit a1d6f67135
23 changed files with 1591 additions and 19 deletions

1
tests/lxd/.gitignore vendored Normal file
View File

@@ -0,0 +1 @@
out/

View File

@@ -0,0 +1,10 @@
# defaults for lxd image creation using
# tests/bin/provision_functions.sh::provision_start()
DEFAULT_LXD_IMAGE="preloaded-ubuntu-jammy"
DEFAULT_LXD_INST_OPTS=(
"-p" "default"
"-p" "bridgenet"
"-c" "limits.cpu=1"
"-c" "limits.memory=2GiB"
)

18
tests/lxd/README.md Normal file
View File

@@ -0,0 +1,18 @@
### To use lxd vm's:
1. run `tests/bin/lx_setup.sh`. This only needs to be run once.
2. run `tests/lxd/preloaded/create_preloaded.sh`. Run this anytime a new base image is updated (usually when ubuntu updates would require a system reboot).
### To bring up a vm:
1. It's helpful to have `vlx` in your path. vlx is a tool that makes `lxc` act a little like vagrant. In bash, create an alias for it: `alias vlx="$(pwd)/tests/bin/vlx"
2. set your working directory to the vm directory you'd like to start (eg. `cd "tests/lxd/vanilla"`), then run `vlx up`
3. to access the vm: `vlx shell` or `vlx ssh`. all vm's have the source root mounted at /cloudinabox or /mailinabox, so you can change files locally and they'll be available on the vm for testing
4. to destroy/delete the vm: `vlx destroy`

View File

@@ -0,0 +1,8 @@
# jammy does not include the dm_crypt kernel module, which is needed
# for luks/encryption-at-rest
# remote-nextcloud-docker-ehdd
remote-nextcloud-docker
upgrade-from-upstream
upgrade
unsetvars

View File

@@ -0,0 +1,3 @@
remote-nextcloud-docker
upgrade-from-upstream
upgrade

119
tests/lxd/parallel.sh Executable file
View File

@@ -0,0 +1,119 @@
#!/bin/bash
#####
##### This file is part of Mail-in-a-Box-LDAP which is released under the
##### terms of the GNU Affero General Public License as published by the
##### Free Software Foundation, either version 3 of the License, or (at
##### your option) any later version. See file LICENSE or go to
##### https://github.com/downtownallday/mailinabox-ldap for full license
##### details.
#####
#
# Parallel provisioning for test vms
#
. "$(dirname "$0")/../bin/lx_functions.sh"
. "$(dirname "$0")/../lib/color-output.sh"
. "$(dirname "$0")/../lib/misc.sh"
boxlist="" # the name of the boxlist or a path to the boxlist file
boxes=() # the contents of the boxlist file
project="$(lx_guess_project_name)"
load_boxlist() {
# sets global variable 'boxlist' and array 'boxes'
boxlist="${1:-default}"
local fn="$boxlist"
if [ ! -f "$fn" ]; then
fn="parallel-boxlist.$boxlist"
fi
if [ ! -f "$fn" ]; then
echo "Could not load boxlist from '${boxlist}'! Failed to find '$fn'."
exit 1
fi
boxes=( $(grep -v '^#' $fn) )
if [ $? -ne 0 ]; then
echo "Could not load boxlist from file '$fn'!"
exit 1
fi
}
# destroy running boxes
if [ "$1" = "-d" ]; then
shift
load_boxlist "$1"
for inst in $(lx_output_inst_list "$project" "n" "csv"); do
if array_contains $inst ${boxes[*]}; then
echo lxc --project "$project" delete $inst --force
lxc --project "$project" delete $inst --force
fi
done
exit 0
elif [ "$1" = "-h" -o "$1" = "--help" ]; then
echo "usage: $0 [-d] [boxlist]"
echo " -d delete/destroy running boxes"
echo " boxlist an file or named boxlist containing a list of instance names. defaults to 'default'"
exit 0
fi
load_boxlist "$1"
# set total parallel vms to (#cores minus 1)
MAX_PROCS=$(cat /proc/cpuinfo | grep processor | wc -l)
let MAX_PROCS-=1
[ $MAX_PROCS -eq 0 ] && MAX_PROCS=1
OUTPUT_DIR=out
#rm -rf "$OUTPUT_DIR"
mkdir -p "$OUTPUT_DIR"
echo "MAX_PROCS=$MAX_PROCS"
echo "OUTPUT_DIR=$OUTPUT_DIR"
start_time="$(date +%s)"
# bring up in parallel
for inst in ${boxes[*]}; do
outfile="$OUTPUT_DIR/$inst.out.txt"
rm -f "$outfile"
echo "Bringing up '$inst'. Output will be in: $outfile" 1>&2
echo $inst
done | xargs -P $MAX_PROCS -I"INSTNAME" \
sh -c '
cd "INSTNAME" &&
./provision.sh >'"../$OUTPUT_DIR/"'INSTNAME.out.txt 2>&1 &&
echo "EXITCODE: 0" >> '"../$OUTPUT_DIR/"'INSTNAME.out.txt ||
echo "EXITCODE: $?" >>'"../$OUTPUT_DIR/"'INSTNAME.out.txt
'
# output overall result"
H1 "Results"
rc=0
for inst in ${boxes[*]}; do
file="$OUTPUT_DIR"/$inst.out.txt
exitcode="$(tail "$file" | grep EXITCODE: | awk '{print $NF}')"
echo -n "$inst: "
if [ -z "$exitcode" ]; then
danger "NO EXITCODE!"
[ $rc -eq 0 ] && rc=2
elif [ "$exitcode" == "0" ]; then
elapsed="$(tail "$file" | grep ^Elapsed | awk -F: '{print $2}')"
success "SUCCESS (${elapsed# })"
else
danger "FAILURE ($exitcode)"
rc=1
fi
done
# output elapsed time
end_time="$(date +%s)"
echo ""
echo "Elapsed time: $(elapsed_pretty $start_time $end_time)"
# exit
echo ""
echo "Guest VMs are running! Destroy them with:"
echo " $0 -d $boxlist"
exit $rc

View File

@@ -0,0 +1,97 @@
#!/bin/bash
#####
##### This file is part of Mail-in-a-Box-LDAP which is released under the
##### terms of the GNU Affero General Public License as published by the
##### Free Software Foundation, either version 3 of the License, or (at
##### your option) any later version. See file LICENSE or go to
##### https://github.com/downtownallday/mailinabox-ldap for full license
##### details.
#####
# this script creates a new lxd image preloaded with software to speed
# up installation.
#
# prerequisites:
#
# tests/bin/lx_setup.sh must be run before running this script. it
# only needs to be run once, or any time the networking setup
# changes (eg. adding a new ethernet card).
#
D="$(dirname "$BASH_SOURCE")"
. "$D/../../bin/lx_functions.sh" || exit 1
project="$(lx_guess_project_name)"
inst_mountpoint=/mailinabox
[ "$project" = "ciab" ] && inst_mountpoint=/cloudinabox
if [ -z "$1" ]; then
imagelist=( $(<./imagelist) )
else
imagelist=( "$@" )
fi
for base_image in "${imagelist[@]}"
do
new_image="preloaded-${base_image/:/-}"
inst_name="preloaded"
echo ""
echo "START: create $new_image using base image $base_image"
echo "Delete existing instance: $inst_name"
lx_delete "$project" "$inst_name" "no-interactive" || exit 1
echo "Create instance '$inst_name' from '$base_image'"
# cloud init configuration creates user 'vmuser' instead of 'ubuntu'
cloud_config_users="#cloud-config
users:
- default
- name: vmuser
gecos: VM user for ssh
primary_group: vmuser
groups: adm, sudo, lxd
shell: /bin/bash
sudo: ALL=(ALL) NOPASSWD:ALL
lock_passwd: true
ssh_authorized_keys:
- $(< $(lx_get_ssh_identity).pub)
"
lx_launch_vm "$project" "$inst_name" "$base_image" "$(lx_project_root_dir)" "$inst_mountpoint" -c cloud-init.user-data="$cloud_config_users" -c limits.cpu=2 -c limits.memory=2GiB -d root,size=30GiB || exit 1
lx_wait_for_boot "$project" "$inst_name"
echo ""
echo "================================================="
echo "Prep the VM instance"
echo "================================================="
lxc --project "$project" exec "$inst_name" --cwd "$inst_mountpoint" -- sudo tests/lxd/preloaded/prepvm.sh --no-dry-run || exit 1
echo ""
echo "================================================="
echo "Create an image from the instance"
echo "================================================="
echo "Stopping instance '$inst_name'"
lxc --project "$project" stop "$inst_name" || exit 1
echo "Create image '$new_image' from instance '$inst_name'"
lxc --project "$project" publish "$inst_name" "local:" --reuse --compression gzip --alias "$new_image" || exit 1 # --compression xz
echo ""
echo "================================================="
echo "Image list ($project)"
echo "================================================="
lx_output_image_list "$project" "ld"
echo ""
echo "================================================="
echo "Delete instance '$inst_name'"
echo "================================================="
lx_delete "$project" "$inst_name" "no-interactive"
echo "Success"
done

View File

@@ -0,0 +1 @@
ubuntu:jammy

230
tests/lxd/preloaded/prepvm.sh Executable file
View File

@@ -0,0 +1,230 @@
#!/bin/bash
#####
##### This file is part of Mail-in-a-Box-LDAP which is released under the
##### terms of the GNU Affero General Public License as published by the
##### Free Software Foundation, either version 3 of the License, or (at
##### your option) any later version. See file LICENSE or go to
##### https://github.com/downtownallday/mailinabox-ldap for full license
##### details.
#####
# Run this on a VM to pre-install all the packages, then
# take a snapshot - it will greatly speed up subsequent
# test installs
#
# What won't be installed:
#
# Nextcloud and Roundcube are downloaded with wget by the setup
# scripts, so they are not included
#
# slapd - we want to test installation with setup/ldap.sh
#
if [ ! -d "setup" ]; then
echo "Run from the miab root directory"
exit 1
fi
source tests/lib/misc.sh
source tests/lib/system.sh
source tests/lib/color-output.sh
dry_run=true
start=$(date +%s)
if [ "$1" == "--no-dry-run" ]; then
dry_run=false
fi
if $dry_run; then
echo "WARNING: dry run is TRUE, no changes will be made"
fi
# prevent apt from running needrestart(1)
export NEEDRESTART_SUSPEND=true
# prevent interaction during package install
export DEBIAN_FRONTEND=noninteractive
# what major version of ubuntu are we installing on?
OS_MAJOR=$(. /etc/os-release; echo $VERSION_ID | awk -F. '{print $1}')
remove_line_continuation() {
local file="$1"
awk '
BEGIN { C=0 }
C==1 && /[^\\]$/ { C=0; print $0; next }
C==1 { printf("%s",substr($0,0,length($0)-1)); next }
/\\$/ { C=1; printf("%s",substr($0,0,length($0)-1)); next }
{ print $0 }' \
"$file"
}
install_packages() {
local return_code=0
while read line; do
pkgs=""
case "$line" in
apt_install* )
pkgs="$(cut -c12- <<<"$line")"
;;
"apt-get install"* )
pkgs="$(cut -c16- <<<"$line")"
;;
"apt install"* )
pkgs="$(cut -c12- <<<"$line")"
;;
esac
# don't install slapd
pkgs="$(sed 's/slapd//g' <<< "$pkgs")"
# manually set PHP_VER if necessary
if grep "PHP_VER" <<<"$pkgs" >/dev/null; then
pkgs="$(sed "s/\"\?\${*PHP_VER}*\"\?/$PHP_VER/g" <<< "$pkgs")"
fi
if [ ! -z "$pkgs" ]; then
H2 "install: $pkgs"
if ! $dry_run; then
exec_no_output apt-get install -y $pkgs
let return_code+=$?
fi
fi
done
return $return_code
}
install_ppas() {
H1 "Add apt repositories"
grep 'hide_output add-apt-repository' setup/system.sh |
while read line; do
line=$(sed 's/^hide_output //' <<< "$line")
H2 "$line"
if ! $dry_run; then
exec_no_output $line
fi
done
}
add_swap() {
H1 "Add a swap file to the system"
if ! $dry_run; then
dd if=/dev/zero of=/swapfile bs=1024 count=$[1024*1024] status=none
chmod 600 /swapfile
mkswap /swapfile
swapon /swapfile
echo "/swapfile none swap sw 0 0" >> /etc/fstab
fi
}
# install PPAs from sources
install_ppas
# add swap file
add_swap
# obtain PHP_VER variable from sources
PHP_VER=$(source setup/functions.sh; echo $PHP_VER)
if ! $dry_run; then
H1 "Upgrade system"
H2 "apt update"
exec_no_output apt-get update -y || exit 1
H2 "apt upgrade"
exec_no_output apt-get upgrade -y --with-new-pkgs || exit 1
H2 "apt autoremove"
exec_no_output apt-get autoremove -y
fi
# without using the same installation order as setup/start.sh, we end
# up with the system's php getting installed in addition to the
# non-system php that may also installed by setup (don't know why,
# probably one of the packages has a dependency). create an ordered
# list of files to process so we get a similar system setup.
setup_files=( $(ls setup/*.sh) )
desired_order=(
setup/functions.sh
setup/preflight.sh
setup/questions.sh
setup/network-checks.sh
setup/system.sh
setup/ssl.sh
setup/dns.sh
setup/ldap.sh
setup/mail-postfix.sh
setup/mail-dovecot.sh
setup/mail-users.sh
setup/dkim.sh
setup/spamassassin.sh
setup/web.sh
setup/webmail.sh
setup/nextcloud.sh
setup/zpush.sh
setup/management.sh
setup/management-capture.sh
setup/munin.sh
setup/firstuser.sh
)
ordered_files=()
for file in "${desired_order[@]}" "${setup_files[@]}"; do
if [ -e "$file" ] && ! array_contains "$file" "${ordered_files[@]}"; then
ordered_files+=( "$file" )
fi
done
failed=0
for file in ${ordered_files[@]}; do
H1 "$file"
remove_line_continuation "$file" | install_packages
[ $? -ne 0 ] && let failed+=1
done
if ! $dry_run; then
# bonus
H1 "install extras"
H2 "openssh, emacs, ntpdate, net-tools, jq"
exec_no_output apt-get install -y openssh-server emacs-nox ntpdate net-tools jq || let failed+=1
# these are added by system-setup scripts and needed for test runner
H2 "python3-dnspython"
exec_no_output apt-get install -y python3-dnspython || let failed+=1
H2 "pyotp(pip)"
exec_no_output python3 -m pip install pyotp --quiet || let failed+=1
# ...and for browser-based tests
#H2 "x11" # needed for chromium w/head (not --headless)
#exec_no_output apt-get install -y xorg openbox xvfb gtk2-engines-pixbuf dbus-x11 xfonts-base xfonts-100dpi xfonts-75dpi xfonts-cyrillic xfonts-scalable x11-apps imagemagick || let failed+=1
H2 "chromium"
#exec_no_output apt-get install -y chromium-browser || let failed+=1
exec_no_output snap install chromium || let failed+=1
H2 "selenium(pip)"
exec_no_output python3 -m pip install selenium --quiet || let failed+=1
# remove apache, which is what setup will do
H2 "remove apache2"
exec_no_output apt-get -y purge apache2 apache2-\*
fi
end=$(date +%s)
echo ""
echo ""
if [ $failed -gt 0 ]; then
echo "$failed failures! ($(elapsed_pretty $start $end))"
echo ""
exit 1
else
echo "Successfully prepped in $(elapsed_pretty $start $end). Take a snapshot...."
echo ""
exit 0
fi

View File

@@ -0,0 +1,35 @@
#!/bin/bash
#####
##### This file is part of Mail-in-a-Box-LDAP which is released under the
##### terms of the GNU Affero General Public License as published by the
##### Free Software Foundation, either version 3 of the License, or (at
##### your option) any later version. See file LICENSE or go to
##### https://github.com/downtownallday/mailinabox-ldap for full license
##### details.
#####
#
# provision a miab-ldap that has a remote nextcloud (using Nextcloud
# from Docker) and an encrypted user-data
#
D=$(dirname "$BASH_SOURCE")
. "$D/../../bin/lx_functions.sh" || exit 1
. "$D/../../bin/provision_functions.sh" || exit 1
# Create the instance (started)
provision_start "" "/mailinabox" || exit 1
# Setup system
provision_shell <<<"
cd /mailinabox
export PRIMARY_HOSTNAME=qa1.abc.com
export FEATURE_MUNIN=false
export EHDD_KEYFILE=\$HOME/keyfile
echo -n 'boo' >\$EHDD_KEYFILE
tests/system-setup/remote-nextcloud-docker.sh || exit 1
tests/runner.sh -no-smtp-remote remote-nextcloud ehdd default || exit 2
"
provision_done $?

View File

@@ -0,0 +1,33 @@
#!/bin/bash
#####
##### This file is part of Mail-in-a-Box-LDAP which is released under the
##### terms of the GNU Affero General Public License as published by the
##### Free Software Foundation, either version 3 of the License, or (at
##### your option) any later version. See file LICENSE or go to
##### https://github.com/downtownallday/mailinabox-ldap for full license
##### details.
#####
#
# provision a miab-ldap that has a remote nextcloud (using Nextcloud
# from Docker)
#
D=$(dirname "$BASH_SOURCE")
. "$D/../../bin/lx_functions.sh" || exit 1
. "$D/../../bin/provision_functions.sh" || exit 1
# Create the instance (started)
provision_start "" "/mailinabox" || exit 1
# Setup system
provision_shell <<<"
cd /mailinabox
export PRIMARY_HOSTNAME=qa2.abc.com
export FEATURE_MUNIN=false
tests/system-setup/remote-nextcloud-docker.sh upgrade --populate=basic || exit 1
tests/runner.sh -no-smtp-remote remote-nextcloud upgrade-basic default || exit 2
"
provision_done $?

View File

@@ -0,0 +1,35 @@
#!/bin/bash
#####
##### This file is part of Mail-in-a-Box-LDAP which is released under the
##### terms of the GNU Affero General Public License as published by the
##### Free Software Foundation, either version 3 of the License, or (at
##### your option) any later version. See file LICENSE or go to
##### https://github.com/downtownallday/mailinabox-ldap for full license
##### details.
#####
#
# provision a miab-ldap that has a remote nextcloud (using Nextcloud
# from Docker)
#
D=$(dirname "$BASH_SOURCE")
. "$D/../../bin/lx_functions.sh" || exit 1
. "$D/../../bin/provision_functions.sh" || exit 1
# Create the instance (started)
provision_start "" "/mailinabox" || exit 1
# Setup system
provision_shell <<<"
export NONINTERACTIVE=1
export PUBLIC_IP=auto
export PUBLIC_IPV6=auto
export PRIMARY_HOSTNAME=mailinabox.lan # auto
export SKIP_NETWORK_CHECKS=1
cd /mailinabox
setup/start.sh
"
provision_done $?

View File

@@ -0,0 +1,33 @@
#!/bin/bash
#####
##### This file is part of Mail-in-a-Box-LDAP which is released under the
##### terms of the GNU Affero General Public License as published by the
##### Free Software Foundation, either version 3 of the License, or (at
##### your option) any later version. See file LICENSE or go to
##### https://github.com/downtownallday/mailinabox-ldap for full license
##### details.
#####
#
# provision a miab-ldap that has a remote nextcloud (using Nextcloud
# from Docker)
#
D=$(dirname "$BASH_SOURCE")
. "$D/../../bin/lx_functions.sh" || exit 1
. "$D/../../bin/provision_functions.sh" || exit 1
# Create the instance (started)
provision_start "" "/mailinabox" || exit 1
# Setup system
provision_shell <<<"
cd /mailinabox
export PRIMARY_HOSTNAME=qa3.abc.com
export UPSTREAM_TAG=main
tests/system-setup/upgrade-from-upstream.sh --populate=basic --populate=totpuser || exit 1
tests/runner.sh -no-smtp-remote upgrade-basic upgrade-totpuser default || exit 2
"
provision_done $?

32
tests/lxd/upgrade/provision.sh Executable file
View File

@@ -0,0 +1,32 @@
#!/bin/bash
#####
##### This file is part of Mail-in-a-Box-LDAP which is released under the
##### terms of the GNU Affero General Public License as published by the
##### Free Software Foundation, either version 3 of the License, or (at
##### your option) any later version. See file LICENSE or go to
##### https://github.com/downtownallday/mailinabox-ldap for full license
##### details.
#####
#
# provision a miab-ldap that has a remote nextcloud (using Nextcloud
# from Docker)
#
D=$(dirname "$BASH_SOURCE")
. "$D/../../bin/lx_functions.sh" || exit 1
. "$D/../../bin/provision_functions.sh" || exit 1
# Create the instance (started)
provision_start "" "/mailinabox" || exit 1
# Setup system
provision_shell <<<"
cd /mailinabox
export PRIMARY_HOSTNAME=upgrade.abc.com
tests/system-setup/upgrade.sh --populate=basic --populate=totpuser || exit 1
tests/runner.sh -no-smtp-remote upgrade-basic upgrade-totpuser default || exit 2
"
provision_done $?

65
tests/lxd/vanilla/provision.sh Executable file
View File

@@ -0,0 +1,65 @@
#!/bin/bash
#####
##### This file is part of Mail-in-a-Box-LDAP which is released under the
##### terms of the GNU Affero General Public License as published by the
##### Free Software Foundation, either version 3 of the License, or (at
##### your option) any later version. See file LICENSE or go to
##### https://github.com/downtownallday/mailinabox-ldap for full license
##### details.
#####
D=$(dirname "$BASH_SOURCE")
. "$D/../../bin/lx_functions.sh" || exit 1
. "$D/../../bin/provision_functions.sh" || exit 1
# Create the instance (started)
provision_start "" "/mailinabox" || exit 1
# Setup system
if [ "$1" = "ciab" ]; then
# use a remote cloudinabox (does not have to be running)
provision_shell <<<"
cd /mailinabox
export PRIMARY_HOSTNAME='${inst}.local'
export NC_PROTO=https
export NC_HOST=vanilla-ciab.local
export NC_PORT=443
export NC_PREFIX=/
export SKIP_SYSTEM_UPDATE=0
tests/system-setup/vanilla.sh --qa-ca --enable-mod=remote-nextcloud
rc=$?
if ! ufw status | grep remote_nextcloud >/dev/null; then
# firewall rules aren't added when ciab is down
echo 'For testing, allow ldaps from anywhere'
ufw allow ldaps
fi
echo 'Add smart host alias - so \$NC_HOST can send mail to/via this host'
(
source tests/lib/all.sh
rest_urlencoded POST /admin/mail/aliases/add qa@abc.com Test_1234 \"address=@\$NC_HOST\" 'description=smart-host' 'permitted_senders=qa@abc.com' 2>/dev/null
echo \"\$REST_HTTP_CODE: \$REST_OUTPUT\"
)
exit $rc
"
provision_done $?
else
# vanilla (default - no miab integration)
provision_shell <<<"
cd /mailinabox
export PRIMARY_HOSTNAME='${inst}.local'
#export FEATURE_MUNIN=false
#export FEATURE_NEXTCLOUD=false
export SKIP_SYSTEM_UPDATE=0
tests/system-setup/vanilla.sh
rc=$?
# --enable-mod=move-postfix-queue-to-user-data
# --enable-mod=roundcube-master
# --enable-mod=roundcube-debug
# --enable-mod=rcmcarddav-composer
exit $rc
"
provision_done $?
fi