external/mailinabox
1
0
Fork 0
mirror of https://github.com/mail-in-a-box/mailinabox.git synced 2025-04-03 00:07:05 +00:00
Code Issues Releases Wiki Activity

Merge pull request #34 from downtownallday/lxd

Browse Source 
Switch to LXD from Vagrant
This commit is contained in:
Downtown Allday 2024-10-04 17:49:17 -04:00 committed by GitHub
commit dd16008fcb
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
38 changed files with 1415 additions and 557 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

37
Vagrantfile vendored
View File

@ -1,37 +0,0 @@
# -*- mode: ruby -*-
#####
##### This file is part of Mail-in-a-Box-LDAP which is released under the
##### terms of the GNU Affero General Public License as published by the
##### Free Software Foundation, either version 3 of the License, or (at
##### your option) any later version. See file LICENSE or go to
##### https://github.com/downtownallday/mailinabox-ldap for full license
##### details.
#####
# vi: set ft=ruby :
Vagrant.configure("2") do |config|
config.vm.box = "ubuntu/jammy64"
# Network config: Since it's a mail server, the machine must be connected
# to the public web. However, we currently don't want to expose SSH since
# the machine's box will let anyone log into it. So instead we'll put the
# machine on a private network.
config.vm.hostname = "mailinabox.lan"
config.vm.network "private_network", ip: "192.168.56.4"
config.vm.provision :shell, :inline => <<-SH
# Set environment variables so that the setup script does
# not ask any questions during provisioning. We'll let the
# machine figure out its own public IP.
export NONINTERACTIVE=1
export PUBLIC_IP=auto
export PUBLIC_IPV6=auto
export PRIMARY_HOSTNAME=auto
#export SKIP_NETWORK_CHECKS=1
# Start the setup script.
cd /vagrant
setup/start.sh
SH
end

1
ehdd/ehdd_funcs.sh
View File

@ -17,6 +17,7 @@ if [ -z "${STORAGE_ROOT:-}" ]; then
fi fi
fi fi
EHDD_IMG="$STORAGE_ROOT.HDD" EHDD_IMG="$STORAGE_ROOT.HDD"
EHDD_MOUNTPOINT="$STORAGE_ROOT" EHDD_MOUNTPOINT="$STORAGE_ROOT"
EHDD_LUKS_NAME="c1" EHDD_LUKS_NAME="c1"

2
tests/assets/.gitignore vendored
View File

@ -1 +1,3 @@
.emacs .emacs
backup/
vm_keys/

251
tests/bin/lx_functions.sh Normal file
View File

@ -0,0 +1,251 @@
#!/bin/bash
#####
##### This file is part of Mail-in-a-Box-LDAP which is released under the
##### terms of the GNU Affero General Public License as published by the
##### Free Software Foundation, either version 3 of the License, or (at
##### your option) any later version. See file LICENSE or go to
##### https://github.com/downtownallday/mailinabox-ldap for full license
##### details.
#####
# source this file
lx_project_root_dir() {
local d="$PWD"
while [ ${#d} -gt 1 ]; do
if [ -e "$d/setup" ]; then
echo "$d"
return 0
fi
d="$(dirname "$d")"
done
return 1
}
lx_guess_project_name() {
local d
d="$(lx_project_root_dir)"
[ $? -ne 0 ] && return 1
if [ -e "$d/setup/redis.sh" ]; then
echo "ciab"
else
echo "miab"
fi
}
# get the interface with the default route (first one)
get_system_default_network_interface() {
ip route | awk '/^default/ {printf "%s", $5; exit 0}'
}
isa_bridge_interface() {
local interface="$1"
if ip --oneline link show type bridge | awk -F: '{print $2}' | grep -q "^ *$interface *\$"; then
echo "yes"
else
echo "no"
fi
}
isa_wifi_device() {
local interface="$1"
local type
type="$(nmcli d show "$interface" | awk '$1=="GENERAL.TYPE:" {print $2}')"
if [ "$type" = "wifi" ]; then
echo "yes"
else
echo "no"
fi
}
# delete an instance
lx_delete() {
local project="${1:-default}"
local inst="$2"
local interactive="${3:-interactive}"
local xargs=""
if [ "$interactive" = "interactive" ]; then
xargs="-i"
fi
# only delete instance if the instance exists
if [ "$(lxc --project "$project" list name="$inst" -c n -f csv)" = "$inst" ]; then
lxc --project "$project" delete "$inst" -f $xargs
fi
}
# create a virtual-machine instance and start it
lx_launch_vm() {
local project="${1:-default}"
local inst_name="$2"
lx_init_vm "$@" || return 1
lxc --project "$project" start "$inst_name" || return 1
}
# create a virtual-machine instance (stopped)
lx_init_vm() {
local project="${1:-default}"
local inst_name="$2"
local image="$3"
local mount_host="$4" # path that you want available in guest
local mount_guest="$5" # mountpoint in guest
shift; shift; shift; shift; shift;
# a storage named the same as project must exist
# e.g. "lxc storage create $project dir" was executed prior.
case "${@}" in
*bridgenet* )
echo "Using network 'bridgenet'"
lxc --project "$project" profile show bridgenet | sed 's/^/ /' || return 1
;;
esac
lxc --project "$project" init "$image" "$inst_name" --vm --storage "$project" "$@" || return 1
if [ ! -z "$mount_host" -a ! -z "$mount_guest" ]; then
echo "adding $mount_guest on $inst_name to refer to $mount_host on host as device '${project}root'"
if [ $EUID -ne 0 ]; then
# so that files created by root on the mount inside the
# guest have the permissions of the current host user and
# not root:root
# see: https://documentation.ubuntu.com/lxd/en/latest/userns-idmap/
local egid="$(id --group)"
local idmap="uid $EUID 0
gid $egid 0"
lxc --project "$project" config set "$inst_name" raw.idmap="$idmap"
fi
lxc --project "$project" config device add "$inst_name" "${project}root" disk source="$(realpath "$mount_host")" path="$mount_guest" || return 1
fi
}
lx_launch_vm_and_wait() {
local project="$1"
local inst="$2"
local base_image="$3"
local mount_project_root_to="$4"
shift; shift; shift; shift;
# Delete existing instance, if it exists
lx_delete "$project" "$inst" interactive || return 1
# Create the instance (started)
lx_launch_vm "$project" "$inst" "$base_image" "$(lx_project_root_dir)" "$mount_project_root_to" "$@" || return 1
lx_wait_for_boot "$project" "$inst" || return 1
}
lx_output_inst_list() {
# Pre-defined column shorthand chars:
# 4 - IPv4 address
# 6 - IPv6 address
# a - Architecture
# b - Storage pool
# c - Creation date
# d - Description
# D - disk usage
# e - Project name
# l - Last used date
# m - Memory usage
# M - Memory usage (%)
# n - Name
# N - Number of Processes
# p - PID of the instance's init process
# P - Profiles
# s - State
# S - Number of snapshots
# t - Type (persistent or ephemeral)
# u - CPU usage (in seconds)
# L - Location of the instance (e.g. its cluster member)
# f - Base Image Fingerprint (short)
# F - Base Image Fingerprint (long)
local project="$1"
local columns="${2:-ns46tSL}"
local format="${3:-table}" # csv|json|table|yaml|compact
lxc --project "$project" list -c "$columns" -f "$format"
}
lx_output_image_list() {
# Column shorthand chars:
# l - Shortest image alias (and optionally number of other aliases)
# L - Newline-separated list of all image aliases
# f - Fingerprint (short)
# F - Fingerprint (long)
# p - Whether image is public
# d - Description
# a - Architecture
# s - Size
# u - Upload date
# t - Type
local project="$1"
local columns="${2:-lfpdatsu}"
local format="${3:-table}" # csv|json|table|yaml|compact
lxc --project "$project" image list -c "$columns" -f "$format"
}
lx_wait_for_boot() {
local project="$1"
local inst="$2"
echo -n "Wait for boot "
while ! lxc --project "$project" exec "$inst" -- ls >/dev/null 2>&1; do
echo -n "."
sleep 1
done
echo ""
echo -n "Wait for cloud-init "
lxc --project "$project" exec "$inst" -- cloud-init status --wait
local rc=$?
if [ $rc -eq 0 ]; then
echo "Wait for ip address "
local ip=""
local count=0
while [ $count -lt 10 ]; do
let count+=1
ip="$(lxc --project "$project" exec "$inst" -- hostname -I | awk '{print $1}')"
rc=$?
echo " [${count}] got: $ip"
if [ $rc -ne 0 -o "$ip" != "" ];then
break
fi
sleep 5
done
fi
echo ""
return $rc
}
lx_get_ssh_identity() {
local keydir="tests/assets/vm_keys"
if [ "$1" != "relative" ]; then
keydir="$(lx_project_root_dir)/$keydir"
fi
echo "$keydir/id_ed25519"
}
lx_get_ssh_known_hosts() {
local id="$(lx_get_ssh_identity)"
local known_hosts="$(dirname "$id")/known_hosts"
echo "$known_hosts"
}
lx_remove_known_host() {
local hostname="$1"
local known_hosts="$(lx_get_ssh_known_hosts)"
ssh-keygen -f "$known_hosts" -R "$hostname"
}
lx_create_ssh_identity() {
local id="$(lx_get_ssh_identity)"
if [ ! -e "$id" ]; then
mkdir -p "$(dirname "$id")"
ssh-keygen -f "$id" -C "vm key" -N "" -t ed25519
fi
}

209
tests/bin/lx_setup.sh Executable file
View File

@ -0,0 +1,209 @@
#!/bin/bash
#####
##### This file is part of Mail-in-a-Box-LDAP which is released under the
##### terms of the GNU Affero General Public License as published by the
##### Free Software Foundation, either version 3 of the License, or (at
##### your option) any later version. See file LICENSE or go to
##### https://github.com/downtownallday/mailinabox-ldap for full license
##### details.
#####
#
# This script configures the system for virtulization by installing
# lxd, adding a host bridge, and configuring some firewall
# rules. Beware that the user running the script will be added to the
# 'sudo' group!
#
# The script can be run anytime, but usually only needs to be run
# once. Once successful, the lxd-based test vms can be run locally.
#
# Limitations:
#
# The host bridge only works with an ethernet interface. Wifi does not
# work, so you must be plugged in to run the vm tests.
#
# Docker cannot be installed alongside as LXD due to conflicts with
# Docker's iptables entries. More on this issue can be found here:
#
# https://github.com/docker/for-linux/issues/103
#
# https://documentation.ubuntu.com/lxd/en/latest/howto/network_bridge_firewalld/#prevent-connectivity-issues-with-lxd-and-docker
#
# Removal:
#
# Run the script with a single "-d" argument to back out the changes.
#
# Helpful tools:
#
# NetworkManager UI: nm-connection-editor
# NetworkManager cli: nmcli
#
D=$(dirname "$BASH_SOURCE")
. "$D/lx_functions.sh" || exit 1
project="$(lx_guess_project_name)"
bridge_yaml="/etc/netplan/51-lxd-bridge.yaml"
install_packages() {
sudo snap install lxd --channel=latest/stable
if [ $EUID -ne 0 ]; then
echo "Add $USER to the 'sudo' group (!!)"
sudo usermod -aG sudo $USER || exit 1
fi
}
remove_packages() {
# note: run 'snap remove --purge lxd' to nuke lxd images,
# otherwise they stay on the system
snap remove lxd
}
create_network_bridge() {
# Create network bridge (we'll bridge vms to the local network)
# On return, sets these variables:
# vm_bridge: to name of bridge interface
# get the interface with the default route (first one)
local default_network_interface="$(get_system_default_network_interface)"
local isa_bridge="$(isa_bridge_interface "$default_network_interface")"
local isa_wifi_device="$(isa_wifi_device "$default_network_interface")"
if [ "$isa_bridge" = "yes" ]; then
vm_bridge="$default_network_interface"
#out_iface="$(ip --oneline link show type bridge_slave | grep -F " $default_network_interface " | awk -F: '{print $2}')"
#out_iface="$default_network_interface"
else
echo "NO HOST BRIDGE FOUND!!! CREATING ONE."
if [ "$isa_wifi_device" = "yes" ]; then
echo "*********************************************************"
echo "UNSUPPORTED: Host bridging is not available with WIFI ! (interface $default_network_interface)"
echo "*********************************************************"
return 1
fi
echo "YOU WILL LOSE NETWORK CONNECTIVITY BRIEFLY"
echo "To remove the host bridge, delete $bridge_yaml, then run netplan apply, or run this script with '-u'"
vm_bridge="br-lxd0"
#out_iface="br-lxd0"
tmp="$(mktemp)"
sudo cat <<EOF > "$tmp"
network:
ethernets:
myeths:
match:
name: $default_network_interface
dhcp4: no
bridges:
$vm_bridge:
dhcp4: yes
interfaces: [ myeths ]
EOF
if [ -e "$bridge_yaml" ]; then
echo "Overwriting netplan $bridge_yaml"
else
echo "Adding netplan $bridge_yaml"
fi
sudo mv "$tmp" "$bridge_yaml" || return 1
sudo chown root:root "$bridge_yaml"
sudo chmod 600 "$bridge_yaml"
sudo netplan apply || return 1
fi
}
remove_network_bridge() {
if [ -e "$bridge_yaml" ]; then
sudo rm -f "$bridge_yaml" || return 1
sudo netplan apply || return 1
else
echo "OK: Bridge configuration does not exist ($bridge_yaml)"
fi
}
install_ufw_rules() {
local bridge="$1"
# per: https://documentation.ubuntu.com/lxd/en/latest/howto/network_bridge_firewalld/
sudo ufw allow in on "$bridge" comment 'for lxd'
sudo ufw route allow in on "$bridge" comment 'for lxd'
sudo ufw route allow out on "$bridge" comment 'for lxd'
if which docker >/dev/null; then
echo "WARNING: docker appears to be installed. Guest VM networking probably won't work as the docker iptables conflict with LXD (guest DHCP broadcasts are dropped)"
fi
}
remove_ufw_rules() {
# 2. Remove all old ufw rules
sudo ufw status numbered | grep -E '(for lxd|for multipass)' | cut -c2-3 | sort -nr |
while read n; do
echo 'y' | sudo ufw delete $n
done
}
create_lxd_project_and_pool() {
local bridge="$1"
# Create project and networking bridge profile. When an instance
# is initialized, include the 'bridgenet' profile (see
# lx_init_vm() in lx_functions.sh)
echo "Create lxd project '$project' with profile 'bridgenet'"
echo " bridge: $bridge"
lxc project create "$project" 2>/dev/null
lxc --project "$project" profile create bridgenet 2>/dev/null
cat <<EOF | lxc --project "$project" profile edit bridgenet
description: Bridged networking LXD profile
devices:
eth0:
name: eth0
nictype: bridged
parent: $bridge
type: nic
EOF
[ $? -ne 0 ] && return 1
# create lxd storage pool for project (our convention is that
# project and storage must have the same name)
if ! lxc storage list -f csv | grep -q "^$project,"; then
echo "Create storage pool for $project project"
lxc storage create "$project" dir || return 1
fi
}
if [ "$1" = "-u" ]; then
# uninstall
echo "Revert ufw rules"
remove_ufw_rules
echo "Remove packages"
remove_packages
echo "Remove network bridge"
remove_network_bridge
else
echo "Install packages"
install_packages || exit 1
echo "Revert ufw rules"
remove_ufw_rules || exit 1
echo "Create network bridge"
create_network_bridge || exit 1 # sets vm_bridge
echo "Install ufw rules"
install_ufw_rules "$vm_bridge" || exit 1
create_lxd_project_and_pool "$vm_bridge" || exit 1
lx_create_ssh_identity || exit 1
fi

58
tests/bin/lx_status.sh Executable file
View File

@ -0,0 +1,58 @@
#!/bin/bash
#####
##### This file is part of Mail-in-a-Box-LDAP which is released under the
##### terms of the GNU Affero General Public License as published by the
##### Free Software Foundation, either version 3 of the License, or (at
##### your option) any later version. See file LICENSE or go to
##### https://github.com/downtownallday/mailinabox-ldap for full license
##### details.
#####
show() {
local project="$1"
local which=$2
if [ -z "$which" -o "$which" = "instances" ]; then
lxc --project "$project" list -c enfsd -f csv | sed "s/^/ /"
fi
if [ -z "$which" -o "$which" = "images" ]; then
lxc --project "$project" image list -c lfsd -f csv | sed "s/^/ $project,/"
fi
}
global="no"
if [ $# -gt 0 ]; then
projects=( "$@" )
else
global="yes"
projects=( $(lxc project list -f csv | awk -F, '{print $1}' | sed 's/ .*$//') )
fi
if [ "$global" = "yes" ]; then
echo "** projects"
idx=0
while [ $idx -lt ${#projects[*]} ]; do
echo " ${projects[$idx]}"
let idx+=1
done
else
echo "Project: ${projects[*]}"
fi
echo "** images"
idx=0
while [ $idx -lt ${#projects[*]} ]; do
project="${projects[$idx]}"
let idx+=1
show "$project" images $verbose
done
echo "** instances"
idx=0
while [ $idx -lt ${#projects[*]} ]; do
project="${projects[$idx]}"
let idx+=1
show "$project" instances $verbose
done

91
tests/bin/provision_functions.sh Normal file
View File

@ -0,0 +1,91 @@
#####
##### This file is part of Mail-in-a-Box-LDAP which is released under the
##### terms of the GNU Affero General Public License as published by the
##### Free Software Foundation, either version 3 of the License, or (at
##### your option) any later version. See file LICENSE or go to
##### https://github.com/downtownallday/mailinabox-ldap for full license
##### details.
#####
# source this file
#
# requires: lx_functions.sh
#
. "$(dirname "$BASH_SOURCE")/../lib/misc.sh"
load_provision_defaults() {
#
# search from the current directory up for a file named
# ".provision_defaults"
#
if [ -z "$PROVISION_DEFAULTS_FILE" ]; then
PROVISION_DEFAULTS_FILE="$(pwd)/.provision_defaults"
while [ "$PROVISION_DEFAULTS_FILE" != "/.provision_defaults" ]; do
[ -e "$PROVISION_DEFAULTS_FILE" ] && break
PROVISION_DEFAULTS_FILE="$(realpath -m "$PROVISION_DEFAULTS_FILE/../..")/.provision_defaults"
done
source "$PROVISION_DEFAULTS_FILE" || return 1
fi
if [ ! -e "$PROVISION_DEFAULTS_FILE" ]; then
return 1
fi
}
provision_start() {
load_provision_defaults || return 1
local base_image="${1:-$DEFAULT_LXD_IMAGE}"
local guest_mount_path="$2"
shift; shift
local opts=( "$@" )
if [ ${#opts[@]} -eq 0 ]; then
opts=( "${DEFAULT_LXD_INST_OPTS[@]}" )
fi
# set these globals
project="$(lx_guess_project_name)"
inst="$(basename "$PWD")"
provision_start_s="$(date +%s)"
echo "Creating instance '$inst' from image $base_image (${opts[@]})"
lx_launch_vm_and_wait \
"$project" "$inst" "$base_image" "$guest_mount_path" \
"${opts[@]}" \
|| return 1
}
provision_shell() {
# provision_start must have been called first!
local remote_path="/tmp/provision.sh"
local lxc_flags="--uid 0 --gid 0 --mode 755 --create-dirs"
if [ ! -z "$1" ]; then
lxc --project "$project" file push "$1" "${inst}${remote_path}" $lxc_flags || return 1
else
local tmp=$(mktemp)
echo "#!/bin/bash" >"$tmp"
cat >>"$tmp"
lxc --project "$project" file push "$tmp" "${inst}${remote_path}" $lxc_flags || return 1
rm -f "$tmp"
fi
lxc --project "$project" exec "$inst" --cwd / --env PROVISION=true \
-- "$remote_path"
}
provision_done() {
local rc="${1:-0}"
echo "Elapsed: $(elapsed_pretty "$provision_start_s" "$(date +%s)")"
if [ $rc -ne 0 ]; then
echo "Failed with code $rc"
return 1
else
echo "Success!"
return 0
fi
}

285
tests/bin/vlx Executable file
View File

@ -0,0 +1,285 @@
#!/bin/bash
#####
##### This file is part of Mail-in-a-Box-LDAP which is released under the
##### terms of the GNU Affero General Public License as published by the
##### Free Software Foundation, either version 3 of the License, or (at
##### your option) any later version. See file LICENSE or go to
##### https://github.com/downtownallday/mailinabox-ldap for full license
##### details.
#####
# This is a helper script to make it easier to interact with lxd by
# calling lxc with arguments derived from conventions we've developed
# in this project.
#
# Those conventions are:
#
# 1. the project name is derived from the working directory. You
# must be anywhere in the source tree. "ciab" for cloudinabox and
# "miab" for mailinabox
#
# 2. the instance name is derived from the base name of the current
# working directory. this means that each instance must have it's
# own directory and have a script within it called "provision.sh"
# that creates the instance.
#
# Run the script with no arguments to see a list of commands.
#
# It's helpful to create a command alias. eg. in bash:
#
# alias vlx="/path/to/tests/bin/vlx"
#
# Add it to your ~/bash_aliases file to make it available for new
# terminal windows.
#
D=$(dirname "$BASH_SOURCE")
. "$D/lx_functions.sh" || exit 1
show_cl="yes"
if [ "$1" = "-q" ]; then
show_cl="no"
shift
fi
vlx_guess() {
if [ $# -eq 2 ]; then
LX_PROJECT="$1"
LX_INST="$2"
elif [ $# -eq 1 ]; then
LX_PROJECT="$(lx_guess_project_name)"
LX_INST="$1"
elif [ $# -eq 0 ]; then
LX_PROJECT="$(lx_guess_project_name)"
LX_INST="$(basename "$PWD")"
else
echo "Invalid number of arguments"
return 1
fi
}
vlx_exec_usage() {
echo "Usage: vlx exec <project> <inst> [<cwd>] -- cmd ..."
echo " or: vlx exec <inst> [<cwd>] -- cmd ..."
echo " or: vlx exec <cwd> -- cmd ..."
echo " or: vlx exec cmd ..."
}
vlx_exec() {
# args
# format 1: project inst [cwd] -- cmd ...
# format 2: inst [cwd] -- cmd ...
# format 3: <cwd> -- cmd ...
# format 4: cmd ...
if [ $# -eq 0 ]; then
vlx_exec_usage
return 1
fi
local args=( "$@" )
local idx=0
while [ $idx -le 3 -a $idx -lt ${#args[*]} ]; do
[ "${args[$idx]}" = "--" ] && break
let idx+=1
done
local wd=""
if [ "${args[$idx]}" = "--" ]; then
if [ $idx -eq 3 ]; then
# format 1 with cwd
wd="$3"
vlx_guess "$1" "$2" || return 1
shift; shift; shift; shift;
elif [ $idx -eq 2 ]; then
# format 1 w/o cwd or 2
if [ "${2#/}" != "$2" ]; then
# wd starts with /, so it's a path
# format 2
wd="$2"
vlx_guess "" "$1" || return 1
else
# format 1 w/o cwd
vlx_guess "$1" "$2" || return 1
fi
shift; shift; shift;
elif [ $idx -eq 1 ]; then
# format 2 w/o cwd or 3
if [ "${1#/}" != "$1" ]; then
# wd starts with /, so it's a path
# format 3
wd="$1"
vlx_guess || return 1
else
# format 2 w/o cwd
vlx_guess "$1" || return 1
fi
shift; shift;
elif [ $idx -eq 0 ]; then
# command line "-- cmd ...", just ignore the leading --
shift;
fi
else
# format 4
vlx_guess || return 1
fi
local xargs=""
if [ ! -z "$wd" ]; then
xargs="--cwd $wd"
fi
[ "$show_cl" = "yes" ] &&
echo lxc --project "$LX_PROJECT" exec "$LX_INST" $xargs -- "$@"
lxc --project "$LX_PROJECT" exec "$LX_INST" $xargs -- "$@"
}
vlx_shell() {
vlx_guess "$@" || return 1
[ "$show_cl" = "yes" ] &&
echo lxc --project "$LX_PROJECT" exec "$LX_INST" -- bash
lxc --project "$LX_PROJECT" exec "$LX_INST" -- bash
}
vlx_hostname() {
vlx_guess "$@" || return 1
lxc --project "$LX_PROJECT" exec "$LX_INST" -- /usr/bin/hostname --fqdn || return 1
}
vlx_ipaddr() {
vlx_guess "$@" || return 1
local hostip
hostip="$(lxc --project "$LX_PROJECT" exec "$LX_INST" -- /usr/bin/hostname -I)"
[ $? -ne 0 -o -z "$hostip" ] && return 1
awk '{print $1}' <<<"$hostip"
}
vlx_ssh() {
local host="$1"
if [ "$host" = "--" ]; then
host=""
else
shift
fi
if [ -z "$host" ]; then
host="$(vlx_ipaddr)"
if [ $? -ne 0 ]; then
echo "Could not determine ip address, please specify"
host=""
fi
if [ -z "$host" ]; then
echo "usage: vlx ssh <vm-hostname>"
return 1
fi
fi
local id="$(lx_get_ssh_identity)"
local known_hosts="$(lx_get_ssh_known_hosts)"
local vmuser="vmuser"
#echo ssh -i "$id" -o UserKnownHostsFile="$known_hosts" -o StrictHostKeyChecking=no "$vmuser@$host" "$@"
echo "Connecting to $vmuser@$host ..."
ssh -i "$id" -o UserKnownHostsFile="$known_hosts" -o StrictHostKeyChecking=no "$vmuser@$host" "$@"
}
vlx_list() {
vlx_guess "$1" || return 1
[ "$show_cl" = "yes" ] &&
echo lxc --project "$LX_PROJECT" list
lxc --project "$LX_PROJECT" list
}
vlx_images() {
vlx_guess "$1" || return 1
[ "$show_cl" = "yes" ] &&
echo lxc --project "$LX_PROJECT" image list
lxc --project "$LX_PROJECT" image list
}
vlx_up() {
if [ -x "./provision.sh" ] ; then
echo "Provision"
./provision.sh "$@" || return 1
else
echo "UP failed: ./provision.sh does not exist or is not executable"
return 1
fi
}
vlx_start() {
vlx_guess "$@" || return 1
[ "$show_cl" = "yes" ] &&
echo lxc --project "$LX_PROJECT" start "$LX_INST"
lxc --project "$LX_PROJECT" start "$LX_INST"
}
vlx_stop() {
vlx_guess "$@" || return 1
[ "$show_cl" = "yes" ] &&
echo lxc --project "$LX_PROJECT" stop "$LX_INST"
lxc --project "$LX_PROJECT" stop "$LX_INST"
}
vlx_delete() {
vlx_guess "$@" || return 1
[ "$show_cl" = "yes" ] &&
echo lxc --project "$LX_PROJECT" delete --force --interactive "$LX_INST"
lxc --project "$LX_PROJECT" delete --force --interactive "$LX_INST"
}
vlx_destroy() {
vlx_delete "$@"
}
vlx_status() {
if [ $# -eq 0 ]; then
vlx_guess || return 1
"$D/lx_status.sh" "$LX_PROJECT"
elif [ "$1" = "-g" ]; then
"$D/lx_status.sh"
else
"$D/lx_status.sh" "$@"
fi
}
vlx_restart() {
vlx_guess "$@" || return 1
[ "$show_cl" = "yes" ] &&
echo lxc --project "$LX_PROJECT" restart "$LX_INST"
lxc --project "$LX_PROJECT" restart "$LX_INST"
}
usage() {
echo "Usage:"
echo "vlx <command> [<arg> ...]"
echo "commands:"
echo " exec <working-directoy> -- command [arg ...]"
echo " exec command [arg ...]"
echo " shell"
echo " ssh"
echo " hostname"
echo " list"
echo " images"
echo " up"
echo " start"
echo " stop"
echo " delete|destroy"
echo " restart"
echo " status"
}
if [ $# -eq 0 ]; then
usage
exit 1
fi
cmd="$1"
handler="vlx_$1"
shift
if [ ! "$(type -t $handler)" = "function" ]; then
echo "Unknown command: $cmd"
exit 1
fi
$handler "$@"

1
tests/lxd/.gitignore vendored Normal file
View File

@ -0,0 +1 @@
out/

10
tests/lxd/.provision_defaults Normal file
View File

@ -0,0 +1,10 @@
# defaults for lxd image creation using
# tests/bin/provision_functions.sh::provision_start()
DEFAULT_LXD_IMAGE="preloaded-ubuntu-jammy"
DEFAULT_LXD_INST_OPTS=(
"-p" "default"
"-p" "bridgenet"
"-c" "limits.cpu=1"
"-c" "limits.memory=2GiB"
)

18
tests/lxd/README.md Normal file
View File

@ -0,0 +1,18 @@
### To use lxd vm's:
1. run `tests/bin/lx_setup.sh`. This only needs to be run once.
2. run `tests/lxd/preloaded/create_preloaded.sh`. Run this anytime a new base image is updated (usually when ubuntu updates would require a system reboot).
### To bring up a vm:
1. It's helpful to have `vlx` in your path. vlx is a tool that makes `lxc` act a little like vagrant. In bash, create an alias for it: `alias vlx="$(pwd)/tests/bin/vlx"
2. set your working directory to the vm directory you'd like to start (eg. `cd "tests/lxd/vanilla"`), then run `vlx up`
3. to access the vm: `vlx shell` or `vlx ssh`. all vm's have the source root mounted at /cloudinabox or /mailinabox, so you can change files locally and they'll be available on the vm for testing
4. to destroy/delete the vm: `vlx destroy`

1
tests/lxd/create-backup/README.md Normal file
View File

@ -0,0 +1 @@
TODO

1
tests/lxd/majorupgrade/README.md Normal file
View File

@ -0,0 +1 @@
TODO - convert to lxd

0
tests/vagrant/majorupgrade/Vagrantfile → tests/lxd/majorupgrade/Vagrantfile vendored
View File

0
tests/vagrant/majorupgrade/majorupgrade.sh → tests/lxd/majorupgrade/majorupgrade.sh
View File

8
tests/lxd/parallel-boxlist.all Normal file
View File

@ -0,0 +1,8 @@
# jammy does not include the dm_crypt kernel module, which is needed
# for luks/encryption-at-rest
# remote-nextcloud-docker-ehdd
remote-nextcloud-docker
upgrade-from-upstream
upgrade
unsetvars

3
tests/lxd/parallel-boxlist.default Normal file
View File

@ -0,0 +1,3 @@
remote-nextcloud-docker
upgrade-from-upstream
upgrade

119
tests/lxd/parallel.sh Executable file
View File

@ -0,0 +1,119 @@
#!/bin/bash
#####
##### This file is part of Mail-in-a-Box-LDAP which is released under the
##### terms of the GNU Affero General Public License as published by the
##### Free Software Foundation, either version 3 of the License, or (at
##### your option) any later version. See file LICENSE or go to
##### https://github.com/downtownallday/mailinabox-ldap for full license
##### details.
#####
#
# Parallel provisioning for test vms
#
. "$(dirname "$0")/../bin/lx_functions.sh"
. "$(dirname "$0")/../lib/color-output.sh"
. "$(dirname "$0")/../lib/misc.sh"
boxlist="" # the name of the boxlist or a path to the boxlist file
boxes=() # the contents of the boxlist file
project="$(lx_guess_project_name)"
load_boxlist() {
# sets global variable 'boxlist' and array 'boxes'
boxlist="${1:-default}"
local fn="$boxlist"
if [ ! -f "$fn" ]; then
fn="parallel-boxlist.$boxlist"
fi
if [ ! -f "$fn" ]; then
echo "Could not load boxlist from '${boxlist}'! Failed to find '$fn'."
exit 1
fi
boxes=( $(grep -v '^#' $fn) )
if [ $? -ne 0 ]; then
echo "Could not load boxlist from file '$fn'!"
exit 1
fi
}
# destroy running boxes
if [ "$1" = "-d" ]; then
shift
load_boxlist "$1"
for inst in $(lx_output_inst_list "$project" "n" "csv"); do
if array_contains $inst ${boxes[*]}; then
echo lxc --project "$project" delete $inst --force
lxc --project "$project" delete $inst --force
fi
done
exit 0
elif [ "$1" = "-h" -o "$1" = "--help" ]; then
echo "usage: $0 [-d] [boxlist]"
echo " -d delete/destroy running boxes"
echo " boxlist an file or named boxlist containing a list of instance names. defaults to 'default'"
exit 0
fi
load_boxlist "$1"
# set total parallel vms to (#cores minus 1)
MAX_PROCS=$(cat /proc/cpuinfo | grep processor | wc -l)
let MAX_PROCS-=1
[ $MAX_PROCS -eq 0 ] && MAX_PROCS=1
OUTPUT_DIR=out
#rm -rf "$OUTPUT_DIR"
mkdir -p "$OUTPUT_DIR"
echo "MAX_PROCS=$MAX_PROCS"
echo "OUTPUT_DIR=$OUTPUT_DIR"
start_time="$(date +%s)"
# bring up in parallel
for inst in ${boxes[*]}; do
outfile="$OUTPUT_DIR/$inst.out.txt"
rm -f "$outfile"
echo "Bringing up '$inst'. Output will be in: $outfile" 1>&2
echo $inst
done | xargs -P $MAX_PROCS -I"INSTNAME" \
sh -c '
cd "INSTNAME" &&
./provision.sh >'"../$OUTPUT_DIR/"'INSTNAME.out.txt 2>&1 &&
echo "EXITCODE: 0" >> '"../$OUTPUT_DIR/"'INSTNAME.out.txt ||
echo "EXITCODE: $?" >>'"../$OUTPUT_DIR/"'INSTNAME.out.txt
'
# output overall result"
H1 "Results"
rc=0
for inst in ${boxes[*]}; do
file="$OUTPUT_DIR"/$inst.out.txt
exitcode="$(tail "$file" | grep EXITCODE: | awk '{print $NF}')"
echo -n "$inst: "
if [ -z "$exitcode" ]; then
danger "NO EXITCODE!"
[ $rc -eq 0 ] && rc=2
elif [ "$exitcode" == "0" ]; then
elapsed="$(tail "$file" | grep ^Elapsed | awk -F: '{print $2}')"
success "SUCCESS (${elapsed# })"
else
danger "FAILURE ($exitcode)"
rc=1
fi
done
# output elapsed time
end_time="$(date +%s)"
echo ""
echo "Elapsed time: $(elapsed_pretty $start_time $end_time)"
# exit
echo ""
echo "Guest VMs are running! Destroy them with:"
echo " $0 -d $boxlist"
exit $rc

101
tests/lxd/preloaded/create_preloaded.sh Executable file
View File

@ -0,0 +1,101 @@
#!/bin/bash
#####
##### This file is part of Mail-in-a-Box-LDAP which is released under the
##### terms of the GNU Affero General Public License as published by the
##### Free Software Foundation, either version 3 of the License, or (at
##### your option) any later version. See file LICENSE or go to
##### https://github.com/downtownallday/mailinabox-ldap for full license
##### details.
#####
# this script creates a new lxd image preloaded with software to speed
# up installation.
#
# prerequisites:
#
# tests/bin/lx_setup.sh must be run before running this script. it
# only needs to be run once, or any time the networking setup
# changes (eg. adding a new ethernet card).
#
D="$(dirname "$BASH_SOURCE")"
. "$D/../../bin/lx_functions.sh" || exit 1
. "$D/../../bin/provision_functions.sh" || exit 1
project="$(lx_guess_project_name)"
inst_mountpoint=/mailinabox
[ "$project" = "ciab" ] && inst_mountpoint=/cloudinabox
load_provision_defaults || exit 1
echo "Provision defaults loaded from: $(realpath --relative-to="$(pwd)" "$PROVISION_DEFAULTS_FILE")"
if [ -z "$1" ]; then
imagelist=( $(<./imagelist) )
else
imagelist=( "$@" )
fi
for base_image in "${imagelist[@]}"
do
new_image="preloaded-${base_image/:/-}"
inst_name="preloaded"
echo ""
echo "START: create $new_image using base image $base_image"
echo "Delete existing instance: $inst_name"
lx_delete "$project" "$inst_name" "no-interactive" || exit 1
echo "Create instance '$inst_name' from '$base_image'"
# cloud init configuration creates user 'vmuser' instead of 'ubuntu'
cloud_config_users="#cloud-config
users:
- default
- name: vmuser
gecos: VM user for ssh
primary_group: vmuser
groups: adm, sudo, lxd
shell: /bin/bash
sudo: ALL=(ALL) NOPASSWD:ALL
lock_passwd: true
ssh_authorized_keys:
- $(< $(lx_get_ssh_identity).pub)
"
lx_launch_vm "$project" "$inst_name" "$base_image" "$(lx_project_root_dir)" "$inst_mountpoint" "${DEFAULT_LXD_INST_OPTS[@]}" -c cloud-init.user-data="$cloud_config_users" -c limits.cpu=2 -c limits.memory=2GiB -d root,size=30GiB || exit 1
lx_wait_for_boot "$project" "$inst_name"
echo ""
echo "================================================="
echo "Prep the VM instance"
echo "================================================="
lxc --project "$project" exec "$inst_name" --cwd "$inst_mountpoint" -- sudo tests/lxd/preloaded/prepvm.sh --no-dry-run || exit 1
echo ""
echo "================================================="
echo "Create an image from the instance"
echo "================================================="
echo "Stopping instance '$inst_name'"
lxc --project "$project" stop "$inst_name" || exit 1
echo "Create image '$new_image' from instance '$inst_name'"
lxc --project "$project" publish "$inst_name" "local:" --reuse --compression gzip --alias "$new_image" || exit 1 # --compression xz
echo ""
echo "================================================="
echo "Image list ($project)"
echo "================================================="
lx_output_image_list "$project" "ld"
echo ""
echo "================================================="
echo "Delete instance '$inst_name'"
echo "================================================="
lx_delete "$project" "$inst_name" "no-interactive"
echo "Success"
done

1
tests/lxd/preloaded/imagelist Normal file
View File

@ -0,0 +1 @@
ubuntu:jammy

0
tests/vagrant/preloaded/prepvm.sh → tests/lxd/preloaded/prepvm.sh
View File

35
tests/lxd/remote-nextcloud-docker-ehdd/provision.sh Executable file
View File

@ -0,0 +1,35 @@
#!/bin/bash
#####
##### This file is part of Mail-in-a-Box-LDAP which is released under the
##### terms of the GNU Affero General Public License as published by the
##### Free Software Foundation, either version 3 of the License, or (at
##### your option) any later version. See file LICENSE or go to
##### https://github.com/downtownallday/mailinabox-ldap for full license
##### details.
#####
#
# provision a miab-ldap that has a remote nextcloud (using Nextcloud
# from Docker) and an encrypted user-data
#
D=$(dirname "$BASH_SOURCE")
. "$D/../../bin/lx_functions.sh" || exit 1
. "$D/../../bin/provision_functions.sh" || exit 1
# Create the instance (started)
provision_start "" "/mailinabox" || exit 1
# Setup system
provision_shell <<<"
cd /mailinabox
export PRIMARY_HOSTNAME=qa1.abc.com
export FEATURE_MUNIN=false
export EHDD_KEYFILE=\$HOME/keyfile
echo -n 'boo' >\$EHDD_KEYFILE
tests/system-setup/remote-nextcloud-docker.sh || exit 1
tests/runner.sh -no-smtp-remote remote-nextcloud ehdd default || exit 2
"
provision_done $?

33
tests/lxd/remote-nextcloud-docker/provision.sh Executable file
View File

@ -0,0 +1,33 @@
#!/bin/bash
#####
##### This file is part of Mail-in-a-Box-LDAP which is released under the
##### terms of the GNU Affero General Public License as published by the
##### Free Software Foundation, either version 3 of the License, or (at
##### your option) any later version. See file LICENSE or go to
##### https://github.com/downtownallday/mailinabox-ldap for full license
##### details.
#####
#
# provision a miab-ldap that has a remote nextcloud (using Nextcloud
# from Docker)
#
D=$(dirname "$BASH_SOURCE")
. "$D/../../bin/lx_functions.sh" || exit 1
. "$D/../../bin/provision_functions.sh" || exit 1
# Create the instance (started)
provision_start "" "/mailinabox" || exit 1
# Setup system
provision_shell <<<"
cd /mailinabox
export PRIMARY_HOSTNAME=qa2.abc.com
export FEATURE_MUNIN=false
tests/system-setup/remote-nextcloud-docker.sh upgrade --populate=basic || exit 1
tests/runner.sh -no-smtp-remote remote-nextcloud upgrade-basic default || exit 2
"
provision_done $?

35
tests/lxd/unsetvars/provision.sh Executable file
View File

@ -0,0 +1,35 @@
#!/bin/bash
#####
##### This file is part of Mail-in-a-Box-LDAP which is released under the
##### terms of the GNU Affero General Public License as published by the
##### Free Software Foundation, either version 3 of the License, or (at
##### your option) any later version. See file LICENSE or go to
##### https://github.com/downtownallday/mailinabox-ldap for full license
##### details.
#####
#
# provision a miab-ldap that has a remote nextcloud (using Nextcloud
# from Docker)
#
D=$(dirname "$BASH_SOURCE")
. "$D/../../bin/lx_functions.sh" || exit 1
. "$D/../../bin/provision_functions.sh" || exit 1
# Create the instance (started)
provision_start "" "/mailinabox" || exit 1
# Setup system
provision_shell <<<"
export NONINTERACTIVE=1
export PUBLIC_IP=auto
export PUBLIC_IPV6=auto
export PRIMARY_HOSTNAME=mailinabox.lan # auto
export SKIP_NETWORK_CHECKS=1
cd /mailinabox
setup/start.sh
"
provision_done $?

33
tests/lxd/upgrade-from-upstream/provision.sh Executable file
View File

@ -0,0 +1,33 @@
#!/bin/bash
#####
##### This file is part of Mail-in-a-Box-LDAP which is released under the
##### terms of the GNU Affero General Public License as published by the
##### Free Software Foundation, either version 3 of the License, or (at
##### your option) any later version. See file LICENSE or go to
##### https://github.com/downtownallday/mailinabox-ldap for full license
##### details.
#####
#
# provision a miab-ldap that has a remote nextcloud (using Nextcloud
# from Docker)
#
D=$(dirname "$BASH_SOURCE")
. "$D/../../bin/lx_functions.sh" || exit 1
. "$D/../../bin/provision_functions.sh" || exit 1
# Create the instance (started)
provision_start "" "/mailinabox" || exit 1
# Setup system
provision_shell <<<"
cd /mailinabox
export PRIMARY_HOSTNAME=qa3.abc.com
export UPSTREAM_TAG=main
tests/system-setup/upgrade-from-upstream.sh --populate=basic --populate=totpuser || exit 1
tests/runner.sh -no-smtp-remote upgrade-basic upgrade-totpuser default || exit 2
"
provision_done $?

32
tests/lxd/upgrade/provision.sh Executable file
View File

@ -0,0 +1,32 @@
#!/bin/bash
#####
##### This file is part of Mail-in-a-Box-LDAP which is released under the
##### terms of the GNU Affero General Public License as published by the
##### Free Software Foundation, either version 3 of the License, or (at
##### your option) any later version. See file LICENSE or go to
##### https://github.com/downtownallday/mailinabox-ldap for full license
##### details.
#####
#
# provision a miab-ldap that has a remote nextcloud (using Nextcloud
# from Docker)
#
D=$(dirname "$BASH_SOURCE")
. "$D/../../bin/lx_functions.sh" || exit 1
. "$D/../../bin/provision_functions.sh" || exit 1
# Create the instance (started)
provision_start "" "/mailinabox" || exit 1
# Setup system
provision_shell <<<"
cd /mailinabox
export PRIMARY_HOSTNAME=upgrade.abc.com
tests/system-setup/upgrade.sh --populate=basic --populate=totpuser || exit 1
tests/runner.sh -no-smtp-remote upgrade-basic upgrade-totpuser default || exit 2
"
provision_done $?

68
tests/lxd/vanilla/provision.sh Executable file
View File

@ -0,0 +1,68 @@
#!/bin/bash
#####
##### This file is part of Mail-in-a-Box-LDAP which is released under the
##### terms of the GNU Affero General Public License as published by the
##### Free Software Foundation, either version 3 of the License, or (at
##### your option) any later version. See file LICENSE or go to
##### https://github.com/downtownallday/mailinabox-ldap for full license
##### details.
#####
D=$(dirname "$BASH_SOURCE")
. "$D/../../bin/lx_functions.sh" || exit 1
. "$D/../../bin/provision_functions.sh" || exit 1
# Create the instance (started)
provision_start "" "/mailinabox" || exit 1
# Setup system
if [ "$TESTS" = "ciab" -o "$1" = "ciab" ]; then
# use a remote cloudinabox (does not have to be running)
provision_shell <<<"
cd /mailinabox
export PRIMARY_HOSTNAME='${inst}.local'
export NC_PROTO=https
export NC_HOST=vanilla-ciab.local
export NC_PORT=443
export NC_PREFIX=/
export SKIP_SYSTEM_UPDATE=0
tests/system-setup/vanilla.sh --qa-ca --enable-mod=remote-nextcloud
rc=\$?
if ! ufw status | grep remote_nextcloud >/dev/null; then
# firewall rules aren't added when ciab is down
echo 'For testing, allow ldaps from anywhere'
ufw allow ldaps
fi
echo 'Add smart host alias - so \$NC_HOST can send mail to/via this host'
(
source tests/lib/all.sh
rest_urlencoded POST /admin/mail/aliases/add qa@abc.com Test_1234 \"address=@\$NC_HOST\" 'description=smart-host' 'permitted_senders=qa@abc.com' 2>/dev/null
echo \"\$REST_HTTP_CODE: \$REST_OUTPUT\"
)
exit \$rc
"
provision_done $?
elif [ -z "$1" ]; then
# vanilla (default - no miab integration)
provision_shell <<<"
cd /mailinabox
export PRIMARY_HOSTNAME='${inst}.local'
#export FEATURE_MUNIN=false
#export FEATURE_NEXTCLOUD=false
export SKIP_SYSTEM_UPDATE=0
tests/system-setup/vanilla.sh
rc=\$?
# --enable-mod=move-postfix-queue-to-user-data
# --enable-mod=roundcube-master
# --enable-mod=roundcube-debug
# --enable-mod=rcmcarddav-composer
exit \$rc
"
provision_done $?
else
echo "Invalid argument: $1"
exit 1
fi

4
tests/system-setup/setup-defaults.sh
View File

@ -22,11 +22,11 @@ export STORAGE_ROOT="${STORAGE_ROOT:-/home/$STORAGE_USER}"
export EMAIL_ADDR="${EMAIL_ADDR:-qa@abc.com}" export EMAIL_ADDR="${EMAIL_ADDR:-qa@abc.com}"
export EMAIL_PW="${EMAIL_PW:-Test_1234}" export EMAIL_PW="${EMAIL_PW:-Test_1234}"
export PUBLIC_IP="${PUBLIC_IP:-$(source ${MIAB_DIR:-.}/setup/functions.sh; get_default_privateip 4)}" export PUBLIC_IP="${PUBLIC_IP:-$(source ${MIAB_DIR:-.}/setup/functions.sh; get_default_privateip 4)}"
if lsmod | grep "^vboxguest[\t ]" >/dev/null; then if lsmod | grep -q "^vboxguest[\t ]" || df | grep -q "^lxd_"; then
# The local mods directory defaults to 'local' (relative to the # The local mods directory defaults to 'local' (relative to the
# source tree, which is a mounted filesystem of the host). This # source tree, which is a mounted filesystem of the host). This
# will keep mods directory out of the source tree when running # will keep mods directory out of the source tree when running
# under virtualbox / vagrant. # under virtualbox / vagrant / lxd.
export LOCAL_MODS_DIR="${LOCAL_MODS_DIR:-/local}" export LOCAL_MODS_DIR="${LOCAL_MODS_DIR:-/local}"
else else
export LOCAL_MODS_DIR="${LOCAL_MODS_DIR:-$(pwd)/local}" export LOCAL_MODS_DIR="${LOCAL_MODS_DIR:-$(pwd)/local}"

34
tests/system-setup/setup-funcs.sh
View File

@ -86,7 +86,7 @@ init_test_system() {
# update system time # update system time
H2 "Set system time" H2 "Set system time"
update_system_time || echo "Ignoring error..." update_system_time || echo "Ignoring error..."
# update package lists before installing anything # update package lists before installing anything
H2 "apt-get update" H2 "apt-get update"
wait_for_apt wait_for_apt
@ -115,7 +115,7 @@ init_test_system() {
| awk '/^(Upgrade|Install): / { print $0 }' | awk '/^(Upgrade|Install): / { print $0 }'
rm -f /tmp/history.log rm -f /tmp/history.log
fi fi
# install avahi if the system dns domain is .local - note that # install avahi if the system dns domain is .local - note that
# /bin/dnsdomainname returns empty string at this point # /bin/dnsdomainname returns empty string at this point
case "$PRIMARY_HOSTNAME" in case "$PRIMARY_HOSTNAME" in
@ -149,7 +149,7 @@ init_miab_testing() {
else else
echo "Not configured for encryption-at-rest" echo "Not configured for encryption-at-rest"
fi fi
H2 "QA prerequisites" H2 "QA prerequisites"
local rc=0 local rc=0
@ -186,7 +186,7 @@ init_miab_testing() {
echo "Copy failed ($?)" echo "Copy failed ($?)"
rc=1 rc=1
fi fi
# create miab_ldap.conf to specify what the Nextcloud LDAP service # create miab_ldap.conf to specify what the Nextcloud LDAP service
# account password will be to avoid a random one created by start.sh # account password will be to avoid a random one created by start.sh
if [ ! -z "$LDAP_NEXTCLOUD_PASSWORD" ]; then if [ ! -z "$LDAP_NEXTCLOUD_PASSWORD" ]; then
@ -228,13 +228,13 @@ init_miab_testing() {
# ignore unknown option - may be interpreted elsewhere # ignore unknown option - may be interpreted elsewhere
shift shift
;; ;;
esac esac
done done
# now that we've copied our files, unmount STORAGE_ROOT if # now that we've copied our files, unmount STORAGE_ROOT if
# encryption-at-rest was enabled # encryption-at-rest was enabled
ehdd/umount.sh ehdd/umount.sh
return $rc return $rc
} }
@ -303,7 +303,7 @@ clone_repo_and_pushd() {
;; ;;
esac esac
done done
if [ -z "$repo" -o -z "$treeish" -o -z "$targetdir" ]; then if [ -z "$repo" -o -z "$treeish" -o -z "$targetdir" ]; then
return 1 return 1
fi fi
@ -329,7 +329,7 @@ upstream_install() {
if clone_repo_and_pushd "$@"; then if clone_repo_and_pushd "$@"; then
need_pop="yes" need_pop="yes"
fi fi
H1 "MIAB UPSTEAM INSTALL [$(git describe 2>/dev/null)]" H1 "MIAB UPSTEAM INSTALL [$(git describe 2>/dev/null)]"
# ensure we're in a MiaB working directory # ensure we're in a MiaB working directory
@ -351,24 +351,24 @@ upstream_install() {
exec_no_output apt-get remove -y nsd exec_no_output apt-get remove -y nsd
#systemctl reset-failed nsd #systemctl reset-failed nsd
fi fi
if ! setup/start.sh; then if ! setup/start.sh; then
echo "$F_WARN" echo "$F_WARN"
dump_file /var/log/syslog 100 dump_file /var/log/syslog 100
echo "$F_RESET" echo "$F_RESET"
die "Upstream setup failed!" die "Upstream setup failed!"
fi fi
H2 "Post-setup actions" H2 "Post-setup actions"
workaround_dovecot_sieve_bug workaround_dovecot_sieve_bug
# set actual STORAGE_ROOT, STORAGE_USER, PRIVATE_IP, etc # set actual STORAGE_ROOT, STORAGE_USER, PRIVATE_IP, etc
. /etc/mailinabox.conf || die "Could not source /etc/mailinabox.conf" . /etc/mailinabox.conf || die "Could not source /etc/mailinabox.conf"
H2 "miab install success" H2 "miab install success"
if [ "$need_pop" = "yes" ]; then if [ "$need_pop" = "yes" ]; then
if [ ! -e tests/vagrant ]; then if [ ! -e tests/lxd ]; then
# if this is an upstream install, then populate using # if this is an upstream install, then populate using
# miabldap's populate scripts (upstream doesn't have any) # miabldap's populate scripts (upstream doesn't have any)
local d local d
@ -397,7 +397,7 @@ miab_ldap_install() {
if clone_repo_and_pushd "$@"; then if clone_repo_and_pushd "$@"; then
need_pop="yes" need_pop="yes"
fi fi
H1 "MIAB-LDAP INSTALL [$(pwd)] [$(git describe 2>/dev/null)]" H1 "MIAB-LDAP INSTALL [$(pwd)] [$(git describe 2>/dev/null)]"
# ensure we're in a MiaB-LDAP working directory # ensure we're in a MiaB-LDAP working directory
if [ ! -e setup/ldap.sh ]; then if [ ! -e setup/ldap.sh ]; then
@ -423,7 +423,7 @@ miab_ldap_install() {
else else
setup/start.sh setup/start.sh
fi fi
if [ $? -ne 0 ]; then if [ $? -ne 0 ]; then
H1 "OUTPUT OF SELECT FILES" H1 "OUTPUT OF SELECT FILES"
dump_file "/var/log/syslog" 100 dump_file "/var/log/syslog" 100
@ -444,11 +444,11 @@ miab_ldap_install() {
fi fi
H2 "miab-ldap install success" H2 "miab-ldap install success"
# populate if specified on command line # populate if specified on command line
populate_by_cli_argument "$@" populate_by_cli_argument "$@"
capture_state_by_cli_argument "$@" capture_state_by_cli_argument "$@"
if [ "$need_pop" = "yes" ]; then if [ "$need_pop" = "yes" ]; then
popd >/dev/null popd >/dev/null
fi fi
@ -457,7 +457,7 @@ miab_ldap_install() {
capture_state_by_cli_argument() { capture_state_by_cli_argument() {
# this must be run with the working directory set to the source # this must be run with the working directory set to the source
# tree corresponding to the the installed state # tree corresponding to the the installed state
# ...ignore unknown options they may be interpreted elsewhere # ...ignore unknown options they may be interpreted elsewhere
local state_dir="" local state_dir=""
for arg; do for arg; do

3
tests/vagrant/.gitignore vendored
View File

@ -1,3 +0,0 @@
.vagrant
out
*-console.log

104
tests/vagrant/Vagrantfile vendored
View File

@ -1,104 +0,0 @@
#####
##### This file is part of Mail-in-a-Box-LDAP which is released under the
##### terms of the GNU Affero General Public License as published by the
##### Free Software Foundation, either version 3 of the License, or (at
##### your option) any later version. See file LICENSE or go to
##### https://github.com/downtownallday/mailinabox-ldap for full license
##### details.
#####
load './funcs.rb'
Vagrant.configure("2") do |config|
config.vm.synced_folder "../..", "/mailinabox", id: "mailinabox", automount: false
use_preloaded_box config, "ubuntu/jammy64"
# fresh install with encryption-at-rest
if ENV['tests']=='all'
config.vm.define "remote-nextcloud-docker-ehdd" do |m1|
m1.vm.provision :shell, :inline => <<-SH
cd /mailinabox
export PRIMARY_HOSTNAME=qa1.abc.com
export FEATURE_MUNIN=false
export EHDD_KEYFILE=$HOME/keyfile
echo -n "boo" >$EHDD_KEYFILE
tests/system-setup/remote-nextcloud-docker.sh || exit 1
tests/runner.sh -no-smtp-remote remote-nextcloud ehdd default || exit 2
SH
end
end
# remote-nextcloud-docker w/basic data
config.vm.define "remote-nextcloud-docker" do |m1|
m1.vm.provision :shell, :inline => <<-SH
cd /mailinabox
export PRIMARY_HOSTNAME=qa2.abc.com
export FEATURE_MUNIN=false
tests/system-setup/remote-nextcloud-docker.sh upgrade --populate=basic || exit 1
tests/runner.sh -no-smtp-remote remote-nextcloud upgrade-basic default || exit 2
SH
end
# upgrade-from-upstream
config.vm.define "upgrade-from-upstream" do |m1|
m1.vm.provision :shell, :inline => <<-SH
cd /mailinabox
export PRIMARY_HOSTNAME=qa3.abc.com
# TODO: change UPSTREAM_TAG to 'main' once upstream is installable
export UPSTREAM_TAG=v67
tests/system-setup/upgrade-from-upstream.sh --populate=basic --populate=totpuser || exit 1
tests/runner.sh -no-smtp-remote upgrade-basic upgrade-totpuser default || exit 2
SH
end
# upgrade
# this test is only needed when testing migrations from miabldap
# to a newer miabldap with a migration step
#
# upgrade will handle testing upgrades of
# miabldap with or without a new migration step
config.vm.define "upgrade" do |m1|
m1.vm.provision :shell, :inline => <<-SH
cd /mailinabox
# TODO: remove DEB_PYTHON_INSTALL_LAYOUT once MIABLDAP_RELEASE_TAG >= v66 (see https://github.com/downtownallday/mailinabox-ldap/commit/371f5bc1b236de40a1ed5d9118140ee13fddf5dc)
export DEB_PYTHON_INSTALL_LAYOUT='deb'
export PRIMARY_HOSTNAME=upgrade.abc.com
tests/system-setup/upgrade.sh --populate=basic --populate=totpuser || exit 1
tests/runner.sh -no-smtp-remote upgrade-basic upgrade-totpuser default || exit 2
SH
end
# unsetvars: because miab sets bash '-e' to fail any setup script
# when a script command returns a non-zero exit code, and more
# importantly '-u' which fails scripts when any unset variable is
# accessed, this definition sets a minimal number of environment
# variables prior to running start.sh. Doing so will test that no
# failures occur during setup in the most common use case because
# other vagrant definitions in this file load
# tests/system-setup/setup-default.sh, which pre-assign a value to
# most variables.
if ENV['tests']=='all' or ENV['tests']=='pre-commit'
config.vm.define "unsetvars" do |m1|
m1.vm.hostname = "mailinabox.lan"
m1.vm.network "private_network", ip: "192.168.56.4"
m1.vm.provision :shell, :inline => <<-SH
export NONINTERACTIVE=1
export PUBLIC_IP=auto
export PUBLIC_IPV6=auto
export PRIMARY_HOSTNAME=auto
export SKIP_NETWORK_CHECKS=1
cd /mailinabox
setup/start.sh
SH
end
end
end

25
tests/vagrant/funcs.rb
View File

@ -1,25 +0,0 @@
#####
##### This file is part of Mail-in-a-Box-LDAP which is released under the
##### terms of the GNU Affero General Public License as published by the
##### Free Software Foundation, either version 3 of the License, or (at
##### your option) any later version. See file LICENSE or go to
##### https://github.com/downtownallday/mailinabox-ldap for full license
##### details.
#####
def use_preloaded_box(obj, name, preloaded_dir=".")
obj.vm.box = String.new(name)
_name=name.sub! '/','-' # ubuntu/bionic64 => ubuntu-bionic64
if File.file?("#{preloaded_dir}/preloaded/preloaded-#{_name}.box")
# box name needs to be unique on the system
obj.vm.box = "preloaded-miabldap-#{_name}"
obj.vm.box_url = "file://" + Dir.pwd + "/#{preloaded_dir}/preloaded/preloaded-#{_name}.box"
if Vagrant.has_plugin?('vagrant-vbguest')
# do not update additions when booting this machine
obj.vbguest.auto_update = false
end
end
end
# Grab the name of the default interface
$default_network_interface = `ip route | awk '/^default/ {printf "%s", $5; exit 0}'`

86
tests/vagrant/parallel.sh
View File

@ -1,86 +0,0 @@
#!/bin/bash
#####
##### This file is part of Mail-in-a-Box-LDAP which is released under the
##### terms of the GNU Affero General Public License as published by the
##### Free Software Foundation, either version 3 of the License, or (at
##### your option) any later version. See file LICENSE or go to
##### https://github.com/downtownallday/mailinabox-ldap for full license
##### details.
#####
# Parallel provisioning for virtualbox because "The Vagrant VirtualBox
# provider does not support parallel execution at this time"
# (https://www.vagrantup.com/docs/providers/virtualbox/usage.html)
#
# Credit to:
# https://dzone.com/articles/parallel-provisioning-speeding
#
. "$(dirname "$0")/../lib/color-output.sh"
. "$(dirname "$0")/../lib/misc.sh"
if [ -z "$tests" ]; then
export tests="pre-commit"
fi
OUTPUT_DIR=out
#rm -rf "$OUTPUT_DIR"
mkdir -p "$OUTPUT_DIR"
# set total parallel vms to (#cores minus 1)
MAX_PROCS=$(cat /proc/cpuinfo | grep processor | wc -l)
let MAX_PROCS-=1
parallel_provision() {
while read box; do
outfile="$OUTPUT_DIR/$box.out.txt"
rm -f "$outfile"
echo "Provisioning '$box'. Output will be in: $outfile" 1>&2
echo $box
done | xargs -P $MAX_PROCS -I"BOXNAME" \
sh -c 'vagrant provision BOXNAME >'"$OUTPUT_DIR/"'BOXNAME.out.txt 2>&1 && echo "EXITCODE: 0" >> '"$OUTPUT_DIR/"'BOXNAME.out.txt || echo "EXITCODE: $?" >>'"$OUTPUT_DIR/"'BOXNAME.out.txt'
}
## -- main -- ##
start_time="$(date +%s)"
# start boxes sequentially to avoid vbox explosions
vagrant up --no-provision
# but run provision tasks in parallel
boxes="$(vagrant status | awk '/running \(/ {print $1}')"
echo "$boxes" | parallel_provision
# output overall result - Vagrantfile script must output "EXITCODE: <num>"
H1 "Results"
rc=0
for box in $boxes; do
file="$OUTPUT_DIR"/$box.out.txt
exitcode="$(tail "$file" | grep EXITCODE: | awk '{print $NF}')"
echo -n "$box: "
if [ -z "$exitcode" ]; then
danger "NO EXITCODE!"
[ $rc -eq 0 ] && rc=2
elif [ "$exitcode" == "0" ]; then
success "SUCCESS"
else
danger "FAILURE ($exitcode)"
rc=1
fi
done
# output elapsed time
end_time="$(date +%s)"
echo ""
echo "Elapsed time: $(elapsed_pretty $start_time $end_time)"
# exit
echo ""
echo "Guest VMs are running! Destroy them with 'vagrant destroy -f'"
exit $rc

2
tests/vagrant/preloaded/.gitignore vendored
View File

@ -1,2 +0,0 @@
*.box
src/

58
tests/vagrant/preloaded/Vagrantfile vendored
View File

@ -1,58 +0,0 @@
#####
##### This file is part of Mail-in-a-Box-LDAP which is released under the
##### terms of the GNU Affero General Public License as published by the
##### Free Software Foundation, either version 3 of the License, or (at
##### your option) any later version. See file LICENSE or go to
##### https://github.com/downtownallday/mailinabox-ldap for full license
##### details.
#####
def checkout_tag_and_mount(obj, tag)
if "#{tag}" == ""
obj.vm.synced_folder "../../..", "/mailinabox", id: "mailinabox", automount: false
else
_srcdir="src/maibldap-#{tag}"
if not Dir.exist?(_srcdir)
puts "Cloning tag #{tag} to #{_srcdir}"
if tag.size==40 and tag.match?(/\A[0-9a-fA-F]+\Z/)
system("git clone #{ENV['MIABLDAP_GIT']} #{_srcdir}")
system("cd #{_srcdir}; git reset --hard #{tag}")
else
system("git clone -b #{tag} --depth 1 #{ENV['MIABLDAP_GIT']} #{_srcdir}")
end
end
obj.vm.synced_folder _srcdir, "/mailinabox", id: "mailinabox", automount: false
end
end
Vagrant.configure("2") do |config|
checkout_tag_and_mount config, ENV['RELEASE_TAG']
config.vm.define "preloaded-ubuntu-bionic64" do |m1|
m1.vm.box = "ubuntu/bionic64"
m1.vm.provision :shell, :inline => <<-SH
cd /mailinabox
tests/vagrant/preloaded/prepvm.sh --no-dry-run
rc=$?
echo "$rc" > "/vagrant/prepcode.txt"
[ $rc -gt 0 ] && exit 1
exit 0
SH
end
config.vm.define "preloaded-ubuntu-jammy64" do |m1|
m1.vm.box = "ubuntu/jammy64"
m1.vm.boot_timeout = 30
m1.vm.provision :shell, :inline => <<-SH
cd /mailinabox
tests/vagrant/preloaded/prepvm.sh --no-dry-run
rc=$?
echo "$rc" > "/vagrant/prepcode.txt"
[ $rc -gt 0 ] && exit 1
exit 0
SH
end
end

134
tests/vagrant/preloaded/create_preloaded.sh
View File

@ -1,134 +0,0 @@
#!/bin/bash
#####
##### This file is part of Mail-in-a-Box-LDAP which is released under the
##### terms of the GNU Affero General Public License as published by the
##### Free Software Foundation, either version 3 of the License, or (at
##### your option) any later version. See file LICENSE or go to
##### https://github.com/downtownallday/mailinabox-ldap for full license
##### details.
#####
# load defaults for MIABLDAP_GIT and MIABLDAP_FINAL_RELEASE_TAG_BIONIC64 (make available to Vagrantfile)
pushd "../../.." >/dev/null
source tests/lib/color-output.sh
source tests/system-setup/setup-defaults.sh || exit 1
popd >/dev/null
H1 "Destroy any running boxes"
vagrant destroy -f
rm -f prepcode.txt
H1 "Ensure plugins are installed"
for plugin in "vagrant-vbguest" "vagrant-reload"
do
if ! vagrant plugin list | grep -F "$plugin" >/dev/null; then
vagrant plugin install "$plugin" || exit 1
fi
done
H1 "Upgrade base boxes"
vagrant box update
boxes=(
"preloaded-ubuntu-jammy64"
"preloaded-ubuntu-bionic64"
)
# preload packages from source of the following git tags. empty string
# means use the current source tree
tags=(
""
"$MIABLDAP_FINAL_RELEASE_TAG_BIONIC64"
)
try_reboot=(
true
false
)
idx=0
for box in "${boxes[@]}"
do
if [ -z "$1" ]; then
# no cli arguments - only process first box
[ $idx -ge 1 ] && break
else
# cli argument specifies "all" or a named box
if [ "$1" != "all" -a "$1" != "$box" ]; then
let idx+=1
continue
fi
fi
H1 "Provision: $box"
export RELEASE_TAG="${tags[$idx]}"
vagrant up $box | tee /tmp/$box.out
upcode=$?
if [ $upcode -eq 0 -a ! -e "./prepcode.txt" ] && ${try_reboot[$idx]} && grep -F 'Authentication failure' /tmp/$box.out >/dev/null; then
# note: upcode is 0 only if config.vm.boot_timeout is set.
# If this works it may be an indication that ruby's internal
# ssh does not support the algorithm required by the server,
# or the public key does not match (vagrant and vm out of
# sync)
echo ""
echo "VAGRANT AUTHENTICATION FAILURE - TRYING LOOSER ALLOWED SSHD ALGS"
if vagrant ssh $box -c "sudo bash -c 'echo PubkeyAcceptedAlgorithms +ssh-rsa > /etc/ssh/sshd_config.d/miabldap.conf; sudo systemctl restart sshd'"; then
vagrant halt $box
vagrant up $box
upcode=$?
fi
fi
if [ $upcode -ne 0 -a ! -e "./prepcode.txt" ] && ${try_reboot[$idx]}
then
# a reboot may be necessary if guest addtions was newly
# compiled by vagrant plugin "vagrant-vbguest"
echo ""
echo "VAGRANT UP RETURNED $upcode -- RETRYING AFTER REBOOT"
vagrant halt $box
vagrant up $box
upcode=$?
fi
rm -f /tmp/$box.out
let idx+=1
prepcode=$(cat "./prepcode.txt")
rm -f prepcode.txt
echo ""
echo "VAGRANT UP RETURNED $upcode"
echo "PREPVM RETURNED $prepcode"
if [ "$prepcode" != "0" -o $upcode -ne 0 ]; then
echo "FAILED!!!!!!!!"
vagrant destroy -f $box
exit 1
fi
if vagrant ssh $box -- cat /var/run/reboot-required >/dev/null 2>&1; then
echo "REBOOT REQUIRED"
vagrant reload $box
else
echo "REBOOT NOT REQUIRED"
fi
vagrant halt $box
vagrant package $box
rm -f $box.box
mv package.box $box.box
vagrant destroy -f $box
cached_name="$(sed 's/preloaded-/preloaded-miabldap-/' <<<"$box")"
echo "Removing cached box $cached_name"
if [ -e "../funcs.rb" ]; then
pushd .. > /dev/null
vagrant box remove $cached_name
code=$?
popd > /dev/null
else
vagrant box remove $cached_name
code=$?
fi
echo "Remove cache box result: $code - ignoring"
done

2
tests/vagrant/vanilla/.gitignore vendored
View File

@ -1,2 +0,0 @@
.vagrant
*-console.log

87
tests/vagrant/vanilla/Vagrantfile vendored
View File

@ -1,87 +0,0 @@
#####
##### This file is part of Mail-in-a-Box-LDAP which is released under the
##### terms of the GNU Affero General Public License as published by the
##### Free Software Foundation, either version 3 of the License, or (at
##### your option) any later version. See file LICENSE or go to
##### https://github.com/downtownallday/mailinabox-ldap for full license
##### details.
#####
load '../funcs.rb'
Vagrant.configure("2") do |config|
config.vm.synced_folder "../../..", "/mailinabox", id: "mailinabox", automount: false
config.vm.network "public_network", bridge: "#$default_network_interface"
use_preloaded_box config, "ubuntu/jammy64", ".."
if ENV['tests']=='ciab'
# vanilla connected to ciab (ciab does not need to be up)
config.vm.define "vanilla" do |m1|
m1.vm.provision :shell, :inline => <<-SH
cat >/tmp/provision.sh <<EOF
#!/bin/bash
if [ \\$EUID -ne 0 ]; then
echo "Must be root"
exit 1
fi
cd /mailinabox
export PRIMARY_HOSTNAME=vanilla.local
export NC_PROTO=https
export NC_HOST=vanilla-ciab.local
export NC_PORT=443
export NC_PREFIX=/
export SKIP_SYSTEM_UPDATE=0
tests/system-setup/vanilla.sh --qa-ca --enable-mod=remote-nextcloud
if ! ufw status | grep remote_nextcloud >/dev/null; then
# firewall rules aren't added when ciab is down
echo "For testing, allow ldaps from anywhere"
ufw allow ldaps
fi
echo "Add smart host alias - so \\$NC_HOST can send mail to/via this host"
(
source tests/lib/all.sh
rest_urlencoded POST /admin/mail/aliases/add qa@abc.com Test_1234 "address=@\\$NC_HOST" "description=smart-host" "permitted_senders=qa@abc.com" 2>/dev/null
echo "\\$REST_HTTP_CODE: \\$REST_OUTPUT"
)
EOF
chmod +x /tmp/provision.sh
/tmp/provision.sh
SH
end # vanilla connected to ciab
else
# vanilla (default) install
config.vm.define "vanilla" do |m1|
m1.vm.provision :shell, :inline => <<-SH
cat >/tmp/provision.sh <<EOF
#!/bin/bash
if [ \\$EUID -ne 0 ]; then
echo "Must be root"
exit 1
fi
start=\\$(date +%s)
cd /mailinabox
export PRIMARY_HOSTNAME=vanilla.local
#export FEATURE_MUNIN=false
#export FEATURE_NEXTCLOUD=false
export SKIP_SYSTEM_UPDATE=0
tests/system-setup/vanilla.sh
# --enable-mod=roundcube-master \
# --enable-mod=roundcube-debug \
# --enable-mod=rcmcarddav-composer
end=\\$(date +%s)
echo "Provisioning took \\$(source tests/lib/misc.sh; elapsed_pretty \\$start \\$end)"
EOF
chmod +x /tmp/provision.sh
/tmp/provision.sh
SH
end # vanilla (default)
end
end