From a79a6c00eb252de8c2581744894c8173a34b2f92 Mon Sep 17 00:00:00 2001 From: downtownallday Date: Mon, 2 Sep 2024 16:11:13 -0400 Subject: [PATCH 01/14] encryption-at-rest: Ensure required kernel modules are installed --- ehdd/create_hdd.sh | 1 + ehdd/ehdd_funcs.sh | 8 ++++++++ ehdd/mount.sh | 1 + 3 files changed, 10 insertions(+) diff --git a/ehdd/create_hdd.sh b/ehdd/create_hdd.sh index 5ed43988..dd327dd0 100755 --- a/ehdd/create_hdd.sh +++ b/ehdd/create_hdd.sh @@ -44,6 +44,7 @@ if [ ! -e "$EHDD_IMG" ]; then echo "" echo "NOTE: You will need to reenter your drive encryption password" fi + assert_kernel_modules cryptsetup luksOpen $(keyfile_option) $loop $EHDD_LUKS_NAME # map device to /dev/mapper/NAME mke2fs -j /dev/mapper/$EHDD_LUKS_NAME # sleep a brief time to avoid "device busy" diff --git a/ehdd/ehdd_funcs.sh b/ehdd/ehdd_funcs.sh index 5593b781..ea605c1d 100644 --- a/ehdd/ehdd_funcs.sh +++ b/ehdd/ehdd_funcs.sh @@ -17,10 +17,18 @@ if [ -z "${STORAGE_ROOT:-}" ]; then fi fi + EHDD_IMG="$STORAGE_ROOT.HDD" EHDD_MOUNTPOINT="$STORAGE_ROOT" EHDD_LUKS_NAME="c1" +assert_kernel_modules() { + local check="$(lsmod | awk '$1=="dm_crypt" {print "yes"}')" + if [ "$check" != "yes" ]; then + echo "Required kernel modules for encryption-at-rest are not loaded. Cannot continue" + exit 1 + fi +} find_unused_loop() { losetup -f diff --git a/ehdd/mount.sh b/ehdd/mount.sh index 5ff44270..f3c59c49 100755 --- a/ehdd/mount.sh +++ b/ehdd/mount.sh @@ -21,6 +21,7 @@ if is_mounted; then exit 0 fi +assert_kernel_modules loop=$(find_unused_loop) losetup $loop "$EHDD_IMG" || exit 1 # map device to /dev/mapper/NAME From a1d6f6713578097b1b68bb0cea80f6327a2c3577 Mon Sep 17 00:00:00 2001 From: downtownallday Date: Mon, 2 Sep 2024 17:09:59 -0400 Subject: [PATCH 02/14] change from vagrant to lxd as the virtualization system --- tests/assets/.gitignore | 2 + tests/bin/lx_functions.sh | 234 ++++++++++++++++ tests/bin/lx_setup.sh | 209 ++++++++++++++ tests/bin/lx_status.sh | 58 ++++ tests/bin/provision_functions.sh | 91 ++++++ tests/bin/vlx | 258 ++++++++++++++++++ tests/lxd/.gitignore | 1 + tests/lxd/.provision_defaults | 10 + tests/lxd/README.md | 18 ++ tests/lxd/parallel-boxlist.all | 8 + tests/lxd/parallel-boxlist.default | 3 + tests/lxd/parallel.sh | 119 ++++++++ tests/lxd/preloaded/create_preloaded.sh | 97 +++++++ tests/lxd/preloaded/imagelist | 1 + tests/lxd/preloaded/prepvm.sh | 230 ++++++++++++++++ .../remote-nextcloud-docker-ehdd/provision.sh | 35 +++ .../lxd/remote-nextcloud-docker/provision.sh | 33 +++ tests/lxd/unsetvars/provision.sh | 35 +++ tests/lxd/upgrade-from-upstream/provision.sh | 33 +++ tests/lxd/upgrade/provision.sh | 32 +++ tests/lxd/vanilla/provision.sh | 65 +++++ tests/system-setup/setup-defaults.sh | 4 +- tests/system-setup/setup-funcs.sh | 34 +-- 23 files changed, 1591 insertions(+), 19 deletions(-) create mode 100644 tests/bin/lx_functions.sh create mode 100755 tests/bin/lx_setup.sh create mode 100755 tests/bin/lx_status.sh create mode 100644 tests/bin/provision_functions.sh create mode 100755 tests/bin/vlx create mode 100644 tests/lxd/.gitignore create mode 100644 tests/lxd/.provision_defaults create mode 100644 tests/lxd/README.md create mode 100644 tests/lxd/parallel-boxlist.all create mode 100644 tests/lxd/parallel-boxlist.default create mode 100755 tests/lxd/parallel.sh create mode 100755 tests/lxd/preloaded/create_preloaded.sh create mode 100644 tests/lxd/preloaded/imagelist create mode 100755 tests/lxd/preloaded/prepvm.sh create mode 100755 tests/lxd/remote-nextcloud-docker-ehdd/provision.sh create mode 100755 tests/lxd/remote-nextcloud-docker/provision.sh create mode 100755 tests/lxd/unsetvars/provision.sh create mode 100755 tests/lxd/upgrade-from-upstream/provision.sh create mode 100755 tests/lxd/upgrade/provision.sh create mode 100755 tests/lxd/vanilla/provision.sh diff --git a/tests/assets/.gitignore b/tests/assets/.gitignore index a47e57a3..57cfb4e3 100644 --- a/tests/assets/.gitignore +++ b/tests/assets/.gitignore @@ -1 +1,3 @@ .emacs +backup/ +vm_keys/ diff --git a/tests/bin/lx_functions.sh b/tests/bin/lx_functions.sh new file mode 100644 index 00000000..4b3890b1 --- /dev/null +++ b/tests/bin/lx_functions.sh @@ -0,0 +1,234 @@ +#!/bin/bash +##### +##### This file is part of Mail-in-a-Box-LDAP which is released under the +##### terms of the GNU Affero General Public License as published by the +##### Free Software Foundation, either version 3 of the License, or (at +##### your option) any later version. See file LICENSE or go to +##### https://github.com/downtownallday/mailinabox-ldap for full license +##### details. +##### + + +# source this file + +lx_project_root_dir() { + local d="$PWD" + while [ ${#d} -gt 1 ]; do + if [ -e "$d/setup" ]; then + echo "$d" + return 0 + fi + d="$(dirname "$d")" + done + return 1 +} + +lx_guess_project_name() { + local d + d="$(lx_project_root_dir)" + [ $? -ne 0 ] && return 1 + if [ -e "$d/setup/redis.sh" ]; then + echo "ciab" + else + echo "miab" + fi +} + + +# get the interface with the default route (first one) +get_system_default_network_interface() { + ip route | awk '/^default/ {printf "%s", $5; exit 0}' +} + +isa_bridge_interface() { + local interface="$1" + if ip --oneline link show type bridge | awk -F: '{print $2}' | grep -q "^ *$interface *\$"; then + echo "yes" + else + echo "no" + fi +} + +isa_wifi_device() { + local interface="$1" + local type + type="$(nmcli d show "$interface" | awk '$1=="GENERAL.TYPE:" {print $2}')" + if [ "$type" = "wifi" ]; then + echo "yes" + else + echo "no" + fi +} + +# delete an instance +lx_delete() { + local project="${1:-default}" + local inst="$2" + local interactive="${3:-interactive}" + + local xargs="" + if [ "$interactive" = "interactive" ]; then + xargs="-i" + fi + + # only delete instance if the instance exists + if [ "$(lxc --project "$project" list name="$inst" -c n -f csv)" = "$inst" ]; then + lxc --project "$project" delete "$inst" -f $xargs + fi +} + + +# create a virtual-machine instance and start it +lx_launch_vm() { + local project="${1:-default}" + local inst_name="$2" + lx_init_vm "$@" || return 1 + lxc --project "$project" start "$inst_name" || return 1 +} + + +# create a virtual-machine instance (stopped) +lx_init_vm() { + local project="${1:-default}" + local inst_name="$2" + local image="$3" + local mount_host="$4" # path that you want available in guest + local mount_guest="$5" # mountpoint in guest + shift; shift; shift; shift; shift; + + # 1. this assumes that a bridge profile has been created called "bridgenet" + # 2. this assumes that storage named the same as project exists, + # e.g. "lxc storage create $project dir" was executed prior. + + case "${@}" in + *bridgenet* ) + echo "Using network 'bridgenet'" + lxc --project "$project" profile show bridgenet | sed 's/^/ /' || return 1 + ;; + esac + + lxc --project "$project" init "$image" "$inst_name" --vm --storage "$project" "$@" || return 1 + + if [ ! -z "$mount_host" -a ! -z "$mount_guest" ]; then + echo "adding $mount_guest on $inst_name to refer to $mount_host on host as device '${project}root'" + if [ $EUID -ne 0 ]; then + # so that files created by root on the mount inside the + # guest have the permissions of the current host user and + # not root:root + # see: https://documentation.ubuntu.com/lxd/en/latest/userns-idmap/ + local egid="$(id --group)" + local idmap="uid $EUID 0 +gid $egid 0" + lxc --project "$project" config set "$inst_name" raw.idmap="$idmap" + fi + lxc --project "$project" config device add "$inst_name" "${project}root" disk source="$(realpath "$mount_host")" path="$mount_guest" || return 1 + fi +} + +lx_launch_vm_and_wait() { + local project="$1" + local inst="$2" + local base_image="$3" + local mount_project_root_to="$4" + shift; shift; shift; shift; + + # Delete existing instance, if it exists + lx_delete "$project" "$inst" interactive || return 1 + + # Create the instance (started) + lx_launch_vm "$project" "$inst" "$base_image" "$(lx_project_root_dir)" "$mount_project_root_to" "$@" || return 1 + + lx_wait_for_boot "$project" "$inst" || return 1 +} + + +lx_output_inst_list() { +# Pre-defined column shorthand chars: +# 4 - IPv4 address +# 6 - IPv6 address +# a - Architecture +# b - Storage pool +# c - Creation date +# d - Description +# D - disk usage +# e - Project name +# l - Last used date +# m - Memory usage +# M - Memory usage (%) +# n - Name +# N - Number of Processes +# p - PID of the instance's init process +# P - Profiles +# s - State +# S - Number of snapshots +# t - Type (persistent or ephemeral) +# u - CPU usage (in seconds) +# L - Location of the instance (e.g. its cluster member) +# f - Base Image Fingerprint (short) +# F - Base Image Fingerprint (long) + local project="$1" + local columns="${2:-ns46tSL}" + local format="${3:-table}" # csv|json|table|yaml|compact + lxc --project "$project" list -c "$columns" -f "$format" +} + +lx_output_image_list() { +# Column shorthand chars: +# l - Shortest image alias (and optionally number of other aliases) +# L - Newline-separated list of all image aliases +# f - Fingerprint (short) +# F - Fingerprint (long) +# p - Whether image is public +# d - Description +# a - Architecture +# s - Size +# u - Upload date +# t - Type + local project="$1" + local columns="${2:-lfpdatsu}" + local format="${3:-table}" # csv|json|table|yaml|compact + lxc --project "$project" image list -c "$columns" -f "$format" +} + +lx_wait_for_boot() { + local project="$1" + local inst="$2" + + echo -n "Wait for boot " + while ! lxc --project "$project" exec "$inst" -- ls >/dev/null 2>&1; do + echo -n "." + sleep 1 + done + echo "" + echo -n "Wait for cloud-init " + lxc --project "$project" exec "$inst" -- cloud-init status --wait + echo "" +} + +lx_get_ssh_identity() { + local keydir="tests/assets/vm_keys" + if [ "$1" != "relative" ]; then + keydir="$(lx_project_root_dir)/$keydir" + fi + echo "$keydir/id_ed25519" +} + +lx_get_ssh_known_hosts() { + local id="$(lx_get_ssh_identity)" + local known_hosts="$(dirname "$id")/known_hosts" + echo "$known_hosts" +} + +lx_remove_known_host() { + local hostname="$1" + local known_hosts="$(lx_get_ssh_known_hosts)" + ssh-keygen -f "$known_hosts" -R "$hostname" +} + +lx_create_ssh_identity() { + local id="$(lx_get_ssh_identity)" + if [ ! -e "$id" ]; then + mkdir -p "$(dirname "$id")" + ssh-keygen -f "$id" -C "vm key" -N "" -t ed25519 + fi +} diff --git a/tests/bin/lx_setup.sh b/tests/bin/lx_setup.sh new file mode 100755 index 00000000..8e080299 --- /dev/null +++ b/tests/bin/lx_setup.sh @@ -0,0 +1,209 @@ +#!/bin/bash +##### +##### This file is part of Mail-in-a-Box-LDAP which is released under the +##### terms of the GNU Affero General Public License as published by the +##### Free Software Foundation, either version 3 of the License, or (at +##### your option) any later version. See file LICENSE or go to +##### https://github.com/downtownallday/mailinabox-ldap for full license +##### details. +##### + + +# +# This script configures the system for virtulization by installing +# lxd, adding a host bridge, and configuring some firewall +# rules. Beware that the user running the script will be added to the +# 'sudo' group! +# +# The script can be run anytime, but usually only needs to be run +# once. Once successful, the lxd-based test vms can be run locally. +# +# Limitations: +# +# The host bridge only works with an ethernet interface. Wifi does not +# work, so you must be plugged in to run the vm tests. +# +# Docker cannot be installed alongside as LXD due to conflicts with +# Docker's iptables entries. More on this issue can be found here: +# +# https://github.com/docker/for-linux/issues/103 +# +# https://documentation.ubuntu.com/lxd/en/latest/howto/network_bridge_firewalld/#prevent-connectivity-issues-with-lxd-and-docker +# +# Removal: +# +# Run the script with a single "-d" argument to back out the changes. +# +# Helpful tools: +# +# NetworkManager UI: nm-connection-editor +# NetworkManager cli: nmcli +# + +D=$(dirname "$BASH_SOURCE") +. "$D/lx_functions.sh" || exit 1 + +project="$(lx_guess_project_name)" +bridge_yaml="/etc/netplan/51-lxd-bridge.yaml" + + +install_packages() { + sudo snap install lxd --channel=latest/stable + if [ $EUID -ne 0 ]; then + echo "Add $USER to the 'sudo' group (!!)" + sudo usermod -aG sudo $USER || exit 1 + fi +} + +remove_packages() { + # note: run 'snap remove --purge lxd' to nuke lxd images, + # otherwise they stay on the system + snap remove lxd +} + +create_network_bridge() { + # Create network bridge (we'll bridge vms to the local network) + # On return, sets these variables: + # vm_bridge: to name of bridge interface + + # get the interface with the default route (first one) + local default_network_interface="$(get_system_default_network_interface)" + local isa_bridge="$(isa_bridge_interface "$default_network_interface")" + local isa_wifi_device="$(isa_wifi_device "$default_network_interface")" + + if [ "$isa_bridge" = "yes" ]; then + vm_bridge="$default_network_interface" + #out_iface="$(ip --oneline link show type bridge_slave | grep -F " $default_network_interface " | awk -F: '{print $2}')" + #out_iface="$default_network_interface" + + else + echo "NO HOST BRIDGE FOUND!!! CREATING ONE." + + if [ "$isa_wifi_device" = "yes" ]; then + echo "*********************************************************" + echo "UNSUPPORTED: Host bridging is not available with WIFI ! (interface $default_network_interface)" + echo "*********************************************************" + return 1 + fi + + echo "YOU WILL LOSE NETWORK CONNECTIVITY BRIEFLY" + echo "To remove the host bridge, delete $bridge_yaml, then run netplan apply, or run this script with '-u'" + vm_bridge="br-lxd0" + #out_iface="br-lxd0" + tmp="$(mktemp)" + sudo cat < "$tmp" +network: + ethernets: + myeths: + match: + name: $default_network_interface + dhcp4: no + bridges: + $vm_bridge: + dhcp4: yes + interfaces: [ myeths ] + +EOF + if [ -e "$bridge_yaml" ]; then + echo "Overwriting netplan $bridge_yaml" + else + echo "Adding netplan $bridge_yaml" + fi + sudo mv "$tmp" "$bridge_yaml" || return 1 + sudo chown root:root "$bridge_yaml" + sudo chmod 600 "$bridge_yaml" + sudo netplan apply || return 1 + fi +} + + +remove_network_bridge() { + if [ -e "$bridge_yaml" ]; then + sudo rm -f "$bridge_yaml" || return 1 + sudo netplan apply || return 1 + else + echo "OK: Bridge configuration does not exist ($bridge_yaml)" + fi +} + +install_ufw_rules() { + local bridge="$1" + # per: https://documentation.ubuntu.com/lxd/en/latest/howto/network_bridge_firewalld/ + sudo ufw allow in on "$bridge" comment 'for lxd' + sudo ufw route allow in on "$bridge" comment 'for lxd' + sudo ufw route allow out on "$bridge" comment 'for lxd' + if which docker >/dev/null; then + echo "WARNING: docker appears to be installed. Guest VM networking probably won't work as the docker iptables conflict with LXD (guest DHCP broadcasts are dropped)" + fi +} + +remove_ufw_rules() { + # 2. Remove all old ufw rules + sudo ufw status numbered | grep -E '(for lxd|for multipass)' | cut -c2-3 | sort -nr | + while read n; do + echo 'y' | sudo ufw delete $n + done +} + +create_lxd_project_and_pool() { + local bridge="$1" + # Create project and networking bridge profile. When an instance + # is initialized, include the 'bridgenet' profile (see + # lx_init_vm() in lx_functions.sh) + + echo "Create lxd project '$project' with profile 'bridgenet'" + echo " bridge: $bridge" + lxc project create "$project" 2>/dev/null + lxc --project "$project" profile create bridgenet 2>/dev/null + cat <"$tmp" + cat >>"$tmp" + lxc --project "$project" file push "$tmp" "${inst}${remote_path}" $lxc_flags || return 1 + rm -f "$tmp" + fi + + lxc --project "$project" exec "$inst" --cwd / --env PROVISION=true \ + -- "$remote_path" +} + + +provision_done() { + local rc="${1:-0}" + echo "Elapsed: $(elapsed_pretty "$provision_start_s" "$(date +%s)")" + if [ $rc -ne 0 ]; then + echo "Failed with code $rc" + return 1 + else + echo "Success!" + return 0 + fi +} diff --git a/tests/bin/vlx b/tests/bin/vlx new file mode 100755 index 00000000..5746f03c --- /dev/null +++ b/tests/bin/vlx @@ -0,0 +1,258 @@ +#!/bin/bash +##### +##### This file is part of Mail-in-a-Box-LDAP which is released under the +##### terms of the GNU Affero General Public License as published by the +##### Free Software Foundation, either version 3 of the License, or (at +##### your option) any later version. See file LICENSE or go to +##### https://github.com/downtownallday/mailinabox-ldap for full license +##### details. +##### + + +# This is a helper script to make it easier to interact with lxd by +# calling lxc with arguments derived from conventions we've developed +# in this project. +# +# Those conventions are: +# +# 1. the project name is derived from the working directory. You +# must be anywhere in the source tree. "ciab" for cloudinabox and +# "miab" for mailinabox +# +# 2. the instance name is derived from the base name of the current +# working directory. this means that each instance must have it's +# own directory and have a script within it called "provision.sh" +# that creates the instance. +# +# Run the script with no arguments to see a list of commands. +# +# It's helpful to create a command alias. eg. in bash: +# +# alias vlx="/path/to/tests/bin/vlx" +# +# Add it to your ~/bash_aliases file to make it available for new +# terminal windows. +# + +D=$(dirname "$BASH_SOURCE") +. "$D/lx_functions.sh" || exit 1 + +vlx_guess() { + if [ $# -eq 2 ]; then + LX_PROJECT="$1" + LX_INST="$2" + elif [ $# -eq 1 ]; then + LX_PROJECT="$(lx_guess_project_name)" + LX_INST="$1" + elif [ $# -eq 0 ]; then + LX_PROJECT="$(lx_guess_project_name)" + LX_INST="$(basename "$PWD")" + else + echo "Invalid number of arguments" + return 1 + fi +} + + +vlx_exec_usage() { + echo "Usage: vlx exec [] -- cmd ..." + echo " or: vlx exec [] -- cmd ..." + echo " or: vlx exec -- cmd ..." + echo " or: vlx exec cmd ..." +} + +vlx_exec() { + # args + # format 1: project inst [cwd] -- cmd ... + # format 2: inst [cwd] -- cmd ... + # format 3: -- cmd ... + # format 4: cmd ... + if [ $# -eq 0 ]; then + vlx_exec_usage + return 1 + fi + + local args=( "$@" ) + local idx=0 + while [ $idx -le 3 -a $idx -lt ${#args[*]} ]; do + [ "${args[$idx]}" = "--" ] && break + let idx+=1 + done + + local wd="" + if [ "${args[$idx]}" = "--" ]; then + if [ $idx -eq 3 ]; then + # format 1 with cwd + echo "f1" + wd="$3" + vlx_guess "$1" "$2" || return 1 + shift; shift; shift; shift; + elif [ $idx -eq 2 ]; then + # format 1 w/o cwd or 2 + if [ "${2#/}" != "$2" ]; then + # wd starts with /, so it's a path + # format 2 + echo "f2" + wd="$2" + vlx_guess "" "$1" || return 1 + else + # format 1 w/o cwd + echo "f1 w/o cwd" + vlx_guess "$1" "$2" || return 1 + fi + shift; shift; shift; + elif [ $idx -eq 1 ]; then + # format 2 w/o cwd or 3 + if [ "${1#/}" != "$1" ]; then + # wd starts with /, so it's a path + # format 3 + echo "f3" + wd="$1" + vlx_guess || return 1 + else + # format 2 w/o cwd + echo "f2 w/o cwd" + vlx_guess "$1" || return 1 + fi + shift; shift; + elif [ $idx -eq 0 ]; then + # command line "-- cmd ...", just ignore the leading -- + shift; + fi + else + # format 4 + echo "f4" + vlx_guess || return 1 + fi + + local xargs="" + if [ ! -z "$wd" ]; then + xargs="--cwd $wd" + fi + echo lxc --project "$LX_PROJECT" exec "$LX_INST" $xargs -- "$@" + lxc --project "$LX_PROJECT" exec "$LX_INST" $xargs -- "$@" +} + +vlx_shell() { + vlx_guess "$@" || return 1 + echo lxc --project "$LX_PROJECT" exec "$LX_INST" -- bash + lxc --project "$LX_PROJECT" exec "$LX_INST" -- bash +} + +vlx_hostname() { + vlx_guess "$@" || return 1 + local host + lxc --project "$LX_PROJECT" exec "$LX_INST" -- /usr/bin/hostname --fqdn || return 1 +} + +vlx_ssh() { + local host="$1" + if [ -z "$host" ]; then + host="$(vlx_hostname)" + if [ $? -ne 0 ]; then + echo "Could not determine hostname, please specify" + host="" + fi + if [ -z "$host" ]; then + echo "usage: vlx ssh " + return 1 + fi + fi + local id="$(lx_get_ssh_identity)" + local known_hosts="$(lx_get_ssh_known_hosts)" + local vmuser="vmuser" + #echo ssh -i "$id" -o UserKnownHostsFile="$known_hosts" -o StrictHostKeyChecking=no "$vmuser@$host" + echo "Connecting to $vmuser@$host ..." + ssh -i "$id" -o UserKnownHostsFile="$known_hosts" -o StrictHostKeyChecking=no "$vmuser@$host" +} + +vlx_list() { + vlx_guess "$1" || return 1 + echo lxc --project "$LX_PROJECT" list + lxc --project "$LX_PROJECT" list +} + +vlx_images() { + vlx_guess "$1" || return 1 + echo lxc --project "$LX_PROJECT" image list + lxc --project "$LX_PROJECT" image list +} + +vlx_up() { + if [ -x "./provision.sh" ] ; then + echo "Provision" + ./provision.sh "$@" || return 1 + else + echo "UP failed: ./provision.sh does not exist or is not executable" + return 1 + fi +} + + +vlx_start() { + vlx_guess "$@" || return 1 + echo lxc --project "$LX_PROJECT" start "$LX_INST" + lxc --project "$LX_PROJECT" start "$LX_INST" +} + +vlx_stop() { + vlx_guess "$@" || return 1 + echo lxc --project "$LX_PROJECT" stop "$LX_INST" + lxc --project "$LX_PROJECT" stop "$LX_INST" +} + +vlx_delete() { + vlx_guess "$@" || return 1 + echo lxc --project "$LX_PROJECT" delete --force --interactive "$LX_INST" + lxc --project "$LX_PROJECT" delete --force --interactive "$LX_INST" +} + +vlx_destroy() { + vlx_delete "$@" +} + +vlx_status() { + if [ $# -eq 0 ]; then + vlx_guess || return 1 + "$D/lx_status.sh" "$LX_PROJECT" + elif [ "$1" = "-g" ]; then + "$D/lx_status.sh" + else + "$D/lx_status.sh" "$@" + fi +} + + +usage() { + echo "Usage:" + echo "vlx [ ...]" + echo "commands:" + echo " exec -- command [arg ...]" + echo " exec command [arg ...]" + echo " shell" + echo " ssh" + echo " hostname" + echo " list" + echo " images" + echo " up" + echo " start" + echo " stop" + echo " delete|destroy" + echo " status" +} + +if [ $# -eq 0 ]; then + usage + exit 1 +fi + +cmd="$1" +handler="vlx_$1" +shift +if [ ! "$(type -t $handler)" = "function" ]; then + echo "Unknown command: $cmd" + exit 1 +fi + +$handler "$@" + diff --git a/tests/lxd/.gitignore b/tests/lxd/.gitignore new file mode 100644 index 00000000..89f9ac04 --- /dev/null +++ b/tests/lxd/.gitignore @@ -0,0 +1 @@ +out/ diff --git a/tests/lxd/.provision_defaults b/tests/lxd/.provision_defaults new file mode 100644 index 00000000..9c47a082 --- /dev/null +++ b/tests/lxd/.provision_defaults @@ -0,0 +1,10 @@ +# defaults for lxd image creation using +# tests/bin/provision_functions.sh::provision_start() + +DEFAULT_LXD_IMAGE="preloaded-ubuntu-jammy" +DEFAULT_LXD_INST_OPTS=( + "-p" "default" + "-p" "bridgenet" + "-c" "limits.cpu=1" + "-c" "limits.memory=2GiB" +) diff --git a/tests/lxd/README.md b/tests/lxd/README.md new file mode 100644 index 00000000..b4a6d9bc --- /dev/null +++ b/tests/lxd/README.md @@ -0,0 +1,18 @@ +### To use lxd vm's: + +1. run `tests/bin/lx_setup.sh`. This only needs to be run once. + +2. run `tests/lxd/preloaded/create_preloaded.sh`. Run this anytime a new base image is updated (usually when ubuntu updates would require a system reboot). + + +### To bring up a vm: + +1. It's helpful to have `vlx` in your path. vlx is a tool that makes `lxc` act a little like vagrant. In bash, create an alias for it: `alias vlx="$(pwd)/tests/bin/vlx" + +2. set your working directory to the vm directory you'd like to start (eg. `cd "tests/lxd/vanilla"`), then run `vlx up` + +3. to access the vm: `vlx shell` or `vlx ssh`. all vm's have the source root mounted at /cloudinabox or /mailinabox, so you can change files locally and they'll be available on the vm for testing + +4. to destroy/delete the vm: `vlx destroy` + + diff --git a/tests/lxd/parallel-boxlist.all b/tests/lxd/parallel-boxlist.all new file mode 100644 index 00000000..1e41b6ed --- /dev/null +++ b/tests/lxd/parallel-boxlist.all @@ -0,0 +1,8 @@ +# jammy does not include the dm_crypt kernel module, which is needed +# for luks/encryption-at-rest +# remote-nextcloud-docker-ehdd + +remote-nextcloud-docker +upgrade-from-upstream +upgrade +unsetvars diff --git a/tests/lxd/parallel-boxlist.default b/tests/lxd/parallel-boxlist.default new file mode 100644 index 00000000..4c1af29e --- /dev/null +++ b/tests/lxd/parallel-boxlist.default @@ -0,0 +1,3 @@ +remote-nextcloud-docker +upgrade-from-upstream +upgrade diff --git a/tests/lxd/parallel.sh b/tests/lxd/parallel.sh new file mode 100755 index 00000000..8447f312 --- /dev/null +++ b/tests/lxd/parallel.sh @@ -0,0 +1,119 @@ +#!/bin/bash +##### +##### This file is part of Mail-in-a-Box-LDAP which is released under the +##### terms of the GNU Affero General Public License as published by the +##### Free Software Foundation, either version 3 of the License, or (at +##### your option) any later version. See file LICENSE or go to +##### https://github.com/downtownallday/mailinabox-ldap for full license +##### details. +##### + + +# +# Parallel provisioning for test vms +# + +. "$(dirname "$0")/../bin/lx_functions.sh" +. "$(dirname "$0")/../lib/color-output.sh" +. "$(dirname "$0")/../lib/misc.sh" + +boxlist="" # the name of the boxlist or a path to the boxlist file +boxes=() # the contents of the boxlist file +project="$(lx_guess_project_name)" + +load_boxlist() { + # sets global variable 'boxlist' and array 'boxes' + boxlist="${1:-default}" + local fn="$boxlist" + if [ ! -f "$fn" ]; then + fn="parallel-boxlist.$boxlist" + fi + if [ ! -f "$fn" ]; then + echo "Could not load boxlist from '${boxlist}'! Failed to find '$fn'." + exit 1 + fi + boxes=( $(grep -v '^#' $fn) ) + if [ $? -ne 0 ]; then + echo "Could not load boxlist from file '$fn'!" + exit 1 + fi +} + +# destroy running boxes +if [ "$1" = "-d" ]; then + shift + load_boxlist "$1" + for inst in $(lx_output_inst_list "$project" "n" "csv"); do + if array_contains $inst ${boxes[*]}; then + echo lxc --project "$project" delete $inst --force + lxc --project "$project" delete $inst --force + fi + done + exit 0 +elif [ "$1" = "-h" -o "$1" = "--help" ]; then + echo "usage: $0 [-d] [boxlist]" + echo " -d delete/destroy running boxes" + echo " boxlist an file or named boxlist containing a list of instance names. defaults to 'default'" + exit 0 +fi + +load_boxlist "$1" + +# set total parallel vms to (#cores minus 1) +MAX_PROCS=$(cat /proc/cpuinfo | grep processor | wc -l) +let MAX_PROCS-=1 +[ $MAX_PROCS -eq 0 ] && MAX_PROCS=1 + +OUTPUT_DIR=out +#rm -rf "$OUTPUT_DIR" +mkdir -p "$OUTPUT_DIR" + +echo "MAX_PROCS=$MAX_PROCS" +echo "OUTPUT_DIR=$OUTPUT_DIR" + +start_time="$(date +%s)" + +# bring up in parallel +for inst in ${boxes[*]}; do + outfile="$OUTPUT_DIR/$inst.out.txt" + rm -f "$outfile" + echo "Bringing up '$inst'. Output will be in: $outfile" 1>&2 + echo $inst +done | xargs -P $MAX_PROCS -I"INSTNAME" \ + sh -c ' +cd "INSTNAME" && +./provision.sh >'"../$OUTPUT_DIR/"'INSTNAME.out.txt 2>&1 && +echo "EXITCODE: 0" >> '"../$OUTPUT_DIR/"'INSTNAME.out.txt || +echo "EXITCODE: $?" >>'"../$OUTPUT_DIR/"'INSTNAME.out.txt +' + +# output overall result" +H1 "Results" + +rc=0 +for inst in ${boxes[*]}; do + file="$OUTPUT_DIR"/$inst.out.txt + exitcode="$(tail "$file" | grep EXITCODE: | awk '{print $NF}')" + echo -n "$inst: " + if [ -z "$exitcode" ]; then + danger "NO EXITCODE!" + [ $rc -eq 0 ] && rc=2 + elif [ "$exitcode" == "0" ]; then + elapsed="$(tail "$file" | grep ^Elapsed | awk -F: '{print $2}')" + success "SUCCESS (${elapsed# })" + else + danger "FAILURE ($exitcode)" + rc=1 + fi +done + +# output elapsed time +end_time="$(date +%s)" +echo "" +echo "Elapsed time: $(elapsed_pretty $start_time $end_time)" + +# exit +echo "" +echo "Guest VMs are running! Destroy them with:" +echo " $0 -d $boxlist" +exit $rc diff --git a/tests/lxd/preloaded/create_preloaded.sh b/tests/lxd/preloaded/create_preloaded.sh new file mode 100755 index 00000000..46b08a0d --- /dev/null +++ b/tests/lxd/preloaded/create_preloaded.sh @@ -0,0 +1,97 @@ +#!/bin/bash +##### +##### This file is part of Mail-in-a-Box-LDAP which is released under the +##### terms of the GNU Affero General Public License as published by the +##### Free Software Foundation, either version 3 of the License, or (at +##### your option) any later version. See file LICENSE or go to +##### https://github.com/downtownallday/mailinabox-ldap for full license +##### details. +##### + + +# this script creates a new lxd image preloaded with software to speed +# up installation. +# +# prerequisites: +# +# tests/bin/lx_setup.sh must be run before running this script. it +# only needs to be run once, or any time the networking setup +# changes (eg. adding a new ethernet card). +# + +D="$(dirname "$BASH_SOURCE")" +. "$D/../../bin/lx_functions.sh" || exit 1 + +project="$(lx_guess_project_name)" +inst_mountpoint=/mailinabox +[ "$project" = "ciab" ] && inst_mountpoint=/cloudinabox + +if [ -z "$1" ]; then + imagelist=( $(<./imagelist) ) +else + imagelist=( "$@" ) +fi + + +for base_image in "${imagelist[@]}" +do + new_image="preloaded-${base_image/:/-}" + inst_name="preloaded" + + echo "" + echo "START: create $new_image using base image $base_image" + echo "Delete existing instance: $inst_name" + lx_delete "$project" "$inst_name" "no-interactive" || exit 1 + + echo "Create instance '$inst_name' from '$base_image'" + + # cloud init configuration creates user 'vmuser' instead of 'ubuntu' + cloud_config_users="#cloud-config +users: + - default + - name: vmuser + gecos: VM user for ssh + primary_group: vmuser + groups: adm, sudo, lxd + shell: /bin/bash + sudo: ALL=(ALL) NOPASSWD:ALL + lock_passwd: true + ssh_authorized_keys: + - $(< $(lx_get_ssh_identity).pub) +" + lx_launch_vm "$project" "$inst_name" "$base_image" "$(lx_project_root_dir)" "$inst_mountpoint" -c cloud-init.user-data="$cloud_config_users" -c limits.cpu=2 -c limits.memory=2GiB -d root,size=30GiB || exit 1 + + lx_wait_for_boot "$project" "$inst_name" + + + echo "" + echo "=================================================" + echo "Prep the VM instance" + echo "=================================================" + lxc --project "$project" exec "$inst_name" --cwd "$inst_mountpoint" -- sudo tests/lxd/preloaded/prepvm.sh --no-dry-run || exit 1 + + echo "" + echo "=================================================" + echo "Create an image from the instance" + echo "=================================================" + echo "Stopping instance '$inst_name'" + lxc --project "$project" stop "$inst_name" || exit 1 + + echo "Create image '$new_image' from instance '$inst_name'" + lxc --project "$project" publish "$inst_name" "local:" --reuse --compression gzip --alias "$new_image" || exit 1 # --compression xz + + echo "" + echo "=================================================" + echo "Image list ($project)" + echo "=================================================" + lx_output_image_list "$project" "ld" + + echo "" + echo "=================================================" + echo "Delete instance '$inst_name'" + echo "=================================================" + lx_delete "$project" "$inst_name" "no-interactive" + + echo "Success" + +done diff --git a/tests/lxd/preloaded/imagelist b/tests/lxd/preloaded/imagelist new file mode 100644 index 00000000..786fd1f7 --- /dev/null +++ b/tests/lxd/preloaded/imagelist @@ -0,0 +1 @@ +ubuntu:jammy diff --git a/tests/lxd/preloaded/prepvm.sh b/tests/lxd/preloaded/prepvm.sh new file mode 100755 index 00000000..e43e2e0b --- /dev/null +++ b/tests/lxd/preloaded/prepvm.sh @@ -0,0 +1,230 @@ +#!/bin/bash +##### +##### This file is part of Mail-in-a-Box-LDAP which is released under the +##### terms of the GNU Affero General Public License as published by the +##### Free Software Foundation, either version 3 of the License, or (at +##### your option) any later version. See file LICENSE or go to +##### https://github.com/downtownallday/mailinabox-ldap for full license +##### details. +##### + + +# Run this on a VM to pre-install all the packages, then +# take a snapshot - it will greatly speed up subsequent +# test installs + +# +# What won't be installed: +# +# Nextcloud and Roundcube are downloaded with wget by the setup +# scripts, so they are not included +# +# slapd - we want to test installation with setup/ldap.sh +# + +if [ ! -d "setup" ]; then + echo "Run from the miab root directory" + exit 1 +fi + +source tests/lib/misc.sh +source tests/lib/system.sh +source tests/lib/color-output.sh + +dry_run=true +start=$(date +%s) + +if [ "$1" == "--no-dry-run" ]; then + dry_run=false +fi + +if $dry_run; then + echo "WARNING: dry run is TRUE, no changes will be made" +fi + + +# prevent apt from running needrestart(1) +export NEEDRESTART_SUSPEND=true + +# prevent interaction during package install +export DEBIAN_FRONTEND=noninteractive + +# what major version of ubuntu are we installing on? +OS_MAJOR=$(. /etc/os-release; echo $VERSION_ID | awk -F. '{print $1}') + + +remove_line_continuation() { + local file="$1" + awk ' +BEGIN { C=0 } +C==1 && /[^\\]$/ { C=0; print $0; next } +C==1 { printf("%s",substr($0,0,length($0)-1)); next } +/\\$/ { C=1; printf("%s",substr($0,0,length($0)-1)); next } + { print $0 }' \ + "$file" +} + +install_packages() { + local return_code=0 + while read line; do + pkgs="" + case "$line" in + apt_install* ) + pkgs="$(cut -c12- <<<"$line")" + ;; + "apt-get install"* ) + pkgs="$(cut -c16- <<<"$line")" + ;; + "apt install"* ) + pkgs="$(cut -c12- <<<"$line")" + ;; + esac + + # don't install slapd + pkgs="$(sed 's/slapd//g' <<< "$pkgs")" + + # manually set PHP_VER if necessary + if grep "PHP_VER" <<<"$pkgs" >/dev/null; then + pkgs="$(sed "s/\"\?\${*PHP_VER}*\"\?/$PHP_VER/g" <<< "$pkgs")" + fi + + if [ ! -z "$pkgs" ]; then + H2 "install: $pkgs" + if ! $dry_run; then + exec_no_output apt-get install -y $pkgs + let return_code+=$? + fi + fi + done + return $return_code +} + +install_ppas() { + H1 "Add apt repositories" + grep 'hide_output add-apt-repository' setup/system.sh | + while read line; do + line=$(sed 's/^hide_output //' <<< "$line") + H2 "$line" + if ! $dry_run; then + exec_no_output $line + fi + done +} + +add_swap() { + H1 "Add a swap file to the system" + if ! $dry_run; then + dd if=/dev/zero of=/swapfile bs=1024 count=$[1024*1024] status=none + chmod 600 /swapfile + mkswap /swapfile + swapon /swapfile + echo "/swapfile none swap sw 0 0" >> /etc/fstab + fi +} + + +# install PPAs from sources +install_ppas + +# add swap file +add_swap + +# obtain PHP_VER variable from sources +PHP_VER=$(source setup/functions.sh; echo $PHP_VER) + + +if ! $dry_run; then + H1 "Upgrade system" + H2 "apt update" + exec_no_output apt-get update -y || exit 1 + H2 "apt upgrade" + exec_no_output apt-get upgrade -y --with-new-pkgs || exit 1 + H2 "apt autoremove" + exec_no_output apt-get autoremove -y +fi + +# without using the same installation order as setup/start.sh, we end +# up with the system's php getting installed in addition to the +# non-system php that may also installed by setup (don't know why, +# probably one of the packages has a dependency). create an ordered +# list of files to process so we get a similar system setup. + +setup_files=( $(ls setup/*.sh) ) +desired_order=( + setup/functions.sh + setup/preflight.sh + setup/questions.sh + setup/network-checks.sh + setup/system.sh + setup/ssl.sh + setup/dns.sh + setup/ldap.sh + setup/mail-postfix.sh + setup/mail-dovecot.sh + setup/mail-users.sh + setup/dkim.sh + setup/spamassassin.sh + setup/web.sh + setup/webmail.sh + setup/nextcloud.sh + setup/zpush.sh + setup/management.sh + setup/management-capture.sh + setup/munin.sh + setup/firstuser.sh +) +ordered_files=() +for file in "${desired_order[@]}" "${setup_files[@]}"; do + if [ -e "$file" ] && ! array_contains "$file" "${ordered_files[@]}"; then + ordered_files+=( "$file" ) + fi +done + +failed=0 + +for file in ${ordered_files[@]}; do + H1 "$file" + remove_line_continuation "$file" | install_packages + [ $? -ne 0 ] && let failed+=1 +done + +if ! $dry_run; then + # bonus + H1 "install extras" + + H2 "openssh, emacs, ntpdate, net-tools, jq" + exec_no_output apt-get install -y openssh-server emacs-nox ntpdate net-tools jq || let failed+=1 + + # these are added by system-setup scripts and needed for test runner + H2 "python3-dnspython" + exec_no_output apt-get install -y python3-dnspython || let failed+=1 + H2 "pyotp(pip)" + exec_no_output python3 -m pip install pyotp --quiet || let failed+=1 + + # ...and for browser-based tests + #H2 "x11" # needed for chromium w/head (not --headless) + #exec_no_output apt-get install -y xorg openbox xvfb gtk2-engines-pixbuf dbus-x11 xfonts-base xfonts-100dpi xfonts-75dpi xfonts-cyrillic xfonts-scalable x11-apps imagemagick || let failed+=1 + H2 "chromium" + #exec_no_output apt-get install -y chromium-browser || let failed+=1 + exec_no_output snap install chromium || let failed+=1 + H2 "selenium(pip)" + exec_no_output python3 -m pip install selenium --quiet || let failed+=1 + + # remove apache, which is what setup will do + H2 "remove apache2" + exec_no_output apt-get -y purge apache2 apache2-\* + +fi + +end=$(date +%s) +echo "" +echo "" +if [ $failed -gt 0 ]; then + echo "$failed failures! ($(elapsed_pretty $start $end))" + echo "" + exit 1 +else + echo "Successfully prepped in $(elapsed_pretty $start $end). Take a snapshot...." + echo "" + exit 0 +fi diff --git a/tests/lxd/remote-nextcloud-docker-ehdd/provision.sh b/tests/lxd/remote-nextcloud-docker-ehdd/provision.sh new file mode 100755 index 00000000..b0038f1a --- /dev/null +++ b/tests/lxd/remote-nextcloud-docker-ehdd/provision.sh @@ -0,0 +1,35 @@ +#!/bin/bash +##### +##### This file is part of Mail-in-a-Box-LDAP which is released under the +##### terms of the GNU Affero General Public License as published by the +##### Free Software Foundation, either version 3 of the License, or (at +##### your option) any later version. See file LICENSE or go to +##### https://github.com/downtownallday/mailinabox-ldap for full license +##### details. +##### + + +# +# provision a miab-ldap that has a remote nextcloud (using Nextcloud +# from Docker) and an encrypted user-data +# + +D=$(dirname "$BASH_SOURCE") +. "$D/../../bin/lx_functions.sh" || exit 1 +. "$D/../../bin/provision_functions.sh" || exit 1 + +# Create the instance (started) +provision_start "" "/mailinabox" || exit 1 + +# Setup system +provision_shell <<<" +cd /mailinabox +export PRIMARY_HOSTNAME=qa1.abc.com +export FEATURE_MUNIN=false +export EHDD_KEYFILE=\$HOME/keyfile +echo -n 'boo' >\$EHDD_KEYFILE +tests/system-setup/remote-nextcloud-docker.sh || exit 1 +tests/runner.sh -no-smtp-remote remote-nextcloud ehdd default || exit 2 +" + +provision_done $? diff --git a/tests/lxd/remote-nextcloud-docker/provision.sh b/tests/lxd/remote-nextcloud-docker/provision.sh new file mode 100755 index 00000000..694bc3c4 --- /dev/null +++ b/tests/lxd/remote-nextcloud-docker/provision.sh @@ -0,0 +1,33 @@ +#!/bin/bash +##### +##### This file is part of Mail-in-a-Box-LDAP which is released under the +##### terms of the GNU Affero General Public License as published by the +##### Free Software Foundation, either version 3 of the License, or (at +##### your option) any later version. See file LICENSE or go to +##### https://github.com/downtownallday/mailinabox-ldap for full license +##### details. +##### + + +# +# provision a miab-ldap that has a remote nextcloud (using Nextcloud +# from Docker) +# + +D=$(dirname "$BASH_SOURCE") +. "$D/../../bin/lx_functions.sh" || exit 1 +. "$D/../../bin/provision_functions.sh" || exit 1 + +# Create the instance (started) +provision_start "" "/mailinabox" || exit 1 + +# Setup system +provision_shell <<<" +cd /mailinabox +export PRIMARY_HOSTNAME=qa2.abc.com +export FEATURE_MUNIN=false +tests/system-setup/remote-nextcloud-docker.sh upgrade --populate=basic || exit 1 +tests/runner.sh -no-smtp-remote remote-nextcloud upgrade-basic default || exit 2 +" + +provision_done $? diff --git a/tests/lxd/unsetvars/provision.sh b/tests/lxd/unsetvars/provision.sh new file mode 100755 index 00000000..bd6638fd --- /dev/null +++ b/tests/lxd/unsetvars/provision.sh @@ -0,0 +1,35 @@ +#!/bin/bash +##### +##### This file is part of Mail-in-a-Box-LDAP which is released under the +##### terms of the GNU Affero General Public License as published by the +##### Free Software Foundation, either version 3 of the License, or (at +##### your option) any later version. See file LICENSE or go to +##### https://github.com/downtownallday/mailinabox-ldap for full license +##### details. +##### + + +# +# provision a miab-ldap that has a remote nextcloud (using Nextcloud +# from Docker) +# + +D=$(dirname "$BASH_SOURCE") +. "$D/../../bin/lx_functions.sh" || exit 1 +. "$D/../../bin/provision_functions.sh" || exit 1 + +# Create the instance (started) +provision_start "" "/mailinabox" || exit 1 + +# Setup system +provision_shell <<<" +export NONINTERACTIVE=1 +export PUBLIC_IP=auto +export PUBLIC_IPV6=auto +export PRIMARY_HOSTNAME=mailinabox.lan # auto +export SKIP_NETWORK_CHECKS=1 +cd /mailinabox +setup/start.sh +" + +provision_done $? diff --git a/tests/lxd/upgrade-from-upstream/provision.sh b/tests/lxd/upgrade-from-upstream/provision.sh new file mode 100755 index 00000000..97728672 --- /dev/null +++ b/tests/lxd/upgrade-from-upstream/provision.sh @@ -0,0 +1,33 @@ +#!/bin/bash +##### +##### This file is part of Mail-in-a-Box-LDAP which is released under the +##### terms of the GNU Affero General Public License as published by the +##### Free Software Foundation, either version 3 of the License, or (at +##### your option) any later version. See file LICENSE or go to +##### https://github.com/downtownallday/mailinabox-ldap for full license +##### details. +##### + + +# +# provision a miab-ldap that has a remote nextcloud (using Nextcloud +# from Docker) +# + +D=$(dirname "$BASH_SOURCE") +. "$D/../../bin/lx_functions.sh" || exit 1 +. "$D/../../bin/provision_functions.sh" || exit 1 + +# Create the instance (started) +provision_start "" "/mailinabox" || exit 1 + +# Setup system +provision_shell <<<" +cd /mailinabox +export PRIMARY_HOSTNAME=qa3.abc.com +export UPSTREAM_TAG=main +tests/system-setup/upgrade-from-upstream.sh --populate=basic --populate=totpuser || exit 1 +tests/runner.sh -no-smtp-remote upgrade-basic upgrade-totpuser default || exit 2 +" + +provision_done $? diff --git a/tests/lxd/upgrade/provision.sh b/tests/lxd/upgrade/provision.sh new file mode 100755 index 00000000..bcbc39fa --- /dev/null +++ b/tests/lxd/upgrade/provision.sh @@ -0,0 +1,32 @@ +#!/bin/bash +##### +##### This file is part of Mail-in-a-Box-LDAP which is released under the +##### terms of the GNU Affero General Public License as published by the +##### Free Software Foundation, either version 3 of the License, or (at +##### your option) any later version. See file LICENSE or go to +##### https://github.com/downtownallday/mailinabox-ldap for full license +##### details. +##### + + +# +# provision a miab-ldap that has a remote nextcloud (using Nextcloud +# from Docker) +# + +D=$(dirname "$BASH_SOURCE") +. "$D/../../bin/lx_functions.sh" || exit 1 +. "$D/../../bin/provision_functions.sh" || exit 1 + +# Create the instance (started) +provision_start "" "/mailinabox" || exit 1 + +# Setup system +provision_shell <<<" +cd /mailinabox +export PRIMARY_HOSTNAME=upgrade.abc.com +tests/system-setup/upgrade.sh --populate=basic --populate=totpuser || exit 1 +tests/runner.sh -no-smtp-remote upgrade-basic upgrade-totpuser default || exit 2 +" + +provision_done $? diff --git a/tests/lxd/vanilla/provision.sh b/tests/lxd/vanilla/provision.sh new file mode 100755 index 00000000..ea28a191 --- /dev/null +++ b/tests/lxd/vanilla/provision.sh @@ -0,0 +1,65 @@ +#!/bin/bash +##### +##### This file is part of Mail-in-a-Box-LDAP which is released under the +##### terms of the GNU Affero General Public License as published by the +##### Free Software Foundation, either version 3 of the License, or (at +##### your option) any later version. See file LICENSE or go to +##### https://github.com/downtownallday/mailinabox-ldap for full license +##### details. +##### + + +D=$(dirname "$BASH_SOURCE") +. "$D/../../bin/lx_functions.sh" || exit 1 +. "$D/../../bin/provision_functions.sh" || exit 1 + +# Create the instance (started) +provision_start "" "/mailinabox" || exit 1 + +# Setup system +if [ "$1" = "ciab" ]; then + # use a remote cloudinabox (does not have to be running) + provision_shell <<<" +cd /mailinabox +export PRIMARY_HOSTNAME='${inst}.local' +export NC_PROTO=https +export NC_HOST=vanilla-ciab.local +export NC_PORT=443 +export NC_PREFIX=/ +export SKIP_SYSTEM_UPDATE=0 +tests/system-setup/vanilla.sh --qa-ca --enable-mod=remote-nextcloud +rc=$? +if ! ufw status | grep remote_nextcloud >/dev/null; then + # firewall rules aren't added when ciab is down + echo 'For testing, allow ldaps from anywhere' + ufw allow ldaps +fi +echo 'Add smart host alias - so \$NC_HOST can send mail to/via this host' +( + source tests/lib/all.sh + rest_urlencoded POST /admin/mail/aliases/add qa@abc.com Test_1234 \"address=@\$NC_HOST\" 'description=smart-host' 'permitted_senders=qa@abc.com' 2>/dev/null + echo \"\$REST_HTTP_CODE: \$REST_OUTPUT\" +) +exit $rc +" + provision_done $? + +else + # vanilla (default - no miab integration) + provision_shell <<<" +cd /mailinabox +export PRIMARY_HOSTNAME='${inst}.local' +#export FEATURE_MUNIN=false +#export FEATURE_NEXTCLOUD=false +export SKIP_SYSTEM_UPDATE=0 +tests/system-setup/vanilla.sh +rc=$? +# --enable-mod=move-postfix-queue-to-user-data +# --enable-mod=roundcube-master +# --enable-mod=roundcube-debug +# --enable-mod=rcmcarddav-composer +exit $rc +" + provision_done $? + +fi diff --git a/tests/system-setup/setup-defaults.sh b/tests/system-setup/setup-defaults.sh index 204b4e74..028325b9 100755 --- a/tests/system-setup/setup-defaults.sh +++ b/tests/system-setup/setup-defaults.sh @@ -22,11 +22,11 @@ export STORAGE_ROOT="${STORAGE_ROOT:-/home/$STORAGE_USER}" export EMAIL_ADDR="${EMAIL_ADDR:-qa@abc.com}" export EMAIL_PW="${EMAIL_PW:-Test_1234}" export PUBLIC_IP="${PUBLIC_IP:-$(source ${MIAB_DIR:-.}/setup/functions.sh; get_default_privateip 4)}" -if lsmod | grep "^vboxguest[\t ]" >/dev/null; then +if lsmod | grep -q "^vboxguest[\t ]" || df | grep -q "^lxd_"; then # The local mods directory defaults to 'local' (relative to the # source tree, which is a mounted filesystem of the host). This # will keep mods directory out of the source tree when running - # under virtualbox / vagrant. + # under virtualbox / vagrant / lxd. export LOCAL_MODS_DIR="${LOCAL_MODS_DIR:-/local}" else export LOCAL_MODS_DIR="${LOCAL_MODS_DIR:-$(pwd)/local}" diff --git a/tests/system-setup/setup-funcs.sh b/tests/system-setup/setup-funcs.sh index fed605fc..63e3b449 100755 --- a/tests/system-setup/setup-funcs.sh +++ b/tests/system-setup/setup-funcs.sh @@ -86,7 +86,7 @@ init_test_system() { # update system time H2 "Set system time" update_system_time || echo "Ignoring error..." - + # update package lists before installing anything H2 "apt-get update" wait_for_apt @@ -115,7 +115,7 @@ init_test_system() { | awk '/^(Upgrade|Install): / { print $0 }' rm -f /tmp/history.log fi - + # install avahi if the system dns domain is .local - note that # /bin/dnsdomainname returns empty string at this point case "$PRIMARY_HOSTNAME" in @@ -149,7 +149,7 @@ init_miab_testing() { else echo "Not configured for encryption-at-rest" fi - + H2 "QA prerequisites" local rc=0 @@ -186,7 +186,7 @@ init_miab_testing() { echo "Copy failed ($?)" rc=1 fi - + # create miab_ldap.conf to specify what the Nextcloud LDAP service # account password will be to avoid a random one created by start.sh if [ ! -z "$LDAP_NEXTCLOUD_PASSWORD" ]; then @@ -228,13 +228,13 @@ init_miab_testing() { # ignore unknown option - may be interpreted elsewhere shift ;; - esac + esac done # now that we've copied our files, unmount STORAGE_ROOT if # encryption-at-rest was enabled ehdd/umount.sh - + return $rc } @@ -303,7 +303,7 @@ clone_repo_and_pushd() { ;; esac done - + if [ -z "$repo" -o -z "$treeish" -o -z "$targetdir" ]; then return 1 fi @@ -329,7 +329,7 @@ upstream_install() { if clone_repo_and_pushd "$@"; then need_pop="yes" fi - + H1 "MIAB UPSTEAM INSTALL [$(git describe 2>/dev/null)]" # ensure we're in a MiaB working directory @@ -351,24 +351,24 @@ upstream_install() { exec_no_output apt-get remove -y nsd #systemctl reset-failed nsd fi - + if ! setup/start.sh; then echo "$F_WARN" dump_file /var/log/syslog 100 echo "$F_RESET" die "Upstream setup failed!" fi - + H2 "Post-setup actions" workaround_dovecot_sieve_bug # set actual STORAGE_ROOT, STORAGE_USER, PRIVATE_IP, etc . /etc/mailinabox.conf || die "Could not source /etc/mailinabox.conf" - + H2 "miab install success" if [ "$need_pop" = "yes" ]; then - if [ ! -e tests/vagrant ]; then + if [ ! -e tests/lxd ]; then # if this is an upstream install, then populate using # miabldap's populate scripts (upstream doesn't have any) local d @@ -397,7 +397,7 @@ miab_ldap_install() { if clone_repo_and_pushd "$@"; then need_pop="yes" fi - + H1 "MIAB-LDAP INSTALL [$(pwd)] [$(git describe 2>/dev/null)]" # ensure we're in a MiaB-LDAP working directory if [ ! -e setup/ldap.sh ]; then @@ -423,7 +423,7 @@ miab_ldap_install() { else setup/start.sh fi - + if [ $? -ne 0 ]; then H1 "OUTPUT OF SELECT FILES" dump_file "/var/log/syslog" 100 @@ -444,11 +444,11 @@ miab_ldap_install() { fi H2 "miab-ldap install success" - + # populate if specified on command line populate_by_cli_argument "$@" capture_state_by_cli_argument "$@" - + if [ "$need_pop" = "yes" ]; then popd >/dev/null fi @@ -457,7 +457,7 @@ miab_ldap_install() { capture_state_by_cli_argument() { # this must be run with the working directory set to the source # tree corresponding to the the installed state - + # ...ignore unknown options they may be interpreted elsewhere local state_dir="" for arg; do From c027db8bf49091e6d4cef214722a062635e80d3c Mon Sep 17 00:00:00 2001 From: downtownallday Date: Tue, 3 Sep 2024 10:49:48 -0400 Subject: [PATCH 03/14] reword comment --- tests/bin/lx_functions.sh | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/bin/lx_functions.sh b/tests/bin/lx_functions.sh index 4b3890b1..2faec0a2 100644 --- a/tests/bin/lx_functions.sh +++ b/tests/bin/lx_functions.sh @@ -96,8 +96,7 @@ lx_init_vm() { local mount_guest="$5" # mountpoint in guest shift; shift; shift; shift; shift; - # 1. this assumes that a bridge profile has been created called "bridgenet" - # 2. this assumes that storage named the same as project exists, + # a storage named the same as project must exist # e.g. "lxc storage create $project dir" was executed prior. case "${@}" in From 446aacb9b6ae4dce8c021c8d1a4b09efefebabdb Mon Sep 17 00:00:00 2001 From: downtownallday Date: Tue, 3 Sep 2024 11:06:58 -0400 Subject: [PATCH 04/14] Don't exit on missing kernel module during non-interactive scenario --- ehdd/ehdd_funcs.sh | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/ehdd/ehdd_funcs.sh b/ehdd/ehdd_funcs.sh index ea605c1d..cee07cbc 100644 --- a/ehdd/ehdd_funcs.sh +++ b/ehdd/ehdd_funcs.sh @@ -25,8 +25,17 @@ EHDD_LUKS_NAME="c1" assert_kernel_modules() { local check="$(lsmod | awk '$1=="dm_crypt" {print "yes"}')" if [ "$check" != "yes" ]; then - echo "Required kernel modules for encryption-at-rest are not loaded. Cannot continue" - exit 1 + if [ ! -z "$EHDD_KEYFILE" ]; then + echo "WARNING: Required kernel modules for encryption-at-rest are not loaded." + # probably testing / virutalization + echo "OUTPUT from lsmod:" + echo "------------------------------------------------------" + lsmod + echo "------------------------------------------------------" + else + echo "Required kernel modules for encryption-at-rest are not loaded. Cannot continue." + exit 1 + fi fi } From 0fce66dbc7e3c46b08d31698deca27badfbb0682 Mon Sep 17 00:00:00 2001 From: downtownallday Date: Tue, 3 Sep 2024 11:11:17 -0400 Subject: [PATCH 05/14] back out assert_kernel_modules --- ehdd/create_hdd.sh | 1 - ehdd/ehdd_funcs.sh | 16 ---------------- ehdd/mount.sh | 1 - 3 files changed, 18 deletions(-) diff --git a/ehdd/create_hdd.sh b/ehdd/create_hdd.sh index dd327dd0..5ed43988 100755 --- a/ehdd/create_hdd.sh +++ b/ehdd/create_hdd.sh @@ -44,7 +44,6 @@ if [ ! -e "$EHDD_IMG" ]; then echo "" echo "NOTE: You will need to reenter your drive encryption password" fi - assert_kernel_modules cryptsetup luksOpen $(keyfile_option) $loop $EHDD_LUKS_NAME # map device to /dev/mapper/NAME mke2fs -j /dev/mapper/$EHDD_LUKS_NAME # sleep a brief time to avoid "device busy" diff --git a/ehdd/ehdd_funcs.sh b/ehdd/ehdd_funcs.sh index cee07cbc..0ed7fa55 100644 --- a/ehdd/ehdd_funcs.sh +++ b/ehdd/ehdd_funcs.sh @@ -22,22 +22,6 @@ EHDD_IMG="$STORAGE_ROOT.HDD" EHDD_MOUNTPOINT="$STORAGE_ROOT" EHDD_LUKS_NAME="c1" -assert_kernel_modules() { - local check="$(lsmod | awk '$1=="dm_crypt" {print "yes"}')" - if [ "$check" != "yes" ]; then - if [ ! -z "$EHDD_KEYFILE" ]; then - echo "WARNING: Required kernel modules for encryption-at-rest are not loaded." - # probably testing / virutalization - echo "OUTPUT from lsmod:" - echo "------------------------------------------------------" - lsmod - echo "------------------------------------------------------" - else - echo "Required kernel modules for encryption-at-rest are not loaded. Cannot continue." - exit 1 - fi - fi -} find_unused_loop() { losetup -f diff --git a/ehdd/mount.sh b/ehdd/mount.sh index f3c59c49..5ff44270 100755 --- a/ehdd/mount.sh +++ b/ehdd/mount.sh @@ -21,7 +21,6 @@ if is_mounted; then exit 0 fi -assert_kernel_modules loop=$(find_unused_loop) losetup $loop "$EHDD_IMG" || exit 1 # map device to /dev/mapper/NAME From 54a3bd100c43710800ce3208acf1380e071bc0a3 Mon Sep 17 00:00:00 2001 From: downtownallday Date: Wed, 4 Sep 2024 08:13:08 -0400 Subject: [PATCH 06/14] Add provision defaults to lxc init --- tests/lxd/preloaded/create_preloaded.sh | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/tests/lxd/preloaded/create_preloaded.sh b/tests/lxd/preloaded/create_preloaded.sh index 46b08a0d..10f186a9 100755 --- a/tests/lxd/preloaded/create_preloaded.sh +++ b/tests/lxd/preloaded/create_preloaded.sh @@ -21,10 +21,14 @@ D="$(dirname "$BASH_SOURCE")" . "$D/../../bin/lx_functions.sh" || exit 1 +. "$D/../../bin/provision_functions.sh" || exit 1 + project="$(lx_guess_project_name)" inst_mountpoint=/mailinabox [ "$project" = "ciab" ] && inst_mountpoint=/cloudinabox +load_provision_defaults || exit 1 +echo "Provision defaults loaded from: $(realpath --relative-to="$(pwd)" "$PROVISION_DEFAULTS_FILE")" if [ -z "$1" ]; then imagelist=( $(<./imagelist) ) @@ -59,7 +63,7 @@ users: ssh_authorized_keys: - $(< $(lx_get_ssh_identity).pub) " - lx_launch_vm "$project" "$inst_name" "$base_image" "$(lx_project_root_dir)" "$inst_mountpoint" -c cloud-init.user-data="$cloud_config_users" -c limits.cpu=2 -c limits.memory=2GiB -d root,size=30GiB || exit 1 + lx_launch_vm "$project" "$inst_name" "$base_image" "$(lx_project_root_dir)" "$inst_mountpoint" "${DEFAULT_LXD_INST_OPTS[@]}" -c cloud-init.user-data="$cloud_config_users" -c limits.cpu=2 -c limits.memory=2GiB -d root,size=30GiB || exit 1 lx_wait_for_boot "$project" "$inst_name" From 6d25bc47bf20ba48c53bc861962e9356713fcbfa Mon Sep 17 00:00:00 2001 From: downtownallday Date: Thu, 5 Sep 2024 12:39:52 -0400 Subject: [PATCH 07/14] add a restart command --- tests/bin/vlx | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/tests/bin/vlx b/tests/bin/vlx index 5746f03c..52cd0191 100755 --- a/tests/bin/vlx +++ b/tests/bin/vlx @@ -71,14 +71,14 @@ vlx_exec() { vlx_exec_usage return 1 fi - + local args=( "$@" ) local idx=0 while [ $idx -le 3 -a $idx -lt ${#args[*]} ]; do [ "${args[$idx]}" = "--" ] && break let idx+=1 done - + local wd="" if [ "${args[$idx]}" = "--" ]; then if [ $idx -eq 3 ]; then @@ -187,7 +187,7 @@ vlx_up() { return 1 fi } - + vlx_start() { vlx_guess "$@" || return 1 @@ -222,6 +222,12 @@ vlx_status() { fi } +vlx_restart() { + vlx_guess "$@" || return 1 + echo lxc --project "$LX_PROJECT" restart "$LX_INST" + lxc --project "$LX_PROJECT" restart "$LX_INST" +} + usage() { echo "Usage:" @@ -238,6 +244,7 @@ usage() { echo " start" echo " stop" echo " delete|destroy" + echo " restart" echo " status" } @@ -255,4 +262,3 @@ if [ ! "$(type -t $handler)" = "function" ]; then fi $handler "$@" - From 2e0b37a09a964c0e7499c2bff7d4f2d25361e9d9 Mon Sep 17 00:00:00 2001 From: downtownallday Date: Sat, 7 Sep 2024 10:52:32 -0400 Subject: [PATCH 08/14] fix syntax error --- tests/lxd/vanilla/provision.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/lxd/vanilla/provision.sh b/tests/lxd/vanilla/provision.sh index ea28a191..6d774d55 100755 --- a/tests/lxd/vanilla/provision.sh +++ b/tests/lxd/vanilla/provision.sh @@ -28,7 +28,7 @@ export NC_PORT=443 export NC_PREFIX=/ export SKIP_SYSTEM_UPDATE=0 tests/system-setup/vanilla.sh --qa-ca --enable-mod=remote-nextcloud -rc=$? +rc=\$? if ! ufw status | grep remote_nextcloud >/dev/null; then # firewall rules aren't added when ciab is down echo 'For testing, allow ldaps from anywhere' @@ -40,7 +40,7 @@ echo 'Add smart host alias - so \$NC_HOST can send mail to/via this host' rest_urlencoded POST /admin/mail/aliases/add qa@abc.com Test_1234 \"address=@\$NC_HOST\" 'description=smart-host' 'permitted_senders=qa@abc.com' 2>/dev/null echo \"\$REST_HTTP_CODE: \$REST_OUTPUT\" ) -exit $rc +exit \$rc " provision_done $? @@ -53,12 +53,12 @@ export PRIMARY_HOSTNAME='${inst}.local' #export FEATURE_NEXTCLOUD=false export SKIP_SYSTEM_UPDATE=0 tests/system-setup/vanilla.sh -rc=$? +rc=\$? # --enable-mod=move-postfix-queue-to-user-data # --enable-mod=roundcube-master # --enable-mod=roundcube-debug # --enable-mod=rcmcarddav-composer -exit $rc +exit \$rc " provision_done $? From 4fedfb377da393d7fe22ca8781a964e9759a18ce Mon Sep 17 00:00:00 2001 From: downtownallday Date: Tue, 10 Sep 2024 14:53:12 -0400 Subject: [PATCH 09/14] during wait for boot, also wait until vm has an ip address --- tests/bin/lx_functions.sh | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/tests/bin/lx_functions.sh b/tests/bin/lx_functions.sh index 2faec0a2..b64a77fc 100644 --- a/tests/bin/lx_functions.sh +++ b/tests/bin/lx_functions.sh @@ -201,7 +201,25 @@ lx_wait_for_boot() { echo "" echo -n "Wait for cloud-init " lxc --project "$project" exec "$inst" -- cloud-init status --wait + local rc=$? + + if [ $rc -eq 0 ]; then + echo "Wait for ip address " + local ip="" + local count=0 + while [ $count -lt 10 ]; do + let count+=1 + ip="$(lxc --project "$project" exec "$inst" -- hostname -I | awk '{print $1}')" + rc=$? + echo " [${count}] got: $ip" + if [ $rc -ne 0 -o "$ip" != "" ];then + break + fi + sleep 5 + done + fi echo "" + return $rc } lx_get_ssh_identity() { From 3e0a6214508724496ce2c629b598cedf4be1b22c Mon Sep 17 00:00:00 2001 From: downtownallday Date: Tue, 10 Sep 2024 14:54:47 -0400 Subject: [PATCH 10/14] allow supplying a command line to execute to ssh remove debugging echo statements add -q argument to suppress outputting lxc command line --- tests/bin/vlx | 59 ++++++++++++++++++++++++++++++++++----------------- 1 file changed, 40 insertions(+), 19 deletions(-) diff --git a/tests/bin/vlx b/tests/bin/vlx index 52cd0191..0ba3436f 100755 --- a/tests/bin/vlx +++ b/tests/bin/vlx @@ -37,6 +37,12 @@ D=$(dirname "$BASH_SOURCE") . "$D/lx_functions.sh" || exit 1 +show_cl="yes" +if [ "$1" = "-q" ]; then + show_cl="no" + shift +fi + vlx_guess() { if [ $# -eq 2 ]; then LX_PROJECT="$1" @@ -83,7 +89,6 @@ vlx_exec() { if [ "${args[$idx]}" = "--" ]; then if [ $idx -eq 3 ]; then # format 1 with cwd - echo "f1" wd="$3" vlx_guess "$1" "$2" || return 1 shift; shift; shift; shift; @@ -92,12 +97,10 @@ vlx_exec() { if [ "${2#/}" != "$2" ]; then # wd starts with /, so it's a path # format 2 - echo "f2" wd="$2" vlx_guess "" "$1" || return 1 else # format 1 w/o cwd - echo "f1 w/o cwd" vlx_guess "$1" "$2" || return 1 fi shift; shift; shift; @@ -106,12 +109,10 @@ vlx_exec() { if [ "${1#/}" != "$1" ]; then # wd starts with /, so it's a path # format 3 - echo "f3" wd="$1" vlx_guess || return 1 else # format 2 w/o cwd - echo "f2 w/o cwd" vlx_guess "$1" || return 1 fi shift; shift; @@ -121,7 +122,6 @@ vlx_exec() { fi else # format 4 - echo "f4" vlx_guess || return 1 fi @@ -129,28 +129,43 @@ vlx_exec() { if [ ! -z "$wd" ]; then xargs="--cwd $wd" fi - echo lxc --project "$LX_PROJECT" exec "$LX_INST" $xargs -- "$@" + [ "$show_cl" = "yes" ] && + echo lxc --project "$LX_PROJECT" exec "$LX_INST" $xargs -- "$@" lxc --project "$LX_PROJECT" exec "$LX_INST" $xargs -- "$@" } vlx_shell() { vlx_guess "$@" || return 1 - echo lxc --project "$LX_PROJECT" exec "$LX_INST" -- bash + [ "$show_cl" = "yes" ] && + echo lxc --project "$LX_PROJECT" exec "$LX_INST" -- bash lxc --project "$LX_PROJECT" exec "$LX_INST" -- bash } vlx_hostname() { vlx_guess "$@" || return 1 - local host lxc --project "$LX_PROJECT" exec "$LX_INST" -- /usr/bin/hostname --fqdn || return 1 } +vlx_ipaddr() { + vlx_guess "$@" || return 1 + local hostip + hostip="$(lxc --project "$LX_PROJECT" exec "$LX_INST" -- /usr/bin/hostname -I)" + [ $? -ne 0 -o -z "$hostip" ] && return 1 + awk '{print $1}' <<<"$hostip" +} + + vlx_ssh() { local host="$1" + if [ "$host" = "--" ]; then + host="" + else + shift + fi if [ -z "$host" ]; then - host="$(vlx_hostname)" + host="$(vlx_ipaddr)" if [ $? -ne 0 ]; then - echo "Could not determine hostname, please specify" + echo "Could not determine ip address, please specify" host="" fi if [ -z "$host" ]; then @@ -161,20 +176,22 @@ vlx_ssh() { local id="$(lx_get_ssh_identity)" local known_hosts="$(lx_get_ssh_known_hosts)" local vmuser="vmuser" - #echo ssh -i "$id" -o UserKnownHostsFile="$known_hosts" -o StrictHostKeyChecking=no "$vmuser@$host" + #echo ssh -i "$id" -o UserKnownHostsFile="$known_hosts" -o StrictHostKeyChecking=no "$vmuser@$host" "$@" echo "Connecting to $vmuser@$host ..." - ssh -i "$id" -o UserKnownHostsFile="$known_hosts" -o StrictHostKeyChecking=no "$vmuser@$host" + ssh -i "$id" -o UserKnownHostsFile="$known_hosts" -o StrictHostKeyChecking=no "$vmuser@$host" "$@" } vlx_list() { vlx_guess "$1" || return 1 - echo lxc --project "$LX_PROJECT" list + [ "$show_cl" = "yes" ] && + echo lxc --project "$LX_PROJECT" list lxc --project "$LX_PROJECT" list } vlx_images() { vlx_guess "$1" || return 1 - echo lxc --project "$LX_PROJECT" image list + [ "$show_cl" = "yes" ] && + echo lxc --project "$LX_PROJECT" image list lxc --project "$LX_PROJECT" image list } @@ -191,19 +208,22 @@ vlx_up() { vlx_start() { vlx_guess "$@" || return 1 - echo lxc --project "$LX_PROJECT" start "$LX_INST" + [ "$show_cl" = "yes" ] && + echo lxc --project "$LX_PROJECT" start "$LX_INST" lxc --project "$LX_PROJECT" start "$LX_INST" } vlx_stop() { vlx_guess "$@" || return 1 - echo lxc --project "$LX_PROJECT" stop "$LX_INST" + [ "$show_cl" = "yes" ] && + echo lxc --project "$LX_PROJECT" stop "$LX_INST" lxc --project "$LX_PROJECT" stop "$LX_INST" } vlx_delete() { vlx_guess "$@" || return 1 - echo lxc --project "$LX_PROJECT" delete --force --interactive "$LX_INST" + [ "$show_cl" = "yes" ] && + echo lxc --project "$LX_PROJECT" delete --force --interactive "$LX_INST" lxc --project "$LX_PROJECT" delete --force --interactive "$LX_INST" } @@ -224,7 +244,8 @@ vlx_status() { vlx_restart() { vlx_guess "$@" || return 1 - echo lxc --project "$LX_PROJECT" restart "$LX_INST" + [ "$show_cl" = "yes" ] && + echo lxc --project "$LX_PROJECT" restart "$LX_INST" lxc --project "$LX_PROJECT" restart "$LX_INST" } From 3b6e6177d03c4fc10d3c3afe07760e8f4b49f181 Mon Sep 17 00:00:00 2001 From: downtownallday Date: Fri, 4 Oct 2024 14:51:25 -0400 Subject: [PATCH 11/14] Remove vagrant references - everything has moved to lxd --- tests/lxd/create-backup/README.md | 1 + tests/lxd/majorupgrade/README.md | 1 + .../{vagrant => lxd}/majorupgrade/Vagrantfile | 0 .../majorupgrade/majorupgrade.sh | 0 tests/lxd/vanilla/provision.sh | 2 +- tests/vagrant/.gitignore | 3 - tests/vagrant/Vagrantfile | 104 -------- tests/vagrant/funcs.rb | 25 -- tests/vagrant/parallel.sh | 86 ------- tests/vagrant/preloaded/.gitignore | 2 - tests/vagrant/preloaded/Vagrantfile | 58 ----- tests/vagrant/preloaded/create_preloaded.sh | 134 ---------- tests/vagrant/preloaded/prepvm.sh | 230 ------------------ tests/vagrant/vanilla/.gitignore | 2 - tests/vagrant/vanilla/Vagrantfile | 87 ------- 15 files changed, 3 insertions(+), 732 deletions(-) create mode 100644 tests/lxd/create-backup/README.md create mode 100644 tests/lxd/majorupgrade/README.md rename tests/{vagrant => lxd}/majorupgrade/Vagrantfile (100%) rename tests/{vagrant => lxd}/majorupgrade/majorupgrade.sh (100%) delete mode 100644 tests/vagrant/.gitignore delete mode 100644 tests/vagrant/Vagrantfile delete mode 100644 tests/vagrant/funcs.rb delete mode 100755 tests/vagrant/parallel.sh delete mode 100644 tests/vagrant/preloaded/.gitignore delete mode 100644 tests/vagrant/preloaded/Vagrantfile delete mode 100755 tests/vagrant/preloaded/create_preloaded.sh delete mode 100755 tests/vagrant/preloaded/prepvm.sh delete mode 100644 tests/vagrant/vanilla/.gitignore delete mode 100644 tests/vagrant/vanilla/Vagrantfile diff --git a/tests/lxd/create-backup/README.md b/tests/lxd/create-backup/README.md new file mode 100644 index 00000000..1333ed77 --- /dev/null +++ b/tests/lxd/create-backup/README.md @@ -0,0 +1 @@ +TODO diff --git a/tests/lxd/majorupgrade/README.md b/tests/lxd/majorupgrade/README.md new file mode 100644 index 00000000..e357eb2d --- /dev/null +++ b/tests/lxd/majorupgrade/README.md @@ -0,0 +1 @@ +TODO - convert to lxd diff --git a/tests/vagrant/majorupgrade/Vagrantfile b/tests/lxd/majorupgrade/Vagrantfile similarity index 100% rename from tests/vagrant/majorupgrade/Vagrantfile rename to tests/lxd/majorupgrade/Vagrantfile diff --git a/tests/vagrant/majorupgrade/majorupgrade.sh b/tests/lxd/majorupgrade/majorupgrade.sh similarity index 100% rename from tests/vagrant/majorupgrade/majorupgrade.sh rename to tests/lxd/majorupgrade/majorupgrade.sh diff --git a/tests/lxd/vanilla/provision.sh b/tests/lxd/vanilla/provision.sh index 6d774d55..3d153e66 100755 --- a/tests/lxd/vanilla/provision.sh +++ b/tests/lxd/vanilla/provision.sh @@ -17,7 +17,7 @@ D=$(dirname "$BASH_SOURCE") provision_start "" "/mailinabox" || exit 1 # Setup system -if [ "$1" = "ciab" ]; then +if [ "$TESTS" = "ciab" -o "$1" = "ciab" ]; then # use a remote cloudinabox (does not have to be running) provision_shell <<<" cd /mailinabox diff --git a/tests/vagrant/.gitignore b/tests/vagrant/.gitignore deleted file mode 100644 index 6cfd7a63..00000000 --- a/tests/vagrant/.gitignore +++ /dev/null @@ -1,3 +0,0 @@ -.vagrant -out -*-console.log diff --git a/tests/vagrant/Vagrantfile b/tests/vagrant/Vagrantfile deleted file mode 100644 index 353bbf3c..00000000 --- a/tests/vagrant/Vagrantfile +++ /dev/null @@ -1,104 +0,0 @@ -##### -##### This file is part of Mail-in-a-Box-LDAP which is released under the -##### terms of the GNU Affero General Public License as published by the -##### Free Software Foundation, either version 3 of the License, or (at -##### your option) any later version. See file LICENSE or go to -##### https://github.com/downtownallday/mailinabox-ldap for full license -##### details. -##### - -load './funcs.rb' - -Vagrant.configure("2") do |config| - - config.vm.synced_folder "../..", "/mailinabox", id: "mailinabox", automount: false - use_preloaded_box config, "ubuntu/jammy64" - - # fresh install with encryption-at-rest - - if ENV['tests']=='all' - config.vm.define "remote-nextcloud-docker-ehdd" do |m1| - m1.vm.provision :shell, :inline => <<-SH -cd /mailinabox -export PRIMARY_HOSTNAME=qa1.abc.com -export FEATURE_MUNIN=false -export EHDD_KEYFILE=$HOME/keyfile -echo -n "boo" >$EHDD_KEYFILE -tests/system-setup/remote-nextcloud-docker.sh || exit 1 -tests/runner.sh -no-smtp-remote remote-nextcloud ehdd default || exit 2 -SH - end - end - - # remote-nextcloud-docker w/basic data - - config.vm.define "remote-nextcloud-docker" do |m1| - m1.vm.provision :shell, :inline => <<-SH -cd /mailinabox -export PRIMARY_HOSTNAME=qa2.abc.com -export FEATURE_MUNIN=false -tests/system-setup/remote-nextcloud-docker.sh upgrade --populate=basic || exit 1 -tests/runner.sh -no-smtp-remote remote-nextcloud upgrade-basic default || exit 2 -SH - end - - - # upgrade-from-upstream - - config.vm.define "upgrade-from-upstream" do |m1| - m1.vm.provision :shell, :inline => <<-SH -cd /mailinabox -export PRIMARY_HOSTNAME=qa3.abc.com -# TODO: change UPSTREAM_TAG to 'main' once upstream is installable -export UPSTREAM_TAG=v67 -tests/system-setup/upgrade-from-upstream.sh --populate=basic --populate=totpuser || exit 1 -tests/runner.sh -no-smtp-remote upgrade-basic upgrade-totpuser default || exit 2 -SH - end - - # upgrade - - # this test is only needed when testing migrations from miabldap - # to a newer miabldap with a migration step - # - # upgrade will handle testing upgrades of - # miabldap with or without a new migration step - config.vm.define "upgrade" do |m1| - m1.vm.provision :shell, :inline => <<-SH -cd /mailinabox -# TODO: remove DEB_PYTHON_INSTALL_LAYOUT once MIABLDAP_RELEASE_TAG >= v66 (see https://github.com/downtownallday/mailinabox-ldap/commit/371f5bc1b236de40a1ed5d9118140ee13fddf5dc) -export DEB_PYTHON_INSTALL_LAYOUT='deb' -export PRIMARY_HOSTNAME=upgrade.abc.com -tests/system-setup/upgrade.sh --populate=basic --populate=totpuser || exit 1 -tests/runner.sh -no-smtp-remote upgrade-basic upgrade-totpuser default || exit 2 -SH - end - - # unsetvars: because miab sets bash '-e' to fail any setup script - # when a script command returns a non-zero exit code, and more - # importantly '-u' which fails scripts when any unset variable is - # accessed, this definition sets a minimal number of environment - # variables prior to running start.sh. Doing so will test that no - # failures occur during setup in the most common use case because - # other vagrant definitions in this file load - # tests/system-setup/setup-default.sh, which pre-assign a value to - # most variables. - - if ENV['tests']=='all' or ENV['tests']=='pre-commit' - config.vm.define "unsetvars" do |m1| - m1.vm.hostname = "mailinabox.lan" - m1.vm.network "private_network", ip: "192.168.56.4" - m1.vm.provision :shell, :inline => <<-SH -export NONINTERACTIVE=1 -export PUBLIC_IP=auto -export PUBLIC_IPV6=auto -export PRIMARY_HOSTNAME=auto -export SKIP_NETWORK_CHECKS=1 -cd /mailinabox -setup/start.sh -SH - end - end - - -end diff --git a/tests/vagrant/funcs.rb b/tests/vagrant/funcs.rb deleted file mode 100644 index b304e242..00000000 --- a/tests/vagrant/funcs.rb +++ /dev/null @@ -1,25 +0,0 @@ -##### -##### This file is part of Mail-in-a-Box-LDAP which is released under the -##### terms of the GNU Affero General Public License as published by the -##### Free Software Foundation, either version 3 of the License, or (at -##### your option) any later version. See file LICENSE or go to -##### https://github.com/downtownallday/mailinabox-ldap for full license -##### details. -##### - -def use_preloaded_box(obj, name, preloaded_dir=".") - obj.vm.box = String.new(name) - _name=name.sub! '/','-' # ubuntu/bionic64 => ubuntu-bionic64 - if File.file?("#{preloaded_dir}/preloaded/preloaded-#{_name}.box") - # box name needs to be unique on the system - obj.vm.box = "preloaded-miabldap-#{_name}" - obj.vm.box_url = "file://" + Dir.pwd + "/#{preloaded_dir}/preloaded/preloaded-#{_name}.box" - if Vagrant.has_plugin?('vagrant-vbguest') - # do not update additions when booting this machine - obj.vbguest.auto_update = false - end - end -end - -# Grab the name of the default interface -$default_network_interface = `ip route | awk '/^default/ {printf "%s", $5; exit 0}'` diff --git a/tests/vagrant/parallel.sh b/tests/vagrant/parallel.sh deleted file mode 100755 index 98fb0c15..00000000 --- a/tests/vagrant/parallel.sh +++ /dev/null @@ -1,86 +0,0 @@ -#!/bin/bash -##### -##### This file is part of Mail-in-a-Box-LDAP which is released under the -##### terms of the GNU Affero General Public License as published by the -##### Free Software Foundation, either version 3 of the License, or (at -##### your option) any later version. See file LICENSE or go to -##### https://github.com/downtownallday/mailinabox-ldap for full license -##### details. -##### - - -# Parallel provisioning for virtualbox because "The Vagrant VirtualBox -# provider does not support parallel execution at this time" -# (https://www.vagrantup.com/docs/providers/virtualbox/usage.html) -# -# Credit to: -# https://dzone.com/articles/parallel-provisioning-speeding -# - -. "$(dirname "$0")/../lib/color-output.sh" -. "$(dirname "$0")/../lib/misc.sh" - -if [ -z "$tests" ]; then - export tests="pre-commit" -fi - - -OUTPUT_DIR=out -#rm -rf "$OUTPUT_DIR" -mkdir -p "$OUTPUT_DIR" - -# set total parallel vms to (#cores minus 1) -MAX_PROCS=$(cat /proc/cpuinfo | grep processor | wc -l) -let MAX_PROCS-=1 - - -parallel_provision() { - while read box; do - outfile="$OUTPUT_DIR/$box.out.txt" - rm -f "$outfile" - echo "Provisioning '$box'. Output will be in: $outfile" 1>&2 - echo $box - done | xargs -P $MAX_PROCS -I"BOXNAME" \ - sh -c 'vagrant provision BOXNAME >'"$OUTPUT_DIR/"'BOXNAME.out.txt 2>&1 && echo "EXITCODE: 0" >> '"$OUTPUT_DIR/"'BOXNAME.out.txt || echo "EXITCODE: $?" >>'"$OUTPUT_DIR/"'BOXNAME.out.txt' -} - -## -- main -- ## - -start_time="$(date +%s)" - -# start boxes sequentially to avoid vbox explosions -vagrant up --no-provision - -# but run provision tasks in parallel -boxes="$(vagrant status | awk '/running \(/ {print $1}')" -echo "$boxes" | parallel_provision - - -# output overall result - Vagrantfile script must output "EXITCODE: " -H1 "Results" - -rc=0 -for box in $boxes; do - file="$OUTPUT_DIR"/$box.out.txt - exitcode="$(tail "$file" | grep EXITCODE: | awk '{print $NF}')" - echo -n "$box: " - if [ -z "$exitcode" ]; then - danger "NO EXITCODE!" - [ $rc -eq 0 ] && rc=2 - elif [ "$exitcode" == "0" ]; then - success "SUCCESS" - else - danger "FAILURE ($exitcode)" - rc=1 - fi -done - -# output elapsed time -end_time="$(date +%s)" -echo "" -echo "Elapsed time: $(elapsed_pretty $start_time $end_time)" - -# exit -echo "" -echo "Guest VMs are running! Destroy them with 'vagrant destroy -f'" -exit $rc diff --git a/tests/vagrant/preloaded/.gitignore b/tests/vagrant/preloaded/.gitignore deleted file mode 100644 index e24e7c27..00000000 --- a/tests/vagrant/preloaded/.gitignore +++ /dev/null @@ -1,2 +0,0 @@ -*.box -src/ diff --git a/tests/vagrant/preloaded/Vagrantfile b/tests/vagrant/preloaded/Vagrantfile deleted file mode 100644 index 453bf80e..00000000 --- a/tests/vagrant/preloaded/Vagrantfile +++ /dev/null @@ -1,58 +0,0 @@ -##### -##### This file is part of Mail-in-a-Box-LDAP which is released under the -##### terms of the GNU Affero General Public License as published by the -##### Free Software Foundation, either version 3 of the License, or (at -##### your option) any later version. See file LICENSE or go to -##### https://github.com/downtownallday/mailinabox-ldap for full license -##### details. -##### - -def checkout_tag_and_mount(obj, tag) - if "#{tag}" == "" - obj.vm.synced_folder "../../..", "/mailinabox", id: "mailinabox", automount: false - else - _srcdir="src/maibldap-#{tag}" - if not Dir.exist?(_srcdir) - puts "Cloning tag #{tag} to #{_srcdir}" - if tag.size==40 and tag.match?(/\A[0-9a-fA-F]+\Z/) - system("git clone #{ENV['MIABLDAP_GIT']} #{_srcdir}") - system("cd #{_srcdir}; git reset --hard #{tag}") - else - system("git clone -b #{tag} --depth 1 #{ENV['MIABLDAP_GIT']} #{_srcdir}") - end - end - obj.vm.synced_folder _srcdir, "/mailinabox", id: "mailinabox", automount: false - end -end - - -Vagrant.configure("2") do |config| - - checkout_tag_and_mount config, ENV['RELEASE_TAG'] - - config.vm.define "preloaded-ubuntu-bionic64" do |m1| - m1.vm.box = "ubuntu/bionic64" - m1.vm.provision :shell, :inline => <<-SH -cd /mailinabox -tests/vagrant/preloaded/prepvm.sh --no-dry-run -rc=$? -echo "$rc" > "/vagrant/prepcode.txt" -[ $rc -gt 0 ] && exit 1 -exit 0 -SH - end - - config.vm.define "preloaded-ubuntu-jammy64" do |m1| - m1.vm.box = "ubuntu/jammy64" - m1.vm.boot_timeout = 30 - m1.vm.provision :shell, :inline => <<-SH -cd /mailinabox -tests/vagrant/preloaded/prepvm.sh --no-dry-run -rc=$? -echo "$rc" > "/vagrant/prepcode.txt" -[ $rc -gt 0 ] && exit 1 -exit 0 -SH - end - -end diff --git a/tests/vagrant/preloaded/create_preloaded.sh b/tests/vagrant/preloaded/create_preloaded.sh deleted file mode 100755 index c9146fd6..00000000 --- a/tests/vagrant/preloaded/create_preloaded.sh +++ /dev/null @@ -1,134 +0,0 @@ -#!/bin/bash -##### -##### This file is part of Mail-in-a-Box-LDAP which is released under the -##### terms of the GNU Affero General Public License as published by the -##### Free Software Foundation, either version 3 of the License, or (at -##### your option) any later version. See file LICENSE or go to -##### https://github.com/downtownallday/mailinabox-ldap for full license -##### details. -##### - - -# load defaults for MIABLDAP_GIT and MIABLDAP_FINAL_RELEASE_TAG_BIONIC64 (make available to Vagrantfile) -pushd "../../.." >/dev/null -source tests/lib/color-output.sh -source tests/system-setup/setup-defaults.sh || exit 1 -popd >/dev/null - -H1 "Destroy any running boxes" -vagrant destroy -f -rm -f prepcode.txt - -H1 "Ensure plugins are installed" -for plugin in "vagrant-vbguest" "vagrant-reload" -do - if ! vagrant plugin list | grep -F "$plugin" >/dev/null; then - vagrant plugin install "$plugin" || exit 1 - fi -done - -H1 "Upgrade base boxes" -vagrant box update - - -boxes=( - "preloaded-ubuntu-jammy64" - "preloaded-ubuntu-bionic64" -) -# preload packages from source of the following git tags. empty string -# means use the current source tree -tags=( - "" - "$MIABLDAP_FINAL_RELEASE_TAG_BIONIC64" -) -try_reboot=( - true - false -) -idx=0 - -for box in "${boxes[@]}" -do - if [ -z "$1" ]; then - # no cli arguments - only process first box - [ $idx -ge 1 ] && break - else - # cli argument specifies "all" or a named box - if [ "$1" != "all" -a "$1" != "$box" ]; then - let idx+=1 - continue - fi - fi - - H1 "Provision: $box" - export RELEASE_TAG="${tags[$idx]}" - vagrant up $box | tee /tmp/$box.out - upcode=$? - - if [ $upcode -eq 0 -a ! -e "./prepcode.txt" ] && ${try_reboot[$idx]} && grep -F 'Authentication failure' /tmp/$box.out >/dev/null; then - # note: upcode is 0 only if config.vm.boot_timeout is set. - # If this works it may be an indication that ruby's internal - # ssh does not support the algorithm required by the server, - # or the public key does not match (vagrant and vm out of - # sync) - echo "" - echo "VAGRANT AUTHENTICATION FAILURE - TRYING LOOSER ALLOWED SSHD ALGS" - if vagrant ssh $box -c "sudo bash -c 'echo PubkeyAcceptedAlgorithms +ssh-rsa > /etc/ssh/sshd_config.d/miabldap.conf; sudo systemctl restart sshd'"; then - vagrant halt $box - vagrant up $box - upcode=$? - fi - fi - - if [ $upcode -ne 0 -a ! -e "./prepcode.txt" ] && ${try_reboot[$idx]} - then - # a reboot may be necessary if guest addtions was newly - # compiled by vagrant plugin "vagrant-vbguest" - echo "" - echo "VAGRANT UP RETURNED $upcode -- RETRYING AFTER REBOOT" - vagrant halt $box - vagrant up $box - upcode=$? - fi - - rm -f /tmp/$box.out - - let idx+=1 - prepcode=$(cat "./prepcode.txt") - rm -f prepcode.txt - echo "" - echo "VAGRANT UP RETURNED $upcode" - echo "PREPVM RETURNED $prepcode" - - if [ "$prepcode" != "0" -o $upcode -ne 0 ]; then - echo "FAILED!!!!!!!!" - vagrant destroy -f $box - exit 1 - fi - - if vagrant ssh $box -- cat /var/run/reboot-required >/dev/null 2>&1; then - echo "REBOOT REQUIRED" - vagrant reload $box - else - echo "REBOOT NOT REQUIRED" - fi - - vagrant halt $box - vagrant package $box - rm -f $box.box - mv package.box $box.box - - vagrant destroy -f $box - cached_name="$(sed 's/preloaded-/preloaded-miabldap-/' <<<"$box")" - echo "Removing cached box $cached_name" - if [ -e "../funcs.rb" ]; then - pushd .. > /dev/null - vagrant box remove $cached_name - code=$? - popd > /dev/null - else - vagrant box remove $cached_name - code=$? - fi - echo "Remove cache box result: $code - ignoring" -done diff --git a/tests/vagrant/preloaded/prepvm.sh b/tests/vagrant/preloaded/prepvm.sh deleted file mode 100755 index e43e2e0b..00000000 --- a/tests/vagrant/preloaded/prepvm.sh +++ /dev/null @@ -1,230 +0,0 @@ -#!/bin/bash -##### -##### This file is part of Mail-in-a-Box-LDAP which is released under the -##### terms of the GNU Affero General Public License as published by the -##### Free Software Foundation, either version 3 of the License, or (at -##### your option) any later version. See file LICENSE or go to -##### https://github.com/downtownallday/mailinabox-ldap for full license -##### details. -##### - - -# Run this on a VM to pre-install all the packages, then -# take a snapshot - it will greatly speed up subsequent -# test installs - -# -# What won't be installed: -# -# Nextcloud and Roundcube are downloaded with wget by the setup -# scripts, so they are not included -# -# slapd - we want to test installation with setup/ldap.sh -# - -if [ ! -d "setup" ]; then - echo "Run from the miab root directory" - exit 1 -fi - -source tests/lib/misc.sh -source tests/lib/system.sh -source tests/lib/color-output.sh - -dry_run=true -start=$(date +%s) - -if [ "$1" == "--no-dry-run" ]; then - dry_run=false -fi - -if $dry_run; then - echo "WARNING: dry run is TRUE, no changes will be made" -fi - - -# prevent apt from running needrestart(1) -export NEEDRESTART_SUSPEND=true - -# prevent interaction during package install -export DEBIAN_FRONTEND=noninteractive - -# what major version of ubuntu are we installing on? -OS_MAJOR=$(. /etc/os-release; echo $VERSION_ID | awk -F. '{print $1}') - - -remove_line_continuation() { - local file="$1" - awk ' -BEGIN { C=0 } -C==1 && /[^\\]$/ { C=0; print $0; next } -C==1 { printf("%s",substr($0,0,length($0)-1)); next } -/\\$/ { C=1; printf("%s",substr($0,0,length($0)-1)); next } - { print $0 }' \ - "$file" -} - -install_packages() { - local return_code=0 - while read line; do - pkgs="" - case "$line" in - apt_install* ) - pkgs="$(cut -c12- <<<"$line")" - ;; - "apt-get install"* ) - pkgs="$(cut -c16- <<<"$line")" - ;; - "apt install"* ) - pkgs="$(cut -c12- <<<"$line")" - ;; - esac - - # don't install slapd - pkgs="$(sed 's/slapd//g' <<< "$pkgs")" - - # manually set PHP_VER if necessary - if grep "PHP_VER" <<<"$pkgs" >/dev/null; then - pkgs="$(sed "s/\"\?\${*PHP_VER}*\"\?/$PHP_VER/g" <<< "$pkgs")" - fi - - if [ ! -z "$pkgs" ]; then - H2 "install: $pkgs" - if ! $dry_run; then - exec_no_output apt-get install -y $pkgs - let return_code+=$? - fi - fi - done - return $return_code -} - -install_ppas() { - H1 "Add apt repositories" - grep 'hide_output add-apt-repository' setup/system.sh | - while read line; do - line=$(sed 's/^hide_output //' <<< "$line") - H2 "$line" - if ! $dry_run; then - exec_no_output $line - fi - done -} - -add_swap() { - H1 "Add a swap file to the system" - if ! $dry_run; then - dd if=/dev/zero of=/swapfile bs=1024 count=$[1024*1024] status=none - chmod 600 /swapfile - mkswap /swapfile - swapon /swapfile - echo "/swapfile none swap sw 0 0" >> /etc/fstab - fi -} - - -# install PPAs from sources -install_ppas - -# add swap file -add_swap - -# obtain PHP_VER variable from sources -PHP_VER=$(source setup/functions.sh; echo $PHP_VER) - - -if ! $dry_run; then - H1 "Upgrade system" - H2 "apt update" - exec_no_output apt-get update -y || exit 1 - H2 "apt upgrade" - exec_no_output apt-get upgrade -y --with-new-pkgs || exit 1 - H2 "apt autoremove" - exec_no_output apt-get autoremove -y -fi - -# without using the same installation order as setup/start.sh, we end -# up with the system's php getting installed in addition to the -# non-system php that may also installed by setup (don't know why, -# probably one of the packages has a dependency). create an ordered -# list of files to process so we get a similar system setup. - -setup_files=( $(ls setup/*.sh) ) -desired_order=( - setup/functions.sh - setup/preflight.sh - setup/questions.sh - setup/network-checks.sh - setup/system.sh - setup/ssl.sh - setup/dns.sh - setup/ldap.sh - setup/mail-postfix.sh - setup/mail-dovecot.sh - setup/mail-users.sh - setup/dkim.sh - setup/spamassassin.sh - setup/web.sh - setup/webmail.sh - setup/nextcloud.sh - setup/zpush.sh - setup/management.sh - setup/management-capture.sh - setup/munin.sh - setup/firstuser.sh -) -ordered_files=() -for file in "${desired_order[@]}" "${setup_files[@]}"; do - if [ -e "$file" ] && ! array_contains "$file" "${ordered_files[@]}"; then - ordered_files+=( "$file" ) - fi -done - -failed=0 - -for file in ${ordered_files[@]}; do - H1 "$file" - remove_line_continuation "$file" | install_packages - [ $? -ne 0 ] && let failed+=1 -done - -if ! $dry_run; then - # bonus - H1 "install extras" - - H2 "openssh, emacs, ntpdate, net-tools, jq" - exec_no_output apt-get install -y openssh-server emacs-nox ntpdate net-tools jq || let failed+=1 - - # these are added by system-setup scripts and needed for test runner - H2 "python3-dnspython" - exec_no_output apt-get install -y python3-dnspython || let failed+=1 - H2 "pyotp(pip)" - exec_no_output python3 -m pip install pyotp --quiet || let failed+=1 - - # ...and for browser-based tests - #H2 "x11" # needed for chromium w/head (not --headless) - #exec_no_output apt-get install -y xorg openbox xvfb gtk2-engines-pixbuf dbus-x11 xfonts-base xfonts-100dpi xfonts-75dpi xfonts-cyrillic xfonts-scalable x11-apps imagemagick || let failed+=1 - H2 "chromium" - #exec_no_output apt-get install -y chromium-browser || let failed+=1 - exec_no_output snap install chromium || let failed+=1 - H2 "selenium(pip)" - exec_no_output python3 -m pip install selenium --quiet || let failed+=1 - - # remove apache, which is what setup will do - H2 "remove apache2" - exec_no_output apt-get -y purge apache2 apache2-\* - -fi - -end=$(date +%s) -echo "" -echo "" -if [ $failed -gt 0 ]; then - echo "$failed failures! ($(elapsed_pretty $start $end))" - echo "" - exit 1 -else - echo "Successfully prepped in $(elapsed_pretty $start $end). Take a snapshot...." - echo "" - exit 0 -fi diff --git a/tests/vagrant/vanilla/.gitignore b/tests/vagrant/vanilla/.gitignore deleted file mode 100644 index 69c190a0..00000000 --- a/tests/vagrant/vanilla/.gitignore +++ /dev/null @@ -1,2 +0,0 @@ -.vagrant -*-console.log diff --git a/tests/vagrant/vanilla/Vagrantfile b/tests/vagrant/vanilla/Vagrantfile deleted file mode 100644 index 98e27d92..00000000 --- a/tests/vagrant/vanilla/Vagrantfile +++ /dev/null @@ -1,87 +0,0 @@ -##### -##### This file is part of Mail-in-a-Box-LDAP which is released under the -##### terms of the GNU Affero General Public License as published by the -##### Free Software Foundation, either version 3 of the License, or (at -##### your option) any later version. See file LICENSE or go to -##### https://github.com/downtownallday/mailinabox-ldap for full license -##### details. -##### - -load '../funcs.rb' - -Vagrant.configure("2") do |config| - - config.vm.synced_folder "../../..", "/mailinabox", id: "mailinabox", automount: false - config.vm.network "public_network", bridge: "#$default_network_interface" - use_preloaded_box config, "ubuntu/jammy64", ".." - - if ENV['tests']=='ciab' - - # vanilla connected to ciab (ciab does not need to be up) - - config.vm.define "vanilla" do |m1| - m1.vm.provision :shell, :inline => <<-SH -cat >/tmp/provision.sh </dev/null; then - # firewall rules aren't added when ciab is down - echo "For testing, allow ldaps from anywhere" - ufw allow ldaps -fi -echo "Add smart host alias - so \\$NC_HOST can send mail to/via this host" -( - source tests/lib/all.sh - rest_urlencoded POST /admin/mail/aliases/add qa@abc.com Test_1234 "address=@\\$NC_HOST" "description=smart-host" "permitted_senders=qa@abc.com" 2>/dev/null - echo "\\$REST_HTTP_CODE: \\$REST_OUTPUT" -) -EOF -chmod +x /tmp/provision.sh -/tmp/provision.sh -SH - end # vanilla connected to ciab - - else - - - # vanilla (default) install - - config.vm.define "vanilla" do |m1| - m1.vm.provision :shell, :inline => <<-SH -cat >/tmp/provision.sh < Date: Fri, 4 Oct 2024 16:26:29 -0400 Subject: [PATCH 12/14] validate argument --- tests/lxd/vanilla/provision.sh | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tests/lxd/vanilla/provision.sh b/tests/lxd/vanilla/provision.sh index 3d153e66..a70c09a9 100755 --- a/tests/lxd/vanilla/provision.sh +++ b/tests/lxd/vanilla/provision.sh @@ -44,7 +44,7 @@ exit \$rc " provision_done $? -else +elif [ -z "$1" ]; then # vanilla (default - no miab integration) provision_shell <<<" cd /mailinabox @@ -62,4 +62,7 @@ exit \$rc " provision_done $? +else + echo "Invalid argument: $1" + exit 1 fi From 696b597a9c4a7beea1e76ff5ade57f94bbd9770e Mon Sep 17 00:00:00 2001 From: downtownallday Date: Fri, 4 Oct 2024 16:27:26 -0400 Subject: [PATCH 13/14] use bash as 'source' is needed in provision scripts --- tests/bin/provision_functions.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/bin/provision_functions.sh b/tests/bin/provision_functions.sh index 68424f0b..1007113f 100644 --- a/tests/bin/provision_functions.sh +++ b/tests/bin/provision_functions.sh @@ -67,7 +67,7 @@ provision_shell() { else local tmp=$(mktemp) - echo "#!/bin/sh" >"$tmp" + echo "#!/bin/bash" >"$tmp" cat >>"$tmp" lxc --project "$project" file push "$tmp" "${inst}${remote_path}" $lxc_flags || return 1 rm -f "$tmp" From 119b11f0227b6565148221bbd0bccb6ef4011a15 Mon Sep 17 00:00:00 2001 From: downtownallday Date: Fri, 4 Oct 2024 17:22:12 -0400 Subject: [PATCH 14/14] remove upstream Vagrantfile --- Vagrantfile | 37 ------------------------------------- 1 file changed, 37 deletions(-) delete mode 100644 Vagrantfile diff --git a/Vagrantfile b/Vagrantfile deleted file mode 100644 index 771f7e27..00000000 --- a/Vagrantfile +++ /dev/null @@ -1,37 +0,0 @@ -# -*- mode: ruby -*- -##### -##### This file is part of Mail-in-a-Box-LDAP which is released under the -##### terms of the GNU Affero General Public License as published by the -##### Free Software Foundation, either version 3 of the License, or (at -##### your option) any later version. See file LICENSE or go to -##### https://github.com/downtownallday/mailinabox-ldap for full license -##### details. -##### - -# vi: set ft=ruby : - -Vagrant.configure("2") do |config| - config.vm.box = "ubuntu/jammy64" - - # Network config: Since it's a mail server, the machine must be connected - # to the public web. However, we currently don't want to expose SSH since - # the machine's box will let anyone log into it. So instead we'll put the - # machine on a private network. - config.vm.hostname = "mailinabox.lan" - config.vm.network "private_network", ip: "192.168.56.4" - - config.vm.provision :shell, :inline => <<-SH - # Set environment variables so that the setup script does - # not ask any questions during provisioning. We'll let the - # machine figure out its own public IP. - export NONINTERACTIVE=1 - export PUBLIC_IP=auto - export PUBLIC_IPV6=auto - export PRIMARY_HOSTNAME=auto - #export SKIP_NETWORK_CHECKS=1 - - # Start the setup script. - cd /vagrant - setup/start.sh -SH -end