Add Vagrant support for running automated tests

setup/start.sh

@@ -150,9 +150,9 @@ fi
 #
 # Run setup mods 
 #
-if [ -d local ]; then
+if [ -d "${LOCAL_MODS_DIR:-local}" ]; then
-    for mod in $(ls local | grep -v '~$'); do
+    for mod in $(ls "${LOCAL_MODS_DIR:-local}" | grep -v '~$'); do
-        local/$mod
+        ${LOCAL_MODS_DIR:-local}/$mod
     done
 fi
 

tests/lib/color-output.sh

@@ -1,9 +1,23 @@
 # ansi escapes for hilighting text
 F_DANGER=$(echo -e "\033[31m")
 F_WARN=$(echo -e "\033[93m")
+F_SUCCESS=$(echo -e "\033[32m")
 F_RESET=$(echo -e "\033[39m")
 
 
+success() {
+    local echoarg
+    case "$1" in
+        -n )
+            echoarg="$1"
+            shift
+            ;;
+        * )
+            echoarg=""
+    esac
+    echo $echoarg "${F_SUCCESS}$1${F_RESET}"
+}
+
 danger() {
     local echoarg
     case "$1" in

tests/lib/misc.sh

@@ -67,3 +67,17 @@ sha1() {
 	python3 -c "import hashlib; m=hashlib.sha1(); m.update(bytearray(r'''$txt''','utf-8')); print(m.hexdigest());" || die "Unable to generate sha1 hash"
 }
 
+elapsed_pretty() {
+    local start_s="$1"
+    local end_s="$2"
+    local elapsed elapsed_m elapsed_s
+    if [ -z "$end_s" ]; then
+        elapsed="$start_s"
+    else
+        let elapsed="$end_s - $start_s"
+    fi
+    
+    let elapsed_m="$elapsed / 60"
+    let elapsed_s="$elapsed % 60"
+    echo "${elapsed_m}m ${elapsed_s}s"
+}

tests/suites/_init.sh

@@ -14,7 +14,7 @@ set +eu
 # globals - all global variables are UPPERCASE
 ASSETS_DIR="assets"
 MIAB_DIR=".."
-BASE_OUTPUTDIR="$(realpath out)"
+BASE_OUTPUTDIR="$(realpath out)/$(hostname | awk -F. '{print $1}')"
 PYMAIL="./test_mail.py"
 declare -i OVERALL_SUCCESSES=0
 declare -i OVERALL_FAILURES=0

tests/system-setup/remote-nextcloud-docker.sh

@@ -84,11 +84,6 @@ install_nextcloud_docker() {
         container_started="false"
     fi
 
-    # H2 "docker: Update /etc/hosts so it can find MiaB-LDAP by name"
-    # echo "$PRIVATE_IP $PRIMARY_HOSTNAME" | \
-    #     docker exec -i NC bash -c 'cat >>/etc/hosts' \
-    #     || die "docker: could not update /etc/hosts"
-    
     # apt-get update
     H2 "docker: apt-get update"
     docker exec NC apt-get update || die "docker: apt-get update failed"
@@ -98,6 +93,7 @@ install_nextcloud_docker() {
     ufw allow ldaps || die "Unable to modify firewall to permit ldaps"
 
     # add MiaB-LDAP's ca_certificate.pem to docker's trusted cert list
+    # (because setup/ssl.sh created its own self-signed ca)
     H2 "docker: update trusted CA list"
     docker cp \
            $STORAGE_ROOT/ssl/ca_certificate.pem \
@@ -146,14 +142,14 @@ install_nextcloud_docker() {
 do_upgrade() {
     local populate_name="$1"
 
-    if [ -e "local/remote-nextcloud.sh" ]; then
+    if [ -e "$LOCAL_MODS_DIR/remote-nextcloud.sh" ]; then
         # we install w/o remote nextcloud first so we can add
         # a user w/contacts and ensure the contact exists in the
         # new system
-        if [ ! -L "local/remote-nextcloud.sh" ]; then
+        if [ ! -L "$LOCAL_MODS_DIR/remote-nextcloud.sh" ]; then
-            echo "Warning: local/remote-nextcloud.sh is a regular file - should be a symlink"
+            echo "Warning: $LOCAL_MODS_DIR/remote-nextcloud.sh is a regular file - should be a symlink"
         fi
-        die "Error: local/remote-nextcloud.sh exists - delete it and try again"
+        die "Error: $LOCAL_MODS_DIR/remote-nextcloud.sh exists - delete it and try again"
     fi
 
     # initialize test system

tests/system-setup/setup-defaults.sh

@@ -8,6 +8,7 @@ export STORAGE_ROOT="${STORAGE_ROOT:-/home/$STORAGE_USER}"
 export EMAIL_ADDR="${EMAIL_ADDR:-qa@abc.com}"
 export EMAIL_PW="${EMAIL_PW:-Test_1234}"
 export PUBLIC_IP="${PUBLIC_IP:-$(source ${MIAB_DIR:-.}/setup/functions.sh; get_default_privateip 4)}"
+export LOCAL_MODS_DIR="${LOCAL_MODS_DIR:-local}"
 
 if [ "$TRAVIS" == "true" ]; then
     export PRIMARY_HOSTNAME=${PRIMARY_HOSTNAME:-box.abc.com}

tests/system-setup/setup-funcs.sh

@@ -135,12 +135,12 @@ init_miab_testing() {
 
 enable_miab_mod() {
     local name="${1}.sh"
-    if [ ! -e "local/$name" ]; then
+    if [ ! -e "$LOCAL_MODS_DIR/$name" ]; then
-        mkdir -p "local"
+        mkdir -p "$LOCAL_MODS_DIR"
-        if ! ln -s "../setup/mods.available/$name" "local/$name"
+        if ! ln -s "$(pwd)/setup/mods.available/$name" "$LOCAL_MODS_DIR/$name"
         then
-            echo "Warning: copying instead of symlinking local/$name"
+            echo "Warning: copying instead of symlinking $LOCAL_MODS_DIR/$name"
-            cp "setup/mods.available/$name" "local/$name"
+            cp "setup/mods.available/$name" "$LOCAL_MODS_DIR/$name"
         fi
     fi
 }

tests/system-setup/upgrade-from-upstream.sh

@@ -125,12 +125,12 @@ populate() {
 case "$1" in
     capture )
         . /etc/mailinabox.conf
-        installed_state_capture "tests/system-setup/state/miab-ldap"
+        installed_state_capture "/tmp/state/miab-ldap"
         exit $?
         ;;
     compare )
         . /etc/mailinabox.conf
-        installed_state_compare "tests/system-setup/state/upstream" "tests/system-setup/state/miab-ldap"
+        installed_state_compare "/tmp/state/upstream" "/tmp/state/miab-ldap"
         exit $?
         ;;
     populate )
@@ -161,15 +161,15 @@ else
     populate_by_name "${1:-basic}"
 
     # capture upstream state
-    installed_state_capture "tests/system-setup/state/upstream"
+    installed_state_capture "/tmp/state/upstream"
 fi
 
 # install miab-ldap and capture state
 miab_ldap_install
-installed_state_capture "tests/system-setup/state/miab-ldap"
+installed_state_capture "/tmp/state/miab-ldap"
 
 # compare states
-if ! installed_state_compare "tests/system-setup/state/upstream" "tests/system-setup/state/miab-ldap"; then
+if ! installed_state_compare "/tmp/state/upstream" "/tmp/state/miab-ldap"; then
     die "Upstream and upgraded states are different !"
 fi
 

tests/vagrant/.gitignore

@@ -0,0 +1,3 @@
+.vagrant
+out
+*-console.log

tests/vagrant/Vagrantfile

@@ -0,0 +1,42 @@
+
+Vagrant.configure("2") do |config|
+
+  config.vm.synced_folder "../..", "/mailinabox", id: "mailinabox", automount: false
+  config.vm.provision "file", source:"globals.sh", destination:"globals.sh"
+
+  # remote-nextcloud-docker
+
+  config.vm.define "remote-nextcloud-docker" do |m1|
+    m1.vm.box = "ubuntu/bionic64"
+    m1.vm.provision :shell, :inline => <<-SH
+source globals.sh || exit 1
+export PRIMARY_HOSTNAME=qa1.abc.com
+export FEATURE_MUNIN=false
+cd /mailinabox
+if tests/system-setup/remote-nextcloud-docker.sh upgrade basic
+then
+   tests/runner.sh default remote-nextcloud upgrade-basic
+fi
+echo "EXITCODE: $?"
+SH
+  end
+  
+
+  # upgrade-from-upstream
+  
+  config.vm.define "upgrade-from-upstream" do |m2|
+    m2.vm.box = "ubuntu/bionic64"
+    m2.vm.provision :shell, :inline => <<-SH
+source globals.sh || exit 1
+export PRIMARY_HOSTNAME=qa2.abc.com
+export UPSTREAM_TAG=master
+cd /mailinabox
+if tests/system-setup/upgrade-from-upstream.sh basic
+then
+  tests/runner.sh default upgrade-basic
+fi
+echo "EXITCODE: $?"
+SH
+  end
+
+end

tests/vagrant/globals.sh

@@ -0,0 +1,2 @@
+export MIAB_LDAP_PROJECT=true
+export LOCAL_MODS_DIR=/local

tests/vagrant/parallel.sh

@@ -0,0 +1,71 @@
+#!/bin/bash
+
+# Parallel provisioning for virtualbox because "The Vagrant VirtualBox
+# provider does not support parallel execution at this time"
+# (https://www.vagrantup.com/docs/providers/virtualbox/usage.html)
+#
+# Credit to:
+#    https://dzone.com/articles/parallel-provisioning-speeding
+#
+
+. "$(dirname "$0")/../lib/color-output.sh"
+. "$(dirname "$0")/../lib/misc.sh"
+
+
+OUTPUT_DIR=out
+rm -rf "$OUTPUT_DIR"
+mkdir -p "$OUTPUT_DIR"
+
+# set total parallel vms to (#cores minus 1)
+MAX_PROCS=$(cat /proc/cpuinfo | grep processor | wc -l)
+let MAX_PROCS-=1
+
+ 
+parallel_provision() {
+    while read box; do
+        outfile="$OUTPUT_DIR/$box.out.txt"
+        echo "Provisioning '$box'. Output will be in: $outfile" 1>&2
+        echo $box
+    done | xargs -P $MAX_PROCS -I"BOXNAME" \
+        sh -c 'vagrant provision BOXNAME >'"$OUTPUT_DIR/"'BOXNAME.out.txt 2>&1 || echo "Error Occurred: BOXNAME"'
+}
+ 
+## -- main -- ##
+
+start_time="$(date +%s)"
+
+# start boxes sequentially to avoid vbox explosions
+vagrant up --no-provision
+ 
+# but run provision tasks in parallel
+vagrant status | grep running | awk '{print $1}' | parallel_provision
+
+
+# output overall result - Vagrantfile script must output "EXITCODE: <num>"
+H1 "Results"
+
+rc=0
+for file in "$OUTPUT_DIR"/*.out.txt; do
+    box=$(basename $file | awk -F. '{print $1}')
+    exitcode="$(tail "$file" | grep EXITCODE: | awk '{print $NF}')"
+    echo -n "$box: "
+    if [ -z "$exitcode" ]; then
+        danger "NO EXITCODE!"
+        [ $rc -eq 0 ] && rc=2
+    elif [ "$exitcode" == "0" ]; then
+        success "SUCCESS"
+    else
+        danger "FAILURE ($exitcode)"
+        rc=1
+    fi
+done
+
+# output elapsed time
+end_time="$(date +%s)"
+echo ""
+echo "Elapsed time: $(elapsed_pretty $start_time $end_time)"
+
+# exit
+echo ""
+echo "Guest VMs are running! Destroy them with 'vagrant destroy -f'"
+exit $rc