syncing up box creation with ciab (prepping for focal)

.gitignore

@@ -5,4 +5,5 @@ tools/__pycache__/
 externals/
 .env
 .vagrant
-api/docs/api-docs.html
+api/docs/api-docs.html
+downloads/

tests/vagrant/Vagrantfile

@@ -1,35 +1,22 @@
+load './funcs.rb'
 
 Vagrant.configure("2") do |config|
 
   config.vm.synced_folder "../..", "/mailinabox", id: "mailinabox", automount: false
-  config.vm.provision "file", source:"globals.sh", destination:"globals.sh"
+  use_preloaded_box config, "ubuntu/bionic64"
-
-  if File.file?("preloaded/preloaded-ubuntu-bionic64.box")
-    config.vm.box = "preloaded-ubuntu-bionic64"
-    config.vm.box_url = "file://" + Dir.pwd + "/preloaded/preloaded-ubuntu-bionic64.box"
-    if Vagrant.has_plugin?('vagrant-vbguest')
-      # do NOT check the correct additions version when booting this machine
-      config.vbguest.auto_update = false
-    end
-  else
-    config.vm.box = "ubuntu/bionic64"
-  end
 
   # fresh install with encryption-at-rest
 
   config.vm.define "remote-nextcloud-docker-ehdd" do |m1|
     m1.vm.provision :shell, :inline => <<-SH
-source globals.sh || exit 1
+cd /mailinabox
+source tests/vagrant/globals.sh || exit 1
 export PRIMARY_HOSTNAME=qa1.abc.com
 export FEATURE_MUNIN=false
 export EHDD_KEYFILE=$HOME/keyfile
 echo -n "boo" >$EHDD_KEYFILE
-cd /mailinabox
+tests/system-setup/remote-nextcloud-docker.sh || exit 1
-tests/system-setup/remote-nextcloud-docker.sh; rc=$?
+tests/runner.sh ehdd default remote-nextcloud || exit 2
-if [ $rc -eq 0 ]; then
-   tests/runner.sh ehdd default remote-nextcloud; rc=$?
-fi
-echo "EXITCODE: $rc"
 SH
   end
 
@@ -37,15 +24,12 @@ SH
 
   config.vm.define "remote-nextcloud-docker" do |m1|
     m1.vm.provision :shell, :inline => <<-SH
-source globals.sh || exit 1
+cd /mailinabox
+source tests/vagrant/globals.sh || exit 1
 export PRIMARY_HOSTNAME=qa2.abc.com
 export FEATURE_MUNIN=false
-cd /mailinabox
+tests/system-setup/remote-nextcloud-docker.sh upgrade basic || exit 1
-tests/system-setup/remote-nextcloud-docker.sh upgrade basic; rc=$?
+tests/runner.sh default remote-nextcloud upgrade-basic || exit 2
-if [ $rc -eq 0 ]; then
-   tests/runner.sh default remote-nextcloud upgrade-basic; rc=$?
-fi
-echo "EXITCODE: $rc"
 SH
   end
 
@@ -54,15 +38,12 @@ SH
   
   config.vm.define "upgrade-from-upstream" do |m1|
     m1.vm.provision :shell, :inline => <<-SH
-source globals.sh || exit 1
+cd /mailinabox
+source tests/vagrant/globals.sh || exit 1
 export PRIMARY_HOSTNAME=qa3.abc.com
 export UPSTREAM_TAG=master
-cd /mailinabox
+tests/system-setup/upgrade-from-upstream.sh basic totpuser || exit 1
-tests/system-setup/upgrade-from-upstream.sh basic totpuser; rc=$?
+tests/runner.sh upgrade-basic upgrade-totpuser default || exit 2
-if [ $rc -eq 0 ]; then
-   tests/runner.sh upgrade-basic upgrade-totpuser default; rc=$?
-fi
-echo "EXITCODE: $rc"
 SH
   end
 
@@ -80,21 +61,14 @@ SH
     config.vm.define "unsetvars" do |m1|
       m1.vm.hostname = "mailinabox.lan"
       m1.vm.network "private_network", ip: "192.168.50.4"
-      
       m1.vm.provision :shell, :inline => <<-SH
-# Set environment variables so that the setup script does
-# not ask any questions during provisioning. We'll let the
-# machine figure out its own public IP.
 export NONINTERACTIVE=1
 export PUBLIC_IP=auto
 export PUBLIC_IPV6=auto
 export PRIMARY_HOSTNAME=auto
 export SKIP_NETWORK_CHECKS=1
-
-# Start the setup script.
 cd /mailinabox
 setup/start.sh
-echo "EXITCODE: $?"
 SH
     end
   end

tests/vagrant/funcs.rb

@@ -0,0 +1,15 @@
+def use_preloaded_box(obj, name, preloaded_dir=".")
+  _name=name.sub! '/','-'  # ubuntu/bionic64 => ubuntu-bionic64
+  if File.file?("#{preloaded_dir}/preloaded/preloaded-#{_name}.box")
+    # box name needs to be unique on the system
+    obj.vm.box = "preloaded-miabldap-#{_name}"
+    obj.vm.box_url = "file://" + Dir.pwd + "/#{preloaded_dir}/preloaded/preloaded-#{_name}.box"
+    if Vagrant.has_plugin?('vagrant-vbguest')
+      # do not update additions when booting this machine
+      obj.vbguest.auto_update = false
+    end
+  else
+    obj.vm.box = name
+  end
+end
+

tests/vagrant/parallel.sh

@@ -13,7 +13,7 @@
 
 
 OUTPUT_DIR=out
-rm -rf "$OUTPUT_DIR"
+#rm -rf "$OUTPUT_DIR"
 mkdir -p "$OUTPUT_DIR"
 
 # set total parallel vms to (#cores minus 1)
@@ -24,10 +24,11 @@ let MAX_PROCS-=1
 parallel_provision() {
     while read box; do
         outfile="$OUTPUT_DIR/$box.out.txt"
+        rm -f "$outfile"
         echo "Provisioning '$box'. Output will be in: $outfile" 1>&2
         echo $box
     done | xargs -P $MAX_PROCS -I"BOXNAME" \
-        sh -c 'vagrant provision BOXNAME >'"$OUTPUT_DIR/"'BOXNAME.out.txt 2>&1 || echo "Error Occurred: BOXNAME"'
+        sh -c 'vagrant provision BOXNAME >'"$OUTPUT_DIR/"'BOXNAME.out.txt 2>&1 && echo "EXITCODE: 0" >> '"$OUTPUT_DIR/"'BOXNAME.out.txt || echo "EXITCODE: $?" >>'"$OUTPUT_DIR/"'BOXNAME.out.txt'
 }
  
 ## -- main -- ##
@@ -38,15 +39,16 @@ start_time="$(date +%s)"
 vagrant up --no-provision
  
 # but run provision tasks in parallel
-vagrant status | grep running | awk '{print $1}' | parallel_provision
+boxes="$(vagrant status | awk '/running \(/ {print $1}')"
+echo "$boxes" | parallel_provision
 
 
 # output overall result - Vagrantfile script must output "EXITCODE: <num>"
 H1 "Results"
 
 rc=0
-for file in "$OUTPUT_DIR"/*.out.txt; do
+for box in $boxes; do
-    box=$(basename $file | awk -F. '{print $1}')
+    file="$OUTPUT_DIR"/$box.out.txt
     exitcode="$(tail "$file" | grep EXITCODE: | awk '{print $NF}')"
     echo -n "$box: "
     if [ -z "$exitcode" ]; then

tests/vagrant/vanilla/Vagrantfile

@@ -1,26 +1,20 @@
+load '../funcs.rb'
 
 Vagrant.configure("2") do |config|
 
   config.vm.synced_folder "../../..", "/mailinabox", id: "mailinabox", automount: false
-  config.vm.provision "file", source:"../globals.sh", destination:"globals.sh"
+  use_preloaded_box config, "ubuntu/bionic64", ".."
 
   # vanilla install
 
   config.vm.define "vanilla" do |m1|
-    if File.file?("../preloaded/preloaded-ubuntu-bionic64.box")
-      m1.vm.box = "preloaded-ubuntu-bionic64"
-      m1.vm.box_url = "file://" + Dir.pwd + "/../preloaded/preloaded-ubuntu-bionic64.box"
-    else
-      m1.vm.box = "ubuntu/bionic64"
-    end
     m1.vm.network "forwarded_port", guest:443, host:8443, protocol:"tcp"
     m1.vm.provision :shell, :inline => <<-SH
-source globals.sh || exit 1
+cd /mailinabox
+source tests/vagrant/globals.sh || exit 1
 export PRIMARY_HOSTNAME=vanilla.local
 export FEATURE_MUNIN=false
-cd /mailinabox
+tests/system-setup/vanilla.sh
-tests/system-setup/vanilla.sh; rc=$?
-echo "EXITCODE: $rc"
 SH
   end