Speed up vm creation

tests/vagrant/Vagrantfile

@@ -4,10 +4,16 @@ Vagrant.configure("2") do |config|
   config.vm.synced_folder "../..", "/mailinabox", id: "mailinabox", automount: false
   config.vm.provision "file", source:"globals.sh", destination:"globals.sh"
 
+  if File.file?("preloaded/preloaded-ubuntu-bionic64.box")
+    config.vm.box = "preloaded-ubuntu-bionic64"
+    config.vm.box_url = "file://" + Dir.pwd + "/preloaded/preloaded-ubuntu-bionic64.box"
+  else
+    config.vm.box = "ubuntu/bionic64"
+  end
+
   # fresh install with encryption-at-rest
 
   config.vm.define "remote-nextcloud-docker-ehdd" do |m1|
-    m1.vm.box = "ubuntu/bionic64"
     m1.vm.provision :shell, :inline => <<-SH
 source globals.sh || exit 1
 export PRIMARY_HOSTNAME=qa1.abc.com
@@ -26,7 +32,6 @@ SH
   # remote-nextcloud-docker w/basic data
 
   config.vm.define "remote-nextcloud-docker" do |m1|
-    m1.vm.box = "ubuntu/bionic64"
     m1.vm.provision :shell, :inline => <<-SH
 source globals.sh || exit 1
 export PRIMARY_HOSTNAME=qa2.abc.com
@@ -43,16 +48,17 @@ SH
 
   # upgrade-from-upstream
   
-  config.vm.define "upgrade-from-upstream" do |m2|
-    m2.vm.box = "ubuntu/bionic64"
-    m2.vm.provision :shell, :inline => <<-SH
+  config.vm.define "upgrade-from-upstream" do |m1|
+    m1.vm.provision :shell, :inline => <<-SH
 source globals.sh || exit 1
 export PRIMARY_HOSTNAME=qa3.abc.com
-export UPSTREAM_TAG=master
+#export UPSTREAM_TAG=master
+export MIAB_UPSTREAM_GIT="https://github.com/fspoettel/mailinabox.git"
+export UPSTREAM_TAG="admin-panel-2fa"
 cd /mailinabox
-tests/system-setup/upgrade-from-upstream.sh basic; rc=$?
+tests/system-setup/upgrade-from-upstream.sh basic totpuser; rc=$?
 if [ $rc -eq 0 ]; then
-   tests/runner.sh default upgrade-basic; rc=$?
+   tests/runner.sh upgrade-basic upgrade-totpuser default; rc=$?
 fi
 echo "EXITCODE: $rc"
 SH

tests/vagrant/preloaded/.gitignore

@@ -0,0 +1 @@
+*.box

tests/vagrant/preloaded/Vagrantfile

@@ -0,0 +1,19 @@
+
+Vagrant.configure("2") do |config|
+
+  config.vm.synced_folder "../../..", "/mailinabox", id: "mailinabox", automount: false
+
+  config.vm.define "preloaded-ubuntu-bionic64" do |m1|
+    m1.vm.box = "ubuntu/bionic64"
+    m1.vm.provision :shell, :inline => <<-SH
+cd /mailinabox
+tests/vagrant/preloaded/prepvm.sh --no-dry-run
+rc=$?
+echo "$rc" > "tests/vagrant/preloaded/prepcode.txt"
+[ $rc -gt 0 ] && exit 1
+exit 0
+SH
+  end
+
+
+end

tests/vagrant/preloaded/create_preloaded.sh

@@ -0,0 +1,25 @@
+#!/bin/bash
+
+vagrant destroy -f
+rm -f prepcode.txt
+
+vagrant up preloaded-ubuntu-bionic64
+upcode=$?
+prepcode=$(cat "./prepcode.txt")
+rm -f prepcode.txt
+echo ""
+echo "VAGRANT UP RETURNED $upcode"
+echo "PREPVM RETURNED $prepcode"
+
+if [ "$prepcode" != "0" -o $upcode -ne 0 ]; then
+    echo "FAILED!!!!!!!!"
+    vagrant destroy -f
+    exit 1
+fi
+
+vagrant halt
+vagrant package
+rm -f preloaded.box
+mv package.box preloaded-ubuntu-bionic64.box
+
+vagrant destroy -f

tests/vagrant/preloaded/prepvm.sh

@@ -0,0 +1,105 @@
+#!/bin/bash
+
+# Run this on a VM to pre-install all the packages, then
+# take a snapshot - it will greatly speed up subsequent
+# test installs
+
+#
+# What won't be installed:
+#
+# Nextcloud and Roundcube are downloaded with wget by the setup
+# scripts, so they are not included
+#
+# postfix, postgrey and slapd because they require terminal input
+#
+
+
+if [ ! -d "setup" ]; then
+    echo "Run from the miab root directory"
+    exit 1
+fi
+
+
+dry_run=true
+
+if [ "$1" == "--no-dry-run" ]; then
+    dry_run=false
+fi
+
+if $dry_run; then
+    echo "WARNING: dry run is TRUE, no changes will be made"
+fi
+
+
+remove_line_continuation() {
+    local file="$1"
+    awk '
+BEGIN            { C=0 } 
+C==1 && /[^\\]$/ { C=0; print $0; next } 
+C==1             { printf("%s",substr($0,0,length($0)-1)); next } 
+/\\$/            { C=1; printf("%s",substr($0,0,length($0)-1)); next } 
+                 { print $0 }' \
+                     "$file"
+}
+
+install_packages() {
+    while read line; do
+        pkgs=""
+        case "$line" in
+             apt_install* )
+                 pkgs="$(cut -c12- <<<"$line")"
+                 ;;
+             "apt-get install"* )
+                 pkgs="$(cut -c16- <<<"$line")"
+                 ;;
+             "apt install"* )
+                 pkgs="$(cut -c12- <<<"$line")"
+                 ;;
+        esac
+        
+        # don't install postfix - causes problems with setup scripts
+        # and requires user input. exclude postgrey because it will
+        # install postfix as a dependency
+        pkgs="$(sed 's/postgrey//g' <<< "$pkgs")"
+        pkgs="$(sed 's/postfix-[^ $]*//g' <<<"$pkgs")"
+        pkgs="$(sed 's/postfix//g' <<<"$pkgs")"
+
+        # don't install slapd - it requires user input
+        pkgs="$(sed 's/slapd//g' <<< "$pkgs")"
+        
+        if [ ! -z "$pkgs" ]; then
+            echo "install: $pkgs"
+            if ! $dry_run; then
+                apt-get install -y -qq $pkgs
+            fi
+        fi
+    done
+}
+
+if ! $dry_run; then
+    apt-get update -y
+    apt-get upgrade -y
+    apt-get autoremove -y
+fi
+
+for file in $(ls setup/*.sh); do
+    remove_line_continuation "$file" | install_packages
+done
+
+if ! $dry_run; then
+    # bonus
+    apt-get install -y -qq openssh-server
+    apt-get install -y -qq emacs-nox
+    apt-get install -y -qq ntpdate
+
+    # these are added by system-setup scripts and needed for test runner
+    apt-get install -y -qq python3-dnspython jq
+
+    # remove apache, which is what setup will do
+    apt-get -y -qq purge apache2 apache2-\*
+
+    echo ""
+    echo ""
+    echo "Done. Take a snapshot...."
+    echo ""
+fi

tests/vagrant/vanilla/Vagrantfile

@@ -7,7 +7,12 @@ Vagrant.configure("2") do |config|
   # vanilla install
 
   config.vm.define "vanilla" do |m1|
+    if File.file?("../preloaded/preloaded-ubuntu-bionic64.box")
+      m1.vm.box = "preloaded-ubuntu-bionic64"
+      m1.vm.box_url = "file://" + Dir.pwd + "/../preloaded/preloaded-ubuntu-bionic64.box"
+    else
       m1.vm.box = "ubuntu/bionic64"
+    end
     m1.vm.network "forwarded_port", guest:443, host:8443, protocol:"tcp"
     m1.vm.provision :shell, :inline => <<-SH
 source globals.sh || exit 1