From a78e6eb3faa4b705198f0c38b1e2715e6da17176 Mon Sep 17 00:00:00 2001 From: downtownallday Date: Wed, 30 Sep 2020 13:33:40 -0400 Subject: [PATCH] Speed up vm creation --- tests/vagrant/Vagrantfile | 22 ++-- tests/vagrant/preloaded/.gitignore | 1 + tests/vagrant/preloaded/Vagrantfile | 19 ++++ tests/vagrant/preloaded/create_preloaded.sh | 25 +++++ tests/vagrant/preloaded/prepvm.sh | 105 ++++++++++++++++++++ tests/vagrant/vanilla/Vagrantfile | 7 +- 6 files changed, 170 insertions(+), 9 deletions(-) create mode 100644 tests/vagrant/preloaded/.gitignore create mode 100644 tests/vagrant/preloaded/Vagrantfile create mode 100755 tests/vagrant/preloaded/create_preloaded.sh create mode 100755 tests/vagrant/preloaded/prepvm.sh diff --git a/tests/vagrant/Vagrantfile b/tests/vagrant/Vagrantfile index 9dc982a7..ac7c429a 100644 --- a/tests/vagrant/Vagrantfile +++ b/tests/vagrant/Vagrantfile @@ -4,10 +4,16 @@ Vagrant.configure("2") do |config| config.vm.synced_folder "../..", "/mailinabox", id: "mailinabox", automount: false config.vm.provision "file", source:"globals.sh", destination:"globals.sh" + if File.file?("preloaded/preloaded-ubuntu-bionic64.box") + config.vm.box = "preloaded-ubuntu-bionic64" + config.vm.box_url = "file://" + Dir.pwd + "/preloaded/preloaded-ubuntu-bionic64.box" + else + config.vm.box = "ubuntu/bionic64" + end + # fresh install with encryption-at-rest config.vm.define "remote-nextcloud-docker-ehdd" do |m1| - m1.vm.box = "ubuntu/bionic64" m1.vm.provision :shell, :inline => <<-SH source globals.sh || exit 1 export PRIMARY_HOSTNAME=qa1.abc.com @@ -26,7 +32,6 @@ SH # remote-nextcloud-docker w/basic data config.vm.define "remote-nextcloud-docker" do |m1| - m1.vm.box = "ubuntu/bionic64" m1.vm.provision :shell, :inline => <<-SH source globals.sh || exit 1 export PRIMARY_HOSTNAME=qa2.abc.com @@ -43,16 +48,17 @@ SH # upgrade-from-upstream - config.vm.define "upgrade-from-upstream" do |m2| - m2.vm.box = "ubuntu/bionic64" - m2.vm.provision :shell, :inline => <<-SH + config.vm.define "upgrade-from-upstream" do |m1| + m1.vm.provision :shell, :inline => <<-SH source globals.sh || exit 1 export PRIMARY_HOSTNAME=qa3.abc.com -export UPSTREAM_TAG=master +#export UPSTREAM_TAG=master +export MIAB_UPSTREAM_GIT="https://github.com/fspoettel/mailinabox.git" +export UPSTREAM_TAG="admin-panel-2fa" cd /mailinabox -tests/system-setup/upgrade-from-upstream.sh basic; rc=$? +tests/system-setup/upgrade-from-upstream.sh basic totpuser; rc=$? if [ $rc -eq 0 ]; then - tests/runner.sh default upgrade-basic; rc=$? + tests/runner.sh upgrade-basic upgrade-totpuser default; rc=$? fi echo "EXITCODE: $rc" SH diff --git a/tests/vagrant/preloaded/.gitignore b/tests/vagrant/preloaded/.gitignore new file mode 100644 index 00000000..fcb502a8 --- /dev/null +++ b/tests/vagrant/preloaded/.gitignore @@ -0,0 +1 @@ +*.box diff --git a/tests/vagrant/preloaded/Vagrantfile b/tests/vagrant/preloaded/Vagrantfile new file mode 100644 index 00000000..d80534b6 --- /dev/null +++ b/tests/vagrant/preloaded/Vagrantfile @@ -0,0 +1,19 @@ + +Vagrant.configure("2") do |config| + + config.vm.synced_folder "../../..", "/mailinabox", id: "mailinabox", automount: false + + config.vm.define "preloaded-ubuntu-bionic64" do |m1| + m1.vm.box = "ubuntu/bionic64" + m1.vm.provision :shell, :inline => <<-SH +cd /mailinabox +tests/vagrant/preloaded/prepvm.sh --no-dry-run +rc=$? +echo "$rc" > "tests/vagrant/preloaded/prepcode.txt" +[ $rc -gt 0 ] && exit 1 +exit 0 +SH + end + + +end diff --git a/tests/vagrant/preloaded/create_preloaded.sh b/tests/vagrant/preloaded/create_preloaded.sh new file mode 100755 index 00000000..4666691f --- /dev/null +++ b/tests/vagrant/preloaded/create_preloaded.sh @@ -0,0 +1,25 @@ +#!/bin/bash + +vagrant destroy -f +rm -f prepcode.txt + +vagrant up preloaded-ubuntu-bionic64 +upcode=$? +prepcode=$(cat "./prepcode.txt") +rm -f prepcode.txt +echo "" +echo "VAGRANT UP RETURNED $upcode" +echo "PREPVM RETURNED $prepcode" + +if [ "$prepcode" != "0" -o $upcode -ne 0 ]; then + echo "FAILED!!!!!!!!" + vagrant destroy -f + exit 1 +fi + +vagrant halt +vagrant package +rm -f preloaded.box +mv package.box preloaded-ubuntu-bionic64.box + +vagrant destroy -f diff --git a/tests/vagrant/preloaded/prepvm.sh b/tests/vagrant/preloaded/prepvm.sh new file mode 100755 index 00000000..2892fe08 --- /dev/null +++ b/tests/vagrant/preloaded/prepvm.sh @@ -0,0 +1,105 @@ +#!/bin/bash + +# Run this on a VM to pre-install all the packages, then +# take a snapshot - it will greatly speed up subsequent +# test installs + +# +# What won't be installed: +# +# Nextcloud and Roundcube are downloaded with wget by the setup +# scripts, so they are not included +# +# postfix, postgrey and slapd because they require terminal input +# + + +if [ ! -d "setup" ]; then + echo "Run from the miab root directory" + exit 1 +fi + + +dry_run=true + +if [ "$1" == "--no-dry-run" ]; then + dry_run=false +fi + +if $dry_run; then + echo "WARNING: dry run is TRUE, no changes will be made" +fi + + +remove_line_continuation() { + local file="$1" + awk ' +BEGIN { C=0 } +C==1 && /[^\\]$/ { C=0; print $0; next } +C==1 { printf("%s",substr($0,0,length($0)-1)); next } +/\\$/ { C=1; printf("%s",substr($0,0,length($0)-1)); next } + { print $0 }' \ + "$file" +} + +install_packages() { + while read line; do + pkgs="" + case "$line" in + apt_install* ) + pkgs="$(cut -c12- <<<"$line")" + ;; + "apt-get install"* ) + pkgs="$(cut -c16- <<<"$line")" + ;; + "apt install"* ) + pkgs="$(cut -c12- <<<"$line")" + ;; + esac + + # don't install postfix - causes problems with setup scripts + # and requires user input. exclude postgrey because it will + # install postfix as a dependency + pkgs="$(sed 's/postgrey//g' <<< "$pkgs")" + pkgs="$(sed 's/postfix-[^ $]*//g' <<<"$pkgs")" + pkgs="$(sed 's/postfix//g' <<<"$pkgs")" + + # don't install slapd - it requires user input + pkgs="$(sed 's/slapd//g' <<< "$pkgs")" + + if [ ! -z "$pkgs" ]; then + echo "install: $pkgs" + if ! $dry_run; then + apt-get install -y -qq $pkgs + fi + fi + done +} + +if ! $dry_run; then + apt-get update -y + apt-get upgrade -y + apt-get autoremove -y +fi + +for file in $(ls setup/*.sh); do + remove_line_continuation "$file" | install_packages +done + +if ! $dry_run; then + # bonus + apt-get install -y -qq openssh-server + apt-get install -y -qq emacs-nox + apt-get install -y -qq ntpdate + + # these are added by system-setup scripts and needed for test runner + apt-get install -y -qq python3-dnspython jq + + # remove apache, which is what setup will do + apt-get -y -qq purge apache2 apache2-\* + + echo "" + echo "" + echo "Done. Take a snapshot...." + echo "" +fi diff --git a/tests/vagrant/vanilla/Vagrantfile b/tests/vagrant/vanilla/Vagrantfile index 5df165bd..8e594afe 100644 --- a/tests/vagrant/vanilla/Vagrantfile +++ b/tests/vagrant/vanilla/Vagrantfile @@ -7,7 +7,12 @@ Vagrant.configure("2") do |config| # vanilla install config.vm.define "vanilla" do |m1| - m1.vm.box = "ubuntu/bionic64" + if File.file?("../preloaded/preloaded-ubuntu-bionic64.box") + m1.vm.box = "preloaded-ubuntu-bionic64" + m1.vm.box_url = "file://" + Dir.pwd + "/../preloaded/preloaded-ubuntu-bionic64.box" + else + m1.vm.box = "ubuntu/bionic64" + end m1.vm.network "forwarded_port", guest:443, host:8443, protocol:"tcp" m1.vm.provision :shell, :inline => <<-SH source globals.sh || exit 1