Simplify alternate repo/branch installation

tests/lib/all.sh

@@ -1,20 +1,20 @@
 #
 # source all lib scripts
 #
-# from your script, supply the path to this directory as the first argument
+#    eg source "tests/lib/all.sh"
-#
-#    eg source "tests/lib/all.sh" "tests/lib"
 #
 # failure to load any script is fatal!
 
-. "$1/color-output.sh" || exit 1
+D=$(dirname "$BASH_SOURCE")
-. "$1/locations.sh"    || exit 2
-. "$1/misc.sh"         || exit 3
-. "$1/rest.sh"         || exit 4
-. "$1/system.sh"       || exit 5
-. "$1/carddav.sh"      || exit 6
 
-. "$1/populate.sh"     || exit 7
+. "$D/color-output.sh" || exit 1
-. "$1/installed-state.sh" || exit 8
+. "$D/locations.sh"    || exit 2
-. "$1/totp.sh"         || exit 9
+. "$D/misc.sh"         || exit 3
+. "$D/rest.sh"         || exit 4
+. "$D/system.sh"       || exit 5
+. "$D/carddav.sh"      || exit 6
+
+. "$D/populate.sh"     || exit 7
+. "$D/installed-state.sh" || exit 8
+. "$D/totp.sh"         || exit 9
 

tests/lib/system.sh

@@ -1,3 +1,6 @@
+#
+# requires:
+#    scripts: [ misc.sh ]
 
 wait_for_apt() {
     # check to see if other package managers have a lock on new
@@ -140,3 +143,32 @@ exec_no_output() {
     [ $code -ne 0 ] && return 1
 	return 0
 }
+
+git_clone() {
+    local REPO="$1"
+    local TREEISH="$2"
+    local TARGETPATH="$3"
+    local OPTIONS="$4"
+
+    if [ ! -x /usr/bin/git ]; then
+        exec_no_output apt-get install -y git || return 1
+    fi
+
+    if ! array_contains "keep-existing" $OPTIONS || \
+            [ ! -d "$TARGETPATH" ] || \
+            [ -z "$(ls -A "$TARGETPATH")" ]
+    then
+        rm -rf "$TARGETPATH"
+        git clone "$REPO" "$TARGETPATH"
+        if [ $? -ne 0 ]; then
+            rm -rf "$TARGETPATH"
+            return 1
+        fi
+    fi
+
+    if [ ! -z "$TREEISH" ]; then
+        pushd "$TARGETPATH" >/dev/null
+        git checkout "$TREEISH" || return 2
+        popd >/dev/null
+    fi
+}

tests/system-setup/setup-funcs.sh

@@ -2,7 +2,7 @@
 #
 # requires:
 #
-#   test scripts: [ lib/misc.sh, lib/system.sh, lib/color-output.sh ]
+#   test scripts: [ lib/misc.sh, lib/system.sh, lib/color-output.sh, lib/installed-state.sh ]
 #
 
 
@@ -255,8 +255,98 @@ workaround_dovecot_sieve_bug() {
 }
 
 
+say_release_info() {
+    H2 "Release info"
+    echo "Code version: $(git describe)"
+    echo "Migration version (miab): $(cat "$STORAGE_ROOT/mailinabox.version")"
+    echo "Migration version (miabldap): $(cat "$STORAGE_ROOT/mailinabox-ldap.version")"
+}
+
+clone_repo_and_pushd() {
+    local repo=""
+    local treeish=""
+    local targetdir=""
+    for arg; do
+        case "$arg" in
+            --checkout-repo=* )
+                repo=$(awk -F= '{print $2}' <<<"$arg")
+                ;;
+            --checkout-treeish=* | --checkout-tag=* )
+                treeish=$(awk -F= '{print $2}' <<<"$arg")
+                ;;
+            --checkout-targetdir=* )
+                targetdir=$(awk -F= '{print $2}' <<<"$arg")
+                ;;
+        esac
+    done
+    
+    if [ -z "$repo" -o -z "$treeish" -o -z "$targetdir" ]; then
+        return 1
+    fi
+
+    H1 "Clone release $treeish from $repo"
+    git_clone \
+        "$repo" \
+        "$treeish" \
+        "$targetdir" \
+        "keep-existing" \
+        || die "could not clone $repo ($treeish) to $targetdir"
+
+    pushd "$targetdir" >/dev/null
+    return 0
+}
+
+
+#
+# install mail-in-a-box (upstream)
+#
+upstream_install() {
+    local need_pop="no"
+    if clone_repo_and_pushd "$@"; then
+        need_pop="yes"
+    fi
+    
+    H1 "MIAB UPSTEAM INSTALL [$(git describe 2>/dev/null)]"
+
+    # ensure we're in a MiaB working directory
+    if [ -e setup/ldap.sh ]; then
+        die "Cannot install: the working directory is MiaB-LDAP!"
+    fi
+    if [ ! -e setup/start.sh ]; then
+        die "Cannot install: the working directory must contain the source"
+    fi
+    
+    if ! setup/start.sh; then
+        echo "$F_WARN"
+        dump_file /var/log/syslog 100
+        echo "$F_RESET"
+        die "Upstream setup failed!"
+    fi
+    
+    H2 "Post-setup actions"
+    workaround_dovecot_sieve_bug
+
+    # set actual STORAGE_ROOT, STORAGE_USER, PRIVATE_IP, etc
+    . /etc/mailinabox.conf || die "Could not source /etc/mailinabox.conf"
+    
+    H2 "miab install success"
+
+    if [ "need_pop" = "yes" ]; then
+        popd >/dev/null
+    fi
+       
+    # populate if specified on command line
+    populate_by_cli_argument "$@"
+}
+
+
 miab_ldap_install() {
-    H1 "MIAB-LDAP INSTALL"
+    local need_pop="no"
+    if clone_repo_and_pushd "$@"; then
+        need_pop="yes"
+    fi
+    
+    H1 "MIAB-LDAP INSTALL [$(git describe 2>/dev/null)]"
     # ensure we're in a MiaB-LDAP working directory
     if [ ! -e setup/ldap.sh ]; then
         die "Cannot install: the working directory is not MiaB-LDAP!"
@@ -302,8 +392,37 @@ miab_ldap_install() {
     fi
 
     H2 "miab-ldap install success"
+
+    if [ "need_pop" = "yes" ]; then
+        popd >/dev/null
+    fi
+       
+    # populate if specified on command line
+    populate_by_cli_argument "$@"
 }
 
+populate_by_cli_argument() {
+    # ...ignore unknown options they may be interpreted elsewhere
+    local populate_names=()
+    local state_dir=""
+    for arg; do
+        case "$arg" in
+            --populate=* )
+                populate_names+=( $(awk -F= '{print $2}' <<<"$arg") )
+                ;;
+            --capture-state=* )
+                state_dir=$(awk -F= '{print $2}' <<<"$arg")
+                ;;
+        esac
+    done
+
+    if [ ${#populate_names} -gt 0 ]; then
+        populate_by_name "${populate_names[@]}"
+    fi
+    if [ ! -z "$state_dir" ]; then
+        installed_state_capture "$state_dir"
+    fi
+}
 
 populate_by_name() {
     local populate_name

tests/system-setup/upgrade-from-upstream.sh

@@ -22,7 +22,7 @@ if [ ! -d "tests/system-setup" ]; then
 fi
 
 # load helper scripts
-. "tests/lib/all.sh" "tests/lib" || die "Could not load lib scripts"
+. "tests/lib/all.sh" || die "Could not load lib scripts"
 . "tests/system-setup/setup-defaults.sh" || die "Could not load setup-defaults"
 . "tests/system-setup/setup-funcs.sh" || die "Could not load setup-funcs"
 
@@ -35,67 +35,9 @@ fi
 init() {
     H1 "INIT"
     init_test_system
-    init_miab_testing || die "Initialization failed"
+    init_miab_testing "$@" || die "Initialization failed"
 }
 
-upstream_install() {
-    local upstream_dir="$1"
-    H1 "INSTALL UPSTREAM"
-    [ ! -x /usr/bin/git ] && apt-get install -y -qq git
-    
-    if [ ! -d "$upstream_dir" ] || [ -z "$(ls -A "$upstream_dir")" ] ; then
-        H2 "Cloning $MIAB_UPSTREAM_GIT"
-        rm -rf "$upstream_dir"
-        git clone "$MIAB_UPSTREAM_GIT" "$upstream_dir"
-        if [ $? -ne 0 ]; then
-            rm -rf "$upstream_dir"
-            die "git clone upstream failed!"
-        fi
-        if [ -z "$UPSTREAM_TAG" ]; then
-            tag_from_readme "$upstream_dir/README.md"
-            if [ $? -ne 0 ]; then
-                rm -rf "$upstream_dir"
-                die "Failed to extract TAG from $upstream_dir/README.md"
-            fi
-        fi
-    fi
-
-    pushd "$upstream_dir" >/dev/null
-    if [ ! -z "$UPSTREAM_TAG" ]; then
-        H2 "Checkout $UPSTREAM_TAG"
-        git checkout "$UPSTREAM_TAG" || die "git checkout $UPSTREAM_TAG failed"
-    fi
-
-    if [ "$TRAVIS" == "true" ]; then
-        # Apply a patch to setup/dns.sh so nsd will start. We must do
-        # it in the script and not after setup.sh runs because part of
-        # setup includes adding a new user via the management
-        # interface and that's where the management daemon crashes:
-        #
-        # "subprocess.CalledProcessError: Command '['/usr/sbin/service', 'nsd', 'restart']' returned non-zero exit status 1"
-        #
-        H2 "Patching upstream setup/dns.sh for Travis-CI"
-        sed -i 's|\(.*include:.*zones\.conf.*\)|cat >> /etc/nsd/nsd.conf <<EOF\n  do-ip4: yes\n  do-ip6: no\nremote-control:\n  control-enable: no\nEOF\n\n\1|' setup/dns.sh \
-            || die "Couldn't patch setup/dns.sh !!"
-    fi
-    
-    H2 "Run upstream setup"
-    if ! setup/start.sh; then
-        echo "$F_WARN"
-        dump_file /var/log/syslog 100
-        echo "$F_RESET"
-        die "Upstream setup failed!"
-    fi
-    
-    workaround_dovecot_sieve_bug
-
-    H2 "Upstream info"
-    echo "Code version: $(git describe)"
-    echo "Migration version: $(cat "$STORAGE_ROOT/mailinabox.version")"
-    popd >/dev/null
-}
-
-
 
 
 
@@ -122,7 +64,7 @@ esac
 
 
 # install basic stuff, set the hostname, time, etc
-init
+init "$@"
 
 # if MiaB-LDAP is already migrated, do not run upstream setup
 [ -e /etc/mailinabox.conf ] && . /etc/mailinabox.conf
@@ -133,23 +75,19 @@ then
 else
     # install upstream
     upstream_dir="$HOME/mailinabox-upstream"
-    upstream_install "$upstream_dir"
+    upstream_install \
-    . /etc/mailinabox.conf
+        "$@" \
-    
+        --checkout-repo="$MIAB_UPSTREAM_GIT" \
-    # populate some data
+        --checkout-treeish="$UPSTREAM_TAG" \
-    if [ $# -gt 0 ]; then
+        --checkout-targetdir="$upstream_dir" \
-        populate_by_name "$@"
+        --capture-state="/tmp/state/upstream"
-    else
+    pushd "$upstream_dir" >/dev/null
-        populate_by_name "basic" "totpuser"
+    say_release_info
-    fi
+    popd >/dev/null
-
-    # capture upstream state
-    installed_state_capture "/tmp/state/upstream" "$upstream_dir"
 fi
 
 # install miab-ldap and capture state
-miab_ldap_install
+miab_ldap_install --capture-state="/tmp/state/miab-ldap"
-installed_state_capture "/tmp/state/miab-ldap"
 
 # compare states
 if ! installed_state_compare "/tmp/state/upstream" "/tmp/state/miab-ldap"; then

tests/system-setup/upgrade.sh

@@ -1,11 +1,11 @@
 #!/bin/bash
 
 # setup MiaB-LDAP by:
-#    1. installing upstream MiaB
+#    1. installing older version of MiaB-LDAP
 #    2. adding some data (users/aliases/etc)
-#    3. upgrading to MiaB-LDAP
+#    3. upgrading to latest MiaB-LDAP
 #
-# See setup-defaults.sh for usernames and passwords.
+# See setup-defaults.sh for usernames and passwords
 #
 
 
@@ -22,7 +22,7 @@ if [ ! -d "tests/system-setup" ]; then
 fi
 
 # load helper scripts
-. "tests/lib/all.sh" "tests/lib" || die "Could not load lib scripts"
+. "tests/lib/all.sh" || die "Could not load lib scripts"
 . "tests/system-setup/setup-defaults.sh" || die "Could not load setup-defaults"
 . "tests/system-setup/setup-funcs.sh" || die "Could not load setup-funcs"
 
@@ -35,68 +35,30 @@ fi
 init() {
     H1 "INIT"
     init_test_system
-    init_miab_testing || die "Initialization failed"
+    init_miab_testing "$@" || die "Initialization failed"
-}
-
-install_release() {
-    install_dir="$1"
-    H1 "INSTALL RELEASE $MIABLDAP_RELEASE_TAG"
-    [ ! -x /usr/bin/git ] && apt-get install -y -qq git
-    
-    if [ ! -d "$install_dir" ] || [ -z "$(ls -A "$install_dir")" ] ; then
-        H2 "Cloning $MIABLDAP_GIT"
-        rm -rf "$install_dir"
-        git clone "$MIABLDAP_GIT" "$install_dir"
-        if [ $? -ne 0 ]; then
-            rm -rf "$install_dir"
-            die "git clone failed!"
-        fi
-    fi
-
-    pushd "$install_dir" >/dev/null
-    H2 "Checkout $MIABLDAP_RELEASE_TAG"
-    git checkout "$MIABLDAP_RELEASE_TAG" || die "git checkout $MIABLDAP_RELEASE_TAG failed"
-    
-    H2 "Run setup"
-    if ! setup/start.sh; then
-        echo "$F_WARN"
-        dump_file /var/log/syslog 100
-        echo "$F_RESET"
-        die "Release $RELEASE_TAG setup failed!"
-    fi
-    
-    workaround_dovecot_sieve_bug
-
-    H2 "Release info"
-    echo "Code version: $(git describe)"
-    echo "Migration version (miabldap): $(cat "$STORAGE_ROOT/mailinabox-ldap.version")"
-    popd >/dev/null
 }
 
 
 # install basic stuff, set the hostname, time, etc
-init
+init "$@"
 
-# install release
+# install tagged release
 release_dir="$HOME/miabldap_$MIABLDAP_RELEASE_TAG"
-install_release "$release_dir"
+miab_ldap_install \
-. /etc/mailinabox.conf
+    "$@" \
-    
+    --checkout-repo="$MIABLDAP_GIT" \
-# populate some data
+    --checkout-treeish="$MIABLDAP_RELEASE_TAG" \
-if [ $# -gt 0 ]; then
+    --checkout-targetdir="$release_dir" \
-    populate_by_name "$@"
+    --capture-state="/tmp/state/release"
-else
-    populate_by_name "basic" "totpuser"
-fi
 
-# capture release state
+pushd "$release_dir" >/dev/null
-installed_state_capture "/tmp/state/release" "$release_dir"
+say_release_info
+popd >/dev/null
 
 # install master miab-ldap and capture state
 H2 "New miabldap"
 echo "git branch: $(git branch | grep '*')"
-miab_ldap_install
+miab_ldap_install --capture-state="/tmp/state/master"
-installed_state_capture "/tmp/state/master"
 
 # compare states
 if ! installed_state_compare "/tmp/state/release" "/tmp/state/master"; then

tests/system-setup/vanilla.sh

@@ -11,7 +11,7 @@ if [ ! -d "tests/system-setup" ]; then
 fi
 
 # load helper scripts
-. "tests/lib/all.sh" "tests/lib" || die "Could not load lib scripts"
+. "tests/lib/all.sh" || die "Could not load lib scripts"
 . "tests/system-setup/setup-defaults.sh" || die "Could not load setup-defaults"
 . "tests/system-setup/setup-funcs.sh" || die "Could not load setup-funcs"
 
@@ -31,14 +31,5 @@ init() {
 # initialize test system
 init "$@"
 
-if array_contains remote-nextcloud "$@"; then
+# run setup
-    H1 "Enable remote-nextcloud mod"
+miab_ldap_install "$@"
-    enable_miab_mod "remote-nextcloud" \
-        || die "Could not enable remote-nextcloud mod"
-else
-    disable_miab_mod "remote-nextcloud"
-fi
-    
-# run setup to use the remote Nextcloud (doesn't need to be available)
-miab_ldap_install
-

tests/vagrant/Vagrantfile

@@ -41,7 +41,7 @@ SH
 cd /mailinabox
 export PRIMARY_HOSTNAME=qa3.abc.com
 export UPSTREAM_TAG=jammyjellyfish2204
-tests/system-setup/upgrade-from-upstream.sh basic totpuser || exit 1
+tests/system-setup/upgrade-from-upstream.sh --populate=basic --populate=totpuser || exit 1
 tests/runner.sh upgrade-basic upgrade-totpuser default || exit 2
 SH
   end
@@ -57,7 +57,7 @@ SH
     m1.vm.provision :shell, :inline => <<-SH
 cd /mailinabox
 export PRIMARY_HOSTNAME=upgrade.abc.com
-tests/system-setup/upgrade.sh basic totpuser || exit 1
+tests/system-setup/upgrade.sh --populate=basic --populate=totpuser || exit 1
 tests/runner.sh upgrade-basic upgrade-totpuser default || exit 2
 SH
   end