Move system-setup directory to tests

Set the hosts FQDN instead of adding hosts entry for private ip

.travis.yml

@@ -23,7 +23,7 @@ before_install:
   - ip add
   
 install:
-  - sudo tests/assets/system-setup/remote-nextcloud-docker.sh
+  - sudo tests/system-setup/remote-nextcloud-docker.sh
 
 script:
   #

tests/system-setup/remote-nextcloud-docker.sh

@@ -7,7 +7,7 @@
 #   on a fresh Ubuntu:
 #      1. checkout or copy the MiaB-LDAP code to ~/mailinabox
 #      2. cd ~/mailinabox
-#      3. sudo tests/assets/system-setup/remote-nextcloud-docker.sh
+#      3. sudo tests/system-setup/remote-nextcloud-docker.sh
 #
 # when complete you should have a working MiaB-LDAP and Nextcloud
 #
@@ -34,15 +34,15 @@ usage() {
 }
 
 # ensure working directory
-if [ ! -d "tests/assets/system-setup" ]; then
+if [ ! -d "tests/system-setup" ]; then
     echo "This script must be run from the MiaB root directory"
     exit 1
 fi
 
 # load helper scripts
-. "tests/assets/system-setup/setup-defaults.sh" \
+. "tests/system-setup/setup-defaults.sh" \
     || die "Could not load setup-defaults"
-. "tests/assets/system-setup/setup-funcs.sh" \
+. "tests/system-setup/setup-funcs.sh" \
     || die "Could not load setup-funcs"
 
 # ensure running as root
@@ -55,36 +55,33 @@ fi
 before_miab_install() {
     H1 "BEFORE MIAB-LDAP INSTALL"
 
-    # create /etc/hosts entry for PRIVATE_IP
     H2 "Update /etc/hosts"
-    update_hosts_for_private_ip || die "Could not update /etc/hosts"
+    #update_hosts_for_private_ip || die "Could not update /etc/hosts"
+    set_system_hostname || die "Could not set hostname"
 
+    # update system time
+    H2 "Set system time"
+    update_system_time || echo "Ignoring error..."
+    
     # update package lists before installing anything
     H2 "apt-get update"
+    wait_for_apt
     apt-get update -qq || die "apt-get update failed!"
+
+    # upgrade packages - if we don't do this and something like bind
+    # is upgraded through automatic upgrades (because maybe MiaB was
+    # previously installed), it may cause problems with the rest of
+    # the setup, such as with name resolution failures
+    if is_false "$TRAVIS"; then
+        H2 "apt-get upgrade"
+        wait_for_apt
+        apt-get upgrade -qq || die "apt-get upgrade failed!"
+    fi
     
     # install prerequisites
     H2 "QA prerequisites"
     install_qa_prerequisites || die "Error installing QA prerequisites"
 
-    # update system time (ignore errors)
-    H2 "Set system time"
-    update_system_time
-
-    # copy in pre-built MiaB-LDAP ssl files
-    #   1. avoid the lengthy generation of DH params
-    H2 "Install QA pre-built"
-    mkdir -p $STORAGE_ROOT/ssl || die "Unable to create $STORAGE_ROOT/ssl"
-    cp tests/assets/ssl/dh2048.pem $STORAGE_ROOT/ssl \
-        || die "Copy dhparams failed"
-
-    # create miab_ldap.conf to specify what the Nextcloud LDAP service
-    # account password will be to avoid a random one created by start.sh
-    mkdir -p $STORAGE_ROOT/ldap
-    [ -e $STORAGE_ROOT/ldap/miab_ldap.conf ] && \
-        echo "Warning: exists: $STORAGE_ROOT/ldap/miab_ldap.conf" 1>&2
-    echo "LDAP_NEXTCLOUD_PASSWORD=\"$LDAP_NEXTCLOUD_PASSWORD\"" >> $STORAGE_ROOT/ldap/miab_ldap.conf
-
     # enable the remote Nextcloud setup mod, which tells MiaB-LDAP to use
     # the remote Nextcloud for calendar and contacts instead of the
     # MiaB-installed one
@@ -118,10 +115,6 @@ after_miab_install() {
     H1 "AFTER MIAB-LDAP INSTALL"
     
     . /etc/mailinabox.conf || die "Could not load /etc/mailinabox.conf"
-
-    # TRAVIS: fix nsd startup problem
-    #H2 "Apply Travis-CI nsd fix"
-    #travis_fix_nsd || die "Could not fix NSD startup issue for TRAVIS-CI"
     
     # run Nextcloud docker image
     H2 "Start Nextcloud docker container"

tests/system-setup/setup-funcs.sh

@@ -6,6 +6,7 @@ die() {
     exit 1
 }
 
+
 H1() {
     local msg="$1"
     echo "----------------------------------------------"
@@ -46,6 +47,7 @@ dump_log() {
 }
 
 is_true() {
+    # empty string is not true
     if [ "$1" == "true" \
               -o "$1" == "TRUE" \
               -o "$1" == "True" \
@@ -59,7 +61,28 @@ is_true() {
         return 1
     fi
 }
-    
+
+is_false() {
+    if is_true $@; then return 1; fi
+    return 0
+}
+
+wait_for_apt() {
+    local count=0
+    while fuser /var/lib/dpkg/lock >/dev/null 2>&1 || fuser /var/lib/dpkg/lock-frontend >/dev/null 2>&1; do
+        sleep 6
+        let count+=1
+        if [ $count -eq 1 ]; then
+            echo -n "Waiting for other package manager to finish..."
+        elif [ $count -gt 100 ]; then
+            echo -n "FAILED"
+            return 1
+        else
+            echo -n "${count}.."
+        fi
+    done
+    [ $count -ge 1 ] && echo ""
+}
 
 dump_conf_files() {
     local skip
@@ -82,16 +105,12 @@ dump_conf_files() {
     fi
 }
 
-install_qa_prerequisites() {
-    # python3-dnspython: is used by the python scripts in 'tests' and is
-    #   not installed by setup
-    # ntpdate: is used by this script
-    apt-get install -y \
-            ntpdate \
-            python3-dnspython
-}
 
 update_system_time() {
+    if [ ! -x /usr/sbin/ntpdate ]; then
+        wait_for_apt
+        apt-get install -y -qq ntpdate || return 1
+    fi
     ntpdate -s ntp.ubuntu.com && echo "System time updated"
 }
 
@@ -119,13 +138,26 @@ update_hosts_for_private_ip() {
     update_hosts "$PRIMARY_HOSTNAME" "$ip4" "$ip6" || return 1
 }
 
+set_system_hostname() {
+    # set the system hostname to the FQDN specified or
+    # PRIMARY_HOSTNAME if no FQDN was given
+    local fqdn="${1:-$PRIMARY_HOSTNAME}"
+    local host="$(awk -F. '{print $1}' <<< "$fqdn")"
+    sed -i 's/^127\.0\.1\.1[ \t].*/127.0.1.1 '"$fqdn $host ip4-loopback/" /etc/hosts || return 1
+    #hostname "$host" || return 1
+    #echo "$host" > /etc/hostname
+    return 0
+}
+
+
 install_docker() {
     if [ -x /usr/bin/docker ]; then
         echo "Docker already installed"
         return 0
     fi
     
-    apt-get install -y \
+    wait_for_apt
+    apt-get install -y -qq \
             apt-transport-https \
             ca-certificates \
             curl \
@@ -133,14 +165,18 @@ install_docker() {
             software-properties-common \
         || return 1
        
+    wait_for_apt
     curl -fsSL https://download.docker.com/linux/ubuntu/gpg | apt-key add - \
         || return 2
     
+    wait_for_apt
     apt-key fingerprint 0EBFCD88 || return 3
     
+    wait_for_apt
     add-apt-repository -y --update "deb [arch=amd64] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" || return 4
     
-    apt-get install -y \
+    wait_for_apt
+    apt-get install -y -qq \
             docker-ce \
             docker-ce-cli \
             containerd.io \
@@ -148,6 +184,41 @@ install_docker() {
 }
 
 
+install_qa_prerequisites() {
+    [ -z "$STORAGE_ROOT" ] \
+        && echo "Error: STORAGE_ROOT not set" 1>&2 \
+        && return 1
+
+    local rc=0
+    
+    # python3-dnspython: is used by the python scripts in 'tests' and is
+    #   not installed by setup
+    wait_for_apt
+    apt-get install -y -qq python3-dnspython
+    
+    # copy in pre-built MiaB-LDAP ssl files
+    #   1. avoid the lengthy generation of DH params
+    mkdir -p $STORAGE_ROOT/ssl \
+        || (echo "Unable to create $STORAGE_ROOT/ssl ($?)" && rc=1)
+    cp tests/assets/ssl/dh2048.pem $STORAGE_ROOT/ssl \
+        || (echo "Copy dhparams failed ($?)" && rc=1)
+
+    # create miab_ldap.conf to specify what the Nextcloud LDAP service
+    # account password will be to avoid a random one created by start.sh
+    if [ ! -z "$LDAP_NEXTCLOUD_PASSWORD" ]; then
+        mkdir -p $STORAGE_ROOT/ldap \
+            || (echo "Could not create $STORAGE_ROOT/ldap" && rc=1)
+        [ -e $STORAGE_ROOT/ldap/miab_ldap.conf ] && \
+            echo "Warning: exists: $STORAGE_ROOT/ldap/miab_ldap.conf" 1>&2
+        touch $STORAGE_ROOT/ldap/miab_ldap.conf || rc=1
+        if ! grep "^LDAP_NEXTCLOUD_PASSWORD=" $STORAGE_ROOT/ldap/miab_ldap.conf >/dev/null; then
+            echo "LDAP_NEXTCLOUD_PASSWORD=\"$LDAP_NEXTCLOUD_PASSWORD\"" >> $STORAGE_ROOT/ldap/miab_ldap.conf
+        fi
+    fi
+    return $rc
+}
+
+
 travis_fix_nsd() {
     if [ "$TRAVIS" != "true" ]; then
         return 0