Automated QA tests for remote nextcloud

tests/assets/mail/roundcube/carddav_refresh.sh

@@ -0,0 +1,136 @@
+#!/usr/bin/env php
+<?php
+
+define('INSTALL_PATH', realpath(__DIR__ . '/..') . '/' );
+
+require_once INSTALL_PATH.'program/include/clisetup.php';
+ini_set('memory_limit', -1);
+
+
+
+function usage()
+{
+    print "Usage: carddav_refresh.sh [-id <number>] username password\n";
+    print "Force a sync of a user's addressbook with the remote server\n";
+    print "Place this script in /path/to/roundcubemail/bin, then change the working directory to /path/to/roundcubemail, then run ./bin/cardav_refresh.sh"; 
+    exit(1);
+}
+
+function _die($msg)
+{
+    fwrite(STDERR, $msg . "\n");
+    exit(1);
+}
+
+$args = rcube_utils::get_opt(array('id' => 'dbid'));
+
+$dbid = 0;
+if (!empty($args['dbid'])) {
+    $dbid = intval($args['dbid']);
+}
+   
+$username = trim($args[0]);
+if (empty($username)) {
+    print "Missing username";
+    usage();
+}
+$password = trim($args[1]);
+if (empty($password)) {
+    usage();
+}
+
+
+// -----
+// From index.php -- initialization and login
+// -----
+
+// init application, start session, init output class, etc.
+$RCMAIL = rcmail::get_instance(0, $GLOBALS['env']);
+
+// trigger startup plugin hook
+$startup = $RCMAIL->plugins->exec_hook('startup', array('task' => $RCMAIL->task, 'action' => $RCMAIL->action));
+$RCMAIL->set_task($startup['task']);
+$RCMAIL->action = $startup['action'];
+$auth = $RCMAIL->plugins->exec_hook('authenticate', array(
+            'host'  => $RCMAIL->autoselect_host(),
+            'user'  => $username,
+            'pass'  => $password,
+            'valid' => true,
+            'cookiecheck' => false,));
+
+// Login
+if ($auth['valid'] && !$auth['abort']
+    && $RCMAIL->login($auth['user'], $auth['pass'], $auth['host'], $auth['cookiecheck']))
+  {
+    print "login ok\n";
+  }
+ else
+   {
+     _die("login failed");
+   }
+
+ 
+// ----------------------------------------------------
+// ensure the carddav tables are created and populated
+// ----------------------------------------------------
+
+require_once('plugins/carddav/carddav_backend.php');
+require_once('plugins/carddav/carddav.php');
+
+$c = new carddav(rcube_plugin_api::get_instance());
+$c->task .= "|cli";
+$c->init();
+print "done: init\n";
+// this ensures the carddav tables are created
+$c->checkMigrations();
+print "done: init tables\n";
+// this populates carddav_addressbooks from config
+$c->init_presets();
+print "done: init addressbooks\n";
+
+// -------------------------------------------------------------
+// Set the last_updated field for addressbooks to an old date.
+// That will force a sync/update
+// -------------------------------------------------------------
+$db = $rcmail->get_dbh();
+$db->db_connect('w');
+if (!$db->is_connected() || $db->is_error()) {
+  _die("No DB connection\n" . $db->is_error());
+}
+print "db connected\n";
+
+$db->query("update " . $db->table_name('carddav_addressbooks') . " set last_updated=? WHERE active=1", '2000-01-01 00:00:00');
+print "update made\n";
+if ($db->is_error()) {
+  _die("DB error occurred: " . $db->is_error());
+}
+
+
+// ------------------------------------------------------
+// Update/sync all active address books
+// ------------------------------------------------------
+
+// first get all row ids
+$dbid=array();
+$sql_result = $db->query('SELECT id FROM ' .
+           $db->table_name('carddav_addressbooks') .
+           ' WHERE active=1');
+if ($db->is_error()) {
+  _die("DB error occurred: " . $db->is_error());
+}
+   
+while ($row = $db->fetch_assoc($sql_result)) {
+  $dbid += array(intval($row['id']));
+  print "carddav_addressbooks id: " . $row['id'] . "\n";
+}
+
+// instantiating carddav_backend causes the update/sync
+foreach($dbid as $id) {
+  $config = carddav_backend::carddavconfig($id);
+  if ($config['needs_update']) {
+    print "instantiating carddav_backend: " . $id . "\n";
+    $b = new carddav_backend($id);
+    print("success\n");
+  }
+}
+

tests/assets/system-setup/remote-nextcloud-docker.sh

@@ -0,0 +1,200 @@
+#!/bin/bash
+
+# setup MiaB-LDAP with a remote Nextcloud running on the same
+# host under Docker exposed as localhost:8000
+#
+# this script must be run with the working directory set to 'tests'
+
+usage() {
+    echo "Usage: $(basename "$0") [\"before-miab-install\"|\"miab-install\"|\"after-miab-install\"]"
+    echo "Install MiaB-LDAP and a remote Nextcloud running under docker exposed as localhost:8000"
+    echo "With no arguments, all three stages are run."
+    exit 1
+}
+
+# ensure working directory
+if [ ! -d "tests/assets/system-setup" ]; then
+    echo "This script must be run from the MiaB root directory"
+    exit 1
+fi
+
+# load helper scripts
+. "tests/assets/system-setup/setup-defaults.sh" \
+    || die "Could not load setup-defaults"
+. "tests/assets/system-setup/setup-funcs.sh" \
+    || die "Could not load setup-funcs"
+
+# ensure running as root
+if [ "$EUID" != "0" ]; then
+    die "This script must be run as root (sudo)"
+fi
+
+
+
+before_miab_install() {
+    H1 "BEFORE MIAB-LDAP INSTALL"
+
+    # create /etc/hosts entry for PRIVATE_IP
+    H2 "Update /etc/hosts"
+    update_hosts_for_private_ip || die "Could not update /etc/hosts"
+
+    # install prerequisites
+    H2 "QA prerequisites"
+    install_qa_prerequisites || die "Error installing QA prerequisites"
+
+    # update system time (ignore errors)
+    H2 "Set system time"
+    update_system_time
+
+    # copy in pre-built MiaB-LDAP ssl files
+    #   1. avoid the lengthy generation of DH params
+    H2 "Install QA pre-built"
+    mkdir -p $STORAGE_ROOT/ssl || die "Unable to create $STORAGE_ROOT/ssl"
+    cp tests/assets/ssl/dh2048.pem $STORAGE_ROOT/ssl \
+        || die "Copy dhparams failed"
+
+    # create miab_ldap.conf to specify what the Nextcloud LDAP service
+    # account password will be to avoid a random one created by start.sh
+    mkdir -p $STORAGE_ROOT/ldap
+    [ -e $STORAGE_ROOT/ldap/miab_ldap.conf ] && \
+        echo "Warning: exists: $STORAGE_ROOT/ldap/miab_ldap.conf" 1>&2
+    echo "LDAP_NEXTCLOUD_PASSWORD=\"$LDAP_NEXTCLOUD_PASSWORD\"" >> $STORAGE_ROOT/ldap/miab_ldap.conf
+
+    # enable the remote Nextcloud setup mod, which tells MiaB-LDAP to use
+    # the remote Nextcloud for calendar and contacts instead of the
+    # MiaB-installed one
+    if [ ! -e "setup/mods.d/remote-nextcloud.sh" ]; then
+        ln -s "../mods.available/remote-nextcloud.sh" "setup/mods.d/remote-nextcloud.sh" || die "Could not create remote-nextcloud.sh symlink"
+    fi
+
+    
+    # install Docker
+    H2 "Install Docker"
+    install_docker || die "Could not install Docker! ($?)"
+}
+
+
+miab_install() {
+    H1 "MIAB-LDAP INSTALL"
+    setup/start.sh -v || die "setup/start.sh failed!"
+}
+
+
+after_miab_install() {
+    H1 "AFTER MIAB-LDAP INSTALL"
+    
+    . /etc/mailinabox.conf || die "Could not load /etc/mailinabox.conf"
+
+    # TRAVIS: fix nsd startup problem
+    H2 "Apply Travis-CI nsd fix"
+    travis_fix_nsd || die "Could not fix NSD startup issue for TRAVIS-CI"
+    
+    # run Nextcloud docker image
+    H2 "Start Nextcloud docker container"
+    docker run -d --name NC -p 8000:80 \
+           --env SQLITE_DATABASE=nextclouddb.sqlite \
+           --env NEXTCLOUD_ADMIN_USER="$NC_ADMIN_USER" \
+           --env NEXTCLOUD_ADMIN_PASSWORD="$NC_ADMIN_PASSWORD" \
+           --env NEXTCLOUD_TRUSTED_DOMAINS="127.0.0.1 ::1" \
+           --env NEXTCLOUD_UPDATE=1 \
+           --env SMTP_HOST="$PRIMARY_HOSTNAME" \
+           --env SMTP_SECURE="tls" \
+           --env SMTP_PORT=587 \
+           --env SMTP_AUTHTYPE="LOGIN" \
+           --env SMTP_NAME="$EMAIL_ADDR" \
+           --env SMTP_PASSWORD="$EMAIL_PW" \
+           --env SMTP_FROM_ADDRESS="$(awk -F@ '{print $1}' <<< "$EMAIL_ADDR")" \
+           --env MAIL_DOMAIN="$(awk -F@ '{print $2}' <<< "$EMAIL_ADDR")" \
+           nextcloud:latest \
+        || die "Docker run failed!"
+
+    H2 "docker: Update /etc/hosts so it can find MiaB-LDAP by name"
+    echo "$PRIVATE_IP $PRIMARY_HOSTNAME" | \
+        docker exec -i NC bash -c 'cat >>/etc/hosts' \
+        || die "docker: could not update /etc/hosts"
+    
+    # apt-get update
+    H2 "docker: apt-get update"
+    docker exec NC apt-get update || die "docker: apt-get update failed"
+
+    # allow LDAP access from docker image
+    H2 "Allow ldaps through firewall so Nextcloud can perform LDAP searches"
+    ufw allow ldaps || die "Unable to modify firewall to permit ldaps"
+
+    # add MiaB-LDAP's ca_certificate.pem to docker's trusted cert list
+    H2 "docker: update trusted CA list"
+    docker cp \
+           $STORAGE_ROOT/ssl/ca_certificate.pem \
+           NC:/usr/local/share/ca-certificates/mailinabox.crt \
+        || die "docker: copy ca_certificate.pem failed"
+    docker exec NC update-ca-certificates \
+        || die "docker: update-ca-certificates failed"
+
+    # wait for Nextcloud installation to complete
+    H2 "Wait for Nextcloud installation to complete"
+    echo -n "Waiting ..."
+    local count=0
+    while true; do
+        if [ $count -ge 10 ]; then
+            echo "FAILED"
+            die "Giving up"
+        fi
+        sleep 6
+        let count+=1
+        if [ $(docker exec NC php -n -r "include 'config/config.php'; print \$CONFIG['installed']?'true':'false';") == "true" ]; then
+            echo "ok"
+            break
+        fi
+        echo -n "${count}..."
+    done
+    
+    # install and enable Nextcloud and apps
+    H2 "docker: install Nextcloud calendar app"
+    docker exec -u www-data NC ./occ app:install calendar \
+        || die "docker: installing calendar app failed"
+    H2 "docker: install Nextcloud contacts app"
+    docker exec -u www-data NC ./occ app:install contacts \
+        || die "docker: installing contacts app failed"
+    H2 "docker: enable user_ldap"
+    docker exec -u www-data NC ./occ app:enable user_ldap \
+        || die "docker: enabling user_ldap failed"
+
+    # integrate Nextcloud with MiaB-LDAP
+    H2 "docker: integrate Nextcloud with MiaB-LDAP"
+    docker cp setup/mods.available/remote-nextcloud-use-miab.sh NC:/tmp \
+        || die "docker: cp remote-nextcloud-use-miab.sh failed"
+    docker exec NC /tmp/remote-nextcloud-use-miab.sh \
+           . \
+           "$NC_ADMIN_USER" \
+           "$NC_ADMIN_PASSWORD" \
+           "$PRIMARY_HOSTNAME" \
+           "$LDAP_NEXTCLOUD_PASSWORD" \
+        || die "docker: error running remote-nextcloud-use-miab.sh"
+}
+
+
+
+#
+# process command line
+#
+
+case "$1" in
+    before-miab-install )
+        before_miab_install
+        ;;
+    after-miab-install )
+        after_miab_install
+        ;;
+    miab-install )
+        miab_install
+        ;;
+    "" )
+        before_miab_install
+        miab_install
+        after_miab_install
+        ;;
+    * )
+        
+        usage
+        ;;
+esac

tests/assets/system-setup/setup-defaults.sh

@@ -0,0 +1,33 @@
+#!/bin/bash
+
+# Used by MiaB-LDAP setup/start.sh
+export NONINTERACTIVE=${NONINTERACTIVE:-1}
+export SKIP_NETWORK_CHECKS=${SKIP_NETWORK_CHECKS:-1}
+export STORAGE_USER="${STORAGE_USER:-user-data}"
+export STORAGE_ROOT="${STORAGE_ROOT:-/home/$STORAGE_USER}"
+export EMAIL_ADDR="${EMAIL_ADDR:-qa@abc.com}"
+export EMAIL_PW="${EMAIL_PW:-Test_1234}"
+export PUBLIC_IP="${PUBLIC_IP:-$(source setup/functions.sh; get_default_privateip 4)}"
+
+if [ "$TRAVIS" == "true" ]; then
+    export PRIMARY_HOSTNAME=${PRIMARY_HOSTNAME:-box.abc.com}
+elif [ -z "$PRIMARY_HOSTNAME" ]; then
+    export PRIMARY_HOSTNAME=$(hostname --fqdn || hostname)
+fi
+
+
+# Placing this var in STORAGE_ROOT/ldap/miab_ldap.conf before running
+# setup/start.sh will avoid a random password from being used for the
+# Nextcloud LDAP service account
+export LDAP_NEXTCLOUD_PASSWORD=${LDAP_NEXTCLOUD_PASSWORD:-Test_LDAP_1234}
+
+# Used by setup/mods.available/remote-nextcloud.sh. These define to
+# MiaB-LDAP the remote Nextcloud that serves calendar and contacts
+export NC_PROTO=${NC_PROTO:-http}
+export NC_HOST=${NC_HOST:-127.0.0.1}
+export NC_PORT=${NC_PORT:-8000}
+export NC_PREFIX=${NC_PREFIX:-/}
+
+# For setup scripts that are installing a remote Nextcloud
+export NC_ADMIN_USER="${NC_ADMIN_USER:-admin}"
+export NC_ADMIN_PASSWORD="${NC_ADMIN_PASSWORD:-Test_1234}"

tests/assets/system-setup/setup-funcs.sh

@@ -0,0 +1,97 @@
+
+die() {
+    local msg="$1"
+    echo "$msg" 1>&2
+    exit 1
+}
+
+H1() {
+    local msg="$1"
+    echo "----------------------------------------------"
+    echo "           $msg"
+    echo "----------------------------------------------"
+}
+
+H2() {
+    local msg="$1"
+    echo "*** $msg ***"
+}
+
+install_qa_prerequisites() {
+    # python3-dnspython: is used by the python scripts in 'tests' and is
+    #   not installed by setup
+    # ntpdate: is used by this script
+    apt-get install -y \
+            ntpdate \
+            python3-dnspython
+}
+
+update_system_time() {
+    ntpdate -s ntp.ubuntu.com && echo "System time updated"
+}
+
+update_hosts() {
+    local host="$1"
+    local ip="$2"
+    local line="$ip $host"
+    if ! grep -F "$line" /etc/hosts 1>/dev/null; then
+        echo "$line" >>/etc/hosts
+    fi
+}
+
+update_hosts_for_private_ip() {
+    # create /etc/hosts entry for PRIVATE_IP
+    # PRIMARY_HOSTNAME must already be set
+    local ip=$(source setup/functions.sh; get_default_privateip 4)
+    [ -z "$ip" ] && return 1
+    update_hosts "$PRIMARY_HOSTNAME" "$ip" || return 1
+}
+
+install_docker() {
+    if [ -x /usr/bin/docker ]; then
+        echo "Docker already installed"
+        return 0
+    fi
+    
+    apt-get install -y \
+            apt-transport-https \
+            ca-certificates \
+            curl \
+            gnupg-agent \
+            software-properties-common \
+        || return 1
+       
+    curl -fsSL https://download.docker.com/linux/ubuntu/gpg | apt-key add - \
+        || return 2
+    
+    apt-key fingerprint 0EBFCD88 || return 3
+    
+    add-apt-repository -y --update "deb [arch=amd64] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" || return 4
+    
+    apt-get install -y \
+            docker-ce \
+            docker-ce-cli \
+            containerd.io \
+        || return 5
+}
+
+
+travis_fix_nsd() {
+    if [ "$TRAVIS" != "true" ]; then
+        return 0
+    fi
+    
+    # nsd won't start on Travis-CI without the changes below: ip6 off and
+    # control-enable set to no. Even though the nsd docs says the
+    # default value for control-enable is no, running "nsd-checkconf -o
+    # control-enable /etc/nsd/nsd.conf" returns "yes", so we explicitly
+    # set it here.
+    #
+    # we're assuming that the "ip-address" line is the last line in the
+    # "server" section of nsd.conf. if this generated file output
+    # changes, the sed command below may need to be adjusted.
+    sed -i 's/ip-address\(.\)\(.*\)/ip-address\1\2\n  do-ip4\1 yes\n  do-ip6\1 no\n  verbosity\1 3\nremote-control\1\n  control-enable\1 no/' /etc/nsd/nsd.conf || return 1
+    cat /etc/nsd/nsd.conf
+    systemctl reset-failed nsd.service || return 2
+    systemctl restart nsd.service || return 3
+}