Separate miab and miab-ldap migrations

Add tests for migrating from miab w/totp (migration 13) to miab-ldap

.travis.yml

@@ -41,11 +41,13 @@ jobs:
   # JOB: Upgrade from upstream install
   - env:
       - PRIMARY_HOSTNAME=box2.abc.com
-      - UPSTREAM_TAG=master
+      #- UPSTREAM_TAG=master
+      - MIAB_UPSTREAM_GIT=https://github.com/fspoettel/mailinabox.git
+      - UPSTREAM_TAG=admin-panel-2fa
     name: upgrade-from-upstream
     install:
-      - sudo tests/system-setup/upgrade-from-upstream.sh basic
+      - sudo tests/system-setup/upgrade-from-upstream.sh basic totpuser
     script:
       # launch automated tests, but skip tests that require remote
       # smtp support because Travis-CI blocks outgoing port 25
-      - sudo tests/runner.sh -dumpoutput -no-smtp-remote default upgrade-basic
+      - sudo tests/runner.sh -dumpoutput -no-smtp-remote default upgrade-basic upgrade-totpuser

setup/migrate.py

@@ -183,6 +183,12 @@ def migration_12(env):
             conn.close()
 
 def migration_13(env):
+	# Add a table for `totp_credentials`
+	db = os.path.join(env["STORAGE_ROOT"], 'mail/users.sqlite')
+	shell("check_call", ["sqlite3", db, "CREATE TABLE IF NOT EXISTS totp_credentials (id INTEGER PRIMARY KEY AUTOINCREMENT, user_email TEXT NOT NULL UNIQUE, secret TEXT NOT NULL, mru_token TEXT, FOREIGN KEY (user_email) REFERENCES users(email) ON DELETE CASCADE);"])
+
+
+def migration_miabldap_1(env):
 	# This migration step moves users from sqlite3 to openldap
 
 	# users table:
@@ -207,7 +213,7 @@ def migration_13(env):
 	#   objectClass: mailGroup
 	#   mail: [source]
 	#   member: [user-dn]         # multi-valued
-
+	
 	print("Migrating users and aliases from sqlite to ldap")
 	
 	# Get the ldap server up and running
@@ -311,11 +317,66 @@ def run_migrations():
 
 		# iterate and try next version...
 
+def run_miabldap_migrations():
+	if not os.access("/etc/mailinabox.conf", os.W_OK, effective_ids=True):
+		print("This script must be run as root.", file=sys.stderr)
+		sys.exit(1)
+
+	env = load_environment()
+
+	migration_id_file = os.path.join(env['STORAGE_ROOT'], 'mailinabox-ldap.version')
+	migration_id = 0
+	if os.path.exists(migration_id_file):
+		with open(migration_id_file) as f:
+			migration_id = f.read().strip();
+
+	ourver = int(migration_id)
+
+	while True:
+		next_ver = (ourver + 1)
+		migration_func = globals().get("migration_miabldap_%d" % next_ver)
+
+		if not migration_func:
+			# No more migrations to run.
+			break
+
+		print()
+		print("Running migration to Mail-in-a-Box LDAP #%d..." % next_ver)
+
+		try:
+			migration_func(env)
+		except Exception as e:
+			print()
+			print("Error running the migration script:")
+			print()
+			print(e)
+			print()
+			print("Your system may be in an inconsistent state now. We're terribly sorry. A re-install from a backup might be the best way to continue.")
+			sys.exit(1)
+
+		ourver = next_ver
+
+		# Write out our current version now. Do this sooner rather than later
+		# in case of any problems.
+		with open(migration_id_file, "w") as f:
+			f.write(str(ourver) + "\n")
+
+		# iterate and try next version...
+
 if __name__ == "__main__":
 	if sys.argv[-1] == "--current":
 		# Return the number of the highest migration.
 		print(str(get_current_migration()))
 	elif sys.argv[-1] == "--migrate":
 		# Perform migrations.
-		run_migrations()
+		env = load_environment()
+		
+		# if miab-ldap already installed, only run miab-ldap migrations
+		if 'LDAP_USERS_BASE' in env:
+			run_miabldap_migrations()
+			
+		# otherwise, run both
+		else:
+			run_migrations()
+			run_miabldap_migrations()
 

tests/suites/_init.sh

@@ -168,7 +168,7 @@ skip_test() {
 	if [ "$SKIP_REMOTE_SMTP_TESTS" == "yes" ] &&
 		   array_contains "remote-smtp" "$@";
 	then
-		test_skip "-no-smtp-remote option given"
+		test_skip "no-smtp-remote option given"
 		return 0
 	fi
 	

tests/suites/upgrade.sh

@@ -38,7 +38,7 @@ verify_populate() {
 
 
 
-suite_start "upgrade"
+suite_start "upgrade-$1"
 
 export ASSETS_DIR
 export MIAB_DIR

tests/system-setup/populate/totpuser-data.sh

@@ -0,0 +1,9 @@
+#
+# requires:
+#    lib scripts: [ misc.sh ]
+#    system-setup scripts: [ setup-defaults.sh ]
+#
+
+TEST_USER="totp_admin@$(email_domainpart "$EMAIL_ADDR")"
+TEST_USER_PASS="$(static_qa_password)"
+TEST_USER_TOTP_SECRET="6VXVWOSCY7JLU4VBZ6LQEJSBN6WYWECU"

tests/system-setup/populate/totpuser-populate.sh

@@ -0,0 +1,37 @@
+#!/bin/bash
+
+. "$(dirname "$0")/../setup-defaults.sh" || exit 1
+. "$(dirname "$0")/../../lib/all.sh" "$(dirname "$0")/../../lib" || exit 1
+. "$(dirname "$0")/totpuser-data.sh" || exit 1
+
+
+url=""
+admin_email="$EMAIL_ADDR"
+admin_pass="$EMAIL_PW"
+
+
+#
+# Add user
+#
+if ! populate_miab_users "$url" "$admin_email" "$admin_pass" "${TEST_USER}:${TEST_USER_PASS}"
+then
+    echo "Unable to add user"
+    exit 1
+fi
+
+# make the user an admin
+if ! rest_urlencoded POST "${url%/}/admin/mail/users/privileges/add" "$admin_email" "$admin_pass" --insecure -- "email=$TEST_USER" "privilege=admin" 2>/dev/null
+then
+    echo "Unable to add 'admin' privilege. err=$REST_ERROR" 1>&2
+    exit 1
+fi
+
+# enable totp
+if ! rest_urlencoded POST "${url%/}/admin/mfa/totp/enable" "$TEST_USER" "$TEST_USER_PASS" --insecure "secret=$TEST_USER_TOTP_SECRET" "token=$(totp_current_token "$TEST_USER_TOTP_SECRET")" 2>/dev/null; then
+    echo "Unable to enable TOTP. err=$REST_ERROR" 1>&2
+    exit 1
+fi
+
+
+exit 0
+

tests/system-setup/populate/totpuser-verify.sh

@@ -0,0 +1,36 @@
+#!/bin/bash
+
+. "$(dirname "$0")/../setup-defaults.sh" || exit 1
+. "$(dirname "$0")/../../lib/all.sh" "$(dirname "$0")/../../lib" || exit 1
+. "$(dirname "$0")/totpuser-data.sh" || exit 1
+
+. /etc/mailinabox.conf || exit 1
+. "${STORAGE_ROOT}/ldap/miab_ldap.conf" || exit 1
+
+
+die() {
+    echo "$1"
+    exit 1
+}
+
+. "$MIAB_DIR/setup/functions-ldap.sh" || exit 1
+
+
+# the user's ldap entry contains the TOTP secret
+# 
+# other tests verify the functioning of totp - just make sure the totp
+# secret was migrated
+#
+get_attribute "$LDAP_USERS_BASE" "(&(mail=$TEST_USER)(objectClass=totpUser))" "totpSecret"
+if [ -z "$ATTR_DN" ]; then
+	echo "totpUser objectClass and secret not present"
+    exit 1
+fi
+
+if [ "$ATTR_VALUE" != "$TEST_USER_TOTP_SECRET" ]; then
+    echo "totpSecret mismatch"
+    exit 1
+fi
+
+echo "OK totpuser-verify passed"
+exit 0

tests/system-setup/setup-defaults.sh

@@ -34,4 +34,5 @@ export NC_ADMIN_USER="${NC_ADMIN_USER:-admin}"
 export NC_ADMIN_PASSWORD="${NC_ADMIN_PASSWORD:-Test_1234}"
 
 # For setup scripts that install upstream versions
-export MIAB_UPSTREAM_GIT="https://github.com/mail-in-a-box/mailinabox.git"
+export MIAB_UPSTREAM_GIT="${MIAB_UPSTREAM_GIT:-https://github.com/mail-in-a-box/mailinabox.git}"
+export UPSTREAM_TAG="${UPSTREAM_TAG:-}"

tests/system-setup/setup-funcs.sh

@@ -235,12 +235,13 @@ miab_ldap_install() {
 
 
 populate_by_name() {
-    local populate_name="$1"
+    local populate_name
-
+    for populate_name; do
-    H1 "Populate Mail-in-a-Box ($populate_name)"
+        H1 "Populate Mail-in-a-Box ($populate_name)"
-    local populate_script="tests/system-setup/populate/${populate_name}-populate.sh"
+        local populate_script="tests/system-setup/populate/${populate_name}-populate.sh"
-    if [ ! -e "$populate_script" ]; then
+        if [ ! -e "$populate_script" ]; then
-        die "Does not exist: $populate_script"
+            die "Does not exist: $populate_script"
-    fi
+        fi
-    "$populate_script" || die "Failed: $populate_script"
+        "$populate_script" || die "Failed: $populate_script"
+    done
 }

tests/system-setup/upgrade-from-upstream.sh

@@ -113,7 +113,7 @@ case "$1" in
         ;;
     populate )
         . /etc/mailinabox.conf
-        populate_by_name "${1:-basic}"
+        populate_by_name "${2:-basic}"
         exit $?
         ;;
 esac
@@ -137,7 +137,11 @@ else
     . /etc/mailinabox.conf
     
     # populate some data
-    populate_by_name "${1:-basic}"
+    if [ $# -gt 0 ]; then
+        populate_by_name "$@"
+    else
+        populate_by_name "basic" "totpuser"
+    fi
 
     # capture upstream state
     pushd "$upstream_dir" >/dev/null