mailinabox/setup/migrate.py

#!/usr/bin/python3 -u
# -*- indent-tabs-mode: t; tab-width: 8; python-indent-offset: 8; -*-
#####
##### This file is part of Mail-in-a-Box-LDAP which is released under the
##### terms of the GNU Affero General Public License as published by the
##### Free Software Foundation, either version 3 of the License, or (at
##### your option) any later version. See file LICENSE or go to
##### https://github.com/downtownallday/mailinabox-ldap for full license
##### details.
#####


# Migrates any file structures, database schemas, etc. between versions of Mail-in-a-Box.

# We have to be careful here that any dependencies are already installed in the previous
# version since this script runs before all other aspects of the setup script.

import sys, os, os.path, glob, re, shutil

sys.path.insert(0, 'management')
from utils import load_environment, load_env_vars_from_file, save_environment, shell
import contextlib

def migration_1(env):
	# Re-arrange where we store SSL certificates. There was a typo also.

	def move_file(fn, domain_name_escaped, filename):
		# Moves an SSL-related file into the right place.
		fn1 = os.path.join( env["STORAGE_ROOT"], 'ssl', domain_name_escaped, file_type)
		os.makedirs(os.path.dirname(fn1), exist_ok=True)
		shutil.move(fn, fn1)

	# Migrate the 'domains' directory.
	for sslfn in glob.glob(os.path.join( env["STORAGE_ROOT"], 'ssl/domains/*' )):
		fn = os.path.basename(sslfn)
		m = re.match("(.*)_(certifiate.pem|cert_sign_req.csr|private_key.pem)$", fn)
		if m:
			# get the new name for the file
			domain_name, file_type = m.groups()
			if file_type == "certifiate.pem": file_type = "ssl_certificate.pem" # typo
			if file_type == "cert_sign_req.csr": file_type = "certificate_signing_request.csr" # nicer
			move_file(sslfn, domain_name, file_type)

	# Move the old domains directory if it is now empty.
	with contextlib.suppress(Exception):
		os.rmdir(os.path.join( env["STORAGE_ROOT"], 'ssl/domains'))

def migration_2(env):
	# Delete the .dovecot_sieve script everywhere. This was formerly a copy of our spam -> Spam
	# script. We now install it as a global script, and we use managesieve, so the old file is
	# irrelevant. Also delete the compiled binary form.
	for fn in glob.glob(os.path.join(env["STORAGE_ROOT"], 'mail/mailboxes/*/*/.dovecot.sieve')):
		os.unlink(fn)
	for fn in glob.glob(os.path.join(env["STORAGE_ROOT"], 'mail/mailboxes/*/*/.dovecot.svbin')):
		os.unlink(fn)

def migration_3(env):
	# Move the migration ID from /etc/mailinabox.conf to $STORAGE_ROOT/mailinabox.version
	# so that the ID stays with the data files that it describes the format of. The writing
	# of the file will be handled by the main function.
	pass

def migration_4(env):
	# Add a new column to the mail users table where we can store administrative privileges.
	db = os.path.join(env["STORAGE_ROOT"], 'mail/users.sqlite')
	shell("check_call", ["sqlite3", db, "ALTER TABLE users ADD privileges TEXT NOT NULL DEFAULT ''"])

def migration_5(env):
	# The secret key for encrypting backups was world readable. Fix here.
	os.chmod(os.path.join(env["STORAGE_ROOT"], 'backup/secret_key.txt'), 0o600)

def migration_6(env):
	# We now will generate multiple DNSSEC keys for different algorithms, since TLDs may
	# not support them all. .email only supports RSA/SHA-256. Rename the keys.conf file
	# to be algorithm-specific.
	basepath = os.path.join(env["STORAGE_ROOT"], 'dns/dnssec')
	shutil.move(os.path.join(basepath, 'keys.conf'), os.path.join(basepath, 'RSASHA1-NSEC3-SHA1.conf'))

def migration_7(env):
	# I previously wanted domain names to be stored in Unicode in the database. Now I want them
	# to be in IDNA. Affects aliases only.
	import sqlite3
	conn = sqlite3.connect(os.path.join(env["STORAGE_ROOT"], "mail/users.sqlite"))

	# Get existing alias source addresses.
	c = conn.cursor()
	c.execute('SELECT source FROM aliases')
	aliases = [ row[0] for row in c.fetchall() ]

	# Update to IDNA-encoded domains.
	for email in aliases:
		try:
			localpart, domainpart = email.split("@")
			domainpart = domainpart.encode("idna").decode("ascii")
			newemail = localpart + "@" + domainpart
			if newemail != email:
				c = conn.cursor()
				c.execute("UPDATE aliases SET source=? WHERE source=?", (newemail, email))
				if c.rowcount != 1: raise ValueError("Alias not found.")
				print("Updated alias", email, "to", newemail)
		except Exception as e:
			print("Error updating IDNA alias", email, e)

	# Save.
	conn.commit()

def migration_8(env):
	# Delete DKIM keys. We had generated 1024-bit DKIM keys.
	# By deleting the key file we'll automatically generate
	# a new key, which will be 2048 bits.
	os.unlink(os.path.join(env['STORAGE_ROOT'], 'mail/dkim/mail.private'))

def migration_9(env):
	# Add a column to the aliases table to store permitted_senders,
	# which is a list of user account email addresses that are
	# permitted to send mail using this alias instead of their own
	# address. This was motivated by the addition of #427 ("Reject
	# outgoing mail if FROM does not match Login") - which introduced
	# the notion of outbound permitted-senders.
	db = os.path.join(env["STORAGE_ROOT"], 'mail/users.sqlite')
	shell("check_call", ["sqlite3", db, "ALTER TABLE aliases ADD permitted_senders TEXT"])

def migration_10(env):
	# Clean up the SSL certificates directory.

	# Move the primary certificate to a new name and then
	# symlink it to the system certificate path.
	import datetime
	system_certificate = os.path.join(env["STORAGE_ROOT"], 'ssl/ssl_certificate.pem')
	if not os.path.islink(system_certificate): # not already a symlink
		new_path = os.path.join(env["STORAGE_ROOT"], 'ssl', env['PRIMARY_HOSTNAME'] + "-" + datetime.datetime.now().date().isoformat().replace("-", "") + ".pem")
		print("Renamed", system_certificate, "to", new_path, "and created a symlink for the original location.")
		shutil.move(system_certificate, new_path)
		os.symlink(new_path, system_certificate)

	# Flatten the directory structure. For any directory
	# that contains a single file named ssl_certificate.pem,
	# move the file out and name it the same as the directory,
	# and remove the directory.
	for sslcert in glob.glob(os.path.join( env["STORAGE_ROOT"], 'ssl/*/ssl_certificate.pem' )):
		d = os.path.dirname(sslcert)
		if len(os.listdir(d)) == 1:
			# This certificate is the only file in that directory.
			newname = os.path.join(env["STORAGE_ROOT"], 'ssl', os.path.basename(d) + '.pem')
			if not os.path.exists(newname):
				shutil.move(sslcert, newname)
				os.rmdir(d)

def migration_11(env):
	# Archive the old Let's Encrypt account directory managed by free_tls_certificates
	# because we'll use that path now for the directory managed by certbot.
	try:
		old_path = os.path.join(env["STORAGE_ROOT"], 'ssl', 'lets_encrypt')
		new_path = os.path.join(env["STORAGE_ROOT"], 'ssl', 'lets_encrypt-old')
		shutil.move(old_path, new_path)
	except:
		# meh
		pass

def migration_12(env):
	# Upgrading to Carddav Roundcube plugin to version 3+, it requires the carddav_*
        # tables to be dropped.
        # Checking that the roundcube database already exists.
        if os.path.exists(os.path.join(env["STORAGE_ROOT"], "mail/roundcube/roundcube.sqlite")):
            import sqlite3
            conn = sqlite3.connect(os.path.join(env["STORAGE_ROOT"], "mail/roundcube/roundcube.sqlite"))
            c = conn.cursor()
            # Get a list of all the tables that begin with 'carddav_'
            c.execute("SELECT name FROM sqlite_master WHERE type = ? AND name LIKE ?", ('table', 'carddav_%'))
            carddav_tables = c.fetchall()
            # If there were tables that begin with 'carddav_', drop them
            if carddav_tables:
                for table in carddav_tables:
                    try:
                        table = table[0]
                        c = conn.cursor()
                        dropcmd = "DROP TABLE %s" % table
                        c.execute(dropcmd)
                    except:
                        print("Failed to drop table", table)
            # Save.
            conn.commit()
            conn.close()

            # Delete all sessions, requiring users to login again to recreate carddav_*
            # databases
            conn = sqlite3.connect(os.path.join(env["STORAGE_ROOT"], "mail/roundcube/roundcube.sqlite"))
            c = conn.cursor()
            c.execute("delete from session;")
            conn.commit()
            conn.close()

def migration_13(env):
	# Add the "mfa" table for configuring MFA for login to the control panel.
	db = os.path.join(env["STORAGE_ROOT"], 'mail/users.sqlite')
	shell("check_call", ["sqlite3", db, "CREATE TABLE mfa (id INTEGER PRIMARY KEY AUTOINCREMENT, user_id INTEGER NOT NULL, type TEXT NOT NULL, secret TEXT NOT NULL, mru_token TEXT, label TEXT, FOREIGN KEY (user_id) REFERENCES users(id) ON DELETE CASCADE);"])

def migration_14(env):
	# Add the "auto_aliases" table.
	db = os.path.join(env["STORAGE_ROOT"], 'mail/users.sqlite')
	shell("check_call", ["sqlite3", db, "CREATE TABLE auto_aliases (id INTEGER PRIMARY KEY AUTOINCREMENT, source TEXT NOT NULL UNIQUE, destination TEXT NOT NULL, permitted_senders TEXT);"])

def migration_15(env):
	# Add a column to the users table to store their quota limit.  Default to '0' for unlimited.
	db = os.path.join(env["STORAGE_ROOT"], 'mail/users.sqlite')
	shell("check_call", ["sqlite3", db, "ALTER TABLE users ADD COLUMN quota TEXT NOT NULL DEFAULT '0';"])


###########################################################


def migration_miabldap_1(env):
	# This migration step moves users from sqlite3 to openldap

	# users table:
	# for each row create an ldap entry of the form:
	#   dn: uid=[uuid],ou=Users,dc=mailinabox
	#   objectClass: inetOrgPerson, mailUser, shadowAccount
	#   mail: [email]
	#   maildrop: [email]
	#   userPassword: [password]
	#   mailaccess: [privilege]   # multi-valued
	#
	# aliases table:
	# for each row create an ldap entry of the form:
	#   dn: cn=[uuid],ou=aliases,ou=Users,dc=mailinabox
	#   objectClass: mailGroup
	#   mail: [source]
	#   member: [destination-dn]   # multi-valued
	#   rfc822MailMember: [email]  # multi-values
	#
	# if the alias has permitted_senders, create:
	#   dn: cn=[uuid],ou=permitted-senders,ou=Config,dc=mailinabox
	#   objectClass: mailGroup
	#   mail: [source]
	#   member: [user-dn]         # multi-valued

	print("Migrating users and aliases from sqlite to ldap")

	# Get the ldap server up and running
	shell("check_call", ["setup/ldap.sh", "-v"])

	import sqlite3, ldap3
	import migration_13 as m13

	# 2. get ldap site details (miab_ldap.conf was created by ldap.sh)
	ldapvars = load_env_vars_from_file(os.path.join(env["STORAGE_ROOT"], "ldap/miab_ldap.conf"), strip_quotes=True)
	ldap_base = ldapvars.LDAP_BASE
	ldap_domains_base = ldapvars.LDAP_DOMAINS_BASE
	ldap_permitted_senders_base = ldapvars.LDAP_PERMITTED_SENDERS_BASE
	ldap_users_base = ldapvars.LDAP_USERS_BASE
	ldap_aliases_base = ldapvars.LDAP_ALIASES_BASE
	ldap_services_base = ldapvars.LDAP_SERVICES_BASE
	ldap_admin_dn = ldapvars.LDAP_ADMIN_DN
	ldap_admin_pass = ldapvars.LDAP_ADMIN_PASSWORD

	# 3. connect
	conn = sqlite3.connect(os.path.join(env["STORAGE_ROOT"], "mail/users.sqlite"))
	ldap = ldap3.Connection('127.0.0.1', ldap_admin_dn, ldap_admin_pass, raise_exceptions=True)
	ldap.bind()

	# 4. perform the migration
	users=m13.create_users(env, conn, ldap, ldap_base, ldap_users_base, ldap_domains_base)
	aliases=m13.create_aliases(env, conn, ldap, ldap_aliases_base)
	permitted=m13.create_permitted_senders(conn, ldap, ldap_users_base, ldap_permitted_senders_base)
	m13.populate_aliases(conn, ldap, users, aliases)

	ldap.unbind()
	conn.close()


def migration_miabldap_2(env):
	# This migration step changes the ldap schema to support utf8 email
	#
	# possible states at this point:
	#   miabldap was installed and is being upgraded
	#      -> old pre-utf8 schema present
	#   a miab install was present and step 1 upgaded it to miabldap
	#      -> new utf8 schema present
	#
	sys.path.append(os.path.realpath(os.path.join(os.path.dirname(__file__), "../management")))
	import ldap3
	from backend import connect
	import migration_14 as m14

	# 1. get ldap site details
	ldapvars = load_env_vars_from_file(os.path.join(env["STORAGE_ROOT"], "ldap/miab_ldap.conf"), strip_quotes=True)
	ldap_domains_base = ldapvars.LDAP_DOMAINS_BASE
	ldap_aliases_base = ldapvars.LDAP_ALIASES_BASE
	ldap_users_base = ldapvars.LDAP_USERS_BASE

	# connect before schema changes to ensure admin password works
	ldap = connect(ldapvars)

	# 2. if this is a miab -> maibldap install, the new schema is
	# already in place and no schema changes are needed. however,
	# if this is a miabldap/1 to miabldap/2 migration, we must
	# upgrade the schema.
	ret = shell("check_output", [
		"ldapsearch",
		"-Q",
		"-Y", "EXTERNAL",
		"-H", "ldapi:///",
		"(&(objectClass=olcSchemaConfig)(cn={*}postfix))",
		"-b", "cn=schema,cn=config",
		"-LLL",
		"olcObjectClasses"
	])

	if "rfc822MailMember" in ret:
		def ldif_change_fn(ldif):
			return ldif.replace("rfc822MailMember: ", "mailMember: ")
		# apply schema changes miabldap/1 -> miabldap/2
		ldap.unbind()
		print("Apply schema changes to support utf8 email addresses")
		m14.apply_schema_changes(env, ldapvars, ldif_change_fn)
		# reconnect
		ldap = connect(ldapvars)

	# 3. migrate to utf8: users, aliases and domains
	print("Create utf8 entries for users and aliases having IDNA domains")
	m14.add_utf8_mail_addresses(env, ldap, ldap_users_base)

	print("Add namedProperties objectclass to aliases")
	m14.add_namedProperties_objectclass(env, ldap, ldap_aliases_base)

	print("Add mailDomain objectclass to domains")
	m14.add_mailDomain_objectclass(env, ldap, ldap_domains_base)

	print("Mark required aliases with 'auto' property")
	m14.add_auto_tag(env, ldap, ldap_aliases_base)

	print("Ensure all required aliases are created")
	m14.ensure_required_aliases(env, ldapvars, ldap)

	ldap.unbind()

def migration_miabldap_3(env):
	# This migration step changes the ldap schema to support quotas
	#
	# possible states at this point:
	#   miabldap was installed and is being upgraded
	#      -> schema update needed
	#   a miab install was present and step 1 upgaded it to miabldap
	#      -> new schema already present
	#
	sys.path.append(os.path.realpath(os.path.join(os.path.dirname(__file__), "../management")))
	import ldap3
	from backend import connect
	import migration_14 as m14

	# 1. get ldap site details
	ldapvars = load_env_vars_from_file(os.path.join(env["STORAGE_ROOT"], "ldap/miab_ldap.conf"), strip_quotes=True)

	# connect before schema changes to ensure admin password works
	ldap = connect(ldapvars)

	# 2. if this is a miab -> maibldap install, the new schema is
	# already in place and no schema changes are needed. however,
	# if this is a miabldap/1 to miabldap/2 migration, we must
	# upgrade the schema.
	ret = shell("check_output", [
		"ldapsearch",
		"-Q",
		"-Y", "EXTERNAL",
		"-H", "ldapi:///",
		"(&(objectClass=olcSchemaConfig)(cn={*}postfix))",
		"-b", "cn=schema,cn=config",
		"-o", "ldif_wrap=no",
		"-LLL",
		"olcObjectClasses"
	])

	ldap.unbind()

	if "mailboxQuota" not in ret:
		def ldif_change_fn(ldif):
			# the schema change we're making does not require any data changes
			return ldif
		# apply schema changes miabldap/2 -> miabldap/3
		print("Apply schema changes to support mailbox quotas")
		m14.apply_schema_changes(env, ldapvars, ldif_change_fn)


def get_current_migration():
	ver = 0
	while True:
		next_ver = (ver + 1)
		migration_func = globals().get("migration_miabldap_%d" % next_ver)
		if not migration_func:
			return ver
		ver = next_ver

def run_migrations():
	if not os.access("/etc/mailinabox.conf", os.W_OK, effective_ids=True):
		print("This script must be run as root.", file=sys.stderr)
		sys.exit(1)

	env = load_environment()

	migration_id_file = os.path.join(env['STORAGE_ROOT'], 'mailinabox.version')
	migration_id = None
	if os.path.exists(migration_id_file):
		with open(migration_id_file, encoding='utf-8') as f:
			migration_id = f.read().strip()

	if migration_id is None:
		# Load the legacy location of the migration ID. We'll drop support
		# for this eventually.
		migration_id = env.get("MIGRATIONID")

	if migration_id is None:
		print()
		print(f"{migration_id_file} file doesn't exists. Skipping migration...")
		return

	ourver = int(migration_id)

	while True:
		next_ver = (ourver + 1)
		migration_func = globals().get("migration_%d" % next_ver)

		if not migration_func:
			# No more migrations to run.
			break

		print()
		print("Running migration to Mail-in-a-Box #%d..." % next_ver)

		try:
			migration_func(env)
		except Exception as e:
			print()
			print("Error running the migration script:")
			print()
			print(e)
			print()
			print("Your system may be in an inconsistent state now. We're terribly sorry. A re-install from a backup might be the best way to continue.")
			sys.exit(1)

		ourver = next_ver

		# Write out our current version now. Do this sooner rather than later
		# in case of any problems.
		with open(migration_id_file, "w", encoding='utf-8') as f:
			f.write(str(ourver) + "\n")

		# Delete the legacy location of this field.
		if "MIGRATIONID" in env:
			del env["MIGRATIONID"]
			save_environment(env)

		# iterate and try next version...

def run_miabldap_migrations():
	if not os.access("/etc/mailinabox.conf", os.W_OK, effective_ids=True):
		print("This script must be run as root.", file=sys.stderr)
		sys.exit(1)

	env = load_environment()

	migration_id_file = os.path.join(env['STORAGE_ROOT'], 'mailinabox-ldap.version')
	migration_id = None
	if os.path.exists(migration_id_file):
		with open(migration_id_file) as f:
			migration_id = f.read().strip();
		if migration_id.strip()=='': migration_id = None

	if migration_id is None:
		if os.path.exists(os.path.join(env['STORAGE_ROOT'], 'mailinabox.version')):
			migration_id = 0
		else:
			print()
			print("%s file doesn't exist. Skipping migration..." % (migration_id_file,))
			return

	ourver = int(migration_id)

	while True:
		next_ver = (ourver + 1)
		migration_func = globals().get("migration_miabldap_%d" % next_ver)

		if not migration_func:
			# No more migrations to run.
			break

		print()
		print("Running migration to Mail-in-a-Box LDAP #%d..." % next_ver)

		try:
			migration_func(env)
		except Exception as e:
			print()
			print("Error running the migration script:")
			print()
			print(e)
			print()
			print("Your system may be in an inconsistent state now. We're terribly sorry. A re-install from a backup might be the best way to continue.")
			#sys.exit(1)
			raise e

		ourver = next_ver

		# Write out our current version now. Do this sooner rather than later
		# in case of any problems.
		with open(migration_id_file, "w") as f:
			f.write(str(ourver) + "\n")

		# iterate and try next version...

if __name__ == "__main__":
	if sys.argv[-1] == "--current":
		# Return the number of the highest migration.
		print(str(get_current_migration()))
	elif sys.argv[-1] == "--migrate":
		# Perform migrations.
		env = load_environment()

		# if miab-ldap already installed, only run miab-ldap migrations
		if 'LDAP_USERS_BASE' in env:
			run_miabldap_migrations()

		# otherwise, run both
		else:
			run_migrations()
			run_miabldap_migrations()