more dockerization work

[From @joshdata: This is part of @toilal's work in #377 and 1eb77c332b. The changes are:

* Separates out the runit configuration from starting Mail-in-a-Box setup so that Mail-in-a-Box setup does not block the starting of runit services and we can assume that runit is running during setup (i.e. we can restart services).
* Adds a SKIP_INSTALL flag so that the container can be restarted without re-running the whole Mail-in-a-Box setup.
* Made containers/docker/run more flexible.
* I'm also adding some "|| exit 0"s to the run script to stop if there are any docker errors.
* I'm also adding the prereqs installs from questions.sh into Dockerfile so we don't have to reinstall each time.

]

Dockerfile

@@ -0,0 +1,57 @@
+# Mail-in-a-Box Dockerfile
+###########################
+#
+# This file lets Mail-in-a-Box run inside of Docker (https://docker.io),
+# a virtualization/containerization manager.
+#
+# Run:
+#   $ containers/docker/run.sh
+# to build the image, launch a storage container, and launch a Mail-in-a-Box
+# container.
+#
+###########################################
+
+# We need a better starting image than docker's ubuntu image because that
+# base image doesn't provide enough to run most Ubuntu services. See
+# http://phusion.github.io/baseimage-docker/ for an explanation.
+
+FROM phusion/baseimage:0.9.16
+
+# Dockerfile metadata.
+MAINTAINER Joshua Tauberer (http://razor.occams.info)
+EXPOSE 25 53/udp 53/tcp 80 443 587 993 4190
+VOLUME /home/user-data
+
+# Use baseimage's init system. A correct init process is required for
+# process #1 in order to have a functioning Linux system.
+CMD ["/sbin/my_init"]
+
+# Create the user-data user, so the start script doesn't have to.
+RUN useradd -m user-data
+
+# Docker has a beautiful way to cache images after each step. The next few
+# steps of installing system packages are very intensive, so we take care
+# of them early and let docker cache the image after that, before doing
+# any Mail-in-a-Box specific system configuration. That makes rebuilds
+# of the image extremely fast.
+
+# Update system packages.
+RUN apt-get update
+RUN DEBIAN_FRONTEND=noninteractive apt-get upgrade -y
+
+# Install packages needed by Mail-in-a-Box.
+ADD containers/docker/apt_package_list.txt /tmp/mailinabox_apt_package_list.txt
+RUN DEBIAN_FRONTEND=noninteractive apt-get install -y $(cat /tmp/mailinabox_apt_package_list.txt)
+
+# from questions.sh -- needs merging into the above line
+RUN DEBIAN_FRONTEND=noninteractive apt-get install -y dialog python3 python3-pip
+RUN pip3 install "email_validator==0.1.0-rc4"
+
+# Now add Mail-in-a-Box to the system.
+ADD . /usr/local/mailinabox
+
+# Configure runit services.
+RUN /usr/local/mailinabox/containers/docker/tools/configure_services.sh
+
+# Add my_init scripts
+ADD containers/docker/my_init.d/* /etc/my_init.d/

containers/docker/apt_package_list.txt

@@ -0,0 +1,86 @@
+bc
+bind9
+ca-certificates
+coreutils
+cron
+curl
+dbconfig-common
+dovecot-antispam
+dovecot-core
+dovecot-imapd
+dovecot-lmtpd
+dovecot-lucene
+dovecot-managesieved
+dovecot-pop3d
+dovecot-sieve
+dovecot-sqlite
+duplicity
+fail2ban
+git
+haveged
+ldnsutils
+libapr1
+libawl-php
+libcurl4-openssl-dev
+libjs-jquery
+libjs-jquery-mousewheel
+libmagic1
+libtool
+libyaml-dev
+links
+memcached
+munin
+munin-node
+nginx
+nsd
+ntp
+opendkim
+opendkim-tools
+opendmarc
+openssh-client
+openssl
+php-apc
+php-auth
+php-crypt-gpg
+php-mail-mime
+php-net-sieve
+php-net-smtp
+php-net-socket
+php-pear
+php-soap
+php-xml-parser
+php5
+php5-cli
+php5-common
+php5-curl
+php5-dev
+php5-fpm
+php5-gd
+php5-imap
+php5-intl
+php5-json
+php5-mcrypt
+php5-memcache
+php5-pspell
+php5-sqlite
+php5-xsl
+postfix
+postfix-pcre
+postgrey
+python3
+python3-dateutil
+python3-dev
+python3-dnspython
+python3-flask
+python3-pip
+pyzor
+razor
+resolvconf
+spampd
+sqlite3
+sudo
+tinymce
+ufw
+unattended-upgrades
+unzip
+wget

containers/docker/my_init.d/10-mailinabox.sh

@@ -0,0 +1,58 @@
+#!/bin/bash
+
+# This script is used within containers to turn it into a Mail-in-a-Box.
+# It is referenced by the Dockerfile. You should not run it directly.
+########################################################################
+
+# Local configuration details were not known at the time the Docker
+# image was created, so all setup is defered until the container
+# is started. That's when this script runs.
+
+# If we're not in an interactive shell, set defaults.
+if [ ! -t 0 ]; then
+	echo '*** Non interactive shell detected...'
+	export PUBLIC_IP=auto
+	export PUBLIC_IPV6=auto
+	export PRIMARY_HOSTNAME=auto
+	export CSR_COUNTRY=US
+	export NONINTERACTIVE=1
+fi
+
+if ([ -z "$FORCE_INSTALL" ] && [ -f /var/lib/mailinabox/api.key ]); then
+	# Mailinabox is already installed and we don't want to reinstall
+	export SKIP_INSTALL=1
+fi
+
+# If we are skipping install, reload from /etc/mailinabox.conf if exists
+if ([ -f /var/lib/mailinabox/api.key ] && [ ! -z "$SKIP_INSTALL" ]); then
+	echo '*** Loading variables from "/etc/mailinabox.conf"...'
+
+	source /etc/mailinabox.conf
+	unset PRIVATE_IP
+	unset PRIVATE_IPV6
+	export SKIP_NETWORK_CHECKS=1
+	export NONINTERACTIVE=1
+fi
+
+export DISABLE_FIREWALL=1
+cd /usr/local/mailinabox
+
+if [ -z "$SKIP_INSTALL" ]; then
+	echo "*** Starting mailinabox installation..."
+	# Run in background to avoid blocking runit initialization while installing.
+	source setup/start.sh &
+else
+	echo "*** Configuring mailinabox..."
+	# Run in foreground for services to be started after configuration is re-written.
+	source setup/questions.sh
+	cat > /etc/mailinabox.conf << EOF;
+STORAGE_USER=$STORAGE_USER
+STORAGE_ROOT=$STORAGE_ROOT
+PRIMARY_HOSTNAME=$PRIMARY_HOSTNAME
+PUBLIC_IP=$PUBLIC_IP
+PUBLIC_IPV6=$PUBLIC_IPV6
+PRIVATE_IP=$PRIVATE_IP
+PRIVATE_IPV6=$PRIVATE_IPV6
+CSR_COUNTRY=$CSR_COUNTRY
+EOF
+fi

containers/docker/run

@@ -0,0 +1,114 @@
+#!/bin/bash
+# Use this script to launch Mail-in-a-Box within a docker container.
+# ==================================================================
+#
+# Run this script from the base directory of the Mail-in-a-Box
+# repository (i.e. run as './containers/docker/run').
+#
+# Set these optional environment variables as needed:
+# * HOST_HTTP_PORT: Host http: port to bind (default: 80).
+# * HOST_HTTPS_PORT: Host https: port to bind (default: 443).
+# * SKIP_BUILD: Skip the build of docker image (default: unset).
+# * NODNS: Skip mapping of DNS ports (53 tcp/upd). They are not always available on host, as another DNS server can be running (default: unset).
+# * CONTAINER_NAME: Name of the main container (default: mailinabox).
+# * CONTAINER_DATA_NAME: Name of the data container (default: mailinabox-data).
+# * NONINTERACTIVE: Use this when mailinabox is already installed on the volume container. Else, it's not recommanded (default: unset).
+#
+# A base image is created first. The base image installs Ubuntu
+# packages and pulls in the Mail-in-a-Box source code. This is
+# defined in Dockerfile at the root of this repository.
+#
+# A mailinabox-data container is created next. This container
+# contains nothing but a shared volume for storing user data.
+# It is segregated from the rest of the live system to make backups
+# easier.
+#
+# The mailinabox container is started last. It is the
+# real thing: it runs the mailinabox image. This container will
+# initialize itself and will initialize the mailinabox-data
+# volume if the volume is new.
+
+# Build or rebuild the image.
+# Rebuilds are very fast.
+
+HOST_HTTP_PORT=${HOST_HTTP_PORT:-80}
+HOST_HTTPS_PORT=${HOST_HTTPS_PORT:-443}
+CONTAINER_NAME=${CONTAINER_NAME:-mailinabox}
+CONTAINER_DATA_NAME=${CONTAINER_DATA_NAME:-${CONTAINER_NAME}-data}
+
+if [ -z "$SKIP_BUILD" ]; then
+	tput setaf 2
+	echo "Building/updating base image (mailinabox)..."
+	tput setaf 7
+
+	docker build -q -t mailinabox . || exit 1
+fi;
+
+if ! docker inspect ${CONTAINER_DATA_NAME} > /dev/null; then
+	tput setaf 2
+	echo
+	echo "Creating a new container for your data (${CONTAINER_DATA_NAME})..."
+	tput setaf 7
+
+	docker create \
+		--name ${CONTAINER_DATA_NAME} \
+		-v /home/user-data \
+		phusion/baseimage:0.9.16 || exit 1
+else
+	tput setaf 2
+	echo
+	echo "Using existing container ${CONTAINER_DATA_NAME} for your data."
+	tput setaf 7
+fi
+
+# End a running container.
+if docker inspect ${CONTAINER_NAME} > /dev/null; then
+	tput setaf 2
+	echo
+	echo "Destroying ${CONTAINER_NAME} container..."
+	tput setaf 7
+
+	docker rm -f ${CONTAINER_NAME}
+fi
+
+# Start container.
+tput setaf 2
+echo
+echo "Starting new container (${CONTAINER_NAME})..."
+tput setaf 7
+
+# Run the services container
+# detached if NONINTERACTIVE is set,
+# interactively if NONINTERACTIVE is not set,
+# Notes:
+# * Passing through SKIP_NETWORK_CHECKS makes it easier to do testing
+#   on a residential network.
+# * --privileged flag cause an issue with bind9/named failing to start in this case
+#   see docker/docker#7318
+docker run \
+	-v /dev/urandom:/dev/random \
+	-p 25:25 \
+	$([ -z "$NODNS" ] && echo "-p 53:53/udp -p 53:53/tcp") \
+	-p $HOST_HTTP_PORT:80 \
+	-p $HOST_HTTPS_PORT:443 \
+	-p 587:587 \
+	-p 993:993 \
+	-p 4190:4190 \
+	--name ${CONTAINER_NAME} \
+	--volumes-from ${CONTAINER_DATA_NAME} \
+	--restart always \
+	$([ ! -z "$NONINTERACTIVE" ] && echo "-d") \
+	-it \
+	-e "IS_DOCKER=1" \
+	-e "SKIP_NETWORK_CHECKS=$SKIP_NETWORK_CHECKS" \
+	mailinabox \
+	|| exit 1
+
+if [ -z "$NONINTERACTIVE" ]; then
+	tput setaf 2
+	echo
+	echo "Restarting container ${CONTAINER_NAME}..."
+	tput setaf 7
+
+	docker restart ${CONTAINER_NAME} || exit 1
+fi

containers/docker/runit/dovecot.sh

@@ -0,0 +1,3 @@
+#!/bin/bash
+
+/usr/sbin/dovecot -F -c /etc/dovecot/dovecot.conf &> /var/log/dovecot.log

containers/docker/tools/configure_services.sh

@@ -0,0 +1,103 @@
+#!/bin/bash
+
+# The phusion/baseimage base image we use for a working Ubuntu
+# replaces the normal Upstart system service management with
+# a ligher-weight service management system called runit that
+# requires a different configuration. We need to create service
+# run files that do not daemonize.
+
+# This removes /etc/init.d service if service exists in runit.
+# It also creates a symlink from /usr/bin/sv to /etc/init.d/$service
+# to support SysV syntax: service $service <command> or /etc/init.d/$service <command>
+SERVICES=/etc/service/*
+for f in $SERVICES
+do
+	service=$(basename "$f")
+	if [ -d /etc/service/$service ]; then
+		if [ -f /etc/init.d/$service ]; then
+			mv /etc/init.d/$service /etc/init.d/$service.lsb
+			chmod -x /etc/init.d/$service.lsb
+		fi
+		ln -s /usr/bin/sv /etc/init.d/$service
+	fi
+done
+
+# Create runit services from sysv services. For most of the services,
+# there is a common pattern we can use: execute the init.d script that
+# the Ubuntu package installs, and then poll for the termination of
+# the daemon.
+function make_runit_service {
+	INITD_NAME=$1
+	WAIT_ON_PROCESS_NAME=$2
+	mkdir -p /etc/service/$INITD_NAME
+	cat > /etc/service/$INITD_NAME/run <<EOF;
+#!/bin/bash
+source /usr/local/mailinabox/setup/functions.sh
+hide_output /etc/init.d/$INITD_NAME restart
+while [ \`ps a -C $WAIT_ON_PROCESS_NAME -o pid= | wc -l\` -gt 0 ]; do
+       sleep 30
+done
+echo $WAIT_ON_PROCESS_NAME died.
+sleep 20
+EOF
+	chmod +x /etc/service/$INITD_NAME/run
+}
+make_runit_service bind9 named
+make_runit_service resolvconf resolvconf
+make_runit_service fail2ban fail2ban
+make_runit_service mailinabox mailinabox-daemon
+make_runit_service memcached memcached
+make_runit_service nginx nginx
+make_runit_service nsd nsd
+make_runit_service opendkim opendkim
+make_runit_service opendmarc opendmarc
+make_runit_service php5-fpm php5-fpm
+make_runit_service postfix postfix
+make_runit_service postgrey postgrey
+make_runit_service spampd spampd
+
+# Dovecot doesn't provide an init.d script, but it does provide
+# a way to launch without daemonization. We wrote a script for
+# that specifically.
+for service in dovecot; do
+	mkdir -p /etc/service/$service
+	cp /usr/local/mailinabox/containers/docker/runit/$service.sh /etc/service/$service/run
+	chmod +x /etc/service/$service/run
+done
+
+# This adds a log/run file on each runit service directory.
+# This file make services stdout/stderr output to svlogd log
+# directory located in /var/log/runit/$service.
+SERVICES=/etc/service/*
+for f in $SERVICES
+do
+	service=$(basename "$f")
+	if [ -d /etc/service/$service ]; then
+		mkdir -p /etc/service/$service/log
+		cat > /etc/service/$service/log/run <<EOF;
+#!/bin/bash
+mkdir -p /var/log/runit
+chmod o-wrx /var/log/runit
+mkdir -p /var/log/runit/$service
+chmod o-wrx /var/log/runit/$service
+exec svlogd -tt /var/log/runit/$service/
+EOF
+		chmod +x /etc/service/$service/log/run
+	fi
+done
+
+# Disable services for now. Until Mail-in-a-Box is installed the
+# services won't be configured right and there would be errors if
+# they got run prematurely.
+SERVICES=/etc/service/*
+for f in $SERVICES
+do
+	service=$(basename "$f")
+	if [ "$service" = "syslog-ng" ]; then continue; fi;
+	if [ "$service" = "syslog-forwarder" ]; then continue; fi;
+	if [ "$service" = "ssh" ]; then continue; fi;
+	if [ "$service" = "cron" ]; then continue; fi;
+	if ([ -d /etc/service/$service ] && [ ! -f /etc/service/$service/down ]); then
+		touch /etc/service/$service/down
+	fi
+done

setup/dns.sh

@@ -127,6 +127,8 @@ EOF
 chmod +x /etc/cron.daily/mailinabox-dnssec
 
 # Permit DNS queries on TCP/UDP in the firewall.
-
 ufw_allow domain
 
+# Start nsd. None of the zones are configured until the management daemon is
+# run later, though.
+restart_service nsd

setup/functions.sh

@@ -39,8 +39,14 @@ function apt_get_quiet {
 }
 
 function apt_install {
-	# Report any packages already installed.
 	PACKAGES=$@
+
+	if [ ! -z "$IS_DOCKER" ]; then
+		# Speed things up because packages are already installed by the image.
+		PACKAGES=""
+	fi
+			
+	# Report any packages already installed.
 	TO_INSTALL=""
 	ALREADY_INSTALLED=""
 	for pkg in $PACKAGES; do
@@ -72,12 +78,20 @@ function get_default_hostname {
 	# Guess the machine's hostname. It should be a fully qualified
 	# domain name suitable for DNS. None of these calls may provide
 	# the right value, but it's the best guess we can make.
-	set -- $(hostname --fqdn      2>/dev/null ||
-                 hostname --all-fqdns 2>/dev/null ||
-                 hostname             2>/dev/null)
+	set -- $(
+		get_hostname_from_reversedns ||
+		hostname --fqdn      2>/dev/null ||
+		hostname --all-fqdns 2>/dev/null ||
+		hostname             2>/dev/null)
 	printf '%s\n' "$1" # return this value
 }
 
+function get_hostname_from_reversedns {
+	# Do a reverse DNS lookup on our public IPv4 address. The output of
+	# `host` is complex -- use sed to get the FDQN.
+	host $(get_publicip_from_web_service 4) | sed "s/.*pointer \(.*\)\./\1/"
+}
+
 function get_publicip_from_web_service {
 	# This seems to be the most reliable way to determine the
 	# machine's public IP address: asking a very nice web API
@@ -153,7 +167,17 @@ function ufw_allow {
 }
 
 function restart_service {
-	hide_output service $1 restart
+	if [ -z "$IS_DOCKER" ]; then
+		# Restart the service.
+		hide_output service $1 restart
+
+	else
+		# In Docker, make sure the service is not disabled by a down file.
+		if [ -f /etc/service/$1/down ]; then
+			rm /etc/service/$1/down
+		fi
+		sv restart $1
+	fi
 }
 
 ## Dialog Functions ##

setup/owncloud.sh

@@ -184,4 +184,5 @@ chmod +x /etc/cron.hourly/mailinabox-owncloud
 
 # Enable PHP modules and restart PHP.
 php5enmod imap
+restart_service memcached
 restart_service php5-fpm

setup/preflight.sh

@@ -4,7 +4,7 @@ if [[ $EUID -ne 0 ]]; then
 	echo
 	echo "sudo $0"
 	echo
-	exit
+	exit 1
 fi
 
 # Check that we are running on Ubuntu 14.04 LTS (or 14.04.xx).
@@ -14,7 +14,7 @@ if [ "`lsb_release -d | sed 's/.*:\s*//' | sed 's/14\.04\.[0-9]/14.04/' `" != "U
 	lsb_release -d | sed 's/.*:\s*//'
 	echo
 	echo "We can't write scripts that run on every possible setup, sorry."
-	exit
+	exit 1
 fi
 
 # Check that we have enough memory.
@@ -30,6 +30,6 @@ if [ ! -d /vagrant ]; then
 	echo "Your Mail-in-a-Box needs more memory (RAM) to function properly."
 	echo "Please provision a machine with at least 768 MB, 1 GB recommended."
 	echo "This machine has $TOTAL_PHYSICAL_MEM MB memory."
-	exit
+	exit 1
 fi
 fi

setup/questions.sh

@@ -12,8 +12,10 @@ if [ -z "$NONINTERACTIVE" ]; then
 		apt_get_quiet install dialog python3 python3-pip  || exit 1
 	fi
 
+	if [ -z "$IS_DOCKER" ]; then
 	# email_validator is repeated in setup/management.sh
 	hide_output pip3 install "email_validator==0.1.0-rc5" || exit 1
+	fi
 
 	message_box "Mail-in-a-Box Installation" \
 		"Hello and thanks for deploying a Mail-in-a-Box!

setup/start.sh

@@ -69,7 +69,7 @@ if [ ! -d $STORAGE_ROOT ]; then
 fi
 if [ ! -f $STORAGE_ROOT/mailinabox.version ]; then
 	echo $(setup/migrate.py --current) > $STORAGE_ROOT/mailinabox.version
-	chown $STORAGE_USER.$STORAGE_USER $STORAGE_ROOT/mailinabox.version
+	chown $STORAGE_USER:$STORAGE_USER $STORAGE_ROOT/mailinabox.version
 fi
 
 
@@ -139,4 +139,4 @@ openssl x509 -in $STORAGE_ROOT/ssl/ssl_certificate.pem -noout -fingerprint \
         | sed "s/SHA1 Fingerprint=//"
 echo
 echo Then you can confirm the security exception and continue.
-echo
+echo

setup/webmail.sh

@@ -126,7 +126,7 @@ EOF
 
 # Create writable directories.
 mkdir -p /var/log/roundcubemail /tmp/roundcubemail $STORAGE_ROOT/mail/roundcube
-chown -R www-data.www-data /var/log/roundcubemail /tmp/roundcubemail $STORAGE_ROOT/mail/roundcube
+chown -R www-data:www-data /var/log/roundcubemail /tmp/roundcubemail $STORAGE_ROOT/mail/roundcube
 
 # Password changing plugin settings
 # The config comes empty by default, so we need the settings 
@@ -147,9 +147,9 @@ usermod -a -G dovecot www-data
 
 # set permissions so that PHP can use users.sqlite
 # could use dovecot instead of www-data, but not sure it matters
-chown root.www-data $STORAGE_ROOT/mail
+chown root:www-data $STORAGE_ROOT/mail
 chmod 775 $STORAGE_ROOT/mail
-chown root.www-data $STORAGE_ROOT/mail/users.sqlite 
+chown root:www-data $STORAGE_ROOT/mail/users.sqlite
 chmod 664 $STORAGE_ROOT/mail/users.sqlite 
 
 # Enable PHP modules.

tools/list_all_packages.py

@@ -0,0 +1,23 @@
+#!/usr/bin/python3
+
+import os.path, glob, re
+
+packages = set()
+
+def add(line):
+	global packages
+	if line.endswith("\\"): line = line[:-1]
+	packages |= set(p for p in line.split(" ") if p not in("", "apt_install"))
+
+for fn in glob.glob(os.path.join(os.path.dirname(__file__), "../setup/*.sh")):
+	with open(fn) as f:
+		in_apt_install = False
+		for line in f:
+			line = line.strip()
+			if line.startswith("apt_install "):
+				in_apt_install = True
+			if in_apt_install:
+				add(line)
+			in_apt_install = in_apt_install and line.endswith("\\")
+
+print("\n".join(sorted(packages)))