From dd5de8acf3e18f11ff86f4cb10b19012654e1542 Mon Sep 17 00:00:00 2001 From: Joshua Tauberer Date: Sun, 17 Aug 2014 08:48:27 -0400 Subject: [PATCH] dockerize (work in progress) Docker support was initially worked on in 2bbb7a5e7ea9ec96650685c288d31449a0cc75a4, but it never really worked. This extends f7d7434012800c3572049af82a501743d4aed583 which was an old branch for docker work. --- Dockerfile | 47 +++++++++++++++ containers/docker/apt_package_list.txt | 82 ++++++++++++++++++++++++++ containers/docker/container_start.sh | 30 ++++++++++ containers/docker/run.sh | 50 ++++++++++++++++ setup/functions.sh | 34 +++++++++-- setup/preflight.sh | 6 +- setup/start.sh | 6 +- tools/list_all_packages.py | 23 ++++++++ 8 files changed, 270 insertions(+), 8 deletions(-) create mode 100644 Dockerfile create mode 100644 containers/docker/apt_package_list.txt create mode 100755 containers/docker/container_start.sh create mode 100755 containers/docker/run.sh create mode 100644 tools/list_all_packages.py diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 00000000..18d8a808 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,47 @@ +# Mail-in-a-Box Dockerfile +########################### +# +# This file lets Mail-in-a-Box run inside of Docker (https://docker.io), +# a virtualization/containerization manager. +# +# Run: +# $ containers/docker/run.sh +# to build the image, launch a storage container, and launch a Mail-in-a-Box +# container. +# +########################################### + +# We need a better starting image than docker's ubuntu image because that +# base image doesn't provide enough to run most Ubuntu services. See +# http://phusion.github.io/baseimage-docker/ for an explanation. + +FROM phusion/baseimage:0.9.15 + +# Dockerfile metadata. +MAINTAINER Joshua Tauberer (http://razor.occams.info) +EXPOSE 22 25 53 80 443 587 993 + +# Docker has a beautiful way to cache images after each step. The next few +# steps of installing system packages are very intensive, so we take care +# of them early and let docker cache the image after that, before doing +# any Mail-in-a-Box specific system configuration. That makes rebuilds +# of the image extremely fast. + +# Update system packages. +RUN apt-get update +RUN DEBIAN_FRONTEND=noninteractive apt-get upgrade -y + +# Install packages needed by Mail-in-a-Box. +ADD containers/docker/apt_package_list.txt /tmp/mailinabox_apt_package_list.txt +RUN DEBIAN_FRONTEND=noninteractive apt-get install -y $(cat /tmp/mailinabox_apt_package_list.txt) +RUN rm -f /tmp/mailinabox_apt_package_list.txt + +# Now add Mail-in-a-Box to the system. +ADD . /usr/local/mailinabox + +# We can't know things like the IP address where the container will eventually +# be deployed until the container is started. We also don't want to create any +# private keys during the creation of the image --- that should wait until the +# container is started too. So our whole setup process is deferred until the +# container is started. +ENTRYPOINT ["/usr/local/mailinabox/containers/docker/container_start.sh"] diff --git a/containers/docker/apt_package_list.txt b/containers/docker/apt_package_list.txt new file mode 100644 index 00000000..96bfdb56 --- /dev/null +++ b/containers/docker/apt_package_list.txt @@ -0,0 +1,82 @@ +bc +bind9 +ca-certificates +coreutils +cron +curl +dbconfig-common +dovecot-antispam +dovecot-core +dovecot-imapd +dovecot-lmtpd +dovecot-managesieved +dovecot-sieve +dovecot-sqlite +duplicity +fail2ban +git +haveged +ldnsutils +libapr1 +libawl-php +libcurl4-openssl-dev +libjs-jquery +libjs-jquery-mousewheel +libmagic1 +libtool +libyaml-dev +links +memcached +nginx +nsd +ntp +opendkim +opendkim-tools +opendmarc +openssh-client +openssl +php-apc +php-auth +php-crypt-gpg +php-mail-mime +php-net-sieve +php-net-smtp +php-net-socket +php-pear +php-soap +php-xml-parser +php5 +php5-cli +php5-common +php5-curl +php5-dev +php5-fpm +php5-gd +php5-imap +php5-intl +php5-json +php5-mcrypt +php5-memcache +php5-pspell +php5-sqlite +php5-xsl +postfix +postfix-pcre +postgrey +python3 +python3-dateutil +python3-dev +python3-dnspython +python3-flask +python3-pip +pyzor +razor +resolvconf +spampd +sqlite3 +sudo +tinymce +ufw +unattended-upgrades +unzip +wget diff --git a/containers/docker/container_start.sh b/containers/docker/container_start.sh new file mode 100755 index 00000000..9e0e9650 --- /dev/null +++ b/containers/docker/container_start.sh @@ -0,0 +1,30 @@ +#!/bin/bash + +# This script is used within containers to turn it into a Mail-in-a-Box. +# It is referenced by the Dockerfile. You should not run it directly. +######################################################################## + +# Local configuration details were not known at the time the Docker +# image was created, so all setup is defered until the container +# is started. That's when this script runs. + +# If we're not in an interactive shell, set defaults. +if [ ! -t 0 ]; then + export PUBLIC_IP=auto + export PUBLIC_IPV6=auto + export PRIMARY_HOSTNAME=auto + export CSR_COUNTRY=US + export NONINTERACTIVE=1 +fi + +# Start configuration. +cd /usr/local/mailinabox +export IS_DOCKER=1 +export DISABLE_FIREWALL=1 +source setup/start.sh # using 'source' means an exit from inside also exits this script and terminates container + +# Once the configuration is complete, start the Unix init process +# provided by the base image. We're running as process 0, and +# /sbin/my_init needs to run as process 0, so use 'exec' to replace +# this shell process and not fork a new one. Nifty right? +exec /sbin/my_init -- bash diff --git a/containers/docker/run.sh b/containers/docker/run.sh new file mode 100755 index 00000000..2816d138 --- /dev/null +++ b/containers/docker/run.sh @@ -0,0 +1,50 @@ +#!/bin/bash +# Use this script to launch Mail-in-a-Box within a docker container. +# ================================================================== +# +# A base image is created first. The base image installs Ubuntu +# packages and pulls in the Mail-in-a-Box source code. This is +# defined in Dockerfile at the root of this repository. +# +# A mailinabox-userdata container is started next. This container +# contains nothing but a shared volume for storing user data. +# It is segregated from the rest of the live system to make backups +# easier. +# +# The mailinabox-services container is started last. It is the +# real thing: it runs the mailinabox image. This container will +# initialize itself and will initialize the mailinabox-userdata +# volume if the volume is new. + + +DOCKER=docker.io + +# Build or rebuild the image. +# Rebuilds are very fast. +$DOCKER build -q -t mailinabox . + +# Start the user-data containerw which is merely to create +# a container that maintains a reference to a volume so that +# we can destroy the main container without losing user data. +if ! $DOCKER ps -a | grep mailinabox-userdata > /dev/null; then + echo Starting user-data volume container... + $DOCKER run -d \ + --name mailinabox-userdata \ + -v /home/user-data \ + scratch bash +fi + +# End a running container. +if $DOCKER ps -a | grep mailinabox-services > /dev/null; then + echo Deleting container... + $DOCKER rm mailinabox-services +fi + +# Start container. +echo Starting new container... +$DOCKER run \ + -p 25 -p 53 -p 80 -p 443 -p 587 -p 993 \ + --volumes-from mailinabox-userdata \ + --name mailinabox-services \ + -t -i \ + mailinabox diff --git a/setup/functions.sh b/setup/functions.sh index 94ea87cd..06809ddc 100644 --- a/setup/functions.sh +++ b/setup/functions.sh @@ -70,12 +70,20 @@ function get_default_hostname { # Guess the machine's hostname. It should be a fully qualified # domain name suitable for DNS. None of these calls may provide # the right value, but it's the best guess we can make. - set -- $(hostname --fqdn 2>/dev/null || - hostname --all-fqdns 2>/dev/null || - hostname 2>/dev/null) + set -- $( + get_hostname_from_reversedns || + hostname --fqdn 2>/dev/null || + hostname --all-fqdns 2>/dev/null || + hostname 2>/dev/null) printf '%s\n' "$1" # return this value } +function get_hostname_from_reversedns { + # Do a reverse DNS lookup on our public IPv4 address. The output of + # `host` is complex -- use sed to get the FDQN. + host $(get_publicip_from_web_service 4) | sed "s/.*pointer \(.*\)\./\1/" +} + function get_publicip_from_web_service { # This seems to be the most reliable way to determine the # machine's public IP address: asking a very nice web API @@ -151,7 +159,25 @@ function ufw_allow { } function restart_service { - hide_output service $1 restart + # Restart a service quietly. + if [ ! "$IS_DOCKER" ]; then + # The normal way to restart a service. + hide_output service $1 restart + else + # On docker, sysvinit is not present. Our base image provides + # a weird way to manage running services. But we're not going + # to use it. Just execute the init.d script directly. + + if [ "$1" == "dovecot" ]; then + # Dovecot does not provide an init.d script. It just provides + # an upstart init configuration. But Docker doesn't provide + # upstart. Start Dovecot specially. + killall dovecot + dovecot -c /etc/dovecot/dovecot.conf + else + hide_output /etc/init.d/$1 restart + fi + fi } ## Dialog Functions ## diff --git a/setup/preflight.sh b/setup/preflight.sh index 90d36cc1..637eef2e 100644 --- a/setup/preflight.sh +++ b/setup/preflight.sh @@ -4,7 +4,7 @@ if [[ $EUID -ne 0 ]]; then echo echo "sudo $0" echo - exit + exit 1 fi # Check that we are running on Ubuntu 14.04 LTS (or 14.04.xx). @@ -14,7 +14,7 @@ if [ "`lsb_release -d | sed 's/.*:\s*//' | sed 's/14\.04\.[0-9]/14.04/' `" != "U lsb_release -d | sed 's/.*:\s*//' echo echo "We can't write scripts that run on every possible setup, sorry." - exit + exit 1 fi # Check that we have enough memory. @@ -30,6 +30,6 @@ if [ ! -d /vagrant ]; then echo "Your Mail-in-a-Box needs more memory (RAM) to function properly." echo "Please provision a machine with at least 768 MB, 1 GB recommended." echo "This machine has $TOTAL_PHYSICAL_MEM MB memory." - exit + exit 1 fi fi diff --git a/setup/start.sh b/setup/start.sh index 9817f49a..2c701f0c 100755 --- a/setup/start.sh +++ b/setup/start.sh @@ -59,7 +59,11 @@ if [ "$PUBLIC_IPV6" = "auto" ]; then # Use a public API to get our public IPv6 address, or fall back to local network configuration. PUBLIC_IPV6=$(get_publicip_from_web_service 6 || get_default_privateip 6) fi -if [ "$PRIMARY_HOSTNAME" = "auto-easy" ]; then +if [ "$PRIMARY_HOSTNAME" = "auto" ]; then + # Use reverse DNS to get this machine's hostname. Install bind9-host early. + hide_output apt-get -y install bind9-host + PRIMARY_HOSTNAME=$(get_default_hostname) +elif [ "$PRIMARY_HOSTNAME" = "auto-easy" ]; then # Generate a probably-unique subdomain under our justtesting.email domain. PRIMARY_HOSTNAME=`echo $PUBLIC_IP | sha1sum | cut -c1-5`.justtesting.email fi diff --git a/tools/list_all_packages.py b/tools/list_all_packages.py new file mode 100644 index 00000000..9b2eab70 --- /dev/null +++ b/tools/list_all_packages.py @@ -0,0 +1,23 @@ +#!/usr/bin/python3 + +import os.path, glob, re + +packages = set() + +def add(line): + global packages + if line.endswith("\\"): line = line[:-1] + packages |= set(p for p in line.split(" ") if p not in("", "apt_install")) + +for fn in glob.glob(os.path.join(os.path.dirname(__file__), "../setup/*.sh")): + with open(fn) as f: + in_apt_install = False + for line in f: + line = line.strip() + if line.startswith("apt_install "): + in_apt_install = True + if in_apt_install: + add(line) + in_apt_install = in_apt_install and line.endswith("\\") + +print("\n".join(sorted(packages)))