improve comments throughout the scripts

scripts/add_mail_user.sh

@@ -1,15 +1,13 @@
-EMAIL_ADDR=$1
+# Create a new email user.
-if [ -z "$EMAIL_ADDR" ]; then
+##########################
-	echo
+
-	echo "Set up your first email account..."
+echo
-        read -e -i "user@`hostname`" -p "Email Address: " EMAIL_ADDR
+echo "Set up your first email account..."
+read -e -i "user@`hostname`" -p "Email Address: " EMAIL_ADDR
+read -e -p "Email Password (blank to skip): " EMAIL_PW
+
+if [ ! -z "$EMAIL_PW" ]; then
+	echo "INSERT INTO users (email, password) VALUES ('$EMAIL_ADDR', '`doveadm pw -s SHA512-CRYPT -p $EMAIL_PW`');" \
+		| sqlite3 $STORAGE_ROOT/mail/users.sqlite
 fi
 
-EMAIL_PW=$2
-if [ -z "$EMAIL_PW" ]; then
-        read -e -p "Email Password: " EMAIL_PW
-fi
-
-echo "INSERT INTO users (email, password) VALUES ('$EMAIL_ADDR', '`doveadm pw -s SHA512-CRYPT -p $EMAIL_PW`');" \
-	| sqlite3 $STORAGE_ROOT/mail/users.sqlite
-

scripts/dkim.sh

@@ -1,18 +1,24 @@
-# Install OpenDKIM.
+# OpenDKIM: Sign outgoing mail with DKIM
-#
+########################################
-# After this, you'll still need to run dns_update to get the DKIM
+
+# After this, you'll still need to run dns_update.sh to get the DKIM
 # signature in the DNS zones.
 
+# Install DKIM
 apt-get install -q -y opendkim opendkim-tools
 
+# Make sure configuration directories exist.
 mkdir -p /etc/opendkim;
 mkdir -p $STORAGE_ROOT/mail/dkim
 
+# Used in InternalHosts and ExternalIgnoreList configuration directives.
+# Not quite sure why.
 echo "127.0.0.1" > /etc/opendkim/TrustedHosts
 
 if grep -q "ExternalIgnoreList" /etc/opendkim.conf; then
 	true; # already done
 else
+	# Add various configuration options to the end.
 	cat >> /etc/opendkim.conf << EOF;
 MinimumKeyBits          1024
 ExternalIgnoreList      refile:/etc/opendkim/TrustedHosts
@@ -24,22 +30,28 @@ RequireSafeKeys         false
 EOF
 fi
 
-# Create a new DKIM key if we don't have one already.
+# Create a new DKIM key if we don't have one already. This creates
+# mail.private and mail.txt in $STORAGE_ROOT/mail/dkim. The former
+# is the actual private key and the latter is the suggested DNS TXT
+# entry which we'll want to include in our DNS setup.
 if [ ! -z "$STORAGE_ROOT/mail/dkim/mail.private" ]; then
 	# Should we specify -h rsa-sha256?
 	opendkim-genkey -r -s mail -D $STORAGE_ROOT/mail/dkim
 fi
 
+# Ensure files are owned by the opendkim user and are private otherwise.
 chown -R opendkim:opendkim $STORAGE_ROOT/mail/dkim
 chmod go-rwx $STORAGE_ROOT/mail/dkim
 
-# add OpenDKIM as a milter to postfix. Be careful. If we add other milters
+# Add OpenDKIM as a milter to postfix, which is how it intercepts outgoing
-# later, it needs to be concatenated on the smtpd_milters line.
+# mail to perform the signing (by adding a mail header).
+# Be careful. If we add other milters later, it needs to be concatenated on the smtpd_milters line.
 tools/editconf.py /etc/postfix/main.cf \
 	smtpd_milters=inet:127.0.0.1:8891 \
 	non_smtpd_milters=\$smtpd_milters \
 	milter_default_action=accept
 
+# Restart services.
 service opendkim restart
 service postfix restart
 

scripts/dns.sh

@@ -1,11 +1,18 @@
-# Configures a DNS server using nsd.
+# DNS: Configure a DNS server using nsd
-#
+#######################################
+
 # After running this script, you also must run scripts/dns_update.sh,
-# and any time a zone file is added/changed/removed. It should be
+# and any time a zone file is added/changed/removed, and any time a
-# run after DKIM is configured, however.
+# new domain name becomes in use by a mail user.
+#
+# This script will turn on DNS for $PUBLIC_HOSTNAME.
+
+# Install nsd3, our DNS server software.
 
 apt-get -qq -y install nsd3
 
+# Get configuraton information.
+
 if [ -z "$PUBLIC_HOSTNAME" ]; then
 	PUBLIC_HOSTNAME=example.org
 fi
@@ -15,6 +22,8 @@ if [ -z "$PUBLIC_IP" ]; then
 	PUBLIC_IP=`wget -q -O- http://instance-data/latest/meta-data/public-ipv4`
 fi
 
+# Prepare nsd3's configuration.
+
 sudo mkdir -p /var/run/nsd3
 mkdir -p "$STORAGE_ROOT/dns";
 
@@ -34,7 +43,11 @@ if [ ! -f "$STORAGE_ROOT/dns/$PUBLIC_HOSTNAME.txt" ]; then
 EOF
 fi
 
-chown -R ubuntu.ubuntu $STORAGE_ROOT/dns
+# Let the storage user own all DNS configuration files.
+
+chown -R $STORAGE_USER.$STORAGE_USER $STORAGE_ROOT/dns
+
+# Permit DNS queries on TCP/UDP in the firewall.
 
 ufw allow domain
 

scripts/dns_update.sh

@@ -1,10 +1,19 @@
+# DNS: Creates DNS zone files
+#############################
+
 # Create nsd.conf and zone files, and updates the OpenDKIM signing tables.
 
+# We set the administrative email address for every domain to domain_contact@[domain.com].
+# You should probably create an alias to your email address.
+
+# This script is safe to run on its own.
+
+# Load $STORAGE_ROOT, $PUBLIC_IP, and $PRIMARY_HOSTNAME.
 source /etc/mailinabox.conf
 PUBLIC_IP=`cat $STORAGE_ROOT/dns/our_ip`
 PRIMARY_HOSTNAME=`cat $STORAGE_ROOT/dns/primary_hostname`
 
-# Ensure a zone file exists for every domain name of a mail user.
+# Ensure a zone file exists for every domain name in use by a mail user.
 for mail_user in `tools/mail.py user`; do
 	domain=`echo $mail_user | sed s/.*@//`
 	if [ ! -f $STORAGE_ROOT/dns/$domain.txt ]; then
@@ -37,12 +46,16 @@ truncate --size 0 /etc/opendkim/KeyTable
 truncate --size 0 /etc/opendkim/SigningTable
 
 for fn in $STORAGE_ROOT/dns/*.txt; do
+	# $fn is the zone configuration file, which is just a placeholder now.
+	#     For every file like mydomain.com.txt we'll create zone information
+	#     for that domain. We don't actually read the file.
+	# $fn2 is the file without the directory.
+	# $zone is the domain name (just mydomain.com).
 	fn2=`basename $fn`
 	zone=`echo $fn2 | sed "s/.txt\$//"`
 	
-	# If the zone file exists, increment the serial number.
+	# If the zone file exists, get the existing zone serial number so we can increment it.
-	# TODO: This needs to be done better so that the existing serial number is
+	# TODO: This needs to be done better so that the existing serial number is persisted in the storage area.
-	# persisted in the storage area.
 	serial=`date +"%Y%m%d00"`
 	if [ -f /etc/nsd3/zones/$fn2 ]; then
 		existing_serial=`grep "serial number" /etc/nsd3/zones/$fn2 | sed "s/; serial number//"`
@@ -51,6 +64,7 @@ for fn in $STORAGE_ROOT/dns/*.txt; do
 		fi
 	fi
 
+	# Create the zone file.
 	cat > /etc/nsd3/zones/$fn2 << EOF;
 \$ORIGIN $zone.    ; default zone domain
 \$TTL 86400           ; default time to live
@@ -76,22 +90,28 @@ mail       IN     A    $PUBLIC_IP
 www        IN     A    $PUBLIC_IP
 EOF
 
-	# If OpenDKIM is set up, append that information to the zone.
+	# If OpenDKIM is set up, append the suggested TXT record to the zone.
 	if [ -f "$STORAGE_ROOT/mail/dkim/mail.txt" ]; then
 		cat "$STORAGE_ROOT/mail/dkim/mail.txt" >> /etc/nsd3/zones/$fn2;
 	fi
 	
+	# Add this zone file to the main nsd configuration file.
 	cat >> /etc/nsd3/nsd.conf << EOF;
 zone:
 	name: $zone
 	zonefile: $fn2
 EOF
 
-	# OpenDKIM
+	# Append a record to OpenDKIM's KeyTable and SigningTable. The SigningTable maps
-	
+	# email addresses to signing information. The KeyTable maps specify the hostname,
-	# For every domain, we sign against the key listed in PRIMARY_HOSTNAME's DNS,
+	# the selector, and the path to the private key.
-	# in case the user is just delegating MX and hasn't set the DKIM info on the
+	#
-	# main DNS record.
+	# Just in case we don't actually host the DNS for all domains of our mail users,
+	# we assume that DKIM is at least configured in the DNS of $PRIMARY_HOSTNAME and
+	# we use that host for all DKIM signatures.
+	#
+	# In SigningTable, we map every email address to a key record called $zone.
+	# Then we specify for the key record named $zone its domain, selector, and key.
 	echo "$zone $PRIMARY_HOSTNAME:mail:$STORAGE_ROOT/mail/dkim/mail.private" >> /etc/opendkim/KeyTable
 	echo "*@$zone $zone" >> /etc/opendkim/SigningTable
 

scripts/mail.sh

@@ -1,7 +1,14 @@
-# Configures a postfix SMTP server and dovecot IMAP server.
+# SMTP/IMAP: Postfix and Dovecot
+################################
+
+# The SMTP server is listening on port 25 for incoming mail (mail for us) and on
+# port 587 for outgoing mail (i.e. mail you send). Port 587 uses STARTTLS (not SSL)
+# and you'll authenticate with your full email address and mail password.
 #
-# We configure these together because postfix delivers mail
+# The IMAP server is listening on port 993 and uses SSL. There is no IMAP server
-# directly to dovecot, so they basically rely on each other.
+# listening on port 143 because it is not encrypted on that port.
+
+# We configure these together because postfix's configuration relies heavily on dovecot.
 
 # Install packages.
 
@@ -9,22 +16,24 @@ DEBIAN_FRONTEND=noninteractive apt-get install -q -y \
 	postfix postgrey \
 	dovecot-core dovecot-imapd dovecot-lmtpd dovecot-sqlite sqlite3
 
-# POSTFIX
-
 mkdir -p $STORAGE_ROOT/mail
 
-# TLS configuration
+# POSTFIX
-sed -i "s/#submission/submission/" /etc/postfix/master.cf # enable submission port (not in Drew Crawford's instructions)
+#########
+
+# Enable the 'submission' port 587 listener.
+sed -i "s/#submission/submission/" /etc/postfix/master.cf
+
+# Enable TLS and require it for all user authentication.
 tools/editconf.py /etc/postfix/main.cf \
 	smtpd_use_tls=yes\
 	smtpd_tls_auth_only=yes \
 	smtp_tls_security_level=may \
 	smtp_tls_loglevel=2 \
 	smtpd_tls_received_header=yes
-	
 	# note: smtpd_use_tls=yes appears to already be the default, but we can never be too sure
 
-# authorization via dovecot
+# Postfix will query dovecot for user authentication.
 tools/editconf.py /etc/postfix/main.cf \
 	smtpd_sasl_type=dovecot \
 	smtpd_sasl_path=private/auth \
@@ -45,39 +54,46 @@ tools/editconf.py /etc/postfix/main.cf \
 tools/editconf.py /etc/postfix/main.cf \
 	smtpd_recipient_restrictions=permit_sasl_authenticated,permit_mynetworks,"reject_rbl_client zen.spamhaus.org","check_policy_service inet:127.0.0.1:10023"
 
+# Have postfix listen on all network interfaces, and set the name of the local machine
+# to localhost for xxx@localhost mail, but I don't think this will have any effect because
+# there is no true local mail delivery.
 tools/editconf.py /etc/postfix/main.cf \
 	inet_interfaces=all \
 	mydestination=localhost
 
-# message delivery is directly to dovecot
+# Handle all local mail delivery by passing it directly to dovecot over LMTP.
 tools/editconf.py /etc/postfix/main.cf virtual_transport=lmtp:unix:private/dovecot-lmtp
 
-# domain and user table is configured in a Sqlite3 database
+# Use a Sqlite3 database to check whether a destination email address exists,
+# and to perform any email alias rewrites.
 tools/editconf.py /etc/postfix/main.cf \
 	virtual_mailbox_domains=sqlite:/etc/postfix/virtual-mailbox-domains.cf \
 	virtual_mailbox_maps=sqlite:/etc/postfix/virtual-mailbox-maps.cf \
 	virtual_alias_maps=sqlite:/etc/postfix/virtual-alias-maps.cf \
 	local_recipient_maps=\$virtual_mailbox_maps
 
+# Here's the path to the database.
 db_path=$STORAGE_ROOT/mail/users.sqlite
 
+# SQL statement to check if we handle mail for a domain.
 cat > /etc/postfix/virtual-mailbox-domains.cf << EOF;
 dbpath=$db_path
 query = SELECT 1 FROM users WHERE email LIKE '%%@%s'
 EOF
 
+# SQL statement to check if we handle mail for a user.
 cat > /etc/postfix/virtual-mailbox-maps.cf << EOF;
 dbpath=$db_path
 query = SELECT 1 FROM users WHERE email='%s'
 EOF
 
+# SQL statement to rewrite an email address if an alias is present.
 cat > /etc/postfix/virtual-alias-maps.cf << EOF;
 dbpath=$db_path
 query = SELECT destination FROM aliases WHERE source='%s'
 EOF
 
-# create an empty mail users database if it doesn't yet exist
+# Create an empty database if it doesn't yet exist.
-
 if [ ! -f $db_path ]; then
 	echo Creating new user database: $db_path;
 	echo "CREATE TABLE users (id INTEGER PRIMARY KEY AUTOINCREMENT, email TEXT NOT NULL UNIQUE, password TEXT NOT NULL, extra);" | sqlite3 $db_path;
@@ -85,25 +101,26 @@ if [ ! -f $db_path ]; then
 fi
 
 # DOVECOT
+#########
 
-# The dovecot-imapd dovecot-lmtpd packages automatically enable those protocols.
+# The dovecot-imapd dovecot-lmtpd packages automatically enable IMAP and LMTP protocols.
 
-# mail storage location
+# Set the location where we'll store user mailboxes.
 tools/editconf.py /etc/dovecot/conf.d/10-mail.conf \
 	mail_location=maildir:$STORAGE_ROOT/mail/mailboxes/%d/%n \
 	mail_privileged_group=mail \
 	first_valid_uid=0
 
-# authentication mechanisms
+# Require that passwords are sent over SSL only, and allow the usual IMAP authentication mechanisms.
 tools/editconf.py /etc/dovecot/conf.d/10-auth.conf \
 	disable_plaintext_auth=yes \
 	"auth_mechanisms=plain login"
 
-# use SQL-based authentication, not the system users
+# Query out Sqlite3 database, and not system users, for authentication.
 sed -i "s/\(\!include auth-system.conf.ext\)/#\1/"  /etc/dovecot/conf.d/10-auth.conf
 sed -i "s/#\(\!include auth-sql.conf.ext\)/\1/"  /etc/dovecot/conf.d/10-auth.conf
 
-# how to access SQL
+# Configure how to access our Sqlite3 database. Not sure what userdb is for.
 cat > /etc/dovecot/conf.d/auth-sql.conf.ext << EOF;
 passdb {
   driver = sql
@@ -114,6 +131,8 @@ userdb {
   args = uid=mail gid=mail home=$STORAGE_ROOT/mail/mailboxes/%d/%n
 }
 EOF
+
+# Configure the SQL to query for a user's password.
 cat > /etc/dovecot/dovecot-sql.conf.ext << EOF;
 driver = sqlite
 connect = $db_path
@@ -121,15 +140,19 @@ default_pass_scheme = SHA512-CRYPT
 password_query = SELECT email as user, password FROM users WHERE email='%u';
 EOF
 
-# disable in-the-clear IMAP and POP because we're paranoid (we haven't even
+# Disable in-the-clear IMAP and POP because we're paranoid (we haven't even
 # enabled POP).
 sed -i "s/#port = 143/port = 0/" /etc/dovecot/conf.d/10-master.conf
 sed -i "s/#port = 110/port = 0/" /etc/dovecot/conf.d/10-master.conf
 
-# Create a Unix domain socket specific for postgres for auth and LMTP because
+# Have dovecot provide authorization and LMTP (local mail delivery) services.
-# postgres is more easily configured to use these locations, and create a TCP socket
+#
-# for spampd to inject mail on (if it's configured later). dovecot's standard
+# We have dovecot listen on a Unix domain socket for these services
-# lmtp unix socket is also listening.
+# in a manner that made postfix configuration above easy.
+#
+# We also have dovecot listen on port 10026 (localhost only) for LMTP
+# in case we have other services that want to deliver local mail, namly
+# spampd.
 cat > /etc/dovecot/conf.d/99-local.conf << EOF;
 service auth {
   unix_listener /var/spool/postfix/private/auth {
@@ -152,7 +175,7 @@ EOF
 
 # Drew Crawford sets the auth-worker process to run as the mail user, but we don't care if it runs as root.
 
-# Enable SSL.
+# Enable SSL and specify the location of the SSL certificate and private key files.
 tools/editconf.py /etc/dovecot/conf.d/10-ssl.conf \
 	ssl=required \
 	"ssl_cert=<$STORAGE_ROOT/ssl/ssl_certificate.pem" \
@@ -160,24 +183,26 @@ tools/editconf.py /etc/dovecot/conf.d/10-ssl.conf \
 	
 # The Dovecot installation already created a self-signed public/private key pair
 # in /etc/dovecot/dovecot.pem and /etc/dovecot/private/dovecot.pem, which we'll
-# use unless certificates already exist.
+# use unless certificates already exist. We'll move them into $STORAGE_ROOT/ssl
+# unless files exist there already.
 mkdir -p $STORAGE_ROOT/ssl
 if [ ! -f $STORAGE_ROOT/ssl/ssl_certificate.pem ]; then cp /etc/dovecot/dovecot.pem $STORAGE_ROOT/ssl/ssl_certificate.pem; fi
 if [ ! -f $STORAGE_ROOT/ssl/ssl_private_key.pem ]; then cp /etc/dovecot/private/dovecot.pem $STORAGE_ROOT/ssl/ssl_private_key.pem; fi
 
+# Ensure configuration files are owned by dovecot and not world readable.
 chown -R mail:dovecot /etc/dovecot
 chmod -R o-rwx /etc/dovecot
 
+# Ensure mailbox files have a directory that exists and are owned by the mail user.
 mkdir -p $STORAGE_ROOT/mail/mailboxes
 chown -R mail.mail $STORAGE_ROOT/mail/mailboxes
 
-# restart services
+# Restart services.
 service postfix restart
 service dovecot restart
 
-# allow mail-related ports in the firewall
+# Allow mail-related ports in the firewall.
 ufw allow smtp
 ufw allow submission
 ufw allow imaps
 
-

scripts/new_volume.sh

@@ -1 +0,0 @@
-

scripts/spamassassin.sh

@@ -1,23 +1,35 @@
-# Spam filtering with spamassassin via spampd.
+# Spam filtering with spamassassin via spampd
+#############################################
 
+# spampd sits between postfix and dovecot. It takes mail from postfix
+# over the LMTP protocol, runs spamassassin on it, and then passes the
+# message over LMTP to dovecot for local delivery.
+
+# In order to move spam automatically into the Spam folder we use the dovecot sieve
+# plugin. Unfortunately, each mail box needs its own sieve script set to do the
+# filtering work. So users_update.sh must be run any time a new mail user is created.
+
+# Install packages.
 apt-get -q -y install spampd dovecot-sieve dovecot-antispam
 
 # Hook into postfix. Replace dovecot with spampd as the mail delivery agent.
 tools/editconf.py /etc/postfix/main.cf virtual_transport=lmtp:[127.0.0.1]:10025
 
-# Hook into dovecot. This is actually the default but we don't want to lose track of it.
+# Pass messages on to docevot on port 10026.
+# This is actually the default setting but we don't want to lose track of it.
 tools/editconf.py /etc/default/spampd DESTPORT=10026
 
-# Automatically move spam into a folder called Spam. Enable the sieve plugin.
+# Enable the sieve plugin which let's us set a script that automatically moves
+# spam into the user's Spam mail filter.
 # (Note: Be careful if we want to use multiple plugins later.)
-# The sieve scripts are installed by users_update.sh.
 sudo sed -i "s/#mail_plugins = .*/mail_plugins = \$mail_plugins sieve/" /etc/dovecot/conf.d/20-lmtp.conf
 
 # Enable the antispam plugin to detect when a message moves between folders so we can
 # pass it to sa-learn for training. (Be careful if we use multiple plugins later.)
 sudo sed -i "s/#mail_plugins = .*/mail_plugins = \$mail_plugins antispam/" /etc/dovecot/conf.d/20-imap.conf
 
-# When mail is moved in or out of the dovecot Spam folder, re-train.
+# When mail is moved in or out of the dovecot Spam folder, re-train using this script
+# that sends the mail to spamassassin.
 # from http://wiki2.dovecot.org/Plugins/Antispam
 cat > /usr/bin/sa-learn-pipe.sh << EOF;
 cat<&0 >> /tmp/sendmail-msg-\$\$.txt
@@ -25,9 +37,9 @@ cat<&0 >> /tmp/sendmail-msg-\$\$.txt
 rm -f /tmp/sendmail-msg-\$\$.txt
 exit 0
 EOF
-
 chmod a+x /usr/bin/sa-learn-pipe.sh
 
+# Configure the antispam plugin to call sa-learn-pipe.sh.
 cat > /etc/dovecot/conf.d/99-local-spampd.conf << EOF;
 plugin {
     antispam_backend = pipe
@@ -43,6 +55,7 @@ EOF
 # sa-learn --ham storage/mail/mailboxes/*/*/cur/
 # sa-learn --spam storage/mail/mailboxes/*/*/.Spam/cur/
 
+# Kick services.
 sudo service spampd restart
 sudo service dovecot restart
 

scripts/start.sh

@@ -1,4 +1,9 @@
+# This is the entry point for configuring the system.
+#####################################################
+
 # Check system setup.
+
+# Check that SSH login with password is disabled. Stop if it's enabled.
 if grep -q "^PasswordAuthentication yes" /etc/ssh/sshd_config \
  || ! grep -q "^PasswordAuthentication no" /etc/ssh/sshd_config ; then
         echo
@@ -10,7 +15,8 @@ if grep -q "^PasswordAuthentication yes" /etc/ssh/sshd_config \
         exit
 fi
 
-# Gather information from the user.
+# Gather information from the user about the hostname and public IP
+# address of this host.
 if [ -z "$PUBLIC_HOSTNAME" ]; then
 	echo
 	echo "Enter the hostname you want to assign to this machine."
@@ -30,17 +36,23 @@ if [ -z "$PUBLIC_IP" ]; then
 	read -e -i "`hostname -i`" -p "Public IP: " PUBLIC_IP
 fi
 
+# Create the user named "userconfig-data" and store all persistent user
+# data (mailboxes, etc.) in that user's home directory.
 if [ -z "$STORAGE_ROOT" ]; then
-	if [ ! -d /home/user-data ]; then useradd -m user-data; fi
+	STORAGE_USER=user-data
-	STORAGE_ROOT=/home/user-data
+	if [ ! -d /home/$STORAGE_USER ]; then useradd -m $STORAGE_USER; fi
+	STORAGE_ROOT=/home/$STORAGE_USER
 	mkdir -p $STORAGE_ROOT
 fi
 
+# Save the global options in /etc/mailinabox.conf so that standalone
+# tools know where to look for data.
 cat > /etc/mailinabox.conf << EOF;
 STORAGE_ROOT=$STORAGE_ROOT
 PUBLIC_HOSTNAME=$PUBLIC_HOSTNAME
 EOF
 
+# Start service configuration.
 . scripts/system.sh
 . scripts/dns.sh
 . scripts/mail.sh
@@ -49,5 +61,4 @@ EOF
 . scripts/dns_update.sh
 . scripts/add_mail_user.sh
 . scripts/users_update.sh
-. scripts/web.sh