more work on making the bash scripts readable

setup/dkim.sh

@@ -1,12 +1,13 @@
-# OpenDKIM: Sign outgoing mail with DKIM
+# OpenDKIM
-########################################
+# ========
-
+#
-# After this, you'll still need to run dns_update.sh to get the DKIM
+# OpenDKIM provides a service that puts a DKIM signature on outbound mail.
-# signature in the DNS zones.
+#
+# The DNS configuration for DKIM is done in the management daemon.
 
 source setup/functions.sh # load our functions
 
-# Install DKIM
+# Install DKIM...
 apt_install opendkim opendkim-tools
 
 # Make sure configuration directories exist.
@@ -18,9 +19,9 @@ mkdir -p $STORAGE_ROOT/mail/dkim
 echo "127.0.0.1" > /etc/opendkim/TrustedHosts
 
 if grep -q "ExternalIgnoreList" /etc/opendkim.conf; then
-	true; # already done
+	true # already done #NODOC
 else
-	# Add various configuration options to the end.
+	# Add various configuration options to the end of `opendkim.conf`.
 	cat >> /etc/opendkim.conf << EOF;
 MinimumKeyBits          1024
 ExternalIgnoreList      refile:/etc/opendkim/TrustedHosts
@@ -32,7 +33,7 @@ RequireSafeKeys         false
 EOF
 fi
 
-# Create a new DKIM key if we don't have one already. This creates
+# Create a new DKIM key. This creates
 # mail.private and mail.txt in $STORAGE_ROOT/mail/dkim. The former
 # is the actual private key and the latter is the suggested DNS TXT
 # entry which we'll want to include in our DNS setup.
@@ -47,7 +48,7 @@ chmod go-rwx $STORAGE_ROOT/mail/dkim
 
 # Add OpenDKIM as a milter to postfix, which is how it intercepts outgoing
 # mail to perform the signing (by adding a mail header).
-# Be careful. If we add other milters later, it needs to be concatenated on the smtpd_milters line.
+# Be careful. If we add other milters later, it needs to be concatenated on the smtpd_milters line. #NODOC
 tools/editconf.py /etc/postfix/main.cf \
 	smtpd_milters=inet:127.0.0.1:8891 \
 	non_smtpd_milters=\$smtpd_milters \

setup/dns.sh

@@ -1,5 +1,5 @@
 #!/bin/bash
-# DNS: Configure a DNS server to host our own DNS
+# DNS
 # -----------------------------------------------
 
 # This script installs packages, but the DNS zone files are only
@@ -14,9 +14,9 @@ source setup/functions.sh # load our functions
 
 # ...but first, we have to create the user because the 
 # current Ubuntu forgets to do so in the .deb
-# see issue #25 and https://bugs.launchpad.net/ubuntu/+source/nsd/+bug/1311886
+# (see issue #25 and https://bugs.launchpad.net/ubuntu/+source/nsd/+bug/1311886)
 if id nsd > /dev/null 2>&1; then
-	true; #echo "nsd user exists... good"; #NODOC
+	true #echo "nsd user exists... good"; #NODOC
 else
 	useradd nsd;
 fi
@@ -40,17 +40,21 @@ mkdir -p "$STORAGE_ROOT/dns/dnssec";
 # TLDs don't all support the same algorithms, so we'll generate keys using a few
 # different algorithms.
 #
-# Supports RSASHA1-NSEC3-SHA1 (didn't test with RSASHA256):
+# Supports `RSASHA1-NSEC3-SHA1` (didn't test with `RSASHA256`):
-#   .info and .me.
 #
-# Requires RSASHA256
+#  * .info
-#   .email
+#  * .me
-FIRST=1
+#
+# Requires `RSASHA256`
+#
+#  * .email
+
+FIRST=1 #NODOC
 for algo in RSASHA1-NSEC3-SHA1 RSASHA256; do
 if [ ! -f "$STORAGE_ROOT/dns/dnssec/$algo.conf" ]; then
 	if [ $FIRST == 1 ]; then
 		echo "Generating DNSSEC signing keys. This may take a few minutes..."
-		FIRST=0
+		FIRST=0 #NODOC
 	fi
 
 	# Create the Key-Signing Key (KSK) (-k) which is the so-called
@@ -58,6 +62,9 @@ if [ ! -f "$STORAGE_ROOT/dns/dnssec/$algo.conf" ]; then
 	# practice), and a nice and long keylength. The domain name we
 	# provide ("_domain_") doesn't matter -- we'll use the same
 	# keys for all our domains.
+	#
+	# `ldns-keygen` outputs the new key's filename to stdout, which
+	# we're capturing into the `KSK` variable.
 	KSK=$(umask 077; cd $STORAGE_ROOT/dns/dnssec; ldns-keygen -a $algo -b 2048 -k _domain_);
 
 	# Now create a Zone-Signing Key (ZSK) which is expected to be
@@ -81,9 +88,13 @@ KSK=$KSK
 ZSK=$ZSK
 EOF
 fi
+
+	# And loop to do the next algorithm...
 done
 
-# Force the dns_update script to be run every day to re-sign zones for DNSSEC.
+# Force the dns_update script to be run every day to re-sign zones for DNSSEC
+# before they expire. When we sign zones (in `dns_update.py`) we specify a
+# 30-day validation window, so we had better re-sign before then.
 cat > /etc/cron.daily/mailinabox-dnssec << EOF;
 #!/bin/bash
 # Mail-in-a-Box

setup/mail-dovecot.sh

@@ -18,15 +18,17 @@
 source setup/functions.sh # load our functions
 source /etc/mailinabox.conf # load global vars
 
-# ### Install packages and basic setup
+# Install packages...
 
 apt_install \
 	dovecot-core dovecot-imapd dovecot-lmtpd dovecot-sqlite sqlite3 \
 	dovecot-sieve dovecot-managesieved
 
-# The dovecot-imapd and dovecot-lmtpd packages automatically enable IMAP and LMTP protocols.
+# The `dovecot-imapd` and `dovecot-lmtpd` packages automatically enable IMAP and LMTP protocols.
 
-# Set the location where we'll store user mailboxes.
+# Set the location where we'll store user mailboxes. '%d' is the domain name and '%n' is the
+# username part of the user's email address. We'll ensure that no bad domains or email addresses
+# are created within the management daemon.
 tools/editconf.py /etc/dovecot/conf.d/10-mail.conf \
 	mail_location=maildir:$STORAGE_ROOT/mail/mailboxes/%d/%n \
 	mail_privileged_group=mail \
@@ -66,7 +68,7 @@ tools/editconf.py /etc/dovecot/conf.d/20-imap.conf \
 # ### LDA (LMTP)
 
 # Enable Dovecot's LDA service with the LMTP protocol. It will listen
-# in port 10026, and Spamassassin will be configured to pass mail there.
+# on port 10026, and Spamassassin will be configured to pass mail there.
 #
 # The disabled unix socket listener is normally how Postfix and Dovecot
 # would communicate (see the Postfix setup script for the corresponding
@@ -91,30 +93,32 @@ protocol imap {
 }
 EOF
 
-# Setting a postmaster_address seems to be required or LMTP won't start.
+# Setting a `postmaster_address` is required or LMTP won't start. An alias
+# will be created automatically by our management daemon.
 tools/editconf.py /etc/dovecot/conf.d/15-lda.conf \
 	postmaster_address=postmaster@$PRIMARY_HOSTNAME
 
 # ### Sieve
 
 # Enable the Dovecot sieve plugin which let's users run scripts that process
-# mail as it comes in. We'll also set a global script that moves mail marked
+# mail as it comes in.
-# as spam by Spamassassin into the user's Spam folder.
 sed -i "s/#mail_plugins = .*/mail_plugins = \$mail_plugins sieve/" /etc/dovecot/conf.d/20-lmtp.conf
 
+# Configure sieve. We'll create a global script that moves mail marked
+# as spam by Spamassassin into the user's Spam folder.
+#
+# * `sieve_before`: The path to our global sieve which handles moving spam to the Spam folder.
+#
+# * `sieve`: The path to the user's main active script. ManageSieve will create a symbolic
+# link here to the actual sieve script. It should not be in the mailbox directory
+# (because then it might appear as a folder) and it should not be in the sieve_dir
+# (because then I suppose it might appear to the user as one of their scripts).
+# * `sieve_dir`: Directory for :personal include scripts for the include extension. This
+# is also where the ManageSieve service stores the user's scripts.
 cat > /etc/dovecot/conf.d/99-local-sieve.conf << EOF;
 plugin {
-  # The path to our global sieve which handles moving spam to the Spam folder.
   sieve_before = /etc/dovecot/sieve-spam.sieve
-
-  # The path to the user's main active script. ManageSieve will create a symbolic
-  # link here to the actual sieve script. It should not be in the mailbox directory
-  # (because then it might appear as a folder) and it should not be in the sieve_dir
-  # (because then I suppose it might appear to the user as one of their scripts).
   sieve = $STORAGE_ROOT/mail/sieve/%d/%n.sieve
-
-  # Directory for :personal include scripts for the include extension. This
-  # is also where the ManageSieve service stores the user's scripts.
   sieve_dir = $STORAGE_ROOT/mail/sieve/%d/%n
 }
 EOF
@@ -122,7 +126,7 @@ EOF
 # Copy the global sieve script into where we've told Dovecot to look for it. Then
 # compile it. Global scripts must be compiled now because Dovecot won't have
 # permission later.
-cp `pwd`/conf/sieve-spam.txt /etc/dovecot/sieve-spam.sieve
+cp conf/sieve-spam.txt /etc/dovecot/sieve-spam.sieve
 sievec /etc/dovecot/sieve-spam.sieve
 
 # PERMISSIONS

setup/mail-postfix.sh

@@ -32,13 +32,26 @@ source /etc/mailinabox.conf # load global vars
 
 # ### Install packages.
 
-apt_install postfix postgrey postfix-pcre ca-certificates
+# Install postfix's packages.
+#
+# * `postfix`: The SMTP server.
+# * `postfix-pcre`: Enables header filtering.
+# * `postgrey`: A mail policy service that soft-rejects mail the first time
+#   it is received. Spammers don't usually try agian. Legitimate mail
+#   always will.
+# * `ca-certificates`: A trust store used to squelch postfix warnings about
+#   untrusted opportunistically-encrypted connections.
+
+apt_install postfix postfix-pcre postgrey ca-certificates
 
 # ### Basic Settings
 
-# Have postfix listen on all network interfaces, set our name (the Debian default seems to be localhost),
+# Set some basic settings...
-# and set the name of the local machine to localhost for xxx@localhost mail (but I don't think this will have any effect because
+#
-# there is no true local mail delivery). Also set the banner (must have the hostname first, then anything).
+# * Have postfix listen on all network interfaces.
+# * Set our name (the Debian default seems to be "localhost" but make it our hostname).
+# * Set the name of the local machine to localhost, which means xxx@localhost is delivered locally, although we don't use it.
+# * Set the SMTP banner (which must have the hostname first, then anything).
 tools/editconf.py /etc/postfix/main.cf \
 	inet_interfaces=all \
 	myhostname=$PRIMARY_HOSTNAME\
@@ -69,7 +82,8 @@ cp conf/postfix_outgoing_mail_header_filters /etc/postfix/outgoing_mail_header_f
 # Enable TLS on these and all other connections (i.e. ports 25 *and* 587) and
 # require TLS before a user is allowed to authenticate. This also makes
 # opportunistic TLS available on *incoming* mail.
-# Set stronger DH parameters, which via openssl tend to default to 1024 bits.
+# Set stronger DH parameters, which via openssl tend to default to 1024 bits
+# (see ssl.sh).
 tools/editconf.py /etc/postfix/main.cf \
 	smtpd_tls_security_level=may\
 	smtpd_tls_auth_only=yes \
@@ -90,25 +104,25 @@ tools/editconf.py /etc/postfix/main.cf \
 
 
 # ### DANE
-#
+
 # When connecting to remote SMTP servers, prefer TLS and use DANE if available.
 #
-# Prefering ("opportunistic") TLS means Postfix will accept whatever SSL certificate the remote
+# Prefering ("opportunistic") TLS means Postfix will use TLS if the remote end
-# end provides, if the remote end offers STARTTLS during the connection. DANE takes this a
+# offers it, otherwise it will transmit the message in the clear. Postfix will
-# step further:
+# accept whatever SSL certificate the remote end provides. Opportunistic TLS
+# protects against passive easvesdropping (but not man-in-the-middle attacks).
+# DANE takes this a step further:
 #
 # Postfix queries DNS for the TLSA record on the destination MX host. If no TLSA records are found,
 # then opportunistic TLS is used. Otherwise the server certificate must match the TLSA records
 # or else the mail bounces. TLSA also requires DNSSEC on the MX host. Postfix doesn't do DNSSEC
 # itself but assumes the system's nameserver does and reports DNSSEC status. Thus this also
-# relies on our local bind9 server being present and smtp_dns_support_level being set to dnssec
+# relies on our local bind9 server being present and `smtp_dns_support_level=dnssec`.
-# to use it.
 #
-# The smtp_tls_CAfile is superflous, but it turns warnings in the logs about untrusted certs
+# The `smtp_tls_CAfile` is superflous, but it eliminates warnings in the logs about untrusted certs,
-# into notices about trusted certs. Since in these cases Postfix is doing opportunistic TLS,
+# which we don't care about seeing because Postfix is doing opportunistic TLS anyway. Better to encrypt,
-# it does not care about whether the remote certificate is trusted. But, looking at the logs,
+# even if we don't know if it's to the right party, than to not encrypt at all. Instead we'll
-# it's nice to be able to see that the connection was in fact encrypted for the right party.
+# now see notices about trusted certs. The CA file is provided by the package `ca-certificates`.
-# The CA file is provided by the package ca-certificates.
 tools/editconf.py /etc/postfix/main.cf \
 	smtp_tls_security_level=dane \
 	smtp_dns_support_level=dnssec \

setup/mail-users.sh

@@ -53,7 +53,6 @@ EOF
 chmod 0600 /etc/dovecot/dovecot-sql.conf.ext # per Dovecot instructions
 
 # Have Dovecot provide an authorization service that Postfix can access & use.
-# Drew Crawford sets the auth-worker process to run as the mail user, but we don't care if it runs as root.
 cat > /etc/dovecot/conf.d/99-local-auth.conf << EOF;
 service auth {
   unix_listener /var/spool/postfix/private/auth {

setup/spamassassin.sh

@@ -1,14 +1,13 @@
 #!/bin/bash
 # Spam filtering with spamassassin via spampd
-#############################################
+# ===========================================
-
+#
 # spampd sits between postfix and dovecot. It takes mail from postfix
 # over the LMTP protocol, runs spamassassin on it, and then passes the
 # message over LMTP to dovecot for local delivery.
-
+#
 # In order to move spam automatically into the Spam folder we use the dovecot sieve
-# plugin. The tools/mail.py tool creates the necessary sieve script for each mail
+# plugin.
-# user when the mail user is created.
 
 source /etc/mailinabox.conf # get global vars
 source setup/functions.sh # load our functions
@@ -29,13 +28,14 @@ hide_output pyzor discover
 tools/editconf.py /etc/default/spampd DESTPORT=10026
 
 # Enable the Dovecot antispam plugin to detect when a message moves between folders so we can
-# pass it to sa-learn for training. (Be careful if we use multiple plugins later.)
+# pass it to sa-learn for training.
+# (Be careful if we use multiple plugins later.) #NODOC
 sed -i "s/#mail_plugins = .*/mail_plugins = \$mail_plugins antispam/" /etc/dovecot/conf.d/20-imap.conf
 
 # When mail is moved in or out of the Dovecot Spam folder, re-train using this script
 # that sends the mail to spamassassin.
 # from http://wiki2.dovecot.org/Plugins/Antispam
-rm -f /usr/bin/sa-learn-pipe.sh # legacy location
+rm -f /usr/bin/sa-learn-pipe.sh # legacy location #NODOC
 cat > /usr/local/bin/sa-learn-pipe.sh << EOF;
 cat<&0 >> /tmp/sendmail-msg-\$\$.txt
 /usr/bin/sa-learn \$* /tmp/sendmail-msg-\$\$.txt > /dev/null

setup/ssl.sh

@@ -2,7 +2,7 @@
 #
 # SSL Certificate
 # ---------------
-#
+
 # Create a self-signed SSL certificate if one has not yet been created.
 #
 # The certificate is for PRIMARY_HOSTNAME specifically and is used for:
@@ -22,29 +22,31 @@ source /etc/mailinabox.conf # load global vars
 apt_install openssl
 
 mkdir -p $STORAGE_ROOT/ssl
-# Generate a new private key if one doesn't already exist.
+# Generate a new private key.
 # Set the umask so the key file is not world-readable.
 if [ ! -f $STORAGE_ROOT/ssl/ssl_private_key.pem ]; then
 	(umask 077; hide_output \
 		openssl genrsa -out $STORAGE_ROOT/ssl/ssl_private_key.pem 2048)
 fi
 
-# Generate a certificate signing request if one doesn't already exist.
+# Generate a certificate signing request.
 if [ ! -f $STORAGE_ROOT/ssl/ssl_cert_sign_req.csr ]; then
 	hide_output \
 	openssl req -new -key $STORAGE_ROOT/ssl/ssl_private_key.pem -out $STORAGE_ROOT/ssl/ssl_cert_sign_req.csr \
 	  -sha256 -subj "/C=$CSR_COUNTRY/ST=/L=/O=/CN=$PRIMARY_HOSTNAME"
 fi
 
-# Generate a SSL certificate by self-signing if a SSL certificate doesn't yet exist.
+# Generate a SSL certificate by self-signing.
 if [ ! -f $STORAGE_ROOT/ssl/ssl_certificate.pem ]; then
 	hide_output \
 	openssl x509 -req -days 365 \
 	  -in $STORAGE_ROOT/ssl/ssl_cert_sign_req.csr -signkey $STORAGE_ROOT/ssl/ssl_private_key.pem -out $STORAGE_ROOT/ssl/ssl_certificate.pem
 fi
 
-# For nginx and postfix, pre-generate some better DH bits. They seem to
+# For nginx and postfix, pre-generate some Diffie-Hellman cipher bits which is
-# each rely on openssl's default of 1024 bits.
+# used when a Diffie-Hellman cipher is selected during TLS negotiation. Diffie-Hellman
+# provides Perfect Forward Security. openssl's default is 1024 bits, but we'll
+# create 2048.
 if [ ! -f $STORAGE_ROOT/ssl/dh2048.pem ]; then
 	openssl dhparam -out $STORAGE_ROOT/ssl/dh2048.pem 2048
 fi

setup/system.sh

@@ -1,11 +1,11 @@
 source setup/functions.sh # load our functions
 
-# Base system configuration
+# Basic System Configuration
 # -------------------------
 
-# ### Base packages
+# ### Install Packages
 
-# Update system packages:
+# Update system packages to make sure we have the latest upstream versions of things from Ubuntu.
 
 echo Updating system packages...
 hide_output apt-get update
@@ -35,8 +35,6 @@ EOF
 
 # ### Firewall
 
-# Turn on the firewall.
-#
 # Various virtualized environments like Docker and some VPSs don't provide #NODOC
 # a kernel that supports iptables. To avoid error-like output in these cases, #NODOC
 # we skip this if the user sets DISABLE_FIREWALL=1. #NODOC
@@ -47,15 +45,15 @@ if [ -z "$DISABLE_FIREWALL" ]; then
 	# Allow incoming connections to SSH.
 	ufw_allow ssh;
 
-	# ssh might be running on an alternate port. Use sshd -T to dump sshd's
+	# ssh might be running on an alternate port. Use sshd -T to dump sshd's #NODOC
-	# settings, find the port it is supposedly running on, and open that port
+	# settings, find the port it is supposedly running on, and open that port #NODOC
-	# too.
+	# too. #NODOC
-	SSH_PORT=$(sshd -T 2>/dev/null | grep "^port " | sed "s/port //")
+	SSH_PORT=$(sshd -T 2>/dev/null | grep "^port " | sed "s/port //") #NODOC
 	if [ ! -z "$SSH_PORT" ]; then
 	if [ "$SSH_PORT" != "22" ]; then
 
-	echo Opening alternate SSH port $SSH_PORT.
+	echo Opening alternate SSH port $SSH_PORT. #NODOC
-	ufw_allow $SSH_PORT;
+	ufw_allow $SSH_PORT #NODOC
 
 	fi
 	fi

setup/web.sh

@@ -5,6 +5,10 @@
 source setup/functions.sh # load our functions
 source /etc/mailinabox.conf # load global vars
 
+# Install nginx and a PHP FastCGI daemon.
+#
+# Turn off nginx's default website.
+
 apt_install nginx php5-fpm
 
 rm -f /etc/nginx/sites-enabled/default
@@ -20,7 +24,7 @@ sed "s#STORAGE_ROOT#$STORAGE_ROOT#" \
 tools/editconf.py /etc/nginx/nginx.conf -s \
 	server_names_hash_bucket_size="64;"
 
-# Bump up max_children to support more concurrent connections
+# Bump up PHP's max_children to support more concurrent connections
 tools/editconf.py /etc/php5/fpm/pool.d/www.conf -c ';' \
 	pm.max_children=8
 
@@ -29,20 +33,20 @@ tools/editconf.py /etc/php5/fpm/pool.d/www.conf -c ';' \
 # until mail accounts have been created.
 
 # make a default homepage
-if [ -d $STORAGE_ROOT/www/static ]; then mv $STORAGE_ROOT/www/static $STORAGE_ROOT/www/default; fi # migration
+if [ -d $STORAGE_ROOT/www/static ]; then mv $STORAGE_ROOT/www/static $STORAGE_ROOT/www/default; fi # migration #NODOC
 mkdir -p $STORAGE_ROOT/www/default
 if [ ! -f $STORAGE_ROOT/www/default/index.html ]; then
 	cp conf/www_default.html $STORAGE_ROOT/www/default/index.html
 fi
 chown -R $STORAGE_USER $STORAGE_ROOT/www
 
-# We previously installed a custom init script to start the PHP FastCGI daemon.
+# We previously installed a custom init script to start the PHP FastCGI daemon. #NODOC
-# Remove it now that we're using php5-fpm.
+# Remove it now that we're using php5-fpm. #NODOC
 if [ -L /etc/init.d/php-fastcgi ]; then
-	echo "Removing /etc/init.d/php-fastcgi, php5-cgi..."
+	echo "Removing /etc/init.d/php-fastcgi, php5-cgi..." #NODOC
-	rm -f /etc/init.d/php-fastcgi
+	rm -f /etc/init.d/php-fastcgi #NODOC
-	hide_output update-rc.d php-fastcgi remove
+	hide_output update-rc.d php-fastcgi remove #NODOC
-	apt-get -y purge php5-cgi
+	apt-get -y purge php5-cgi #NODOC
 fi
 
 # Put our webfinger script into a well-known location.
@@ -51,11 +55,11 @@ for f in webfinger; do
 	chown www-data.www-data /usr/local/bin/mailinabox-$f.php
 done
 
-# Remove obsoleted scripts.
+# Remove obsoleted scripts. #NODOC
-# exchange-autodiscover is now handled by Z-Push.
+# exchange-autodiscover is now handled by Z-Push. #NODOC
-for f in exchange-autodiscover; do
+for f in exchange-autodiscover; do #NODOC
-	rm -f /usr/local/bin/mailinabox-$f.php
+	rm -f /usr/local/bin/mailinabox-$f.php #NODOC
-done
+done #NODOC
 
 # Make some space for users to customize their webfinger responses.
 mkdir -p $STORAGE_ROOT/webfinger/acct;

setup/webmail.sh

@@ -23,16 +23,16 @@ apt_install \
 	php5 php5-sqlite php5-mcrypt php5-intl php5-json php5-common php-auth php-net-smtp php-net-socket php-net-sieve php-mail-mime php-crypt-gpg php5-gd php5-pspell \
 	tinymce libjs-jquery libjs-jquery-mousewheel libmagic1
 
-# We used to install Roundcube from Ubuntu, without triggering the dependencies
+# We used to install Roundcube from Ubuntu, without triggering the dependencies #NODOC
-# on Apache and MySQL, by downloading the debs and installing them manually.
+# on Apache and MySQL, by downloading the debs and installing them manually. #NODOC
-# Now that we're beyond that, get rid of those debs before installing from source.
+# Now that we're beyond that, get rid of those debs before installing from source. #NODOC
-apt-get purge -qq -y roundcube*
+apt-get purge -qq -y roundcube* #NODOC
 
 # Install Roundcube from source if it is not already present or if it is out of date.
 VERSION=1.0.2
 needs_update=0 #NODOC
 if [ ! -f /usr/local/lib/roundcubemail/version ]; then
-	# not installed yet
+	# not installed yet #NODOC
 	needs_update=1 #NODOC
 elif [[ $VERSION != `cat /usr/local/lib/roundcubemail/version` ]]; then
 	# checks if the version is what we want

tools/readable_bash.py

@@ -30,33 +30,74 @@ def generate_documentation():
 					color: #555;
 	    	}
 	    	h2, h3 {
-	    		margin-bottom: 1em;
+	    		margin-top: .25em;
+	    		margin-bottom: .75em;
 	    	}
 	    	p {
 	    		margin-bottom: 1em;
 	    	}
+		    	.intro p {
+		    		margin: 1.5em 0;
+		    	}
+		    li {
+		    	margin-bottom: .33em;
+		    }
+
+			.sourcefile {
+	    		padding-top: 1.5em;
+	    		padding-bottom: 1em;
+	    		font-size: 90%;
+	    		text-align: right;
+			}
+				.sourcefile a {
+					color: red;
+				}
+
+	    	.instructions .row.contd {
+	    		border-top: 1px solid #E0E0E0;
+	    	}
+
+	    	.prose {
+	    		padding-top: 1em;    	
+	    		padding-bottom: 1em;
+	    	}
+	    	.terminal {
+	    		background-color: #EEE;
+	    		padding-top: 1em;
+	    		padding-bottom: 1em;
+	    	}
 
 	    	pre {
-	    		margin: 1em 1em 1.5em 1em;
 	    		color: black;
+	    		border: 0;
+	    		background: none;
+	    		font-size: 100%;
 	    	}
 
 	    	div.write-to {
-	    		margin: 1em;
+	    		margin: 0 0 1em .5em;
-	    		border: 1px solid #999;
 	    	}
 	    	div.write-to p {
 	    		padding: .5em;
 	    		margin: 0;
 	    	}
 	    	div.write-to .filename {
-	    		background-color: #EEE;
+	    		padding: .25em;
-	    		padding: .5em;
+	    		background-color: #666;
+	    		color: white;
+	    		font-family: monospace;
 	    		font-weight: bold;
 	    	}
+	    	div.write-to .filename span {
+	    		font-family: sans-serif;
+	    		font-weight: normal;
+	    	}
 	    	div.write-to pre {
-	    		padding: .5em;
 	    		margin: 0;
+	    		padding: .25em;
+	    		border: 1px solid #999;
+	    		border-radius: 0;
+	    		font-size: 90%;
 	    	}
 
 	    	pre.shell > div:before {
@@ -67,11 +108,15 @@ def generate_documentation():
     </head>
     <body>
     <div class="container">
-      <div class="row">
+      <div class="row intro">
         <div class="col-xs-12">
         <h1>Build Your Own Mail Server From Scratch</h1>
-        <p>Here&rsquo;s how you can build your own mail server from scratch. This document is generated automatically from our setup script.</p>
+        <p>Here&rsquo;s how you can build your own mail server from scratch.</p>
+        <p>This document is generated automatically from <a href="https://mailinabox.email">Mail-in-a-Box</a>&rsquo;s setup script <a href="https://github.com/mail-in-a-box/mailinabox">source code</a>.</p>
         <hr>
+      </div>
+    </div>
+    <div class="container instructions">
  """)
 
 	parser = Source.parser()
@@ -80,7 +125,7 @@ def generate_documentation():
 			fn = parser.parse_string(line).filename()
 		except:
 			continue
-		if fn in ("setup/preflight.sh", "setup/questions.sh", "setup/firstuser.sh", "setup/management.sh"):
+		if fn in ("setup/start.sh", "setup/preflight.sh", "setup/questions.sh", "setup/firstuser.sh", "setup/management.sh"):
 			continue
 
 		import sys
@@ -91,6 +136,13 @@ def generate_documentation():
 	print("""
         <script src="https://ajax.googleapis.com/ajax/libs/jquery/1.10.1/jquery.min.js"></script>
         <script src="https://maxcdn.bootstrapcdn.com/bootstrap/3.2.0/js/bootstrap.min.js"></script>
+        <script>
+        $(function() {
+			$('.terminal').each(function() {
+			  $(this).outerHeight( $(this).parent().innerHeight() );
+			});
+        })
+		</script>
     </body>
 </html>
 """)
@@ -101,8 +153,13 @@ class HashBang(Grammar):
 		return ""
 
 def strip_indent(s):
+	s = s.replace("\t", "    ")
 	lines = s.split("\n")
-	min_indent = min(len(re.match(r"\s*", line).group(0)) for line in lines if len(line) > 0)
+	try:
+		min_indent = min(len(re.match(r"\s*", line).group(0)) for line in lines if len(line) > 0)
+	except ValueError:
+		# No non-empty lines.
+		min_indent = 0
 	lines = [line[min_indent:] for line in lines]
 	return "\n".join(lines)
 
@@ -126,11 +183,14 @@ class Source(Grammar):
 		return BashScript.parse(self.filename())
 
 class CatEOF(Grammar):
-	grammar = (ZERO_OR_MORE(SPACE), L('cat > '), ANY_EXCEPT(WHITESPACE), L(" <<"), OPTIONAL(SPACE), L("EOF;"), EOL, REPEAT(ANY, greedy=False), EOL, L("EOF"), EOL)
+	grammar = (ZERO_OR_MORE(SPACE), L('cat '), L('>') | L('>>'), L(' '), ANY_EXCEPT(WHITESPACE), L(" <<"), OPTIONAL(SPACE), L("EOF"), EOL, REPEAT(ANY, greedy=False), EOL, L("EOF"), EOL)
 	def value(self):
-		content = self[7].string
+		content = self[9].string
 		content = re.sub(r"\\([$])", r"\1", content) # un-escape bash-escaped characters
-		return "<div class='write-to'><div class='filename'>overwrite<br>" + self[2].string + "</div><pre>" + cgi.escape(content) + "</pre></div>\n"
+		return "<div class='write-to'><div class='filename'>%s <span>(%s)</span></div><pre>%s</pre></div>\n" \
+			% (self[4].string,
+			   "overwrite" if ">>" not in self[2].string else "append to",
+			   cgi.escape(content))
 
 class HideOutput(Grammar):
 	grammar = (L("hide_output "), REF("BashElement"))
@@ -150,7 +210,7 @@ class EditConf(Grammar):
 		FILENAME,
 		SPACE,
 		OPTIONAL((LIST_OF(
-			L("-w") | L("-s") | L("-c ';'"),
+			L("-w") | L("-s") | L("-c ;"),
 			sep=SPACE,
 		), SPACE)),
 		REST_OF_LINE,
@@ -159,27 +219,14 @@ class EditConf(Grammar):
 		)
 	def value(self):
 		conffile = self[1]
-		options = [""]
+		options = []
-		mode = 1
+		eq = "="
-		for c in self[4].string:
+		if self[3] and "-s" in self[3].string: eq = " "
-			if mode == 1 and c in (" ", "\t") and options[-1] != "":
+		for opt in re.split("\s+", self[4].string):
-				# new word
+			k, v = opt.split("=", 1)
-				options.append("")
+			v = re.sub(r"\n+", "", fixup_tokens(v)) # not sure why newlines are getting doubled
-			elif mode < 0:
+			options.append("%s%s%s" % (k, eq, v))
-				# escaped character
+		return "<div class='write-to'><div class='filename'>" + self[1].string + " <span>(change settings)</span></div><pre>" + "\n".join(cgi.escape(s) for s in options) + "</pre></div>\n"
-				options[-1] += c
-				mode = -mode
-			elif c == "\\":
-				# escape next character
-				mode = -mode
-			elif mode == 1 and c == '"':
-				mode = 2
-			elif mode == 2 and c == '"':
-				mode = 1
-			else:
-				options[-1] += c
-		if options[-1] == "": options.pop(-1)
-		return "<div class='write-to'><div class='filename'>additional settings for<br>" + self[1].string + "</div><pre>" + "\n".join(cgi.escape(s) for s in options) + "</pre></div>\n"
 
 class CaptureOutput(Grammar):
 	grammar = OPTIONAL(SPACE), WORD("A-Za-z_"), L('=$('), REST_OF_LINE, L(")"), OPTIONAL(L(';')), EOL
@@ -193,8 +240,14 @@ class SedReplace(Grammar):
 	def value(self):
 		return "<div class='write-to'><div class='filename'>edit<br>" + self[8].string + "</div><p>replace</p><pre>" + cgi.escape(self[3].string.replace(".*", ". . .")) + "</pre><p>with</p><pre>" + cgi.escape(self[5].string.replace("\\n", "\n").replace("\\t", "\t")) + "</pre></div>\n"
 
+class EchoPipe(Grammar):
+	grammar = OPTIONAL(SPACE), L("echo "), REST_OF_LINE, L(' | '), REST_OF_LINE, EOL
+	def value(self):
+		text = " ".join("\"%s\"" % s for s in self[2].string.split(" "))
+		return "<pre class='shell'><div>echo " + cgi.escape(text) + " \<br> | " + self[4].string + "</div></pre>\n"
+
 def shell_line(bash):
-	return "<pre class='shell'><div>" + cgi.escape(wrap_lines(bash.strip())) + "</div></pre>\n"
+	return "<pre class='shell'><div>" + cgi.escape(bash.strip()) + "</div></pre>\n"
 
 class AptGet(Grammar):
 	grammar = (ZERO_OR_MORE(SPACE), L("apt_install "), REST_OF_LINE, EOL)
@@ -213,13 +266,92 @@ class OtherLine(Grammar):
 	grammar = (REST_OF_LINE, EOL)
 	def value(self):
 		if self.string.strip() == "": return ""
-		return "<pre class='shell'><div>" + cgi.escape(self.string.rstrip()) + "</div></pre>\n"
+		if "source setup/functions.sh" in self.string: return ""
+		if "source /etc/mailinabox.conf" in self.string: return ""
+		return "<pre class='shell'><div>" + cgi.escape(self.string.strip()) + "</div></pre>\n"
 
 class BashElement(Grammar):
-	grammar = Comment | Source | CatEOF | SuppressedLine | HideOutput | EditConf | CaptureOutput | SedReplace | AptGet | UfwAllow | RestartService | OtherLine
+	grammar = Comment | CatEOF | EchoPipe | SuppressedLine | HideOutput | EditConf | SedReplace | AptGet | UfwAllow | RestartService | OtherLine
 	def value(self):
 		return self[0].value()
 
+# Make some special characters to private use Unicode code points.
+bash_special_characters = {
+	"\n": "\uE000",
+	" ": "\uE001",
+}
+
+def quasitokenize(bashscript):
+	# Make a parse of bash easier by making the tokenization easy.
+	newscript = ""
+	quote_mode = None
+	escape_next = False
+	line_comment = False
+	subshell = 0
+	for c in bashscript:
+		if line_comment:
+			# We're in a comment until the end of the line.
+			newscript += c
+			if c == '\n':
+				line_comment = False
+		elif escape_next:
+			# Previous character was a \. Normally the next character
+			# comes through literally, but escaped newlines are line
+			# continuations.
+			if c == "\n":
+				c = " "
+			else:
+				newscript += c
+			escape_next = False
+		elif c == "\\":
+			# Escaping next character.
+			escape_next = True
+		elif quote_mode is None and c in ('"', "'"):
+			# Starting a quoted word.
+			quote_mode = c
+		elif c == quote_mode:
+			# Ending a quoted word.
+			quote_mode = None
+		elif quote_mode is not None and quote_mode != "EOF" and c in bash_special_characters:
+			# Replace special tokens within quoted words so that they
+			# don't interfere with tokenization later.
+			newscript += bash_special_characters[c]
+		elif quote_mode is None and c == '#':
+			# Start of a line comment.
+			newscript += c
+			line_comment = True
+		elif quote_mode is None and c == ';' and subshell == 0:
+			# End of a statement.
+			newscript += "\n"
+		elif quote_mode is None and c == '(':
+			# Start of a subshell.
+			newscript += c
+			subshell += 1
+		elif quote_mode is None and c == ')':
+			# End of a subshell.
+			newscript += c
+			subshell -= 1
+		elif quote_mode is None and c == '\t':
+			# Make these just spaces.
+			if newscript[-1] != " ":
+				newscript += " "
+		else:
+			# All other characters.
+			newscript += c
+
+		# "<< EOF" escaping.
+		if quote_mode is None and re.search("<<\s*EOF\n$", newscript):
+			quote_mode = "EOF"
+		elif quote_mode == "EOF" and re.search("\nEOF\n$", newscript):
+			quote_mode = None
+
+	return newscript
+
+def fixup_tokens(s):
+	for c, enc in bash_special_characters.items():
+		s = s.replace(enc, c)
+	return s
+
 class BashScript(Grammar):
 	grammar = (OPTIONAL(HashBang), REPEAT(BashElement))
 	def value(self):
@@ -228,22 +360,68 @@ class BashScript(Grammar):
 	@staticmethod
 	def parse(fn):
 		if fn in ("setup/functions.sh", "/etc/mailinabox.conf"): return ""
-		parser = BashScript.parser()
 		string = open(fn).read()
-		string = re.sub(r"\s*\\\n\s*", " ", string)
+
+		# tokenize
 		string = re.sub(".* #NODOC\n", "", string)
-		string = re.sub("\n\s*if .*|\n\s*fi|\n\s*else|\n\s*elif .*", "", string)
+		string = re.sub("\n\s*if .*\n.*then.*|\n\s*fi|\n\s*else|\n\s*elif .*", "", string)
+		string = quasitokenize(string)
 		string = re.sub("hide_output ", "", string)
+
+		parser = BashScript.parser()
 		result = parser.parse_string(string)
-	
+
-		v = "<div class='sourcefile'><a href=\"%s\">%s</a></div>\n" % ("https://github.com/mail-in-a-box/mailinabox/tree/master/" + fn, fn)
+		v = "<div class='row'><div class='col-xs-12 sourcefile'>view the bash source for the following section at <a href=\"%s\">%s</a></div></div>\n" \
-		v += "".join(result.value())
+			 % ("https://github.com/mail-in-a-box/mailinabox/tree/master/" + fn, fn)
+
+		mode = 0
+		for item in result.value():
+			if item.strip() == "":
+				pass
+			elif item.startswith("<p") and not item.startswith("<pre"):
+				clz = ""
+				if mode == 2:
+					v += "</div>\n" # col
+					v += "</div>\n" # row
+					mode = 0
+					clz = "contd"
+				if mode == 0:
+					v += "<div class='row %s'>\n" % clz
+					v += "<div class='col-md-6 prose'>\n"
+				v += item
+				mode = 1
+			elif item.startswith("<h"):
+				if mode != 0:
+					v += "</div>\n" # col
+					v += "</div>\n" # row
+				v += "<div class='row'>\n"
+				v += "<div class='col-md-6 header'>\n"
+				v += item
+				v += "</div>\n" # col
+				v += "<div class='col-md-6 terminal'> </div>\n"
+				v += "</div>\n" # row
+				mode = 0
+			else:
+				if mode == 0:
+					v += "<div class='row'>\n"
+					v += "<div class='col-md-offset-6 col-md-6 terminal'>\n"
+				elif mode == 1:
+					v += "</div>\n"
+					v += "<div class='col-md-6 terminal'>\n"
+				mode = 2
+				v += item
+
+		v += "</div>\n" # col
+		v += "</div>\n" # row
+
+		v = fixup_tokens(v)
 
 		v = v.replace("</pre>\n<pre class='shell'>", "")
 		v = re.sub("<pre>([\w\W]*?)</pre>", lambda m : "<pre>" + strip_indent(m.group(1)) + "</pre>", v)
 
-		v = re.sub(r"\$?PRIMARY_HOSTNAME", "<b>box.yourdomain.com</b>", v)
+		v = re.sub(r"(\$?)PRIMARY_HOSTNAME", r"<b>box.yourdomain.com</b>", v)
-		v = re.sub(r"\$?STORAGE_ROOT", "<code><b>/path/to/user-data</b></code>", v)
+		v = re.sub(r"\$STORAGE_ROOT", r"<b>$STORE</b>", v)
+		v = re.sub(r"\$CSR_COUNTRY", r"<b>US</b>", v)
 		v = v.replace("`pwd`",  "<code><b>/path/to/mailinabox</b></code>")
 
 		return v