various improvements in bash comments

setup/dkim.sh

@@ -1,5 +1,5 @@
 # OpenDKIM
-# ========
+# --------
 #
 # OpenDKIM provides a service that puts a DKIM signature on outbound mail.
 #

setup/dns.sh

@@ -39,7 +39,9 @@ mkdir -p /var/run/nsd
 mkdir -p "$STORAGE_ROOT/dns/dnssec";
 
 # TLDs don't all support the same algorithms, so we'll generate keys using a few
-# different algorithms.
+# different algorithms. RSASHA1-NSEC3-SHA1 was possibly the first widely used
+# algorithm that supported NSEC3, which is a security best practice. However TLDs
+# will probably be moving away from it to a a SHA256-based algorithm.
 #
 # Supports `RSASHA1-NSEC3-SHA1` (didn't test with `RSASHA256`):
 #
@@ -58,11 +60,9 @@ if [ ! -f "$STORAGE_ROOT/dns/dnssec/$algo.conf" ]; then
 		FIRST=0 #NODOC
 	fi
 
-	# Create the Key-Signing Key (KSK) (-k) which is the so-called
+	# Create the Key-Signing Key (KSK) (with `-k`) which is the so-called
-	# Secure Entry Point. Use a NSEC3-compatible algorithm (best
+	# Secure Entry Point. The domain name we provide ("_domain_") doesn't
-	# practice), and a nice and long keylength. The domain name we
+	#  matter -- we'll use the same keys for all our domains.
-	# provide ("_domain_") doesn't matter -- we'll use the same
-	# keys for all our domains.
 	#
 	# `ldns-keygen` outputs the new key's filename to stdout, which
 	# we're capturing into the `KSK` variable.
@@ -71,13 +71,13 @@ if [ ! -f "$STORAGE_ROOT/dns/dnssec/$algo.conf" ]; then
 	# Now create a Zone-Signing Key (ZSK) which is expected to be
 	# rotated more often than a KSK, although we have no plans to
 	# rotate it (and doing so would be difficult to do without
-	# disturbing DNS availability.) Omit '-k' and use a shorter key.
+	# disturbing DNS availability.) Omit `-k` and use a shorter key length.
 	ZSK=$(umask 077; cd $STORAGE_ROOT/dns/dnssec; ldns-keygen -a $algo -b 1024 _domain_);
 
 	# These generate two sets of files like:
 	#
-	# * `K_domain_.+007+08882.ds`: DS record normally provided to domain name registrar (but it's actually invalid with "_domain_")
+	# * `K_domain_.+007+08882.ds`: DS record normally provided to domain name registrar (but it's actually invalid with `_domain_`)
-	# * `K_domain_.+007+08882.key`: public key (goes into DS record & upstream DNS provider like your registrar)
+	# * `K_domain_.+007+08882.key`: public key
 	# * `K_domain_.+007+08882.private`: private key (secret!)
 
 	# The filenames are unpredictable and encode the key generation

setup/mail-dovecot.sh

@@ -52,8 +52,9 @@ tools/editconf.py /etc/dovecot/conf.d/10-ssl.conf \
 	"ssl_protocols=!SSLv3 !SSLv2" \
 	"ssl_cipher_list=TLSv1+HIGH !SSLv2 !RC4 !aNULL !eNULL !3DES @STRENGTH"
 
-# Disable in-the-clear IMAP and POP because we're paranoid (we haven't even
+# Disable in-the-clear IMAP because there is no reason for a user to transmit
-# enabled POP).
+# login credentials outside of an encrypted connection. Although we haven't
+# even installed the POP server, ensure it is disabled too.
 sed -i "s/#port = 143/port = 0/" /etc/dovecot/conf.d/10-master.conf
 sed -i "s/#port = 110/port = 0/" /etc/dovecot/conf.d/10-master.conf
 

setup/mail-postfix.sh

@@ -63,7 +63,7 @@ tools/editconf.py /etc/postfix/main.cf \
 # Enable the 'submission' port 587 smtpd server and tweak its settings.
 #
 # * Require the best ciphers for incoming connections per http://baldric.net/2013/12/07/tls-ciphers-in-postfix-and-dovecot/.
-#   but without affecting opportunistic TLS on incoming mail, which will allow any cipher (it's better than none).
+#   By putting this setting here we leave opportunistic TLS on incoming mail at default cipher settings (any cipher is better than none).
 # * Give it a different name in syslog to distinguish it from the port 25 smtpd server.
 # * Add a new cleanup service specific to the submission service ('authclean')
 #   that filters out privacy-sensitive headers on mail being sent out by
@@ -96,9 +96,9 @@ tools/editconf.py /etc/postfix/main.cf \
 # relayed elsewhere. We don't want to be an "open relay". On outbound
 # mail, require one of:
 #
-# * permit_sasl_authenticated: Authenticated users (i.e. on port 587).
+# * `permit_sasl_authenticated`: Authenticated users (i.e. on port 587).
-# * permit_mynetworks: Mail that originates locally.
+# * `permit_mynetworks`: Mail that originates locally.
-# * reject_unauth_destination: No one else. (Permits mail whose destination is local and rejects other mail.)
+# * `reject_unauth_destination`: No one else. (Permits mail whose destination is local and rejects other mail.)
 tools/editconf.py /etc/postfix/main.cf \
 	smtpd_relay_restrictions=permit_sasl_authenticated,permit_mynetworks,reject_unauth_destination
 
@@ -142,20 +142,20 @@ tools/editconf.py /etc/postfix/main.cf virtual_transport=lmtp:[127.0.0.1]:10025
 
 # Who can send mail to us? Some basic filters.
 #
-# * reject_non_fqdn_sender: Reject not-nice-looking return paths.
+# * `reject_non_fqdn_sender`: Reject not-nice-looking return paths.
-# * reject_unknown_sender_domain: Reject return paths with invalid domains.
+# * `reject_unknown_sender_domain`: Reject return paths with invalid domains.
-# * reject_rhsbl_sender: Reject return paths that use blacklisted domains.
+# * `reject_rhsbl_sender`: Reject return paths that use blacklisted domains.
-# * permit_sasl_authenticated: Authenticated users (i.e. on port 587) can skip further checks.
+# * `permit_sasl_authenticated`: Authenticated users (i.e. on port 587) can skip further checks.
-# * permit_mynetworks: Mail that originates locally can skip further checks.
+# * `permit_mynetworks`: Mail that originates locally can skip further checks.
-# * reject_rbl_client: Reject connections from IP addresses blacklisted in zen.spamhaus.org
+# * `reject_rbl_client`: Reject connections from IP addresses blacklisted in zen.spamhaus.org
-# * reject_unlisted_recipient: Although Postfix will reject mail to unknown recipients, it's nicer to reject such mail ahead of greylisting rather than after.
+# * `reject_unlisted_recipient`: Although Postfix will reject mail to unknown recipients, it's nicer to reject such mail ahead of greylisting rather than after.
-# * check_policy_service: Apply greylisting using postgrey.
+# * `check_policy_service`: Apply greylisting using postgrey.
 #
-# Notes:
+# Notes: #NODOC
-# permit_dnswl_client can pass through mail from whitelisted IP addresses, which would be good to put before greylisting
+# permit_dnswl_client can pass through mail from whitelisted IP addresses, which would be good to put before greylisting #NODOC
-# so these IPs get mail delivered quickly. But when an IP is not listed in the permit_dnswl_client list (i.e. it is not
+# so these IPs get mail delivered quickly. But when an IP is not listed in the permit_dnswl_client list (i.e. it is not #NODOC
-# whitelisted) then postfix does a DEFER_IF_REJECT, which results in all "unknown user" sorts of messages turning into
+# whitelisted) then postfix does a DEFER_IF_REJECT, which results in all "unknown user" sorts of messages turning into #NODOC
-# "450 4.7.1 Client host rejected: Service unavailable". This is a retry code, so the mail doesn't properly bounce.
+# "450 4.7.1 Client host rejected: Service unavailable". This is a retry code, so the mail doesn't properly bounce. #NODOC
 tools/editconf.py /etc/postfix/main.cf \
 	smtpd_sender_restrictions="reject_non_fqdn_sender,reject_unknown_sender_domain,reject_rhsbl_sender dbl.spamhaus.org" \
 	smtpd_recipient_restrictions=permit_sasl_authenticated,permit_mynetworks,"reject_rbl_client zen.spamhaus.org",reject_unlisted_recipient,"check_policy_service inet:127.0.0.1:10023"

setup/spamassassin.sh

@@ -1,6 +1,6 @@
 #!/bin/bash
 # Spam filtering with spamassassin via spampd
-# ===========================================
+# -------------------------------------------
 #
 # spampd sits between postfix and dovecot. It takes mail from postfix
 # over the LMTP protocol, runs spamassassin on it, and then passes the

tools/readable_bash.py

@@ -67,6 +67,10 @@ def generate_documentation():
 	    		padding-bottom: 1em;
 	    	}
 
+	    	ul {
+	    		padding-left: 1.25em;
+	    	}
+
 	    	pre {
 	    		color: black;
 	    		border: 0;
@@ -82,7 +86,7 @@ def generate_documentation():
 	    		margin: 0;
 	    	}
 	    	div.write-to .filename {
-	    		padding: .25em;
+	    		padding: .25em .5em;
 	    		background-color: #666;
 	    		color: white;
 	    		font-family: monospace;
@@ -94,7 +98,7 @@ def generate_documentation():
 	    	}
 	    	div.write-to pre {
 	    		margin: 0;
-	    		padding: .25em;
+	    		padding: .5em;
 	    		border: 1px solid #999;
 	    		border-radius: 0;
 	    		font-size: 90%;
@@ -197,11 +201,11 @@ class HideOutput(Grammar):
 	def value(self):
 		return self[1].value()
 
-class SuppressedLine(Grammar):
+class EchoLine(Grammar):
 	grammar = (OPTIONAL(SPACE), L("echo "), REST_OF_LINE, EOL)
 	def value(self):
 		if "|" in self.string  or ">" in self.string:
-			return "<pre class='shell'><div>" + cgi.escape(self.string.strip()) + "</div></pre>\n"
+			return "<pre class='shell'><div>" + recode_bash(self.string.strip()) + "</div></pre>\n"
 		return ""
 
 class EditConf(Grammar):
@@ -244,10 +248,10 @@ class EchoPipe(Grammar):
 	grammar = OPTIONAL(SPACE), L("echo "), REST_OF_LINE, L(' | '), REST_OF_LINE, EOL
 	def value(self):
 		text = " ".join("\"%s\"" % s for s in self[2].string.split(" "))
-		return "<pre class='shell'><div>echo " + cgi.escape(text) + " \<br> | " + self[4].string + "</div></pre>\n"
+		return "<pre class='shell'><div>echo " + recode_bash(text) + " \<br> | " + recode_bash(self[4].string) + "</div></pre>\n"
 
 def shell_line(bash):
-	return "<pre class='shell'><div>" + cgi.escape(bash.strip()) + "</div></pre>\n"
+	return "<pre class='shell'><div>" + recode_bash(bash.strip()) + "</div></pre>\n"
 
 class AptGet(Grammar):
 	grammar = (ZERO_OR_MORE(SPACE), L("apt_install "), REST_OF_LINE, EOL)
@@ -268,18 +272,25 @@ class OtherLine(Grammar):
 		if self.string.strip() == "": return ""
 		if "source setup/functions.sh" in self.string: return ""
 		if "source /etc/mailinabox.conf" in self.string: return ""
-		return "<pre class='shell'><div>" + cgi.escape(self.string.strip()) + "</div></pre>\n"
+		return "<pre class='shell'><div>" + recode_bash(self.string.strip()) + "</div></pre>\n"
 
 class BashElement(Grammar):
-	grammar = Comment | CatEOF | EchoPipe | SuppressedLine | HideOutput | EditConf | SedReplace | AptGet | UfwAllow | RestartService | OtherLine
+	grammar = Comment | CatEOF | EchoPipe | EchoLine | HideOutput | EditConf | SedReplace | AptGet | UfwAllow | RestartService | OtherLine
 	def value(self):
 		return self[0].value()
 
 # Make some special characters to private use Unicode code points.
-bash_special_characters = {
+bash_special_characters1 = {
 	"\n": "\uE000",
 	" ": "\uE001",
 }
+bash_special_characters2 = {
+	"$": "\uE010",
+}
+bash_escapes = {
+	"n": "\uE020",
+	"t": "\uE021",
+}
 
 def quasitokenize(bashscript):
 	# Make a parse of bash easier by making the tokenization easy.
@@ -297,10 +308,12 @@ def quasitokenize(bashscript):
 		elif escape_next:
 			# Previous character was a \. Normally the next character
 			# comes through literally, but escaped newlines are line
-			# continuations.
+			# continuations and some escapes are for special characters
+			# which we'll recode and then turn back into escapes later.
 			if c == "\n":
 				c = " "
-			else:
+			elif c in bash_escapes:
+				c = bash_escapes[c]
 			newscript += c
 			escape_next = False
 		elif c == "\\":
@@ -312,10 +325,10 @@ def quasitokenize(bashscript):
 		elif c == quote_mode:
 			# Ending a quoted word.
 			quote_mode = None
-		elif quote_mode is not None and quote_mode != "EOF" and c in bash_special_characters:
+		elif quote_mode is not None and quote_mode != "EOF" and c in bash_special_characters1:
 			# Replace special tokens within quoted words so that they
 			# don't interfere with tokenization later.
-			newscript += bash_special_characters[c]
+			newscript += bash_special_characters1[c]
 		elif quote_mode is None and c == '#':
 			# Start of a line comment.
 			newscript += c
@@ -335,6 +348,12 @@ def quasitokenize(bashscript):
 			# Make these just spaces.
 			if newscript[-1] != " ":
 				newscript += " "
+		elif quote_mode is None and c == ' ':
+			# Collapse consecutive spaces.
+			if newscript[-1] != " ":
+				newscript += " "
+		elif c in bash_special_characters2:
+			newscript += bash_special_characters2[c]
 		else:
 			# All other characters.
 			newscript += c
@@ -347,9 +366,27 @@ def quasitokenize(bashscript):
 
 	return newscript
 
+def recode_bash(s):
+	def requote(tok):
+		tok = tok.replace("\\", "\\\\")
+		for c in bash_special_characters2:
+			tok = tok.replace(c, "\\" + c)
+		tok = fixup_tokens(tok)
+		if " " in tok or '"' in tok:
+			tok = tok.replace("\"", "\\\"")
+			tok = '"' + tok +'"'
+		else:
+			tok = tok.replace("'", "\\'")
+		return tok
+	return cgi.escape(" ".join(requote(tok) for tok in s.split(" ")))
+
 def fixup_tokens(s):
-	for c, enc in bash_special_characters.items():
+	for c, enc in bash_special_characters1.items():
 		s = s.replace(enc, c)
+	for c, enc in bash_special_characters2.items():
+		s = s.replace(enc, c)
+	for esc, c in bash_escapes.items():
+		s = s.replace(c, "\\" + esc)
 	return s
 
 class BashScript(Grammar):
@@ -364,7 +401,7 @@ class BashScript(Grammar):
 
 		# tokenize
 		string = re.sub(".* #NODOC\n", "", string)
-		string = re.sub("\n\s*if .*\n.*then.*|\n\s*fi|\n\s*else|\n\s*elif .*", "", string)
+		string = re.sub("\n\s*if .*then.*|\n\s*fi|\n\s*else|\n\s*elif .*", "", string)
 		string = quasitokenize(string)
 		string = re.sub("hide_output ", "", string)