more work on making the bash scripts readable

2026-03-05 15:57:23 +01:00 · 2014-10-04 17:57:26 -04:00
parent db0967446b
commit 5fd107cae5
11 changed files with 354 additions and 143 deletions
--- a/tools/readable_bash.py
+++ b/tools/readable_bash.py
@@ -30,33 +30,74 @@ def generate_documentation():
 					color: #555;
 	    	}
 	    	h2, h3 {
-	    		margin-bottom: 1em;
+	    		margin-top: .25em;
+	    		margin-bottom: .75em;
 	    	}
 	    	p {
 	    		margin-bottom: 1em;
 	    	}
+		    	.intro p {
+		    		margin: 1.5em 0;
+		    	}
+		    li {
+		    	margin-bottom: .33em;
+		    }
+
+			.sourcefile {
+	    		padding-top: 1.5em;
+	    		padding-bottom: 1em;
+	    		font-size: 90%;
+	    		text-align: right;
+			}
+				.sourcefile a {
+					color: red;
+				}
+
+	    	.instructions .row.contd {
+	    		border-top: 1px solid #E0E0E0;
+	    	}
+
+	    	.prose {
+	    		padding-top: 1em;    	
+	    		padding-bottom: 1em;
+	    	}
+	    	.terminal {
+	    		background-color: #EEE;
+	    		padding-top: 1em;
+	    		padding-bottom: 1em;
+	    	}

 	    	pre {
-	    		margin: 1em 1em 1.5em 1em;
 	    		color: black;
+	    		border: 0;
+	    		background: none;
+	    		font-size: 100%;
 	    	}

 	    	div.write-to {
-	    		margin: 1em;
-	    		border: 1px solid #999;
+	    		margin: 0 0 1em .5em;
 	    	}
 	    	div.write-to p {
 	    		padding: .5em;
 	    		margin: 0;
 	    	}
 	    	div.write-to .filename {
-	    		background-color: #EEE;
-	    		padding: .5em;
+	    		padding: .25em;
+	    		background-color: #666;
+	    		color: white;
+	    		font-family: monospace;
 	    		font-weight: bold;
 	    	}
+	    	div.write-to .filename span {
+	    		font-family: sans-serif;
+	    		font-weight: normal;
+	    	}
 	    	div.write-to pre {
-	    		padding: .5em;
 	    		margin: 0;
+	    		padding: .25em;
+	    		border: 1px solid #999;
+	    		border-radius: 0;
+	    		font-size: 90%;
 	    	}

 	    	pre.shell > div:before {
@@ -67,11 +108,15 @@ def generate_documentation():
    </head>
    <body>
    <div class="container">
-      <div class="row">
+      <div class="row intro">
        <div class="col-xs-12">
        <h1>Build Your Own Mail Server From Scratch</h1>
-        <p>Here&rsquo;s how you can build your own mail server from scratch. This document is generated automatically from our setup script.</p>
+        <p>Here&rsquo;s how you can build your own mail server from scratch.</p>
+        <p>This document is generated automatically from <a href="https://mailinabox.email">Mail-in-a-Box</a>&rsquo;s setup script <a href="https://github.com/mail-in-a-box/mailinabox">source code</a>.</p>
        <hr>
+      </div>
+    </div>
+    <div class="container instructions">
 """)

 	parser = Source.parser()
@@ -80,7 +125,7 @@ def generate_documentation():
 			fn = parser.parse_string(line).filename()
 		except:
 			continue
-		if fn in ("setup/preflight.sh", "setup/questions.sh", "setup/firstuser.sh", "setup/management.sh"):
+		if fn in ("setup/start.sh", "setup/preflight.sh", "setup/questions.sh", "setup/firstuser.sh", "setup/management.sh"):
 			continue

 		import sys
@@ -91,6 +136,13 @@ def generate_documentation():
 	print("""
        <script src="https://ajax.googleapis.com/ajax/libs/jquery/1.10.1/jquery.min.js"></script>
        <script src="https://maxcdn.bootstrapcdn.com/bootstrap/3.2.0/js/bootstrap.min.js"></script>
+        <script>
+        $(function() {
+			$('.terminal').each(function() {
+			  $(this).outerHeight( $(this).parent().innerHeight() );
+			});
+        })
+		</script>
    </body>
 </html>
 """)
@@ -101,8 +153,13 @@ class HashBang(Grammar):
 		return ""

 def strip_indent(s):
+	s = s.replace("\t", "    ")
 	lines = s.split("\n")
-	min_indent = min(len(re.match(r"\s*", line).group(0)) for line in lines if len(line) > 0)
+	try:
+		min_indent = min(len(re.match(r"\s*", line).group(0)) for line in lines if len(line) > 0)
+	except ValueError:
+		# No non-empty lines.
+		min_indent = 0
 	lines = [line[min_indent:] for line in lines]
 	return "\n".join(lines)

@@ -126,11 +183,14 @@ class Source(Grammar):
 		return BashScript.parse(self.filename())

 class CatEOF(Grammar):
-	grammar = (ZERO_OR_MORE(SPACE), L('cat > '), ANY_EXCEPT(WHITESPACE), L(" <<"), OPTIONAL(SPACE), L("EOF;"), EOL, REPEAT(ANY, greedy=False), EOL, L("EOF"), EOL)
+	grammar = (ZERO_OR_MORE(SPACE), L('cat '), L('>') | L('>>'), L(' '), ANY_EXCEPT(WHITESPACE), L(" <<"), OPTIONAL(SPACE), L("EOF"), EOL, REPEAT(ANY, greedy=False), EOL, L("EOF"), EOL)
 	def value(self):
-		content = self[7].string
+		content = self[9].string
 		content = re.sub(r"\\([$])", r"\1", content) # un-escape bash-escaped characters
-		return "<div class='write-to'><div class='filename'>overwrite<br>" + self[2].string + "</div><pre>" + cgi.escape(content) + "</pre></div>\n"
+		return "<div class='write-to'><div class='filename'>%s <span>(%s)</span></div><pre>%s</pre></div>\n" \
+			% (self[4].string,
+			   "overwrite" if ">>" not in self[2].string else "append to",
+			   cgi.escape(content))

 class HideOutput(Grammar):
 	grammar = (L("hide_output "), REF("BashElement"))
@@ -150,7 +210,7 @@ class EditConf(Grammar):
 		FILENAME,
 		SPACE,
 		OPTIONAL((LIST_OF(
-			L("-w") | L("-s") | L("-c ';'"),
+			L("-w") | L("-s") | L("-c ;"),
 			sep=SPACE,
 		), SPACE)),
 		REST_OF_LINE,
@@ -159,27 +219,14 @@ class EditConf(Grammar):
 		)
 	def value(self):
 		conffile = self[1]
-		options = [""]
-		mode = 1
-		for c in self[4].string:
-			if mode == 1 and c in (" ", "\t") and options[-1] != "":
-				# new word
-				options.append("")
-			elif mode < 0:
-				# escaped character
-				options[-1] += c
-				mode = -mode
-			elif c == "\\":
-				# escape next character
-				mode = -mode
-			elif mode == 1 and c == '"':
-				mode = 2
-			elif mode == 2 and c == '"':
-				mode = 1
-			else:
-				options[-1] += c
-		if options[-1] == "": options.pop(-1)
-		return "<div class='write-to'><div class='filename'>additional settings for<br>" + self[1].string + "</div><pre>" + "\n".join(cgi.escape(s) for s in options) + "</pre></div>\n"
+		options = []
+		eq = "="
+		if self[3] and "-s" in self[3].string: eq = " "
+		for opt in re.split("\s+", self[4].string):
+			k, v = opt.split("=", 1)
+			v = re.sub(r"\n+", "", fixup_tokens(v)) # not sure why newlines are getting doubled
+			options.append("%s%s%s" % (k, eq, v))
+		return "<div class='write-to'><div class='filename'>" + self[1].string + " <span>(change settings)</span></div><pre>" + "\n".join(cgi.escape(s) for s in options) + "</pre></div>\n"

 class CaptureOutput(Grammar):
 	grammar = OPTIONAL(SPACE), WORD("A-Za-z_"), L('=$('), REST_OF_LINE, L(")"), OPTIONAL(L(';')), EOL
@@ -193,8 +240,14 @@ class SedReplace(Grammar):
 	def value(self):
 		return "<div class='write-to'><div class='filename'>edit<br>" + self[8].string + "</div><p>replace</p><pre>" + cgi.escape(self[3].string.replace(".*", ". . .")) + "</pre><p>with</p><pre>" + cgi.escape(self[5].string.replace("\\n", "\n").replace("\\t", "\t")) + "</pre></div>\n"

+class EchoPipe(Grammar):
+	grammar = OPTIONAL(SPACE), L("echo "), REST_OF_LINE, L(' | '), REST_OF_LINE, EOL
+	def value(self):
+		text = " ".join("\"%s\"" % s for s in self[2].string.split(" "))
+		return "<pre class='shell'><div>echo " + cgi.escape(text) + " \<br> | " + self[4].string + "</div></pre>\n"
+
 def shell_line(bash):
-	return "<pre class='shell'><div>" + cgi.escape(wrap_lines(bash.strip())) + "</div></pre>\n"
+	return "<pre class='shell'><div>" + cgi.escape(bash.strip()) + "</div></pre>\n"

 class AptGet(Grammar):
 	grammar = (ZERO_OR_MORE(SPACE), L("apt_install "), REST_OF_LINE, EOL)
@@ -213,13 +266,92 @@ class OtherLine(Grammar):
 	grammar = (REST_OF_LINE, EOL)
 	def value(self):
 		if self.string.strip() == "": return ""
-		return "<pre class='shell'><div>" + cgi.escape(self.string.rstrip()) + "</div></pre>\n"
+		if "source setup/functions.sh" in self.string: return ""
+		if "source /etc/mailinabox.conf" in self.string: return ""
+		return "<pre class='shell'><div>" + cgi.escape(self.string.strip()) + "</div></pre>\n"

 class BashElement(Grammar):
-	grammar = Comment | Source | CatEOF | SuppressedLine | HideOutput | EditConf | CaptureOutput | SedReplace | AptGet | UfwAllow | RestartService | OtherLine
+	grammar = Comment | CatEOF | EchoPipe | SuppressedLine | HideOutput | EditConf | SedReplace | AptGet | UfwAllow | RestartService | OtherLine
 	def value(self):
 		return self[0].value()

+# Make some special characters to private use Unicode code points.
+bash_special_characters = {
+	"\n": "\uE000",
+	" ": "\uE001",
+}
+
+def quasitokenize(bashscript):
+	# Make a parse of bash easier by making the tokenization easy.
+	newscript = ""
+	quote_mode = None
+	escape_next = False
+	line_comment = False
+	subshell = 0
+	for c in bashscript:
+		if line_comment:
+			# We're in a comment until the end of the line.
+			newscript += c
+			if c == '\n':
+				line_comment = False
+		elif escape_next:
+			# Previous character was a \. Normally the next character
+			# comes through literally, but escaped newlines are line
+			# continuations.
+			if c == "\n":
+				c = " "
+			else:
+				newscript += c
+			escape_next = False
+		elif c == "\\":
+			# Escaping next character.
+			escape_next = True
+		elif quote_mode is None and c in ('"', "'"):
+			# Starting a quoted word.
+			quote_mode = c
+		elif c == quote_mode:
+			# Ending a quoted word.
+			quote_mode = None
+		elif quote_mode is not None and quote_mode != "EOF" and c in bash_special_characters:
+			# Replace special tokens within quoted words so that they
+			# don't interfere with tokenization later.
+			newscript += bash_special_characters[c]
+		elif quote_mode is None and c == '#':
+			# Start of a line comment.
+			newscript += c
+			line_comment = True
+		elif quote_mode is None and c == ';' and subshell == 0:
+			# End of a statement.
+			newscript += "\n"
+		elif quote_mode is None and c == '(':
+			# Start of a subshell.
+			newscript += c
+			subshell += 1
+		elif quote_mode is None and c == ')':
+			# End of a subshell.
+			newscript += c
+			subshell -= 1
+		elif quote_mode is None and c == '\t':
+			# Make these just spaces.
+			if newscript[-1] != " ":
+				newscript += " "
+		else:
+			# All other characters.
+			newscript += c
+
+		# "<< EOF" escaping.
+		if quote_mode is None and re.search("<<\s*EOF\n$", newscript):
+			quote_mode = "EOF"
+		elif quote_mode == "EOF" and re.search("\nEOF\n$", newscript):
+			quote_mode = None
+
+	return newscript
+
+def fixup_tokens(s):
+	for c, enc in bash_special_characters.items():
+		s = s.replace(enc, c)
+	return s
+
 class BashScript(Grammar):
 	grammar = (OPTIONAL(HashBang), REPEAT(BashElement))
 	def value(self):
@@ -228,22 +360,68 @@ class BashScript(Grammar):
 	@staticmethod
 	def parse(fn):
 		if fn in ("setup/functions.sh", "/etc/mailinabox.conf"): return ""
-		parser = BashScript.parser()
 		string = open(fn).read()
-		string = re.sub(r"\s*\\\n\s*", " ", string)
+
+		# tokenize
 		string = re.sub(".* #NODOC\n", "", string)
-		string = re.sub("\n\s*if .*|\n\s*fi|\n\s*else|\n\s*elif .*", "", string)
+		string = re.sub("\n\s*if .*\n.*then.*|\n\s*fi|\n\s*else|\n\s*elif .*", "", string)
+		string = quasitokenize(string)
 		string = re.sub("hide_output ", "", string)
+
+		parser = BashScript.parser()
 		result = parser.parse_string(string)
-	
-		v = "<div class='sourcefile'><a href=\"%s\">%s</a></div>\n" % ("https://github.com/mail-in-a-box/mailinabox/tree/master/" + fn, fn)
-		v += "".join(result.value())
+
+		v = "<div class='row'><div class='col-xs-12 sourcefile'>view the bash source for the following section at <a href=\"%s\">%s</a></div></div>\n" \
+			 % ("https://github.com/mail-in-a-box/mailinabox/tree/master/" + fn, fn)
+
+		mode = 0
+		for item in result.value():
+			if item.strip() == "":
+				pass
+			elif item.startswith("<p") and not item.startswith("<pre"):
+				clz = ""
+				if mode == 2:
+					v += "</div>\n" # col
+					v += "</div>\n" # row
+					mode = 0
+					clz = "contd"
+				if mode == 0:
+					v += "<div class='row %s'>\n" % clz
+					v += "<div class='col-md-6 prose'>\n"
+				v += item
+				mode = 1
+			elif item.startswith("<h"):
+				if mode != 0:
+					v += "</div>\n" # col
+					v += "</div>\n" # row
+				v += "<div class='row'>\n"
+				v += "<div class='col-md-6 header'>\n"
+				v += item
+				v += "</div>\n" # col
+				v += "<div class='col-md-6 terminal'> </div>\n"
+				v += "</div>\n" # row
+				mode = 0
+			else:
+				if mode == 0:
+					v += "<div class='row'>\n"
+					v += "<div class='col-md-offset-6 col-md-6 terminal'>\n"
+				elif mode == 1:
+					v += "</div>\n"
+					v += "<div class='col-md-6 terminal'>\n"
+				mode = 2
+				v += item
+
+		v += "</div>\n" # col
+		v += "</div>\n" # row
+
+		v = fixup_tokens(v)

 		v = v.replace("</pre>\n<pre class='shell'>", "")
 		v = re.sub("<pre>([\w\W]*?)</pre>", lambda m : "<pre>" + strip_indent(m.group(1)) + "</pre>", v)

-		v = re.sub(r"\$?PRIMARY_HOSTNAME", "<b>box.yourdomain.com</b>", v)
-		v = re.sub(r"\$?STORAGE_ROOT", "<code><b>/path/to/user-data</b></code>", v)
+		v = re.sub(r"(\$?)PRIMARY_HOSTNAME", r"<b>box.yourdomain.com</b>", v)
+		v = re.sub(r"\$STORAGE_ROOT", r"<b>$STORE</b>", v)
+		v = re.sub(r"\$CSR_COUNTRY", r"<b>US</b>", v)
 		v = v.replace("`pwd`",  "<code><b>/path/to/mailinabox</b></code>")

 		return v