#!/usr/bin/python3 # # Generate documentation for how this machine works by # parsing our bash scripts! import cgi, re import markdown from modgrammar import * def generate_documentation(): print(""" Build Your Own Mail Server From Scratch

Build Your Own Mail Server From Scratch

Here’s how you can build your own mail server from scratch.

This document is generated automatically from Mail-in-a-Box’s setup script source code.

""") parser = Source.parser() for line in open("setup/start.sh"): try: fn = parser.parse_string(line).filename() except: continue if fn in ("setup/start.sh", "setup/preflight.sh", "setup/questions.sh", "setup/firstuser.sh", "setup/management.sh"): continue import sys print(fn, file=sys.stderr) print(BashScript.parse(fn)) print(""" """) class HashBang(Grammar): grammar = (L('#!'), REST_OF_LINE, EOL) def value(self): return "" def strip_indent(s): s = s.replace("\t", " ") lines = s.split("\n") try: min_indent = min(len(re.match(r"\s*", line).group(0)) for line in lines if len(line) > 0) except ValueError: # No non-empty lines. min_indent = 0 lines = [line[min_indent:] for line in lines] return "\n".join(lines) class Comment(Grammar): grammar = ONE_OR_MORE(ZERO_OR_MORE(SPACE), L('#'), REST_OF_LINE, EOL) def value(self): if self.string.replace("#", "").strip() == "": return "\n" lines = [x[2].string for x in self[0]] content = "\n".join(lines) content = strip_indent(content) return markdown.markdown(content, output_format="html4") + "\n\n" FILENAME = WORD('a-z0-9-/.') class Source(Grammar): grammar = ((L('.') | L('source')), L(' '), FILENAME, Comment | EOL) def filename(self): return self[2].string.strip() def value(self): return BashScript.parse(self.filename()) class CatEOF(Grammar): grammar = (ZERO_OR_MORE(SPACE), L('cat '), L('>') | L('>>'), L(' '), ANY_EXCEPT(WHITESPACE), L(" <<"), OPTIONAL(SPACE), L("EOF"), EOL, REPEAT(ANY, greedy=False), EOL, L("EOF"), EOL) def value(self): content = self[9].string content = re.sub(r"\\([$])", r"\1", content) # un-escape bash-escaped characters return "
%s (%s)
%s
\n" \ % (self[4].string, "overwrite" if ">>" not in self[2].string else "append to", cgi.escape(content)) class HideOutput(Grammar): grammar = (L("hide_output "), REF("BashElement")) def value(self): return self[1].value() class SuppressedLine(Grammar): grammar = (OPTIONAL(SPACE), L("echo "), REST_OF_LINE, EOL) def value(self): if "|" in self.string or ">" in self.string: return "
" + cgi.escape(self.string.strip()) + "
\n" return "" class EditConf(Grammar): grammar = ( L('tools/editconf.py '), FILENAME, SPACE, OPTIONAL((LIST_OF( L("-w") | L("-s") | L("-c ;"), sep=SPACE, ), SPACE)), REST_OF_LINE, OPTIONAL(SPACE), EOL ) def value(self): conffile = self[1] options = [] eq = "=" if self[3] and "-s" in self[3].string: eq = " " for opt in re.split("\s+", self[4].string): k, v = opt.split("=", 1) v = re.sub(r"\n+", "", fixup_tokens(v)) # not sure why newlines are getting doubled options.append("%s%s%s" % (k, eq, v)) return "
" + self[1].string + " (change settings)
" + "\n".join(cgi.escape(s) for s in options) + "
\n" class CaptureOutput(Grammar): grammar = OPTIONAL(SPACE), WORD("A-Za-z_"), L('=$('), REST_OF_LINE, L(")"), OPTIONAL(L(';')), EOL def value(self): cmd = self[3].string cmd = cmd.replace("; ", "\n") return "
$" + self[1].string + "=
" + cgi.escape(cmd) + "
\n" class SedReplace(Grammar): grammar = OPTIONAL(SPACE), L('sed -i "s/'), OPTIONAL(L('^')), ONE_OR_MORE(WORD("-A-Za-z0-9 #=\\{};.*$_!()")), L('/'), ONE_OR_MORE(WORD("-A-Za-z0-9 #=\\{};.*$_!()")), L('/"'), SPACE, FILENAME, EOL def value(self): return "
edit
" + self[8].string + "

replace

" + cgi.escape(self[3].string.replace(".*", ". . .")) + "

with

" + cgi.escape(self[5].string.replace("\\n", "\n").replace("\\t", "\t")) + "
\n" class EchoPipe(Grammar): grammar = OPTIONAL(SPACE), L("echo "), REST_OF_LINE, L(' | '), REST_OF_LINE, EOL def value(self): text = " ".join("\"%s\"" % s for s in self[2].string.split(" ")) return "
echo " + cgi.escape(text) + " \
 | " + self[4].string + "
\n" def shell_line(bash): return "
" + cgi.escape(bash.strip()) + "
\n" class AptGet(Grammar): grammar = (ZERO_OR_MORE(SPACE), L("apt_install "), REST_OF_LINE, EOL) def value(self): return shell_line("apt-get install -y " + re.sub(r"\s+", " ", self[2].string)) class UfwAllow(Grammar): grammar = (ZERO_OR_MORE(SPACE), L("ufw_allow "), REST_OF_LINE, EOL) def value(self): return shell_line("ufw allow " + self[2].string) class RestartService(Grammar): grammar = (ZERO_OR_MORE(SPACE), L("restart_service "), REST_OF_LINE, EOL) def value(self): return shell_line("service " + self[2].string + " restart") class OtherLine(Grammar): grammar = (REST_OF_LINE, EOL) def value(self): if self.string.strip() == "": return "" if "source setup/functions.sh" in self.string: return "" if "source /etc/mailinabox.conf" in self.string: return "" return "
" + cgi.escape(self.string.strip()) + "
\n" class BashElement(Grammar): grammar = Comment | CatEOF | EchoPipe | SuppressedLine | HideOutput | EditConf | SedReplace | AptGet | UfwAllow | RestartService | OtherLine def value(self): return self[0].value() # Make some special characters to private use Unicode code points. bash_special_characters = { "\n": "\uE000", " ": "\uE001", } def quasitokenize(bashscript): # Make a parse of bash easier by making the tokenization easy. newscript = "" quote_mode = None escape_next = False line_comment = False subshell = 0 for c in bashscript: if line_comment: # We're in a comment until the end of the line. newscript += c if c == '\n': line_comment = False elif escape_next: # Previous character was a \. Normally the next character # comes through literally, but escaped newlines are line # continuations. if c == "\n": c = " " else: newscript += c escape_next = False elif c == "\\": # Escaping next character. escape_next = True elif quote_mode is None and c in ('"', "'"): # Starting a quoted word. quote_mode = c elif c == quote_mode: # Ending a quoted word. quote_mode = None elif quote_mode is not None and quote_mode != "EOF" and c in bash_special_characters: # Replace special tokens within quoted words so that they # don't interfere with tokenization later. newscript += bash_special_characters[c] elif quote_mode is None and c == '#': # Start of a line comment. newscript += c line_comment = True elif quote_mode is None and c == ';' and subshell == 0: # End of a statement. newscript += "\n" elif quote_mode is None and c == '(': # Start of a subshell. newscript += c subshell += 1 elif quote_mode is None and c == ')': # End of a subshell. newscript += c subshell -= 1 elif quote_mode is None and c == '\t': # Make these just spaces. if newscript[-1] != " ": newscript += " " else: # All other characters. newscript += c # "<< EOF" escaping. if quote_mode is None and re.search("<<\s*EOF\n$", newscript): quote_mode = "EOF" elif quote_mode == "EOF" and re.search("\nEOF\n$", newscript): quote_mode = None return newscript def fixup_tokens(s): for c, enc in bash_special_characters.items(): s = s.replace(enc, c) return s class BashScript(Grammar): grammar = (OPTIONAL(HashBang), REPEAT(BashElement)) def value(self): return [line.value() for line in self[1]] @staticmethod def parse(fn): if fn in ("setup/functions.sh", "/etc/mailinabox.conf"): return "" string = open(fn).read() # tokenize string = re.sub(".* #NODOC\n", "", string) string = re.sub("\n\s*if .*\n.*then.*|\n\s*fi|\n\s*else|\n\s*elif .*", "", string) string = quasitokenize(string) string = re.sub("hide_output ", "", string) parser = BashScript.parser() result = parser.parse_string(string) v = "
view the bash source for the following section at %s
\n" \ % ("https://github.com/mail-in-a-box/mailinabox/tree/master/" + fn, fn) mode = 0 for item in result.value(): if item.strip() == "": pass elif item.startswith("\n
", "")
		v = re.sub("
([\w\W]*?)
", lambda m : "
" + strip_indent(m.group(1)) + "
", v)

		v = re.sub(r"(\$?)PRIMARY_HOSTNAME", r"box.yourdomain.com", v)
		v = re.sub(r"\$STORAGE_ROOT", r"$STORE", v)
		v = re.sub(r"\$CSR_COUNTRY", r"US", v)
		v = v.replace("`pwd`",  "/path/to/mailinabox")

		return v

def wrap_lines(text, cols=60):
	ret = ""
	words = re.split("(\s+)", text)
	linelen = 0
	for w in words:
		if linelen + len(w) > cols-1:
			ret += " \\\n"
			ret += "   "
			linelen = 0
		if linelen == 0 and w.strip() == "": continue
		ret += w
		linelen += len(w)
	return ret

if __name__ == '__main__':
	generate_documentation()