#!/usr/bin/python3 # # Generate documentation for how this machine works by # parsing our bash scripts! import cgi import re import markdown from modgrammar import * def generate_documentation(): print(""" Build Your Own Mail Server From Scratch

Build Your Own Mail Server From Scratch

Here’s how you can build your own mail server from scratch.

This document is generated automatically from Mail-in-a-Box’s setup script source code.

""") parser = Source.parser() for line in open("setup/start.sh"): try: fn = parser.parse_string(line).filename() except: continue if fn in ("setup/start.sh", "setup/preflight.sh", "setup/questions.sh", "setup/firstuser.sh", "setup/management.sh"): continue import sys print(fn, file=sys.stderr) print(BashScript.parse(fn)) print(""" """) class HashBang(Grammar): grammar = (L('#!'), REST_OF_LINE, EOL) def value(self): return "" def strip_indent(s): s = s.replace("\t", " ") lines = s.split("\n") try: min_indent = min(len(re.match(r"\s*", line).group(0)) for line in lines if len(line) > 0) except ValueError: # No non-empty lines. min_indent = 0 lines = [line[min_indent:] for line in lines] return "\n".join(lines) class Comment(Grammar): grammar = ONE_OR_MORE(ZERO_OR_MORE(SPACE), L('#'), REST_OF_LINE, EOL) def value(self): if self.string.replace("#", "").strip() == "": return "\n" lines = [x[2].string for x in self[0]] content = "\n".join(lines) content = strip_indent(content) return markdown.markdown(content, output_format="html4") + "\n\n" FILENAME = WORD('a-z0-9-/.') class Source(Grammar): grammar = ((L('.') | L('source')), L(' '), FILENAME, Comment | EOL) def filename(self): return self[2].string.strip() def value(self): return BashScript.parse(self.filename()) class CatEOF(Grammar): grammar = (ZERO_OR_MORE(SPACE), L('cat '), L('>') | L('>>'), L(' '), ANY_EXCEPT(WHITESPACE), L(" <<"), OPTIONAL(SPACE), L("EOF"), EOL, REPEAT(ANY, greedy=False), EOL, L("EOF"), EOL) def value(self): content = self[9].string # un-escape bash-escaped characters content = re.sub(r"\\([$])", r"\1", content) return "
%s (%s)
%s
\n" \ % (self[4].string, "overwrite" if ">>" not in self[2].string else "append to", cgi.escape(content)) class HideOutput(Grammar): grammar = (L("hide_output "), REF("BashElement")) def value(self): return self[1].value() class EchoLine(Grammar): grammar = (OPTIONAL(SPACE), L("echo "), REST_OF_LINE, EOL) def value(self): if "|" in self.string or ">" in self.string: return "
" + recode_bash(self.string.strip()) + "
\n" return "" class EditConf(Grammar): grammar = ( L('tools/editconf.py '), FILENAME, SPACE, OPTIONAL((LIST_OF( L("-w") | L("-s") | L("-c ;"), sep=SPACE, ), SPACE)), REST_OF_LINE, OPTIONAL(SPACE), EOL ) def value(self): conffile = self[1] options = [] eq = "=" if self[3] and "-s" in self[3].string: eq = " " for opt in re.split("\s+", self[4].string): k, v = opt.split("=", 1) # not sure why newlines are getting doubled v = re.sub(r"\n+", "", fixup_tokens(v)) options.append("%s%s%s" % (k, eq, v)) return "
" + self[1].string + " (change settings)
" + "\n".join(cgi.escape(s) for s in options) + "
\n" class CaptureOutput(Grammar): grammar = OPTIONAL(SPACE), WORD("A-Za-z_"), L('=$('), REST_OF_LINE, L(")"), OPTIONAL(L(';')), EOL def value(self): cmd = self[3].string cmd = cmd.replace("; ", "\n") return "
$" + self[1].string + "=
" + cgi.escape(cmd) + "
\n" class SedReplace(Grammar): grammar = OPTIONAL(SPACE), L('sed -i "s/'), OPTIONAL(L('^')), ONE_OR_MORE(WORD("-A-Za-z0-9 #=\\{};.*$_!()")), L('/'), ONE_OR_MORE(WORD("-A-Za-z0-9 #=\\{};.*$_!()")), L('/"'), SPACE, FILENAME, EOL def value(self): return "
edit
" + self[8].string + "

replace

" + cgi.escape(self[3].string.replace(".*", ". . .")) + "

with

" + cgi.escape(self[5].string.replace("\\n", "\n").replace("\\t", "\t")) + "
\n" class EchoPipe(Grammar): grammar = OPTIONAL(SPACE), L("echo "), REST_OF_LINE, L(' | '), REST_OF_LINE, EOL def value(self): text = " ".join("\"%s\"" % s for s in self[2].string.split(" ")) return "
echo " + recode_bash(text) + " \
 | " + recode_bash(self[4].string) + "
\n" def shell_line(bash): return "
" + recode_bash(bash.strip()) + "
\n" class AptGet(Grammar): grammar = (ZERO_OR_MORE(SPACE), L("apt_install "), REST_OF_LINE, EOL) def value(self): return shell_line("apt-get install -y " + re.sub(r"\s+", " ", self[2].string)) class UfwAllow(Grammar): grammar = (ZERO_OR_MORE(SPACE), L("ufw_allow "), REST_OF_LINE, EOL) def value(self): return shell_line("ufw allow " + self[2].string) class RestartService(Grammar): grammar = (ZERO_OR_MORE(SPACE), L("restart_service "), REST_OF_LINE, EOL) def value(self): return shell_line("service " + self[2].string + " restart") class OtherLine(Grammar): grammar = (REST_OF_LINE, EOL) def value(self): if self.string.strip() == "": return "" if "source setup/functions.sh" in self.string: return "" if "source /etc/mailinabox.conf" in self.string: return "" return "
" + recode_bash(self.string.strip()) + "
\n" class BashElement(Grammar): grammar = Comment | CatEOF | EchoPipe | EchoLine | HideOutput | EditConf | SedReplace | AptGet | UfwAllow | RestartService | OtherLine def value(self): return self[0].value() # Make some special characters to private use Unicode code points. bash_special_characters1 = { "\n": "\uE000", " ": "\uE001", } bash_special_characters2 = { "$": "\uE010", } bash_escapes = { "n": "\uE020", "t": "\uE021", } def quasitokenize(bashscript): # Make a parse of bash easier by making the tokenization easy. newscript = "" quote_mode = None escape_next = False line_comment = False subshell = 0 for c in bashscript: if line_comment: # We're in a comment until the end of the line. newscript += c if c == '\n': line_comment = False elif escape_next: # Previous character was a \. Normally the next character # comes through literally, but escaped newlines are line # continuations and some escapes are for special characters # which we'll recode and then turn back into escapes later. if c == "\n": c = " " elif c in bash_escapes: c = bash_escapes[c] newscript += c escape_next = False elif c == "\\": # Escaping next character. escape_next = True elif quote_mode is None and c in ('"', "'"): # Starting a quoted word. quote_mode = c elif c == quote_mode: # Ending a quoted word. quote_mode = None elif quote_mode is not None and quote_mode != "EOF" and c in bash_special_characters1: # Replace special tokens within quoted words so that they # don't interfere with tokenization later. newscript += bash_special_characters1[c] elif quote_mode is None and c == '#': # Start of a line comment. newscript += c line_comment = True elif quote_mode is None and c == ';' and subshell == 0: # End of a statement. newscript += "\n" elif quote_mode is None and c == '(': # Start of a subshell. newscript += c subshell += 1 elif quote_mode is None and c == ')': # End of a subshell. newscript += c subshell -= 1 elif quote_mode is None and c == '\t': # Make these just spaces. if newscript[-1] != " ": newscript += " " elif quote_mode is None and c == ' ': # Collapse consecutive spaces. if newscript[-1] != " ": newscript += " " elif c in bash_special_characters2: newscript += bash_special_characters2[c] else: # All other characters. newscript += c # "<< EOF" escaping. if quote_mode is None and re.search("<<\s*EOF\n$", newscript): quote_mode = "EOF" elif quote_mode == "EOF" and re.search("\nEOF\n$", newscript): quote_mode = None return newscript def recode_bash(s): def requote(tok): tok = tok.replace("\\", "\\\\") for c in bash_special_characters2: tok = tok.replace(c, "\\" + c) tok = fixup_tokens(tok) if " " in tok or '"' in tok: tok = tok.replace("\"", "\\\"") tok = '"' + tok + '"' else: tok = tok.replace("'", "\\'") return tok return cgi.escape(" ".join(requote(tok) for tok in s.split(" "))) def fixup_tokens(s): for c, enc in bash_special_characters1.items(): s = s.replace(enc, c) for c, enc in bash_special_characters2.items(): s = s.replace(enc, c) for esc, c in bash_escapes.items(): s = s.replace(c, "\\" + esc) return s class BashScript(Grammar): grammar = (OPTIONAL(HashBang), REPEAT(BashElement)) def value(self): return [line.value() for line in self[1]] @staticmethod def parse(fn): if fn in ("setup/functions.sh", "/etc/mailinabox.conf"): return "" string = open(fn).read() # tokenize string = re.sub(".* #NODOC\n", "", string) string = re.sub("\n\s*if .*then.*|\n\s*fi|\n\s*else|\n\s*elif .*", "", string) string = quasitokenize(string) string = re.sub("hide_output ", "", string) parser = BashScript.parser() result = parser.parse_string(string) v = "
view the bash source for the following section at %s
\n" \ % ("https://github.com/mail-in-a-box/mailinabox/tree/master/" + fn, fn) mode = 0 for item in result.value(): if item.strip() == "": pass elif item.startswith("\n
", "")
        v = re.sub("
([\w\W]*?)
", lambda m: "
" + strip_indent(m.group(1)) + "
", v)

        v = re.sub(r"(\$?)PRIMARY_HOSTNAME", r"box.yourdomain.com", v)
        v = re.sub(r"\$STORAGE_ROOT", r"$STORE", v)
        v = re.sub(r"\$CSR_COUNTRY", r"US", v)
        v = v.replace("`pwd`",  "/path/to/mailinabox")

        return v


def wrap_lines(text, cols=60):
    ret = ""
    words = re.split("(\s+)", text)
    linelen = 0
    for w in words:
        if linelen + len(w) > cols-1:
            ret += " \\\n"
            ret += "   "
            linelen = 0
        if linelen == 0 and w.strip() == "":
            continue
        ret += w
        linelen += len(w)
    return ret

if __name__ == '__main__':
    generate_documentation()