2014-09-21 17:43:21 +00:00
#!/usr/bin/python3
#
# Generate documentation for how this machine works by
# parsing our bash scripts!
import cgi , re
import markdown
from modgrammar import *
def generate_documentation ( ) :
print ( """ <!DOCTYPE html>
< html >
< head >
< meta charset = " utf-8 " >
< meta http - equiv = " X-UA-Compatible " content = " IE=edge,chrome=1 " >
< meta name = " viewport " content = " width=device-width " >
< title > Build Your Own Mail Server From Scratch < / title >
< link rel = " stylesheet " href = " https://maxcdn.bootstrapcdn.com/bootstrap/3.2.0/css/bootstrap.min.css " >
< link rel = " stylesheet " href = " https://maxcdn.bootstrapcdn.com/bootstrap/3.2.0/css/bootstrap-theme.min.css " >
< style >
@import url ( https : / / fonts . googleapis . com / css ? family = Iceland ) ;
@import url ( https : / / fonts . googleapis . com / css ? family = Raleway : 400 , 700 ) ;
@import url ( https : / / fonts . googleapis . com / css ? family = Ubuntu : 300 , 500 ) ;
body {
font - family : Raleway , sans - serif ;
font - size : 16 px ;
color : #555;
}
h2 , h3 {
2014-10-04 21:57:26 +00:00
margin - top : .25 em ;
margin - bottom : .75 em ;
2014-09-21 17:43:21 +00:00
}
p {
margin - bottom : 1 em ;
}
2014-10-04 21:57:26 +00:00
. intro p {
margin : 1.5 em 0 ;
}
li {
margin - bottom : .33 em ;
}
. sourcefile {
padding - top : 1.5 em ;
padding - bottom : 1 em ;
font - size : 90 % ;
text - align : right ;
}
. sourcefile a {
color : red ;
}
. instructions . row . contd {
border - top : 1 px solid #E0E0E0;
}
. prose {
2020-06-07 13:47:51 +00:00
padding - top : 1 em ;
2014-10-04 21:57:26 +00:00
padding - bottom : 1 em ;
}
. terminal {
background - color : #EEE;
padding - top : 1 em ;
padding - bottom : 1 em ;
}
2014-09-21 17:43:21 +00:00
2014-10-13 14:00:26 +00:00
ul {
padding - left : 1.25 em ;
}
2014-09-21 17:43:21 +00:00
pre {
color : black ;
2014-10-04 21:57:26 +00:00
border : 0 ;
background : none ;
font - size : 100 % ;
2014-09-21 17:43:21 +00:00
}
div . write - to {
2014-10-04 21:57:26 +00:00
margin : 0 0 1 em .5 em ;
2014-09-21 17:43:21 +00:00
}
div . write - to p {
padding : .5 em ;
margin : 0 ;
}
div . write - to . filename {
2014-10-13 14:00:26 +00:00
padding : .25 em .5 em ;
2014-10-04 21:57:26 +00:00
background - color : #666;
color : white ;
font - family : monospace ;
2014-09-21 17:43:21 +00:00
font - weight : bold ;
}
2014-10-04 21:57:26 +00:00
div . write - to . filename span {
font - family : sans - serif ;
font - weight : normal ;
}
2014-09-21 17:43:21 +00:00
div . write - to pre {
margin : 0 ;
2014-10-13 14:00:26 +00:00
padding : .5 em ;
2014-10-04 21:57:26 +00:00
border : 1 px solid #999;
border - radius : 0 ;
font - size : 90 % ;
2014-09-21 17:43:21 +00:00
}
2014-09-21 20:05:11 +00:00
pre . shell > div : before {
content : " $ " ;
color : #666;
}
2014-09-21 17:43:21 +00:00
< / style >
< / head >
< body >
< div class = " container " >
2014-10-04 21:57:26 +00:00
< div class = " row intro " >
2014-09-21 17:43:21 +00:00
< div class = " col-xs-12 " >
< h1 > Build Your Own Mail Server From Scratch < / h1 >
2014-10-04 21:57:26 +00:00
< p > Here & rsquo ; s how you can build your own mail server from scratch . < / p >
< p > This document is generated automatically from < a href = " https://mailinabox.email " > Mail - in - a - Box < / a > & rsquo ; s setup script < a href = " https://github.com/mail-in-a-box/mailinabox " > source code < / a > . < / p >
2014-09-21 17:43:21 +00:00
< hr >
2014-10-04 21:57:26 +00:00
< / div >
< / div >
< div class = " container instructions " >
2014-09-21 17:43:21 +00:00
""" )
parser = Source . parser ( )
2025-01-08 13:14:02 +00:00
with open ( " setup/start.sh " ) as start_file :
2024-01-27 16:32:32 +00:00
for line in start_file :
try :
fn = parser . parse_string ( line ) . filename ( )
except :
continue
if fn in ( " setup/start.sh " , " setup/preflight.sh " , " setup/questions.sh " , " setup/firstuser.sh " , " setup/management.sh " ) :
continue
2014-09-21 17:43:21 +00:00
import sys
print ( fn , file = sys . stderr )
print ( BashScript . parse ( fn ) )
print ( """
< script src = " https://ajax.googleapis.com/ajax/libs/jquery/1.10.1/jquery.min.js " > < / script >
< script src = " https://maxcdn.bootstrapcdn.com/bootstrap/3.2.0/js/bootstrap.min.js " > < / script >
2014-10-04 21:57:26 +00:00
< script >
$ ( function ( ) {
$ ( ' .terminal ' ) . each ( function ( ) {
$ ( this ) . outerHeight ( $ ( this ) . parent ( ) . innerHeight ( ) ) ;
} ) ;
} )
< / script >
2014-09-21 17:43:21 +00:00
< / body >
< / html >
""" )
class HashBang ( Grammar ) :
grammar = ( L ( ' #! ' ) , REST_OF_LINE , EOL )
def value ( self ) :
return " "
def strip_indent ( s ) :
2014-10-04 21:57:26 +00:00
s = s . replace ( " \t " , " " )
2014-09-21 17:43:21 +00:00
lines = s . split ( " \n " )
2014-10-04 21:57:26 +00:00
try :
min_indent = min ( len ( re . match ( r " \ s* " , line ) . group ( 0 ) ) for line in lines if len ( line ) > 0 )
except ValueError :
# No non-empty lines.
min_indent = 0
2014-09-21 17:43:21 +00:00
lines = [ line [ min_indent : ] for line in lines ]
return " \n " . join ( lines )
class Comment ( Grammar ) :
grammar = ONE_OR_MORE ( ZERO_OR_MORE ( SPACE ) , L ( ' # ' ) , REST_OF_LINE , EOL )
def value ( self ) :
if self . string . replace ( " # " , " " ) . strip ( ) == " " :
return " \n "
lines = [ x [ 2 ] . string for x in self [ 0 ] ]
content = " \n " . join ( lines )
content = strip_indent ( content )
return markdown . markdown ( content , output_format = " html4 " ) + " \n \n "
FILENAME = WORD ( ' a-z0-9-/. ' )
class Source ( Grammar ) :
grammar = ( ( L ( ' . ' ) | L ( ' source ' ) ) , L ( ' ' ) , FILENAME , Comment | EOL )
def filename ( self ) :
return self [ 2 ] . string . strip ( )
def value ( self ) :
return BashScript . parse ( self . filename ( ) )
class CatEOF ( Grammar ) :
2014-10-04 21:57:26 +00:00
grammar = ( ZERO_OR_MORE ( SPACE ) , L ( ' cat ' ) , L ( ' > ' ) | L ( ' >> ' ) , L ( ' ' ) , ANY_EXCEPT ( WHITESPACE ) , L ( " << " ) , OPTIONAL ( SPACE ) , L ( " EOF " ) , EOL , REPEAT ( ANY , greedy = False ) , EOL , L ( " EOF " ) , EOL )
2014-09-21 17:43:21 +00:00
def value ( self ) :
2014-10-04 21:57:26 +00:00
content = self [ 9 ] . string
2014-09-21 20:05:11 +00:00
content = re . sub ( r " \\ ([$]) " , r " \ 1 " , content ) # un-escape bash-escaped characters
2025-01-08 13:09:05 +00:00
return " <div class= ' write-to ' ><div class= ' filename ' > {} <span>( {} )</span></div><pre> {} </pre></div> \n " . format ( self [ 4 ] . string ,
2014-10-04 21:57:26 +00:00
" overwrite " if " >> " not in self [ 2 ] . string else " append to " ,
cgi . escape ( content ) )
2014-09-21 17:43:21 +00:00
class HideOutput ( Grammar ) :
grammar = ( L ( " hide_output " ) , REF ( " BashElement " ) )
def value ( self ) :
return self [ 1 ] . value ( )
2014-10-13 14:00:26 +00:00
class EchoLine ( Grammar ) :
2014-09-21 17:43:21 +00:00
grammar = ( OPTIONAL ( SPACE ) , L ( " echo " ) , REST_OF_LINE , EOL )
def value ( self ) :
if " | " in self . string or " > " in self . string :
2014-10-13 14:00:26 +00:00
return " <pre class= ' shell ' ><div> " + recode_bash ( self . string . strip ( ) ) + " </div></pre> \n "
2014-09-21 17:43:21 +00:00
return " "
class EditConf ( Grammar ) :
grammar = (
L ( ' tools/editconf.py ' ) ,
FILENAME ,
SPACE ,
OPTIONAL ( ( LIST_OF (
2014-10-04 21:57:26 +00:00
L ( " -w " ) | L ( " -s " ) | L ( " -c ; " ) ,
2014-09-21 17:43:21 +00:00
sep = SPACE ,
) , SPACE ) ) ,
REST_OF_LINE ,
OPTIONAL ( SPACE ) ,
EOL
)
def value ( self ) :
conffile = self [ 1 ]
2014-10-04 21:57:26 +00:00
options = [ ]
eq = " = "
if self [ 3 ] and " -s " in self [ 3 ] . string : eq = " "
for opt in re . split ( " \ s+ " , self [ 4 ] . string ) :
k , v = opt . split ( " = " , 1 )
v = re . sub ( r " \ n+ " , " " , fixup_tokens ( v ) ) # not sure why newlines are getting doubled
2025-01-08 13:13:33 +00:00
options . append ( f " { k } { eq } { v } " )
2014-10-04 21:57:26 +00:00
return " <div class= ' write-to ' ><div class= ' filename ' > " + self [ 1 ] . string + " <span>(change settings)</span></div><pre> " + " \n " . join ( cgi . escape ( s ) for s in options ) + " </pre></div> \n "
2014-09-21 17:43:21 +00:00
class CaptureOutput ( Grammar ) :
grammar = OPTIONAL ( SPACE ) , WORD ( " A-Za-z_ " ) , L ( ' =$( ' ) , REST_OF_LINE , L ( " ) " ) , OPTIONAL ( L ( ' ; ' ) ) , EOL
def value ( self ) :
cmd = self [ 3 ] . string
cmd = cmd . replace ( " ; " , " \n " )
return " <div class= ' write-to ' ><div class= ' filename ' >$ " + self [ 1 ] . string + " =</div><pre> " + cgi . escape ( cmd ) + " </pre></div> \n "
class SedReplace ( Grammar ) :
grammar = OPTIONAL ( SPACE ) , L ( ' sed -i " s/ ' ) , OPTIONAL ( L ( ' ^ ' ) ) , ONE_OR_MORE ( WORD ( " -A-Za-z0-9 #= \\ {} ;.*$_!() " ) ) , L ( ' / ' ) , ONE_OR_MORE ( WORD ( " -A-Za-z0-9 #= \\ {} ;.*$_!() " ) ) , L ( ' / " ' ) , SPACE , FILENAME , EOL
def value ( self ) :
2014-09-21 20:05:11 +00:00
return " <div class= ' write-to ' ><div class= ' filename ' >edit<br> " + self [ 8 ] . string + " </div><p>replace</p><pre> " + cgi . escape ( self [ 3 ] . string . replace ( " .* " , " . . . " ) ) + " </pre><p>with</p><pre> " + cgi . escape ( self [ 5 ] . string . replace ( " \\ n " , " \n " ) . replace ( " \\ t " , " \t " ) ) + " </pre></div> \n "
2014-10-04 21:57:26 +00:00
class EchoPipe ( Grammar ) :
grammar = OPTIONAL ( SPACE ) , L ( " echo " ) , REST_OF_LINE , L ( ' | ' ) , REST_OF_LINE , EOL
def value ( self ) :
2025-01-08 13:13:33 +00:00
text = " " . join ( f " \" { s } \" " for s in self [ 2 ] . string . split ( " " ) )
2014-10-13 14:00:26 +00:00
return " <pre class= ' shell ' ><div>echo " + recode_bash ( text ) + " \ <br> | " + recode_bash ( self [ 4 ] . string ) + " </div></pre> \n "
2014-10-04 21:57:26 +00:00
2014-09-21 20:05:11 +00:00
def shell_line ( bash ) :
2014-10-13 14:00:26 +00:00
return " <pre class= ' shell ' ><div> " + recode_bash ( bash . strip ( ) ) + " </div></pre> \n "
2014-09-21 17:43:21 +00:00
class AptGet ( Grammar ) :
grammar = ( ZERO_OR_MORE ( SPACE ) , L ( " apt_install " ) , REST_OF_LINE , EOL )
def value ( self ) :
2014-09-21 20:05:11 +00:00
return shell_line ( " apt-get install -y " + re . sub ( r " \ s+ " , " " , self [ 2 ] . string ) )
2014-09-21 17:43:21 +00:00
class UfwAllow ( Grammar ) :
grammar = ( ZERO_OR_MORE ( SPACE ) , L ( " ufw_allow " ) , REST_OF_LINE , EOL )
def value ( self ) :
2014-09-21 20:05:11 +00:00
return shell_line ( " ufw allow " + self [ 2 ] . string )
2020-06-07 13:47:51 +00:00
class UfwLimit ( Grammar ) :
grammar = ( ZERO_OR_MORE ( SPACE ) , L ( " ufw_limit " ) , REST_OF_LINE , EOL )
def value ( self ) :
return shell_line ( " ufw limit " + self [ 2 ] . string )
2014-09-21 20:05:11 +00:00
class RestartService ( Grammar ) :
grammar = ( ZERO_OR_MORE ( SPACE ) , L ( " restart_service " ) , REST_OF_LINE , EOL )
def value ( self ) :
return shell_line ( " service " + self [ 2 ] . string + " restart " )
2014-09-21 17:43:21 +00:00
class OtherLine ( Grammar ) :
grammar = ( REST_OF_LINE , EOL )
def value ( self ) :
if self . string . strip ( ) == " " : return " "
2014-10-04 21:57:26 +00:00
if " source setup/functions.sh " in self . string : return " "
if " source /etc/mailinabox.conf " in self . string : return " "
2014-10-13 14:00:26 +00:00
return " <pre class= ' shell ' ><div> " + recode_bash ( self . string . strip ( ) ) + " </div></pre> \n "
2014-09-21 17:43:21 +00:00
class BashElement ( Grammar ) :
2020-06-07 13:47:51 +00:00
grammar = Comment | CatEOF | EchoPipe | EchoLine | HideOutput | EditConf | SedReplace | AptGet | UfwAllow | UfwLimit | RestartService | OtherLine
2014-09-21 17:43:21 +00:00
def value ( self ) :
return self [ 0 ] . value ( )
2014-10-04 21:57:26 +00:00
# Make some special characters to private use Unicode code points.
2014-10-13 14:00:26 +00:00
bash_special_characters1 = {
2014-10-04 21:57:26 +00:00
" \n " : " \uE000 " ,
" " : " \uE001 " ,
}
2014-10-13 14:00:26 +00:00
bash_special_characters2 = {
" $ " : " \uE010 " ,
}
bash_escapes = {
" n " : " \uE020 " ,
" t " : " \uE021 " ,
}
2014-10-04 21:57:26 +00:00
def quasitokenize ( bashscript ) :
# Make a parse of bash easier by making the tokenization easy.
newscript = " "
quote_mode = None
escape_next = False
line_comment = False
subshell = 0
for c in bashscript :
if line_comment :
# We're in a comment until the end of the line.
newscript + = c
if c == ' \n ' :
line_comment = False
elif escape_next :
# Previous character was a \. Normally the next character
# comes through literally, but escaped newlines are line
2014-10-13 14:00:26 +00:00
# continuations and some escapes are for special characters
# which we'll recode and then turn back into escapes later.
2014-10-04 21:57:26 +00:00
if c == " \n " :
c = " "
2014-10-13 14:00:26 +00:00
elif c in bash_escapes :
c = bash_escapes [ c ]
newscript + = c
2014-10-04 21:57:26 +00:00
escape_next = False
elif c == " \\ " :
# Escaping next character.
escape_next = True
elif quote_mode is None and c in ( ' " ' , " ' " ) :
# Starting a quoted word.
quote_mode = c
elif c == quote_mode :
# Ending a quoted word.
quote_mode = None
2014-10-13 14:00:26 +00:00
elif quote_mode is not None and quote_mode != " EOF " and c in bash_special_characters1 :
2014-10-04 21:57:26 +00:00
# Replace special tokens within quoted words so that they
# don't interfere with tokenization later.
2014-10-13 14:00:26 +00:00
newscript + = bash_special_characters1 [ c ]
2014-10-04 21:57:26 +00:00
elif quote_mode is None and c == ' # ' :
# Start of a line comment.
newscript + = c
line_comment = True
elif quote_mode is None and c == ' ; ' and subshell == 0 :
# End of a statement.
newscript + = " \n "
elif quote_mode is None and c == ' ( ' :
# Start of a subshell.
newscript + = c
subshell + = 1
elif quote_mode is None and c == ' ) ' :
# End of a subshell.
newscript + = c
subshell - = 1
elif quote_mode is None and c == ' \t ' :
# Make these just spaces.
if newscript [ - 1 ] != " " :
newscript + = " "
2014-10-13 14:00:26 +00:00
elif quote_mode is None and c == ' ' :
# Collapse consecutive spaces.
if newscript [ - 1 ] != " " :
newscript + = " "
elif c in bash_special_characters2 :
newscript + = bash_special_characters2 [ c ]
2014-10-04 21:57:26 +00:00
else :
# All other characters.
newscript + = c
# "<< EOF" escaping.
if quote_mode is None and re . search ( " << \ s*EOF \n $ " , newscript ) :
quote_mode = " EOF "
elif quote_mode == " EOF " and re . search ( " \n EOF \n $ " , newscript ) :
quote_mode = None
return newscript
2014-10-13 14:00:26 +00:00
def recode_bash ( s ) :
def requote ( tok ) :
tok = tok . replace ( " \\ " , " \\ \\ " )
for c in bash_special_characters2 :
tok = tok . replace ( c , " \\ " + c )
tok = fixup_tokens ( tok )
if " " in tok or ' " ' in tok :
tok = tok . replace ( " \" " , " \\ \" " )
tok = ' " ' + tok + ' " '
else :
tok = tok . replace ( " ' " , " \\ ' " )
return tok
return cgi . escape ( " " . join ( requote ( tok ) for tok in s . split ( " " ) ) )
2014-10-04 21:57:26 +00:00
def fixup_tokens ( s ) :
2014-10-13 14:00:26 +00:00
for c , enc in bash_special_characters1 . items ( ) :
s = s . replace ( enc , c )
for c , enc in bash_special_characters2 . items ( ) :
2014-10-04 21:57:26 +00:00
s = s . replace ( enc , c )
2014-10-13 14:00:26 +00:00
for esc , c in bash_escapes . items ( ) :
s = s . replace ( c , " \\ " + esc )
2014-10-04 21:57:26 +00:00
return s
2014-09-21 17:43:21 +00:00
class BashScript ( Grammar ) :
grammar = ( OPTIONAL ( HashBang ) , REPEAT ( BashElement ) )
def value ( self ) :
return [ line . value ( ) for line in self [ 1 ] ]
@staticmethod
def parse ( fn ) :
if fn in ( " setup/functions.sh " , " /etc/mailinabox.conf " ) : return " "
2025-01-08 13:14:02 +00:00
with open ( fn ) as f :
2023-01-15 13:28:43 +00:00
string = f . read ( )
2014-10-04 21:57:26 +00:00
# tokenize
2014-09-21 17:43:21 +00:00
string = re . sub ( " .* #NODOC \n " , " " , string )
2014-10-13 14:00:26 +00:00
string = re . sub ( " \n \ s*if .*then.*| \n \ s*fi| \n \ s*else| \n \ s*elif .* " , " " , string )
2014-10-04 21:57:26 +00:00
string = quasitokenize ( string )
2025-01-08 13:12:17 +00:00
string = re . sub ( r " hide_output " , " " , string )
2014-10-04 21:57:26 +00:00
parser = BashScript . parser ( )
2014-09-21 17:43:21 +00:00
result = parser . parse_string ( string )
2014-10-04 21:57:26 +00:00
2025-01-08 13:09:05 +00:00
v = " <div class= ' row ' ><div class= ' col-xs-12 sourcefile ' >view the bash source for the following section at <a href= \" {} \" > {} </a></div></div> \n " . format ( " https://github.com/mail-in-a-box/mailinabox/tree/master/ " + fn , fn )
2014-10-04 21:57:26 +00:00
mode = 0
for item in result . value ( ) :
if item . strip ( ) == " " :
pass
elif item . startswith ( " <p " ) and not item . startswith ( " <pre " ) :
clz = " "
if mode == 2 :
v + = " </div> \n " # col
v + = " </div> \n " # row
mode = 0
clz = " contd "
if mode == 0 :
2025-01-08 13:13:33 +00:00
v + = f " <div class= ' row { clz } ' > \n "
2014-10-04 21:57:26 +00:00
v + = " <div class= ' col-md-6 prose ' > \n "
v + = item
mode = 1
elif item . startswith ( " <h " ) :
if mode != 0 :
v + = " </div> \n " # col
v + = " </div> \n " # row
v + = " <div class= ' row ' > \n "
v + = " <div class= ' col-md-6 header ' > \n "
v + = item
v + = " </div> \n " # col
v + = " <div class= ' col-md-6 terminal ' > </div> \n "
v + = " </div> \n " # row
mode = 0
else :
if mode == 0 :
v + = " <div class= ' row ' > \n "
v + = " <div class= ' col-md-offset-6 col-md-6 terminal ' > \n "
elif mode == 1 :
v + = " </div> \n "
v + = " <div class= ' col-md-6 terminal ' > \n "
mode = 2
v + = item
v + = " </div> \n " # col
v + = " </div> \n " # row
v = fixup_tokens ( v )
2014-09-21 17:43:21 +00:00
2014-09-21 20:05:11 +00:00
v = v . replace ( " </pre> \n <pre class= ' shell ' > " , " " )
2014-09-21 17:43:21 +00:00
v = re . sub ( " <pre>([ \ w \ W]*?)</pre> " , lambda m : " <pre> " + strip_indent ( m . group ( 1 ) ) + " </pre> " , v )
2014-10-04 21:57:26 +00:00
v = re . sub ( r " ( \ $?)PRIMARY_HOSTNAME " , r " <b>box.yourdomain.com</b> " , v )
v = re . sub ( r " \ $STORAGE_ROOT " , r " <b>$STORE</b> " , v )
2014-09-21 17:43:21 +00:00
v = v . replace ( " `pwd` " , " <code><b>/path/to/mailinabox</b></code> " )
return v
2014-09-21 20:05:11 +00:00
def wrap_lines ( text , cols = 60 ) :
ret = " "
words = re . split ( " ( \ s+) " , text )
linelen = 0
for w in words :
if linelen + len ( w ) > cols - 1 :
ret + = " \\ \n "
ret + = " "
linelen = 0
if linelen == 0 and w . strip ( ) == " " : continue
ret + = w
linelen + = len ( w )
return ret
2014-09-21 17:43:21 +00:00
if __name__ == ' __main__ ' :
generate_documentation ( )
