From 9d40a12f44fbd4b8b46f72a5c642e2e2d4054bc9 Mon Sep 17 00:00:00 2001 From: Joshua Tauberer Date: Sun, 21 Sep 2014 13:43:21 -0400 Subject: [PATCH] first pass at making readable documentation by parsing the bash scripts --- setup/dns.sh | 21 ++-- setup/mail-dovecot.sh | 13 +-- setup/mail-postfix.sh | 68 ++++++------ setup/mail-users.sh | 9 +- setup/ssl.sh | 11 +- setup/system.sh | 71 +++++++----- tools/readable_bash.py | 238 +++++++++++++++++++++++++++++++++++++++++ 7 files changed, 347 insertions(+), 84 deletions(-) create mode 100644 tools/readable_bash.py diff --git a/setup/dns.sh b/setup/dns.sh index c9220a04..d1ca7304 100755 --- a/setup/dns.sh +++ b/setup/dns.sh @@ -1,6 +1,6 @@ #!/bin/bash -# DNS: Configure a DNS server using nsd -####################################### +# DNS: Configure a DNS server to host our own DNS +# ----------------------------------------------- # This script installs packages, but the DNS zone files are only # created by the /dns/update API in the management server because @@ -9,23 +9,23 @@ source setup/functions.sh # load our functions -# Install nsd, our DNS server software, and ldnsutils which helps +# Install `nsd`, our DNS server software, and `ldnsutils` which helps # us sign zones for DNSSEC. # ...but first, we have to create the user because the # current Ubuntu forgets to do so in the .deb # see issue #25 and https://bugs.launchpad.net/ubuntu/+source/nsd/+bug/1311886 if id nsd > /dev/null 2>&1; then - true; #echo "nsd user exists... good"; + true; #echo "nsd user exists... good"; #NODOC else useradd nsd; fi # Okay now install the packages. # -# nsd: The non-recursive nameserver that publishes our DNS records. -# ldnsutils: Helper utilities for signing DNSSEC zones. -# openssh-client: Provides ssh-keyscan which we use to create SSHFP records. +# * nsd: The non-recursive nameserver that publishes our DNS records. +# * ldnsutils: Helper utilities for signing DNSSEC zones. +# * openssh-client: Provides ssh-keyscan which we use to create SSHFP records. apt_install nsd ldnsutils openssh-client @@ -53,9 +53,10 @@ if [ ! -f "$STORAGE_ROOT/dns/dnssec/keys.conf" ]; then ZSK=$(umask 077; cd $STORAGE_ROOT/dns/dnssec; ldns-keygen -a RSASHA1-NSEC3-SHA1 -b 1024 _domain_); # These generate two sets of files like: - # K_domain_.+007+08882.ds <- DS record for adding to NSD configuration files - # K_domain_.+007+08882.key <- public key (goes into DS record & upstream DNS provider like your registrar) - # K_domain_.+007+08882.private <- private key (secret!) + # + # * `K_domain_.+007+08882.ds`: DS record to provide to domain name registrar + # * `K_domain_.+007+08882.key`: public key (goes into DS record & upstream DNS provider like your registrar) + # * `K_domain_.+007+08882.private`: private key (secret!) # The filenames are unpredictable and encode the key generation # options. So we'll store the names of the files we just generated. diff --git a/setup/mail-dovecot.sh b/setup/mail-dovecot.sh index 2ce75fb1..f415d01d 100755 --- a/setup/mail-dovecot.sh +++ b/setup/mail-dovecot.sh @@ -1,6 +1,7 @@ #!/bin/bash # # Dovecot (IMAP and LDA) +# ---------------------- # # Dovecot is *both* the IMAP server (the protocol that email applications # use to query a mailbox) as well as the local delivery agent (LDA), @@ -17,13 +18,13 @@ source setup/functions.sh # load our functions source /etc/mailinabox.conf # load global vars -# Install packages. +# ### Install packages and basic setup apt_install \ dovecot-core dovecot-imapd dovecot-lmtpd dovecot-sqlite sqlite3 \ dovecot-sieve dovecot-managesieved -# The dovecot-imapd dovecot-lmtpd packages automatically enable IMAP and LMTP protocols. +# The dovecot-imapd and dovecot-lmtpd packages automatically enable IMAP and LMTP protocols. # Set the location where we'll store user mailboxes. tools/editconf.py /etc/dovecot/conf.d/10-mail.conf \ @@ -31,7 +32,7 @@ tools/editconf.py /etc/dovecot/conf.d/10-mail.conf \ mail_privileged_group=mail \ first_valid_uid=0 -# IMAP +# ### IMAP # Require that passwords are sent over SSL only, and allow the usual IMAP authentication mechanisms. # The LOGIN mechanism is supposedly for Microsoft products like Outlook to do SMTP login (I guess @@ -62,7 +63,7 @@ sed -i "s/#port = 110/port = 0/" /etc/dovecot/conf.d/10-master.conf tools/editconf.py /etc/dovecot/conf.d/20-imap.conf \ imap_idle_notify_interval="4 mins" -# LDA (LMTP) +# ### LDA (LMTP) # Enable Dovecot's LDA service with the LMTP protocol. It will listen # in port 10026, and Spamassassin will be configured to pass mail there. @@ -94,12 +95,12 @@ EOF tools/editconf.py /etc/dovecot/conf.d/15-lda.conf \ postmaster_address=postmaster@$PRIMARY_HOSTNAME -# SIEVE +# ### Sieve # Enable the Dovecot sieve plugin which let's users run scripts that process # mail as it comes in. We'll also set a global script that moves mail marked # as spam by Spamassassin into the user's Spam folder. -sudo sed -i "s/#mail_plugins = .*/mail_plugins = \$mail_plugins sieve/" /etc/dovecot/conf.d/20-lmtp.conf +sed -i "s/#mail_plugins = .*/mail_plugins = \$mail_plugins sieve/" /etc/dovecot/conf.d/20-lmtp.conf cat > /etc/dovecot/conf.d/99-local-sieve.conf << EOF; plugin { diff --git a/setup/mail-postfix.sh b/setup/mail-postfix.sh index 35567230..7514dba8 100755 --- a/setup/mail-postfix.sh +++ b/setup/mail-postfix.sh @@ -1,6 +1,7 @@ #!/bin/bash # # Postfix (SMTP) +# -------------- # # Postfix handles the transmission of email between servers # using the SMTP protocol. It is a Mail Transfer Agent (MTA). @@ -29,11 +30,11 @@ source setup/functions.sh # load our functions source /etc/mailinabox.conf # load global vars -# Install packages. +# ### Install packages. apt_install postfix postgrey postfix-pcre ca-certificates -# Basic Settings +# ### Basic Settings # Have postfix listen on all network interfaces, set our name (the Debian default seems to be localhost), # and set the name of the local machine to localhost for xxx@localhost mail (but I don't think this will have any effect because @@ -44,15 +45,16 @@ tools/editconf.py /etc/postfix/main.cf \ smtpd_banner="\$myhostname ESMTP Hi, I'm a Mail-in-a-Box (Ubuntu/Postfix; see https://mailinabox.email/)" \ mydestination=localhost -# Outgoing Mail +# ### Outgoing Mail # Enable the 'submission' port 587 smtpd server and tweak its settings. -# a) Require the best ciphers for incoming connections per http://baldric.net/2013/12/07/tls-ciphers-in-postfix-and-dovecot/. -# but without affecting opportunistic TLS on incoming mail, which will allow any cipher (it's better than none). -# b) Give it a different name in syslog to distinguish it from the port 25 smtpd server. -# c) Add a new cleanup service specific to the submission service ('authclean') -# that filters out privacy-sensitive headers on mail being sent out by -# authenticated users. +# +# * Require the best ciphers for incoming connections per http://baldric.net/2013/12/07/tls-ciphers-in-postfix-and-dovecot/. +# but without affecting opportunistic TLS on incoming mail, which will allow any cipher (it's better than none). +# * Give it a different name in syslog to distinguish it from the port 25 smtpd server. +# * Add a new cleanup service specific to the submission service ('authclean') +# that filters out privacy-sensitive headers on mail being sent out by +# authenticated users. tools/editconf.py /etc/postfix/master.cf -s -w \ "submission=inet n - - - - smtpd -o syslog_name=postfix/submission @@ -64,7 +66,7 @@ tools/editconf.py /etc/postfix/master.cf -s -w \ # Install the `outgoing_mail_header_filters` file required by the new 'authclean' service. cp conf/postfix_outgoing_mail_header_filters /etc/postfix/outgoing_mail_header_filters -# Enable TLS on incoming connections (i.e. ports 25 *and* 587) and +# Enable TLS on these and all other connections (i.e. ports 25 *and* 587) and # require TLS before a user is allowed to authenticate. This also makes # opportunistic TLS available on *incoming* mail. tools/editconf.py /etc/postfix/main.cf \ @@ -74,6 +76,19 @@ tools/editconf.py /etc/postfix/main.cf \ smtpd_tls_key_file=$STORAGE_ROOT/ssl/ssl_private_key.pem \ smtpd_tls_received_header=yes +# Prevent non-authenticated users from sending mail that requires being +# relayed elsewhere. We don't want to be an "open relay". On outbound +# mail, require one of: +# +# * permit_sasl_authenticated: Authenticated users (i.e. on port 587). +# * permit_mynetworks: Mail that originates locally. +# * reject_unauth_destination: No one else. (Permits mail whose destination is local and rejects other mail.) +tools/editconf.py /etc/postfix/main.cf \ + smtpd_relay_restrictions=permit_sasl_authenticated,permit_mynetworks,reject_unauth_destination + + +# ### DANE +# # When connecting to remote SMTP servers, prefer TLS and use DANE if available. # # Prefering ("opportunistic") TLS means Postfix will accept whatever SSL certificate the remote @@ -98,38 +113,27 @@ tools/editconf.py /etc/postfix/main.cf \ smtp_tls_CAfile=/etc/ssl/certs/ca-certificates.crt \ smtp_tls_loglevel=2 -# Incoming Mail +# ### Incoming Mail # Pass any incoming mail over to a local delivery agent. Spamassassin # will act as the LDA agent at first. It is listening on port 10025 # with LMTP. Spamassassin will pass the mail over to Dovecot after. # -# In a basic setup we would pass mail directly to Dovecot like so: -# tools/editconf.py /etc/postfix/main.cf virtual_transport=lmtp:unix:private/dovecot-lmtp +# In a basic setup we would pass mail directly to Dovecot by setting +# virtual_transport to `lmtp:unix:private/dovecot-lmtp`. # tools/editconf.py /etc/postfix/main.cf virtual_transport=lmtp:[127.0.0.1]:10025 -# Who can send outbound mail? The purpose of this is to prevent -# non-authenticated users from sending mail that requires being -# relayed elsewhere. We don't want to be an "open relay". -# -# permit_sasl_authenticated: Authenticated users (i.e. on port 587). -# permit_mynetworks: Mail that originates locally. -# reject_unauth_destination: No one else. (Permits mail whose destination is local and rejects other mail.) -tools/editconf.py /etc/postfix/main.cf \ - smtpd_relay_restrictions=permit_sasl_authenticated,permit_mynetworks,reject_unauth_destination - # Who can send mail to us? Some basic filters. # -# reject_non_fqdn_sender: Reject not-nice-looking return paths. -# reject_unknown_sender_domain: Reject return paths with invalid domains. -# reject_rhsbl_sender: Reject return paths that use blacklisted domains. -# -# permit_sasl_authenticated: Authenticated users (i.e. on port 587) can skip further checks. -# permit_mynetworks: Mail that originates locally can skip further checks. -# reject_rbl_client: Reject connections from IP addresses blacklisted in zen.spamhaus.org -# reject_unlisted_recipient: Although Postfix will reject mail to unknown recipients, it's nicer to reject such mail ahead of greylisting rather than after. -# check_policy_service: Apply greylisting using postgrey. +# * reject_non_fqdn_sender: Reject not-nice-looking return paths. +# * reject_unknown_sender_domain: Reject return paths with invalid domains. +# * reject_rhsbl_sender: Reject return paths that use blacklisted domains. +# * permit_sasl_authenticated: Authenticated users (i.e. on port 587) can skip further checks. +# * permit_mynetworks: Mail that originates locally can skip further checks. +# * reject_rbl_client: Reject connections from IP addresses blacklisted in zen.spamhaus.org +# * reject_unlisted_recipient: Although Postfix will reject mail to unknown recipients, it's nicer to reject such mail ahead of greylisting rather than after. +# * check_policy_service: Apply greylisting using postgrey. # # Notes: # permit_dnswl_client can pass through mail from whitelisted IP addresses, which would be good to put before greylisting diff --git a/setup/mail-users.sh b/setup/mail-users.sh index 6e0abcd4..e5289057 100755 --- a/setup/mail-users.sh +++ b/setup/mail-users.sh @@ -1,6 +1,7 @@ #!/bin/bash # # User Authentication and Destination Validation +# ---------------------------------------------- # # This script configures user authentication for Dovecot # and Postfix (which relies on Dovecot) and destination @@ -9,6 +10,8 @@ source setup/functions.sh # load our functions source /etc/mailinabox.conf # load global vars +# ### User and Alias Database + # The database of mail users (i.e. authenticated users, who have mailboxes) # and aliases (forwarders). @@ -21,8 +24,7 @@ if [ ! -f $db_path ]; then echo "CREATE TABLE aliases (id INTEGER PRIMARY KEY AUTOINCREMENT, source TEXT NOT NULL UNIQUE, destination TEXT NOT NULL);" | sqlite3 $db_path; fi -# User Authentication -##################### +# ### User Authentication # Have Dovecot query our database, and not system users, for authentication. sed -i "s/#*\(\!include auth-system.conf.ext\)/#\1/" /etc/dovecot/conf.d/10-auth.conf @@ -68,8 +70,7 @@ tools/editconf.py /etc/postfix/main.cf \ smtpd_sasl_path=private/auth \ smtpd_sasl_auth_enable=yes -# Destination Validation -######################## +# ### Destination Validation # Use a Sqlite3 database to check whether a destination email address exists, # and to perform any email alias rewrites in Postfix. diff --git a/setup/ssl.sh b/setup/ssl.sh index 5c2280c3..d66ac68e 100755 --- a/setup/ssl.sh +++ b/setup/ssl.sh @@ -1,6 +1,7 @@ #!/bin/bash # # SSL Certificate +# --------------- # # Create a self-signed SSL certificate if one has not yet been created. # @@ -21,20 +22,22 @@ source /etc/mailinabox.conf # load global vars apt_install openssl mkdir -p $STORAGE_ROOT/ssl +# Generate a new private key if one doesn't already exist. +# Set the umask so the key file is not world-readable. if [ ! -f $STORAGE_ROOT/ssl/ssl_private_key.pem ]; then - # Generate a new private key if one doesn't already exist. - # Set the umask so the key file is not world-readable. (umask 077; hide_output \ openssl genrsa -out $STORAGE_ROOT/ssl/ssl_private_key.pem 2048) fi + +# Generate a certificate signing request if one doesn't already exist. if [ ! -f $STORAGE_ROOT/ssl/ssl_cert_sign_req.csr ]; then - # Generate a certificate signing request if one doesn't already exist. hide_output \ openssl req -new -key $STORAGE_ROOT/ssl/ssl_private_key.pem -out $STORAGE_ROOT/ssl/ssl_cert_sign_req.csr \ -sha256 -subj "/C=$CSR_COUNTRY/ST=/L=/O=/CN=$PRIMARY_HOSTNAME" fi + +# Generate a SSL certificate by self-signing if a SSL certificate doesn't yet exist. if [ ! -f $STORAGE_ROOT/ssl/ssl_certificate.pem ]; then - # Generate a SSL certificate by self-signing if a SSL certificate doesn't yet exist. hide_output \ openssl x509 -req -days 365 \ -in $STORAGE_ROOT/ssl/ssl_cert_sign_req.csr -signkey $STORAGE_ROOT/ssl/ssl_private_key.pem -out $STORAGE_ROOT/ssl/ssl_certificate.pem diff --git a/setup/system.sh b/setup/system.sh index dd25ab42..92ac4a67 100755 --- a/setup/system.sh +++ b/setup/system.sh @@ -1,6 +1,11 @@ source setup/functions.sh # load our functions -# Base system configuration. +# Base system configuration +# ------------------------- + +# ### Base packages + +# Update system packages: echo Updating system packages... hide_output apt-get update @@ -8,12 +13,12 @@ hide_output apt-get -y upgrade # Install basic utilities. # -# haveged: Provides extra entropy to /dev/random so it doesn't stall +# * haveged: Provides extra entropy to /dev/random so it doesn't stall # when generating random numbers for private keys (e.g. during # ldns-keygen). -# unattended-upgrades: Apt tool to install security updates automatically. -# ntp: keeps the system time correct -# fail2ban: scans log files for repeated failed login attempts and blocks the remote IP at the firewall +# * unattended-upgrades: Apt tool to install security updates automatically. +# * ntp: keeps the system time correct +# * fail2ban: scans log files for repeated failed login attempts and blocks the remote IP at the firewall apt_install python3 python3-dev python3-pip \ wget curl \ @@ -28,16 +33,18 @@ APT::Periodic::Unattended-Upgrade "1"; APT::Periodic::Verbose "1"; EOF -if [ -z "$DISABLE_FIREWALL" ]; then - # Turn on the firewall. First allow incoming SSH, then turn on the firewall. - # Other ports will be opened at the point where we set up those services. - # - # Various virtualized environments like Docker and some VPSs don't provide - # a kernel that supports iptables. To avoid error-like output in these cases, - # let us disable the firewall. +# ### Firewall +# Turn on the firewall. +# +# Various virtualized environments like Docker and some VPSs don't provide #NODOC +# a kernel that supports iptables. To avoid error-like output in these cases, #NODOC +# we skip this if the user sets DISABLE_FIREWALL=1. #NODOC +if [ -z "$DISABLE_FIREWALL" ]; then + # Install `ufw` which provides a simple firewall configuration. apt_install ufw + # Allow incoming connections to SSH. ufw_allow ssh; # ssh might be running on an alternate port. Use sshd -T to dump sshd's @@ -45,34 +52,40 @@ if [ -z "$DISABLE_FIREWALL" ]; then # too. SSH_PORT=$(sshd -T 2>/dev/null | grep "^port " | sed "s/port //") if [ ! -z "$SSH_PORT" ]; then - if [ "$SSH_PORT" != "22" ]; then - echo Opening alternate SSH port $SSH_PORT. - ufw_allow $SSH_PORT; - fi + if [ "$SSH_PORT" != "22" ]; then + + echo Opening alternate SSH port $SSH_PORT. + ufw_allow $SSH_PORT; + + fi fi ufw --force enable; -fi +fi #NODOC -# Resolve DNS using bind9 locally, rather than whatever DNS server is supplied -# by the machine's network configuration. We do this to ensure that DNS queries +# ### Local DNS Service + +# Install a local DNS server, rather than using the DNS server provided by the +# ISP's network configuration. +# +# We do this to ensure that DNS queries # that *we* make (i.e. looking up other external domains) perform DNSSEC checks. # We could use Google's Public DNS, but we don't want to create a dependency on -# Google per our goals of decentralization. bind9, as packaged for Ubuntu, has +# Google per our goals of decentralization. `bind9`, as packaged for Ubuntu, has # DNSSEC enabled by default via "dnssec-validation auto". # -# So we'll be running bind9 bound to 127.0.0.1 for locally-issued DNS queries -# and nsd bound to the public ethernet interface for remote DNS queries asking -# about our domain names. nsd is configured in dns.sh. +# So we'll be running `bind9` bound to 127.0.0.1 for locally-issued DNS queries +# and `nsd` bound to the public ethernet interface for remote DNS queries asking +# about our domain names. `nsd` is configured later. # # About the settings: # -# * RESOLVCONF=yes will have bind9 take over /etc/resolv.conf to tell +# * RESOLVCONF=yes will have `bind9` take over /etc/resolv.conf to tell # local services that DNS queries are handled on localhost. -# * Adding -4 to OPTIONS will have bind9 not listen on IPv6 addresses +# * Adding -4 to OPTIONS will have `bind9` not listen on IPv6 addresses # so that we're sure there's no conflict with nsd, our public domain # name server, on IPV6. -# * The listen-on directive in named.conf.options restricts bind9 to +# * The listen-on directive in named.conf.options restricts `bind9` to # binding to the loopback interface instead of all interfaces. apt_install bind9 resolvconf tools/editconf.py /etc/default/bind9 \ @@ -83,9 +96,11 @@ if ! grep -q "listen-on " /etc/bind/named.conf.options; then sed -i "s/^}/\n\tlisten-on { 127.0.0.1; };\n}/" /etc/bind/named.conf.options fi if [ -f /etc/resolvconf/resolv.conf.d/original ]; then - echo "Archiving old resolv.conf (was /etc/resolvconf/resolv.conf.d/original, now /etc/resolvconf/resolv.conf.original)." - mv /etc/resolvconf/resolv.conf.d/original /etc/resolvconf/resolv.conf.original + echo "Archiving old resolv.conf (was /etc/resolvconf/resolv.conf.d/original, now /etc/resolvconf/resolv.conf.original)." #NODOC + mv /etc/resolvconf/resolv.conf.d/original /etc/resolvconf/resolv.conf.original #NODOC fi +# Restart the DNS services. + restart_service bind9 restart_service resolvconf diff --git a/tools/readable_bash.py b/tools/readable_bash.py new file mode 100644 index 00000000..c5877b40 --- /dev/null +++ b/tools/readable_bash.py @@ -0,0 +1,238 @@ +#!/usr/bin/python3 +# +# Generate documentation for how this machine works by +# parsing our bash scripts! + +import cgi, re +import markdown +from modgrammar import * + +def generate_documentation(): + print(""" + + + + + + + Build Your Own Mail Server From Scratch + + + + + + + +
+
+
+

Build Your Own Mail Server From Scratch

+

Here’s how you can build your own mail server from scratch. This document is generated automatically from our setup script.

+
+ """) + + parser = Source.parser() + for line in open("setup/start.sh"): + try: + fn = parser.parse_string(line).filename() + except: + continue + if fn in ("setup/preflight.sh", "setup/questions.sh", "setup/firstuser.sh", "setup/management.sh"): + continue + + import sys + print(fn, file=sys.stderr) + + print(BashScript.parse(fn)) + + print(""" + + + + +""") + +class HashBang(Grammar): + grammar = (L('#!'), REST_OF_LINE, EOL) + def value(self): + return "" + +def strip_indent(s): + lines = s.split("\n") + min_indent = min(len(re.match(r"\s*", line).group(0)) for line in lines if len(line) > 0) + lines = [line[min_indent:] for line in lines] + return "\n".join(lines) + +class Comment(Grammar): + grammar = ONE_OR_MORE(ZERO_OR_MORE(SPACE), L('#'), REST_OF_LINE, EOL) + def value(self): + if self.string.replace("#", "").strip() == "": + return "\n" + lines = [x[2].string for x in self[0]] + content = "\n".join(lines) + content = strip_indent(content) + return markdown.markdown(content, output_format="html4") + "\n\n" + +FILENAME = WORD('a-z0-9-/.') + +class Source(Grammar): + grammar = ((L('.') | L('source')), L(' '), FILENAME, Comment | EOL) + def filename(self): + return self[2].string.strip() + def value(self): + return BashScript.parse(self.filename()) + +class CatEOF(Grammar): + grammar = (ZERO_OR_MORE(SPACE), L('cat > '), ANY_EXCEPT(WHITESPACE), L(" <<"), OPTIONAL(SPACE), L("EOF;"), EOL, REPEAT(ANY, greedy=False), EOL, L("EOF"), EOL) + def value(self): + return "
" + self[2].string + "
" + cgi.escape(self[7].string) + "
\n" + +class HideOutput(Grammar): + grammar = (L("hide_output "), REF("BashElement")) + def value(self): + return self[1].value() + +class SuppressedLine(Grammar): + grammar = (OPTIONAL(SPACE), L("echo "), REST_OF_LINE, EOL) + def value(self): + if "|" in self.string or ">" in self.string: + return "
" + cgi.escape(self.string) + "
\n" + return "" + +class EditConf(Grammar): + grammar = ( + L('tools/editconf.py '), + FILENAME, + SPACE, + OPTIONAL((LIST_OF( + L("-w") | L("-s"), + sep=SPACE, + ), SPACE)), + REST_OF_LINE, + OPTIONAL(SPACE), + EOL + ) + def value(self): + conffile = self[1] + options = [""] + mode = 1 + for c in self[4].string: + if mode == 1 and c in (" ", "\t") and options[-1] != "": + # new word + options.append("") + elif mode < 0: + # escaped character + options[-1] += c + mode = -mode + elif c == "\\": + # escape next character + mode = -mode + elif mode == 1 and c == '"': + mode = 2 + elif mode == 2 and c == '"': + mode = 1 + else: + options[-1] += c + if options[-1] == "": options.pop(-1) + return "
" + self[1].string + "
" + "\n".join(cgi.escape(s) for s in options) + "
\n" + +class CaptureOutput(Grammar): + grammar = OPTIONAL(SPACE), WORD("A-Za-z_"), L('=$('), REST_OF_LINE, L(")"), OPTIONAL(L(';')), EOL + def value(self): + cmd = self[3].string + cmd = cmd.replace("; ", "\n") + return "
$" + self[1].string + "=
" + cgi.escape(cmd) + "
\n" + +class SedReplace(Grammar): + grammar = OPTIONAL(SPACE), L('sed -i "s/'), OPTIONAL(L('^')), ONE_OR_MORE(WORD("-A-Za-z0-9 #=\\{};.*$_!()")), L('/'), ONE_OR_MORE(WORD("-A-Za-z0-9 #=\\{};.*$_!()")), L('/"'), SPACE, FILENAME, EOL + def value(self): + return "
" + self[8].string + "

replace

" + cgi.escape(self[3].string.replace(".*", ". . .")) + "

with

" + cgi.escape(self[5].string.replace("\\n", "\n").replace("\\t", "\t")) + "
\n" + +class AptGet(Grammar): + grammar = (ZERO_OR_MORE(SPACE), L("apt_install "), REST_OF_LINE, EOL) + def value(self): + return "
" + self[0].string + "apt-get install -y " + cgi.escape(re.sub(r"\s+", " ", self[2].string)) + "
\n" +class UfwAllow(Grammar): + grammar = (ZERO_OR_MORE(SPACE), L("ufw_allow "), REST_OF_LINE, EOL) + def value(self): + return "
" + self[0].string + "ufw allow " + cgi.escape(self[2].string) + "
\n" + +class OtherLine(Grammar): + grammar = (REST_OF_LINE, EOL) + def value(self): + if self.string.strip() == "": return "" + return "
" + cgi.escape(self.string.rstrip()) + "
\n" + +class BashElement(Grammar): + grammar = Comment | Source | CatEOF | SuppressedLine | HideOutput | EditConf | CaptureOutput | SedReplace | AptGet | UfwAllow | OtherLine + def value(self): + return self[0].value() + +class BashScript(Grammar): + grammar = (OPTIONAL(HashBang), REPEAT(BashElement)) + def value(self): + return [line.value() for line in self[1]] + + @staticmethod + def parse(fn): + if fn in ("setup/functions.sh", "/etc/mailinabox.conf"): return "" + parser = BashScript.parser() + string = open(fn).read() + string = re.sub(r"\s*\\\n\s*", " ", string) + string = re.sub(".* #NODOC\n", "", string) + string = re.sub("\n\s*if .*|\n\s*fi|\n\s*else", "", string) + string = re.sub("hide_output ", "", string) + result = parser.parse_string(string) + + v = "
%s
\n" % ("https://github.com/mail-in-a-box/mailinabox/tree/master/" + fn, fn) + v += "".join(result.value()) + + v = v.replace("\n
", "\n")
+		v = re.sub("
([\w\W]*?)
", lambda m : "
" + strip_indent(m.group(1)) + "
", v)
+
+		v = re.sub(r"\$?PRIMARY_HOSTNAME", "box.yourdomain.com", v)
+		v = re.sub(r"\$?STORAGE_ROOT", "/path/to/user-data", v)
+		v = v.replace("`pwd`",  "/path/to/mailinabox")
+
+		return v
+
+if __name__ == '__main__':
+	generate_documentation()