From 5fd107cae5f8cde29c076c2ed5c3c754c40438a6 Mon Sep 17 00:00:00 2001 From: Joshua Tauberer Date: Sat, 4 Oct 2014 17:57:26 -0400 Subject: [PATCH] more work on making the bash scripts readable --- setup/dkim.sh | 21 ++-- setup/dns.sh | 31 +++-- setup/mail-dovecot.sh | 38 +++--- setup/mail-postfix.sh | 46 ++++--- setup/mail-users.sh | 1 - setup/spamassassin.sh | 14 +-- setup/ssl.sh | 14 ++- setup/system.sh | 20 ++- setup/web.sh | 30 +++-- setup/webmail.sh | 10 +- tools/readable_bash.py | 272 ++++++++++++++++++++++++++++++++++------- 11 files changed, 354 insertions(+), 143 deletions(-) diff --git a/setup/dkim.sh b/setup/dkim.sh index 06aa5ece..e3e24195 100644 --- a/setup/dkim.sh +++ b/setup/dkim.sh @@ -1,12 +1,13 @@ -# OpenDKIM: Sign outgoing mail with DKIM -######################################## - -# After this, you'll still need to run dns_update.sh to get the DKIM -# signature in the DNS zones. +# OpenDKIM +# ======== +# +# OpenDKIM provides a service that puts a DKIM signature on outbound mail. +# +# The DNS configuration for DKIM is done in the management daemon. source setup/functions.sh # load our functions -# Install DKIM +# Install DKIM... apt_install opendkim opendkim-tools # Make sure configuration directories exist. @@ -18,9 +19,9 @@ mkdir -p $STORAGE_ROOT/mail/dkim echo "127.0.0.1" > /etc/opendkim/TrustedHosts if grep -q "ExternalIgnoreList" /etc/opendkim.conf; then - true; # already done + true # already done #NODOC else - # Add various configuration options to the end. + # Add various configuration options to the end of `opendkim.conf`. cat >> /etc/opendkim.conf << EOF; MinimumKeyBits 1024 ExternalIgnoreList refile:/etc/opendkim/TrustedHosts @@ -32,7 +33,7 @@ RequireSafeKeys false EOF fi -# Create a new DKIM key if we don't have one already. This creates +# Create a new DKIM key. This creates # mail.private and mail.txt in $STORAGE_ROOT/mail/dkim. The former # is the actual private key and the latter is the suggested DNS TXT # entry which we'll want to include in our DNS setup. @@ -47,7 +48,7 @@ chmod go-rwx $STORAGE_ROOT/mail/dkim # Add OpenDKIM as a milter to postfix, which is how it intercepts outgoing # mail to perform the signing (by adding a mail header). -# Be careful. If we add other milters later, it needs to be concatenated on the smtpd_milters line. +# Be careful. If we add other milters later, it needs to be concatenated on the smtpd_milters line. #NODOC tools/editconf.py /etc/postfix/main.cf \ smtpd_milters=inet:127.0.0.1:8891 \ non_smtpd_milters=\$smtpd_milters \ diff --git a/setup/dns.sh b/setup/dns.sh index 8b49482c..e92a5b82 100755 --- a/setup/dns.sh +++ b/setup/dns.sh @@ -1,5 +1,5 @@ #!/bin/bash -# DNS: Configure a DNS server to host our own DNS +# DNS # ----------------------------------------------- # This script installs packages, but the DNS zone files are only @@ -14,9 +14,9 @@ source setup/functions.sh # load our functions # ...but first, we have to create the user because the # current Ubuntu forgets to do so in the .deb -# see issue #25 and https://bugs.launchpad.net/ubuntu/+source/nsd/+bug/1311886 +# (see issue #25 and https://bugs.launchpad.net/ubuntu/+source/nsd/+bug/1311886) if id nsd > /dev/null 2>&1; then - true; #echo "nsd user exists... good"; #NODOC + true #echo "nsd user exists... good"; #NODOC else useradd nsd; fi @@ -40,17 +40,21 @@ mkdir -p "$STORAGE_ROOT/dns/dnssec"; # TLDs don't all support the same algorithms, so we'll generate keys using a few # different algorithms. # -# Supports RSASHA1-NSEC3-SHA1 (didn't test with RSASHA256): -# .info and .me. +# Supports `RSASHA1-NSEC3-SHA1` (didn't test with `RSASHA256`): # -# Requires RSASHA256 -# .email -FIRST=1 +# * .info +# * .me +# +# Requires `RSASHA256` +# +# * .email + +FIRST=1 #NODOC for algo in RSASHA1-NSEC3-SHA1 RSASHA256; do if [ ! -f "$STORAGE_ROOT/dns/dnssec/$algo.conf" ]; then if [ $FIRST == 1 ]; then echo "Generating DNSSEC signing keys. This may take a few minutes..." - FIRST=0 + FIRST=0 #NODOC fi # Create the Key-Signing Key (KSK) (-k) which is the so-called @@ -58,6 +62,9 @@ if [ ! -f "$STORAGE_ROOT/dns/dnssec/$algo.conf" ]; then # practice), and a nice and long keylength. The domain name we # provide ("_domain_") doesn't matter -- we'll use the same # keys for all our domains. + # + # `ldns-keygen` outputs the new key's filename to stdout, which + # we're capturing into the `KSK` variable. KSK=$(umask 077; cd $STORAGE_ROOT/dns/dnssec; ldns-keygen -a $algo -b 2048 -k _domain_); # Now create a Zone-Signing Key (ZSK) which is expected to be @@ -81,9 +88,13 @@ KSK=$KSK ZSK=$ZSK EOF fi + + # And loop to do the next algorithm... done -# Force the dns_update script to be run every day to re-sign zones for DNSSEC. +# Force the dns_update script to be run every day to re-sign zones for DNSSEC +# before they expire. When we sign zones (in `dns_update.py`) we specify a +# 30-day validation window, so we had better re-sign before then. cat > /etc/cron.daily/mailinabox-dnssec << EOF; #!/bin/bash # Mail-in-a-Box diff --git a/setup/mail-dovecot.sh b/setup/mail-dovecot.sh index f415d01d..c45e2c4a 100755 --- a/setup/mail-dovecot.sh +++ b/setup/mail-dovecot.sh @@ -18,15 +18,17 @@ source setup/functions.sh # load our functions source /etc/mailinabox.conf # load global vars -# ### Install packages and basic setup +# Install packages... apt_install \ dovecot-core dovecot-imapd dovecot-lmtpd dovecot-sqlite sqlite3 \ dovecot-sieve dovecot-managesieved -# The dovecot-imapd and dovecot-lmtpd packages automatically enable IMAP and LMTP protocols. +# The `dovecot-imapd` and `dovecot-lmtpd` packages automatically enable IMAP and LMTP protocols. -# Set the location where we'll store user mailboxes. +# Set the location where we'll store user mailboxes. '%d' is the domain name and '%n' is the +# username part of the user's email address. We'll ensure that no bad domains or email addresses +# are created within the management daemon. tools/editconf.py /etc/dovecot/conf.d/10-mail.conf \ mail_location=maildir:$STORAGE_ROOT/mail/mailboxes/%d/%n \ mail_privileged_group=mail \ @@ -66,7 +68,7 @@ tools/editconf.py /etc/dovecot/conf.d/20-imap.conf \ # ### LDA (LMTP) # Enable Dovecot's LDA service with the LMTP protocol. It will listen -# in port 10026, and Spamassassin will be configured to pass mail there. +# on port 10026, and Spamassassin will be configured to pass mail there. # # The disabled unix socket listener is normally how Postfix and Dovecot # would communicate (see the Postfix setup script for the corresponding @@ -91,30 +93,32 @@ protocol imap { } EOF -# Setting a postmaster_address seems to be required or LMTP won't start. +# Setting a `postmaster_address` is required or LMTP won't start. An alias +# will be created automatically by our management daemon. tools/editconf.py /etc/dovecot/conf.d/15-lda.conf \ postmaster_address=postmaster@$PRIMARY_HOSTNAME # ### Sieve # Enable the Dovecot sieve plugin which let's users run scripts that process -# mail as it comes in. We'll also set a global script that moves mail marked -# as spam by Spamassassin into the user's Spam folder. +# mail as it comes in. sed -i "s/#mail_plugins = .*/mail_plugins = \$mail_plugins sieve/" /etc/dovecot/conf.d/20-lmtp.conf +# Configure sieve. We'll create a global script that moves mail marked +# as spam by Spamassassin into the user's Spam folder. +# +# * `sieve_before`: The path to our global sieve which handles moving spam to the Spam folder. +# +# * `sieve`: The path to the user's main active script. ManageSieve will create a symbolic +# link here to the actual sieve script. It should not be in the mailbox directory +# (because then it might appear as a folder) and it should not be in the sieve_dir +# (because then I suppose it might appear to the user as one of their scripts). +# * `sieve_dir`: Directory for :personal include scripts for the include extension. This +# is also where the ManageSieve service stores the user's scripts. cat > /etc/dovecot/conf.d/99-local-sieve.conf << EOF; plugin { - # The path to our global sieve which handles moving spam to the Spam folder. sieve_before = /etc/dovecot/sieve-spam.sieve - - # The path to the user's main active script. ManageSieve will create a symbolic - # link here to the actual sieve script. It should not be in the mailbox directory - # (because then it might appear as a folder) and it should not be in the sieve_dir - # (because then I suppose it might appear to the user as one of their scripts). sieve = $STORAGE_ROOT/mail/sieve/%d/%n.sieve - - # Directory for :personal include scripts for the include extension. This - # is also where the ManageSieve service stores the user's scripts. sieve_dir = $STORAGE_ROOT/mail/sieve/%d/%n } EOF @@ -122,7 +126,7 @@ EOF # Copy the global sieve script into where we've told Dovecot to look for it. Then # compile it. Global scripts must be compiled now because Dovecot won't have # permission later. -cp `pwd`/conf/sieve-spam.txt /etc/dovecot/sieve-spam.sieve +cp conf/sieve-spam.txt /etc/dovecot/sieve-spam.sieve sievec /etc/dovecot/sieve-spam.sieve # PERMISSIONS diff --git a/setup/mail-postfix.sh b/setup/mail-postfix.sh index 5aa8b903..c189deff 100755 --- a/setup/mail-postfix.sh +++ b/setup/mail-postfix.sh @@ -32,13 +32,26 @@ source /etc/mailinabox.conf # load global vars # ### Install packages. -apt_install postfix postgrey postfix-pcre ca-certificates +# Install postfix's packages. +# +# * `postfix`: The SMTP server. +# * `postfix-pcre`: Enables header filtering. +# * `postgrey`: A mail policy service that soft-rejects mail the first time +# it is received. Spammers don't usually try agian. Legitimate mail +# always will. +# * `ca-certificates`: A trust store used to squelch postfix warnings about +# untrusted opportunistically-encrypted connections. + +apt_install postfix postfix-pcre postgrey ca-certificates # ### Basic Settings -# Have postfix listen on all network interfaces, set our name (the Debian default seems to be localhost), -# and set the name of the local machine to localhost for xxx@localhost mail (but I don't think this will have any effect because -# there is no true local mail delivery). Also set the banner (must have the hostname first, then anything). +# Set some basic settings... +# +# * Have postfix listen on all network interfaces. +# * Set our name (the Debian default seems to be "localhost" but make it our hostname). +# * Set the name of the local machine to localhost, which means xxx@localhost is delivered locally, although we don't use it. +# * Set the SMTP banner (which must have the hostname first, then anything). tools/editconf.py /etc/postfix/main.cf \ inet_interfaces=all \ myhostname=$PRIMARY_HOSTNAME\ @@ -69,7 +82,8 @@ cp conf/postfix_outgoing_mail_header_filters /etc/postfix/outgoing_mail_header_f # Enable TLS on these and all other connections (i.e. ports 25 *and* 587) and # require TLS before a user is allowed to authenticate. This also makes # opportunistic TLS available on *incoming* mail. -# Set stronger DH parameters, which via openssl tend to default to 1024 bits. +# Set stronger DH parameters, which via openssl tend to default to 1024 bits +# (see ssl.sh). tools/editconf.py /etc/postfix/main.cf \ smtpd_tls_security_level=may\ smtpd_tls_auth_only=yes \ @@ -90,25 +104,25 @@ tools/editconf.py /etc/postfix/main.cf \ # ### DANE -# + # When connecting to remote SMTP servers, prefer TLS and use DANE if available. # -# Prefering ("opportunistic") TLS means Postfix will accept whatever SSL certificate the remote -# end provides, if the remote end offers STARTTLS during the connection. DANE takes this a -# step further: +# Prefering ("opportunistic") TLS means Postfix will use TLS if the remote end +# offers it, otherwise it will transmit the message in the clear. Postfix will +# accept whatever SSL certificate the remote end provides. Opportunistic TLS +# protects against passive easvesdropping (but not man-in-the-middle attacks). +# DANE takes this a step further: # # Postfix queries DNS for the TLSA record on the destination MX host. If no TLSA records are found, # then opportunistic TLS is used. Otherwise the server certificate must match the TLSA records # or else the mail bounces. TLSA also requires DNSSEC on the MX host. Postfix doesn't do DNSSEC # itself but assumes the system's nameserver does and reports DNSSEC status. Thus this also -# relies on our local bind9 server being present and smtp_dns_support_level being set to dnssec -# to use it. +# relies on our local bind9 server being present and `smtp_dns_support_level=dnssec`. # -# The smtp_tls_CAfile is superflous, but it turns warnings in the logs about untrusted certs -# into notices about trusted certs. Since in these cases Postfix is doing opportunistic TLS, -# it does not care about whether the remote certificate is trusted. But, looking at the logs, -# it's nice to be able to see that the connection was in fact encrypted for the right party. -# The CA file is provided by the package ca-certificates. +# The `smtp_tls_CAfile` is superflous, but it eliminates warnings in the logs about untrusted certs, +# which we don't care about seeing because Postfix is doing opportunistic TLS anyway. Better to encrypt, +# even if we don't know if it's to the right party, than to not encrypt at all. Instead we'll +# now see notices about trusted certs. The CA file is provided by the package `ca-certificates`. tools/editconf.py /etc/postfix/main.cf \ smtp_tls_security_level=dane \ smtp_dns_support_level=dnssec \ diff --git a/setup/mail-users.sh b/setup/mail-users.sh index d18b6e04..387ce698 100755 --- a/setup/mail-users.sh +++ b/setup/mail-users.sh @@ -53,7 +53,6 @@ EOF chmod 0600 /etc/dovecot/dovecot-sql.conf.ext # per Dovecot instructions # Have Dovecot provide an authorization service that Postfix can access & use. -# Drew Crawford sets the auth-worker process to run as the mail user, but we don't care if it runs as root. cat > /etc/dovecot/conf.d/99-local-auth.conf << EOF; service auth { unix_listener /var/spool/postfix/private/auth { diff --git a/setup/spamassassin.sh b/setup/spamassassin.sh index 298a4f25..62de55f6 100755 --- a/setup/spamassassin.sh +++ b/setup/spamassassin.sh @@ -1,14 +1,13 @@ #!/bin/bash # Spam filtering with spamassassin via spampd -############################################# - +# =========================================== +# # spampd sits between postfix and dovecot. It takes mail from postfix # over the LMTP protocol, runs spamassassin on it, and then passes the # message over LMTP to dovecot for local delivery. - +# # In order to move spam automatically into the Spam folder we use the dovecot sieve -# plugin. The tools/mail.py tool creates the necessary sieve script for each mail -# user when the mail user is created. +# plugin. source /etc/mailinabox.conf # get global vars source setup/functions.sh # load our functions @@ -29,13 +28,14 @@ hide_output pyzor discover tools/editconf.py /etc/default/spampd DESTPORT=10026 # Enable the Dovecot antispam plugin to detect when a message moves between folders so we can -# pass it to sa-learn for training. (Be careful if we use multiple plugins later.) +# pass it to sa-learn for training. +# (Be careful if we use multiple plugins later.) #NODOC sed -i "s/#mail_plugins = .*/mail_plugins = \$mail_plugins antispam/" /etc/dovecot/conf.d/20-imap.conf # When mail is moved in or out of the Dovecot Spam folder, re-train using this script # that sends the mail to spamassassin. # from http://wiki2.dovecot.org/Plugins/Antispam -rm -f /usr/bin/sa-learn-pipe.sh # legacy location +rm -f /usr/bin/sa-learn-pipe.sh # legacy location #NODOC cat > /usr/local/bin/sa-learn-pipe.sh << EOF; cat<&0 >> /tmp/sendmail-msg-\$\$.txt /usr/bin/sa-learn \$* /tmp/sendmail-msg-\$\$.txt > /dev/null diff --git a/setup/ssl.sh b/setup/ssl.sh index 53654d7e..f0ac2e8e 100755 --- a/setup/ssl.sh +++ b/setup/ssl.sh @@ -2,7 +2,7 @@ # # SSL Certificate # --------------- -# + # Create a self-signed SSL certificate if one has not yet been created. # # The certificate is for PRIMARY_HOSTNAME specifically and is used for: @@ -22,29 +22,31 @@ source /etc/mailinabox.conf # load global vars apt_install openssl mkdir -p $STORAGE_ROOT/ssl -# Generate a new private key if one doesn't already exist. +# Generate a new private key. # Set the umask so the key file is not world-readable. if [ ! -f $STORAGE_ROOT/ssl/ssl_private_key.pem ]; then (umask 077; hide_output \ openssl genrsa -out $STORAGE_ROOT/ssl/ssl_private_key.pem 2048) fi -# Generate a certificate signing request if one doesn't already exist. +# Generate a certificate signing request. if [ ! -f $STORAGE_ROOT/ssl/ssl_cert_sign_req.csr ]; then hide_output \ openssl req -new -key $STORAGE_ROOT/ssl/ssl_private_key.pem -out $STORAGE_ROOT/ssl/ssl_cert_sign_req.csr \ -sha256 -subj "/C=$CSR_COUNTRY/ST=/L=/O=/CN=$PRIMARY_HOSTNAME" fi -# Generate a SSL certificate by self-signing if a SSL certificate doesn't yet exist. +# Generate a SSL certificate by self-signing. if [ ! -f $STORAGE_ROOT/ssl/ssl_certificate.pem ]; then hide_output \ openssl x509 -req -days 365 \ -in $STORAGE_ROOT/ssl/ssl_cert_sign_req.csr -signkey $STORAGE_ROOT/ssl/ssl_private_key.pem -out $STORAGE_ROOT/ssl/ssl_certificate.pem fi -# For nginx and postfix, pre-generate some better DH bits. They seem to -# each rely on openssl's default of 1024 bits. +# For nginx and postfix, pre-generate some Diffie-Hellman cipher bits which is +# used when a Diffie-Hellman cipher is selected during TLS negotiation. Diffie-Hellman +# provides Perfect Forward Security. openssl's default is 1024 bits, but we'll +# create 2048. if [ ! -f $STORAGE_ROOT/ssl/dh2048.pem ]; then openssl dhparam -out $STORAGE_ROOT/ssl/dh2048.pem 2048 fi diff --git a/setup/system.sh b/setup/system.sh index 92ac4a67..5cbda2e2 100755 --- a/setup/system.sh +++ b/setup/system.sh @@ -1,11 +1,11 @@ source setup/functions.sh # load our functions -# Base system configuration +# Basic System Configuration # ------------------------- -# ### Base packages +# ### Install Packages -# Update system packages: +# Update system packages to make sure we have the latest upstream versions of things from Ubuntu. echo Updating system packages... hide_output apt-get update @@ -35,8 +35,6 @@ EOF # ### Firewall -# Turn on the firewall. -# # Various virtualized environments like Docker and some VPSs don't provide #NODOC # a kernel that supports iptables. To avoid error-like output in these cases, #NODOC # we skip this if the user sets DISABLE_FIREWALL=1. #NODOC @@ -47,15 +45,15 @@ if [ -z "$DISABLE_FIREWALL" ]; then # Allow incoming connections to SSH. ufw_allow ssh; - # ssh might be running on an alternate port. Use sshd -T to dump sshd's - # settings, find the port it is supposedly running on, and open that port - # too. - SSH_PORT=$(sshd -T 2>/dev/null | grep "^port " | sed "s/port //") + # ssh might be running on an alternate port. Use sshd -T to dump sshd's #NODOC + # settings, find the port it is supposedly running on, and open that port #NODOC + # too. #NODOC + SSH_PORT=$(sshd -T 2>/dev/null | grep "^port " | sed "s/port //") #NODOC if [ ! -z "$SSH_PORT" ]; then if [ "$SSH_PORT" != "22" ]; then - echo Opening alternate SSH port $SSH_PORT. - ufw_allow $SSH_PORT; + echo Opening alternate SSH port $SSH_PORT. #NODOC + ufw_allow $SSH_PORT #NODOC fi fi diff --git a/setup/web.sh b/setup/web.sh index ec96d5af..60c21fd0 100755 --- a/setup/web.sh +++ b/setup/web.sh @@ -5,6 +5,10 @@ source setup/functions.sh # load our functions source /etc/mailinabox.conf # load global vars +# Install nginx and a PHP FastCGI daemon. +# +# Turn off nginx's default website. + apt_install nginx php5-fpm rm -f /etc/nginx/sites-enabled/default @@ -20,7 +24,7 @@ sed "s#STORAGE_ROOT#$STORAGE_ROOT#" \ tools/editconf.py /etc/nginx/nginx.conf -s \ server_names_hash_bucket_size="64;" -# Bump up max_children to support more concurrent connections +# Bump up PHP's max_children to support more concurrent connections tools/editconf.py /etc/php5/fpm/pool.d/www.conf -c ';' \ pm.max_children=8 @@ -29,20 +33,20 @@ tools/editconf.py /etc/php5/fpm/pool.d/www.conf -c ';' \ # until mail accounts have been created. # make a default homepage -if [ -d $STORAGE_ROOT/www/static ]; then mv $STORAGE_ROOT/www/static $STORAGE_ROOT/www/default; fi # migration +if [ -d $STORAGE_ROOT/www/static ]; then mv $STORAGE_ROOT/www/static $STORAGE_ROOT/www/default; fi # migration #NODOC mkdir -p $STORAGE_ROOT/www/default if [ ! -f $STORAGE_ROOT/www/default/index.html ]; then cp conf/www_default.html $STORAGE_ROOT/www/default/index.html fi chown -R $STORAGE_USER $STORAGE_ROOT/www -# We previously installed a custom init script to start the PHP FastCGI daemon. -# Remove it now that we're using php5-fpm. +# We previously installed a custom init script to start the PHP FastCGI daemon. #NODOC +# Remove it now that we're using php5-fpm. #NODOC if [ -L /etc/init.d/php-fastcgi ]; then - echo "Removing /etc/init.d/php-fastcgi, php5-cgi..." - rm -f /etc/init.d/php-fastcgi - hide_output update-rc.d php-fastcgi remove - apt-get -y purge php5-cgi + echo "Removing /etc/init.d/php-fastcgi, php5-cgi..." #NODOC + rm -f /etc/init.d/php-fastcgi #NODOC + hide_output update-rc.d php-fastcgi remove #NODOC + apt-get -y purge php5-cgi #NODOC fi # Put our webfinger script into a well-known location. @@ -51,11 +55,11 @@ for f in webfinger; do chown www-data.www-data /usr/local/bin/mailinabox-$f.php done -# Remove obsoleted scripts. -# exchange-autodiscover is now handled by Z-Push. -for f in exchange-autodiscover; do - rm -f /usr/local/bin/mailinabox-$f.php -done +# Remove obsoleted scripts. #NODOC +# exchange-autodiscover is now handled by Z-Push. #NODOC +for f in exchange-autodiscover; do #NODOC + rm -f /usr/local/bin/mailinabox-$f.php #NODOC +done #NODOC # Make some space for users to customize their webfinger responses. mkdir -p $STORAGE_ROOT/webfinger/acct; diff --git a/setup/webmail.sh b/setup/webmail.sh index 1d896642..3db2e26a 100755 --- a/setup/webmail.sh +++ b/setup/webmail.sh @@ -23,16 +23,16 @@ apt_install \ php5 php5-sqlite php5-mcrypt php5-intl php5-json php5-common php-auth php-net-smtp php-net-socket php-net-sieve php-mail-mime php-crypt-gpg php5-gd php5-pspell \ tinymce libjs-jquery libjs-jquery-mousewheel libmagic1 -# We used to install Roundcube from Ubuntu, without triggering the dependencies -# on Apache and MySQL, by downloading the debs and installing them manually. -# Now that we're beyond that, get rid of those debs before installing from source. -apt-get purge -qq -y roundcube* +# We used to install Roundcube from Ubuntu, without triggering the dependencies #NODOC +# on Apache and MySQL, by downloading the debs and installing them manually. #NODOC +# Now that we're beyond that, get rid of those debs before installing from source. #NODOC +apt-get purge -qq -y roundcube* #NODOC # Install Roundcube from source if it is not already present or if it is out of date. VERSION=1.0.2 needs_update=0 #NODOC if [ ! -f /usr/local/lib/roundcubemail/version ]; then - # not installed yet + # not installed yet #NODOC needs_update=1 #NODOC elif [[ $VERSION != `cat /usr/local/lib/roundcubemail/version` ]]; then # checks if the version is what we want diff --git a/tools/readable_bash.py b/tools/readable_bash.py index 137e05ac..30e39e41 100644 --- a/tools/readable_bash.py +++ b/tools/readable_bash.py @@ -30,33 +30,74 @@ def generate_documentation(): color: #555; } h2, h3 { - margin-bottom: 1em; + margin-top: .25em; + margin-bottom: .75em; } p { margin-bottom: 1em; } + .intro p { + margin: 1.5em 0; + } + li { + margin-bottom: .33em; + } + + .sourcefile { + padding-top: 1.5em; + padding-bottom: 1em; + font-size: 90%; + text-align: right; + } + .sourcefile a { + color: red; + } + + .instructions .row.contd { + border-top: 1px solid #E0E0E0; + } + + .prose { + padding-top: 1em; + padding-bottom: 1em; + } + .terminal { + background-color: #EEE; + padding-top: 1em; + padding-bottom: 1em; + } pre { - margin: 1em 1em 1.5em 1em; color: black; + border: 0; + background: none; + font-size: 100%; } div.write-to { - margin: 1em; - border: 1px solid #999; + margin: 0 0 1em .5em; } div.write-to p { padding: .5em; margin: 0; } div.write-to .filename { - background-color: #EEE; - padding: .5em; + padding: .25em; + background-color: #666; + color: white; + font-family: monospace; font-weight: bold; } + div.write-to .filename span { + font-family: sans-serif; + font-weight: normal; + } div.write-to pre { - padding: .5em; margin: 0; + padding: .25em; + border: 1px solid #999; + border-radius: 0; + font-size: 90%; } pre.shell > div:before { @@ -67,11 +108,15 @@ def generate_documentation():
-
+

Build Your Own Mail Server From Scratch

-

Here’s how you can build your own mail server from scratch. This document is generated automatically from our setup script.

+

Here’s how you can build your own mail server from scratch.

+

This document is generated automatically from Mail-in-a-Box’s setup script source code.

+
+
+
""") parser = Source.parser() @@ -80,7 +125,7 @@ def generate_documentation(): fn = parser.parse_string(line).filename() except: continue - if fn in ("setup/preflight.sh", "setup/questions.sh", "setup/firstuser.sh", "setup/management.sh"): + if fn in ("setup/start.sh", "setup/preflight.sh", "setup/questions.sh", "setup/firstuser.sh", "setup/management.sh"): continue import sys @@ -91,6 +136,13 @@ def generate_documentation(): print(""" + """) @@ -101,8 +153,13 @@ class HashBang(Grammar): return "" def strip_indent(s): + s = s.replace("\t", " ") lines = s.split("\n") - min_indent = min(len(re.match(r"\s*", line).group(0)) for line in lines if len(line) > 0) + try: + min_indent = min(len(re.match(r"\s*", line).group(0)) for line in lines if len(line) > 0) + except ValueError: + # No non-empty lines. + min_indent = 0 lines = [line[min_indent:] for line in lines] return "\n".join(lines) @@ -126,11 +183,14 @@ class Source(Grammar): return BashScript.parse(self.filename()) class CatEOF(Grammar): - grammar = (ZERO_OR_MORE(SPACE), L('cat > '), ANY_EXCEPT(WHITESPACE), L(" <<"), OPTIONAL(SPACE), L("EOF;"), EOL, REPEAT(ANY, greedy=False), EOL, L("EOF"), EOL) + grammar = (ZERO_OR_MORE(SPACE), L('cat '), L('>') | L('>>'), L(' '), ANY_EXCEPT(WHITESPACE), L(" <<"), OPTIONAL(SPACE), L("EOF"), EOL, REPEAT(ANY, greedy=False), EOL, L("EOF"), EOL) def value(self): - content = self[7].string + content = self[9].string content = re.sub(r"\\([$])", r"\1", content) # un-escape bash-escaped characters - return "
overwrite
" + self[2].string + "
" + cgi.escape(content) + "
\n" + return "
%s (%s)
%s
\n" \ + % (self[4].string, + "overwrite" if ">>" not in self[2].string else "append to", + cgi.escape(content)) class HideOutput(Grammar): grammar = (L("hide_output "), REF("BashElement")) @@ -150,7 +210,7 @@ class EditConf(Grammar): FILENAME, SPACE, OPTIONAL((LIST_OF( - L("-w") | L("-s") | L("-c ';'"), + L("-w") | L("-s") | L("-c ;"), sep=SPACE, ), SPACE)), REST_OF_LINE, @@ -159,27 +219,14 @@ class EditConf(Grammar): ) def value(self): conffile = self[1] - options = [""] - mode = 1 - for c in self[4].string: - if mode == 1 and c in (" ", "\t") and options[-1] != "": - # new word - options.append("") - elif mode < 0: - # escaped character - options[-1] += c - mode = -mode - elif c == "\\": - # escape next character - mode = -mode - elif mode == 1 and c == '"': - mode = 2 - elif mode == 2 and c == '"': - mode = 1 - else: - options[-1] += c - if options[-1] == "": options.pop(-1) - return "
additional settings for
" + self[1].string + "
" + "\n".join(cgi.escape(s) for s in options) + "
\n" + options = [] + eq = "=" + if self[3] and "-s" in self[3].string: eq = " " + for opt in re.split("\s+", self[4].string): + k, v = opt.split("=", 1) + v = re.sub(r"\n+", "", fixup_tokens(v)) # not sure why newlines are getting doubled + options.append("%s%s%s" % (k, eq, v)) + return "
" + self[1].string + " (change settings)
" + "\n".join(cgi.escape(s) for s in options) + "
\n" class CaptureOutput(Grammar): grammar = OPTIONAL(SPACE), WORD("A-Za-z_"), L('=$('), REST_OF_LINE, L(")"), OPTIONAL(L(';')), EOL @@ -193,8 +240,14 @@ class SedReplace(Grammar): def value(self): return "
edit
" + self[8].string + "

replace

" + cgi.escape(self[3].string.replace(".*", ". . .")) + "

with

" + cgi.escape(self[5].string.replace("\\n", "\n").replace("\\t", "\t")) + "
\n" +class EchoPipe(Grammar): + grammar = OPTIONAL(SPACE), L("echo "), REST_OF_LINE, L(' | '), REST_OF_LINE, EOL + def value(self): + text = " ".join("\"%s\"" % s for s in self[2].string.split(" ")) + return "
echo " + cgi.escape(text) + " \
 | " + self[4].string + "
\n" + def shell_line(bash): - return "
" + cgi.escape(wrap_lines(bash.strip())) + "
\n" + return "
" + cgi.escape(bash.strip()) + "
\n" class AptGet(Grammar): grammar = (ZERO_OR_MORE(SPACE), L("apt_install "), REST_OF_LINE, EOL) @@ -213,13 +266,92 @@ class OtherLine(Grammar): grammar = (REST_OF_LINE, EOL) def value(self): if self.string.strip() == "": return "" - return "
" + cgi.escape(self.string.rstrip()) + "
\n" + if "source setup/functions.sh" in self.string: return "" + if "source /etc/mailinabox.conf" in self.string: return "" + return "
" + cgi.escape(self.string.strip()) + "
\n" class BashElement(Grammar): - grammar = Comment | Source | CatEOF | SuppressedLine | HideOutput | EditConf | CaptureOutput | SedReplace | AptGet | UfwAllow | RestartService | OtherLine + grammar = Comment | CatEOF | EchoPipe | SuppressedLine | HideOutput | EditConf | SedReplace | AptGet | UfwAllow | RestartService | OtherLine def value(self): return self[0].value() +# Make some special characters to private use Unicode code points. +bash_special_characters = { + "\n": "\uE000", + " ": "\uE001", +} + +def quasitokenize(bashscript): + # Make a parse of bash easier by making the tokenization easy. + newscript = "" + quote_mode = None + escape_next = False + line_comment = False + subshell = 0 + for c in bashscript: + if line_comment: + # We're in a comment until the end of the line. + newscript += c + if c == '\n': + line_comment = False + elif escape_next: + # Previous character was a \. Normally the next character + # comes through literally, but escaped newlines are line + # continuations. + if c == "\n": + c = " " + else: + newscript += c + escape_next = False + elif c == "\\": + # Escaping next character. + escape_next = True + elif quote_mode is None and c in ('"', "'"): + # Starting a quoted word. + quote_mode = c + elif c == quote_mode: + # Ending a quoted word. + quote_mode = None + elif quote_mode is not None and quote_mode != "EOF" and c in bash_special_characters: + # Replace special tokens within quoted words so that they + # don't interfere with tokenization later. + newscript += bash_special_characters[c] + elif quote_mode is None and c == '#': + # Start of a line comment. + newscript += c + line_comment = True + elif quote_mode is None and c == ';' and subshell == 0: + # End of a statement. + newscript += "\n" + elif quote_mode is None and c == '(': + # Start of a subshell. + newscript += c + subshell += 1 + elif quote_mode is None and c == ')': + # End of a subshell. + newscript += c + subshell -= 1 + elif quote_mode is None and c == '\t': + # Make these just spaces. + if newscript[-1] != " ": + newscript += " " + else: + # All other characters. + newscript += c + + # "<< EOF" escaping. + if quote_mode is None and re.search("<<\s*EOF\n$", newscript): + quote_mode = "EOF" + elif quote_mode == "EOF" and re.search("\nEOF\n$", newscript): + quote_mode = None + + return newscript + +def fixup_tokens(s): + for c, enc in bash_special_characters.items(): + s = s.replace(enc, c) + return s + class BashScript(Grammar): grammar = (OPTIONAL(HashBang), REPEAT(BashElement)) def value(self): @@ -228,22 +360,68 @@ class BashScript(Grammar): @staticmethod def parse(fn): if fn in ("setup/functions.sh", "/etc/mailinabox.conf"): return "" - parser = BashScript.parser() string = open(fn).read() - string = re.sub(r"\s*\\\n\s*", " ", string) + + # tokenize string = re.sub(".* #NODOC\n", "", string) - string = re.sub("\n\s*if .*|\n\s*fi|\n\s*else|\n\s*elif .*", "", string) + string = re.sub("\n\s*if .*\n.*then.*|\n\s*fi|\n\s*else|\n\s*elif .*", "", string) + string = quasitokenize(string) string = re.sub("hide_output ", "", string) + + parser = BashScript.parser() result = parser.parse_string(string) - - v = "
%s
\n" % ("https://github.com/mail-in-a-box/mailinabox/tree/master/" + fn, fn) - v += "".join(result.value()) + + v = "
view the bash source for the following section at %s
\n" \ + % ("https://github.com/mail-in-a-box/mailinabox/tree/master/" + fn, fn) + + mode = 0 + for item in result.value(): + if item.strip() == "": + pass + elif item.startswith("\n
", "")
 		v = re.sub("
([\w\W]*?)
", lambda m : "
" + strip_indent(m.group(1)) + "
", v)
 
-		v = re.sub(r"\$?PRIMARY_HOSTNAME", "box.yourdomain.com", v)
-		v = re.sub(r"\$?STORAGE_ROOT", "/path/to/user-data", v)
+		v = re.sub(r"(\$?)PRIMARY_HOSTNAME", r"box.yourdomain.com", v)
+		v = re.sub(r"\$STORAGE_ROOT", r"$STORE", v)
+		v = re.sub(r"\$CSR_COUNTRY", r"US", v)
 		v = v.replace("`pwd`",  "/path/to/mailinabox")
 
 		return v