external
/
mailinabox
mirror of https://github.com/mail-in-a-box/mailinabox.git
1
0
Fork 0
Code Issues Releases Wiki Activity

pep8 cleanup of Python source files 

pep8 (https://www.python.org/dev/peps/pep-0008/) is the commonly accepted and widely adopted code style convention for Python.
I used pycodestyle (https://pycodestyle.readthedocs.io/en/latest/) to check for pep8 compatibility.
Especially the mix of tabs and spaces in the Python files makes it hard to work with. I switched to spaces, because that's what
pep8 expects and the majority of Python programmers use.
This commit is contained in:
Wolfgang Steitz 2016-11-20 13:23:46 +01:00
parent c3605f6211
commit 8cea79de8b
5 changed files with 650 additions and 561 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

167
tools/editconf.py
View File

@ -14,20 +14,21 @@
# #
# NAME VALUE # NAME VALUE
# #
# If the -c option is given, then the supplied character becomes the comment character # If the -c option is given, then the supplied character becomes the commentcharacter
# #
# If the -w option is given, then setting lines continue onto following # If the -w option is given, then setting lines continue onto following
# lines while the lines start with whitespace, e.g.: # lines while the lines start with whitespace, e.g.:
# #
# NAME VAL # NAME VAL
# UE # UE
import sys, re import sys
import re
# sanity check # sanity check
if len(sys.argv) < 3: if len(sys.argv) < 3:
print("usage: python3 editconf.py /etc/file.conf [-s] [-w] [-c <CHARACTER>] [-t] NAME=VAL [NAME=VAL ...]") print("usage: python3 editconf.py /etc/file.conf [-s] [-w] [-c <CHARACTER>] [-t] NAME=VAL [NAME=VAL ...]")
sys.exit(1) sys.exit(1)
# parse command line arguments # parse command line arguments
filename = sys.argv[1] filename = sys.argv[1]
@ -39,30 +40,30 @@ comment_char = "#"
folded_lines = False folded_lines = False
testing = False testing = False
while settings[0][0] == "-" and settings[0] != "--": while settings[0][0] == "-" and settings[0] != "--":
opt = settings.pop(0) opt = settings.pop(0)
if opt == "-s": if opt == "-s":
# Space is the delimiter # Space is the delimiter
delimiter = " " delimiter = " "
delimiter_re = r"\s+" delimiter_re = r"\s+"
elif opt == "-w": elif opt == "-w":
# Line folding is possible in this file. # Line folding is possible in this file.
folded_lines = True folded_lines = True
elif opt == "-c": elif opt == "-c":
# Specifies a different comment character. # Specifies a different comment character.
comment_char = settings.pop(0) comment_char = settings.pop(0)
elif opt == "-t": elif opt == "-t":
testing = True testing = True
else: else:
print("Invalid option.") print("Invalid option.")
sys.exit(1) sys.exit(1)
# sanity check command line # sanity check command line
for setting in settings: for setting in settings:
try: try:
name, value = setting.split("=", 1) name, value = setting.split("=", 1)
except: except:
import subprocess import subprocess
print("Invalid command line: ", subprocess.list2cmdline(sys.argv)) print("Invalid command line: ", subprocess.list2cmdline(sys.argv))
# create the new config file in memory # create the new config file in memory
@ -71,67 +72,69 @@ buf = ""
input_lines = list(open(filename)) input_lines = list(open(filename))
while len(input_lines) > 0: while len(input_lines) > 0:
line = input_lines.pop(0) line = input_lines.pop(0)
# If this configuration file uses folded lines, append any folded lines # If this configuration file uses folded lines, append any folded lines
# into our input buffer. # into our input buffer.
if folded_lines and line[0] not in (comment_char, " ", ""): if folded_lines and line[0] not in (comment_char, " ", ""):
while len(input_lines) > 0 and input_lines[0][0] in " \t": while len(input_lines) > 0 and input_lines[0][0] in " \t":
line += input_lines.pop(0) line += input_lines.pop(0)
# See if this line is for any settings passed on the command line. # See if this line is for any settings passed on the command line.
for i in range(len(settings)): for i in range(len(settings)):
# Check that this line contain this setting from the command-line arguments. # Check that this line contain this setting from the command-line arguments.
name, val = settings[i].split("=", 1) name, val = settings[i].split("=", 1)
m = re.match( m = re.match(
"(\s*)" "(\s*)" +
+ "(" + re.escape(comment_char) + "\s*)?" "(" + re.escape(comment_char) + "\s*)?" +
+ re.escape(name) + delimiter_re + "(.*?)\s*$", re.escape(name) + delimiter_re + "(.*?)\s*$",
line, re.S) line, re.S)
if not m: continue if not m:
indent, is_comment, existing_val = m.groups() continue
indent, is_comment, existing_val = m.groups()
# If this is already the setting, do nothing.
if is_comment is None and existing_val == val:
# It may be that we've already inserted this setting higher
# in the file so check for that first.
if i in found:
break
buf += line
found.add(i)
break
# comment-out the existing line (also comment any folded lines)
if is_comment is None:
buf += comment_char + line.rstrip().replace("\n", "\n" + comment_char) + "\n"
else:
# the line is already commented, pass it through
buf += line
# if this option oddly appears more than once, don't add the setting again
if i in found:
break
# add the new setting
buf += indent + name + delimiter + val + "\n"
# note that we've applied this option
found.add(i)
break
else:
# If did not match any setting names, pass this line through.
buf += line
# If this is already the setting, do nothing.
if is_comment is None and existing_val == val:
# It may be that we've already inserted this setting higher
# in the file so check for that first.
if i in found: break
buf += line
found.add(i)
break
# comment-out the existing line (also comment any folded lines)
if is_comment is None:
buf += comment_char + line.rstrip().replace("\n", "\n" + comment_char) + "\n"
else:
# the line is already commented, pass it through
buf += line
# if this option oddly appears more than once, don't add the setting again
if i in found:
break
# add the new setting
buf += indent + name + delimiter + val + "\n"
# note that we've applied this option
found.add(i)
break
else:
# If did not match any setting names, pass this line through.
buf += line
# Put any settings we didn't see at the end of the file. # Put any settings we didn't see at the end of the file.
for i in range(len(settings)): for i in range(len(settings)):
if i not in found: if i not in found:
name, val = settings[i].split("=", 1) name, val = settings[i].split("=", 1)
buf += name + delimiter + val + "\n" buf += name + delimiter + val + "\n"
if not testing: if not testing:
# Write out the new file. # Write out the new file.
with open(filename, "w") as f: with open(filename, "w") as f:
f.write(buf) f.write(buf)
else: else:
# Just print the new file to stdout. # Just print the new file to stdout.
print(buf) print(buf)

183
tools/mail.py
View File

@ -1,31 +1,40 @@
#!/usr/bin/python3 #!/usr/bin/python3
import sys, getpass, urllib.request, urllib.error, json, re import sys
import getpass
import urllib.request
import urllib.error
import json
import re
def mgmt(cmd, data=None, is_json=False): def mgmt(cmd, data=None, is_json=False):
# The base URL for the management daemon. (Listens on IPv4 only.) # The base URL for the management daemon. (Listens on IPv4 only.)
mgmt_uri = 'http://127.0.0.1:10222' mgmt_uri = 'http://127.0.0.1:10222'
setup_key_auth(mgmt_uri) setup_key_auth(mgmt_uri)
req = urllib.request.Request(mgmt_uri + cmd, urllib.parse.urlencode(data).encode("utf8") if data else None)
try:
response = urllib.request.urlopen(req)
except urllib.error.HTTPError as e:
if e.code == 401:
try:
print(e.read().decode("utf8"))
except:
pass
print("The management daemon refused access. The API key file may be out of sync. \
Try 'service mailinabox restart'.", file=sys.stderr)
elif hasattr(e, 'read'):
print(e.read().decode('utf8'), file=sys.stderr)
else:
print(e, file=sys.stderr)
sys.exit(1)
resp = response.read().decode('utf8')
if is_json:
resp = json.loads(resp)
return resp
req = urllib.request.Request(mgmt_uri + cmd, urllib.parse.urlencode(data).encode("utf8") if data else None)
try:
response = urllib.request.urlopen(req)
except urllib.error.HTTPError as e:
if e.code == 401:
try:
print(e.read().decode("utf8"))
except:
pass
print("The management daemon refused access. The API key file may be out of sync. Try 'service mailinabox restart'.", file=sys.stderr)
elif hasattr(e, 'read'):
print(e.read().decode('utf8'), file=sys.stderr)
else:
print(e, file=sys.stderr)
sys.exit(1)
resp = response.read().decode('utf8')
if is_json: resp = json.loads(resp)
return resp
def read_password(): def read_password():
while True: while True:
@ -43,89 +52,91 @@ def read_password():
break break
return first return first
def setup_key_auth(mgmt_uri):
key = open('/var/lib/mailinabox/api.key').read().strip()
auth_handler = urllib.request.HTTPBasicAuthHandler() def setup_key_auth(mgmt_uri):
auth_handler.add_password( key = open('/var/lib/mailinabox/api.key').read().strip()
realm='Mail-in-a-Box Management Server',
uri=mgmt_uri, auth_handler = urllib.request.HTTPBasicAuthHandler()
user=key, auth_handler.add_password(
passwd='') realm='Mail-in-a-Box Management Server',
opener = urllib.request.build_opener(auth_handler) uri=mgmt_uri,
urllib.request.install_opener(opener) user=key,
passwd='')
opener = urllib.request.build_opener(auth_handler)
urllib.request.install_opener(opener)
if len(sys.argv) < 2: if len(sys.argv) < 2:
print("Usage: ") print("Usage: ")
print(" tools/mail.py user (lists users)") print(" tools/mail.py user (lists users)")
print(" tools/mail.py user add user@domain.com [password]") print(" tools/mail.py user add user@domain.com [password]")
print(" tools/mail.py user password user@domain.com [password]") print(" tools/mail.py user password user@domain.com [password]")
print(" tools/mail.py user remove user@domain.com") print(" tools/mail.py user remove user@domain.com")
print(" tools/mail.py user make-admin user@domain.com") print(" tools/mail.py user make-admin user@domain.com")
print(" tools/mail.py user remove-admin user@domain.com") print(" tools/mail.py user remove-admin user@domain.com")
print(" tools/mail.py user admins (lists admins)") print(" tools/mail.py user admins (lists admins)")
print(" tools/mail.py alias (lists aliases)") print(" tools/mail.py alias (lists aliases)")
print(" tools/mail.py alias add incoming.name@domain.com sent.to@other.domain.com") print(" tools/mail.py alias add incoming.name@domain.com sent.to@other.domain.com")
print(" tools/mail.py alias add incoming.name@domain.com 'sent.to@other.domain.com, multiple.people@other.domain.com'") print(" tools/mail.py alias add incoming.name@domain.com 'sent.to@other.domain.com, multiple.people@other.domain.com'")
print(" tools/mail.py alias remove incoming.name@domain.com") print(" tools/mail.py alias remove incoming.name@domain.com")
print() print()
print("Removing a mail user does not delete their mail folders on disk. It only prevents IMAP/SMTP login.") print("Removing a mail user does not delete their mail folders on disk. It only prevents IMAP/SMTP login.")
print() print()
elif sys.argv[1] == "user" and len(sys.argv) == 2: elif sys.argv[1] == "user" and len(sys.argv) == 2:
# Dump a list of users, one per line. Mark admins with an asterisk. # Dump a list of users, one per line. Mark admins with an asterisk.
users = mgmt("/mail/users?format=json", is_json=True) users = mgmt("/mail/users?format=json", is_json=True)
for domain in users: for domain in users:
for user in domain["users"]: for user in domain["users"]:
if user['status'] == 'inactive': continue if user['status'] == 'inactive':
print(user['email'], end='') continue
if "admin" in user['privileges']: print(user['email'], end='')
print("*", end='') if "admin" in user['privileges']:
print() print("*", end='')
print()
elif sys.argv[1] == "user" and sys.argv[2] in ("add", "password"): elif sys.argv[1] == "user" and sys.argv[2] in ("add", "password"):
if len(sys.argv) < 5: if len(sys.argv) < 5:
if len(sys.argv) < 4: if len(sys.argv) < 4:
email = input("email: ") email = input("email: ")
else: else:
email = sys.argv[3] email = sys.argv[3]
pw = read_password() pw = read_password()
else: else:
email, pw = sys.argv[3:5] email, pw = sys.argv[3:5]
if sys.argv[2] == "add": if sys.argv[2] == "add":
print(mgmt("/mail/users/add", { "email": email, "password": pw })) print(mgmt("/mail/users/add", {"email": email, "password": pw}))
elif sys.argv[2] == "password": elif sys.argv[2] == "password":
print(mgmt("/mail/users/password", { "email": email, "password": pw })) print(mgmt("/mail/users/password", {"email": email, "password": pw}))
elif sys.argv[1] == "user" and sys.argv[2] == "remove" and len(sys.argv) == 4: elif sys.argv[1] == "user" and sys.argv[2] == "remove" and len(sys.argv) == 4:
print(mgmt("/mail/users/remove", { "email": sys.argv[3] })) print(mgmt("/mail/users/remove", {"email": sys.argv[3]}))
elif sys.argv[1] == "user" and sys.argv[2] in ("make-admin", "remove-admin") and len(sys.argv) == 4: elif sys.argv[1] == "user" and sys.argv[2] in ("make-admin", "remove-admin") and len(sys.argv) == 4:
if sys.argv[2] == "make-admin": if sys.argv[2] == "make-admin":
action = "add" action = "add"
else: else:
action = "remove" action = "remove"
print(mgmt("/mail/users/privileges/" + action, { "email": sys.argv[3], "privilege": "admin" })) print(mgmt("/mail/users/privileges/" + action, {"email": sys.argv[3], "privilege": "admin"}))
elif sys.argv[1] == "user" and sys.argv[2] == "admins": elif sys.argv[1] == "user" and sys.argv[2] == "admins":
# Dump a list of admin users. # Dump a list of admin users.
users = mgmt("/mail/users?format=json", is_json=True) users = mgmt("/mail/users?format=json", is_json=True)
for domain in users: for domain in users:
for user in domain["users"]: for user in domain["users"]:
if "admin" in user['privileges']: if "admin" in user['privileges']:
print(user['email']) print(user['email'])
elif sys.argv[1] == "alias" and len(sys.argv) == 2: elif sys.argv[1] == "alias" and len(sys.argv) == 2:
print(mgmt("/mail/aliases")) print(mgmt("/mail/aliases"))
elif sys.argv[1] == "alias" and sys.argv[2] == "add" and len(sys.argv) == 5: elif sys.argv[1] == "alias" and sys.argv[2] == "add" and len(sys.argv) == 5:
print(mgmt("/mail/aliases/add", { "address": sys.argv[3], "forwards_to": sys.argv[4] })) print(mgmt("/mail/aliases/add", {"address": sys.argv[3], "forwards_to": sys.argv[4]}))
elif sys.argv[1] == "alias" and sys.argv[2] == "remove" and len(sys.argv) == 4: elif sys.argv[1] == "alias" and sys.argv[2] == "remove" and len(sys.argv) == 4:
print(mgmt("/mail/aliases/remove", { "address": sys.argv[3] })) print(mgmt("/mail/aliases/remove", {"address": sys.argv[3]}))
else: else:
print("Invalid command-line arguments.") print("Invalid command-line arguments.")
sys.exit(1) sys.exit(1)

55
tools/parse-nginx-log-bootstrap-accesses.py
View File

@ -5,7 +5,11 @@
# looking at accesses to the bootstrap.sh script (which is currently at the URL # looking at accesses to the bootstrap.sh script (which is currently at the URL
# .../setup.sh). # .../setup.sh).
import re, glob, gzip, os.path, json import re
import glob
import gzip
import os.path
import json
import dateutil.parser import dateutil.parser
outfn = "/home/user-data/www/mailinabox.email/install-stats.json" outfn = "/home/user-data/www/mailinabox.email/install-stats.json"
@ -16,37 +20,38 @@ accesses = set()
# Scan the current and rotated access logs. # Scan the current and rotated access logs.
for fn in glob.glob("/var/log/nginx/access.log*"): for fn in glob.glob("/var/log/nginx/access.log*"):
# Gunzip if necessary. # Gunzip if necessary.
if fn.endswith(".gz"): if fn.endswith(".gz"):
f = gzip.open(fn) f = gzip.open(fn)
else: else:
f = open(fn, "rb") f = open(fn, "rb")
# Loop through the lines in the access log. # Loop through the lines in the access log.
with f: with f:
for line in f: for line in f:
# Find lines that are GETs on the bootstrap script by either curl or wget. # Find lines that are GETs on the bootstrap script by either curl or wget.
# (Note that we purposely skip ...?ping=1 requests which is the admin panel querying us for updates.) # (Note that we purposely skip ...?ping=1 requests which is the admin
# (Also, the URL changed in January 2016, but we'll accept both.) # panel querying us for updates.)
m = re.match(rb"(?P<ip>\S+) - - \[(?P<date>.*?)\] \"GET /(bootstrap.sh|setup.sh) HTTP/.*\" 200 \d+ .* \"(?:curl|wget)", line, re.I) # (Also, the URL changed in January 2016, but we'll accept both.)
if m: m = re.match(rb"(?P<ip>\S+) - - \[(?P<date>.*?)\] \"GET /(bootstrap.sh|setup.sh) HTTP/.*\" 200 \d+ .* \"(?:curl|wget)", line, re.I)
date, time = m.group("date").decode("ascii").split(":", 1) if m:
date = dateutil.parser.parse(date).date().isoformat() date, time = m.group("date").decode("ascii").split(":", 1)
ip = m.group("ip").decode("ascii") date = dateutil.parser.parse(date).date().isoformat()
accesses.add( (date, ip) ) ip = m.group("ip").decode("ascii")
accesses.add((date, ip))
# Aggregate by date. # Aggregate by date.
by_date = { } by_date = {}
for date, ip in accesses: for date, ip in accesses:
by_date[date] = by_date.get(date, 0) + 1 by_date[date] = by_date.get(date, 0) + 1
# Since logs are rotated, store the statistics permanently in a JSON file. # Since logs are rotated, store the statistics permanently in a JSON file.
# Load in the stats from an existing file. # Load in the stats from an existing file.
if os.path.exists(outfn): if os.path.exists(outfn):
existing_data = json.load(open(outfn)) existing_data = json.load(open(outfn))
for date, count in existing_data: for date, count in existing_data:
if date not in by_date: if date not in by_date:
by_date[date] = count by_date[date] = count
# Turn into a list rather than a dict structure to make it ordered. # Turn into a list rather than a dict structure to make it ordered.
by_date = sorted(by_date.items()) by_date = sorted(by_date.items())
@ -56,4 +61,4 @@ by_date.pop(-1)
# Write out. # Write out.
with open(outfn, "w") as f: with open(outfn, "w") as f:
json.dump(by_date, f, sort_keys=True, indent=True) json.dump(by_date, f, sort_keys=True, indent=True)

785
tools/readable_bash.py
View File

@ -3,12 +3,14 @@
# Generate documentation for how this machine works by # Generate documentation for how this machine works by
# parsing our bash scripts! # parsing our bash scripts!
import cgi, re import cgi
import re
import markdown import markdown
from modgrammar import * from modgrammar import *
def generate_documentation(): def generate_documentation():
print("""<!DOCTYPE html> print("""<!DOCTYPE html>
<html> <html>
<head> <head>
<meta charset="utf-8"> <meta charset="utf-8">
@ -21,93 +23,93 @@ def generate_documentation():
<link rel="stylesheet" href="https://maxcdn.bootstrapcdn.com/bootstrap/3.2.0/css/bootstrap-theme.min.css"> <link rel="stylesheet" href="https://maxcdn.bootstrapcdn.com/bootstrap/3.2.0/css/bootstrap-theme.min.css">
<style> <style>
@import url(https://fonts.googleapis.com/css?family=Iceland); @import url(https://fonts.googleapis.com/css?family=Iceland);
@import url(https://fonts.googleapis.com/css?family=Raleway:400,700); @import url(https://fonts.googleapis.com/css?family=Raleway:400,700);
@import url(https://fonts.googleapis.com/css?family=Ubuntu:300,500); @import url(https://fonts.googleapis.com/css?family=Ubuntu:300,500);
body { body {
font-family: Raleway, sans-serif; font-family: Raleway, sans-serif;
font-size: 16px; font-size: 16px;
color: #555; color: #555;
} }
h2, h3 { h2, h3 {
margin-top: .25em; margin-top: .25em;
margin-bottom: .75em; margin-bottom: .75em;
} }
p { p {
margin-bottom: 1em; margin-bottom: 1em;
} }
.intro p { .intro p {
margin: 1.5em 0; margin: 1.5em 0;
} }
li { li {
margin-bottom: .33em; margin-bottom: .33em;
} }
.sourcefile { .sourcefile {
padding-top: 1.5em; padding-top: 1.5em;
padding-bottom: 1em; padding-bottom: 1em;
font-size: 90%; font-size: 90%;
text-align: right; text-align: right;
} }
.sourcefile a { .sourcefile a {
color: red; color: red;
} }
.instructions .row.contd { .instructions .row.contd {
border-top: 1px solid #E0E0E0; border-top: 1px solid #E0E0E0;
} }
.prose { .prose {
padding-top: 1em; padding-top: 1em;
padding-bottom: 1em; padding-bottom: 1em;
} }
.terminal { .terminal {
background-color: #EEE; background-color: #EEE;
padding-top: 1em; padding-top: 1em;
padding-bottom: 1em; padding-bottom: 1em;
} }
ul { ul {
padding-left: 1.25em; padding-left: 1.25em;
} }
pre { pre {
color: black; color: black;
border: 0; border: 0;
background: none; background: none;
font-size: 100%; font-size: 100%;
} }
div.write-to { div.write-to {
margin: 0 0 1em .5em; margin: 0 0 1em .5em;
} }
div.write-to p { div.write-to p {
padding: .5em; padding: .5em;
margin: 0; margin: 0;
} }
div.write-to .filename { div.write-to .filename {
padding: .25em .5em; padding: .25em .5em;
background-color: #666; background-color: #666;
color: white; color: white;
font-family: monospace; font-family: monospace;
font-weight: bold; font-weight: bold;
} }
div.write-to .filename span { div.write-to .filename span {
font-family: sans-serif; font-family: sans-serif;
font-weight: normal; font-weight: normal;
} }
div.write-to pre { div.write-to pre {
margin: 0; margin: 0;
padding: .5em; padding: .5em;
border: 1px solid #999; border: 1px solid #999;
border-radius: 0; border-radius: 0;
font-size: 90%; font-size: 90%;
} }
pre.shell > div:before { pre.shell > div:before {
content: "$ "; content: "$ ";
color: #666; color: #666;
} }
</style> </style>
</head> </head>
<body> <body>
@ -123,358 +125,421 @@ def generate_documentation():
<div class="container instructions"> <div class="container instructions">
""") """)
parser = Source.parser() parser = Source.parser()
for line in open("setup/start.sh"): for line in open("setup/start.sh"):
try: try:
fn = parser.parse_string(line).filename() fn = parser.parse_string(line).filename()
except: except:
continue continue
if fn in ("setup/start.sh", "setup/preflight.sh", "setup/questions.sh", "setup/firstuser.sh", "setup/management.sh"): if fn in ("setup/start.sh", "setup/preflight.sh", "setup/questions.sh",
continue "setup/firstuser.sh", "setup/management.sh"):
continue
import sys import sys
print(fn, file=sys.stderr) print(fn, file=sys.stderr)
print(BashScript.parse(fn)) print(BashScript.parse(fn))
print(""" print("""
<script src="https://ajax.googleapis.com/ajax/libs/jquery/1.10.1/jquery.min.js"></script> <script src="https://ajax.googleapis.com/ajax/libs/jquery/1.10.1/jquery.min.js"></script>
<script src="https://maxcdn.bootstrapcdn.com/bootstrap/3.2.0/js/bootstrap.min.js"></script> <script src="https://maxcdn.bootstrapcdn.com/bootstrap/3.2.0/js/bootstrap.min.js"></script>
<script> <script>
$(function() { $(function() {
$('.terminal').each(function() { $('.terminal').each(function() {
$(this).outerHeight( $(this).parent().innerHeight() ); $(this).outerHeight( $(this).parent().innerHeight() );
}); });
}) })
</script> </script>
</body> </body>
</html> </html>
""") """)
class HashBang(Grammar): class HashBang(Grammar):
grammar = (L('#!'), REST_OF_LINE, EOL) grammar = (L('#!'), REST_OF_LINE, EOL)
def value(self):
return "" def value(self):
return ""
def strip_indent(s): def strip_indent(s):
s = s.replace("\t", " ") s = s.replace("\t", " ")
lines = s.split("\n") lines = s.split("\n")
try: try:
min_indent = min(len(re.match(r"\s*", line).group(0)) for line in lines if len(line) > 0) min_indent = min(len(re.match(r"\s*", line).group(0)) for line in lines if len(line) > 0)
except ValueError: except ValueError:
# No non-empty lines. # No non-empty lines.
min_indent = 0 min_indent = 0
lines = [line[min_indent:] for line in lines] lines = [line[min_indent:] for line in lines]
return "\n".join(lines) return "\n".join(lines)
class Comment(Grammar): class Comment(Grammar):
grammar = ONE_OR_MORE(ZERO_OR_MORE(SPACE), L('#'), REST_OF_LINE, EOL) grammar = ONE_OR_MORE(ZERO_OR_MORE(SPACE), L('#'), REST_OF_LINE, EOL)
def value(self):
if self.string.replace("#", "").strip() == "": def value(self):
return "\n" if self.string.replace("#", "").strip() == "":
lines = [x[2].string for x in self[0]] return "\n"
content = "\n".join(lines) lines = [x[2].string for x in self[0]]
content = strip_indent(content) content = "\n".join(lines)
return markdown.markdown(content, output_format="html4") + "\n\n" content = strip_indent(content)
return markdown.markdown(content, output_format="html4") + "\n\n"
FILENAME = WORD('a-z0-9-/.') FILENAME = WORD('a-z0-9-/.')
class Source(Grammar): class Source(Grammar):
grammar = ((L('.') | L('source')), L(' '), FILENAME, Comment | EOL) grammar = ((L('.') | L('source')), L(' '), FILENAME, Comment | EOL)
def filename(self):
return self[2].string.strip() def filename(self):
def value(self): return self[2].string.strip()
return BashScript.parse(self.filename())
def value(self):
return BashScript.parse(self.filename())
class CatEOF(Grammar): class CatEOF(Grammar):
grammar = (ZERO_OR_MORE(SPACE), L('cat '), L('>') | L('>>'), L(' '), ANY_EXCEPT(WHITESPACE), L(" <<"), OPTIONAL(SPACE), L("EOF"), EOL, REPEAT(ANY, greedy=False), EOL, L("EOF"), EOL) grammar = (ZERO_OR_MORE(SPACE), L('cat '), L('>') | L('>>'), L(' '), ANY_EXCEPT(WHITESPACE),
def value(self): L(" <<"), OPTIONAL(SPACE), L("EOF"), EOL, REPEAT(ANY, greedy=False), EOL, L("EOF"), EOL)
content = self[9].string
content = re.sub(r"\\([$])", r"\1", content) # un-escape bash-escaped characters def value(self):
return "<div class='write-to'><div class='filename'>%s <span>(%s)</span></div><pre>%s</pre></div>\n" \ content = self[9].string
% (self[4].string, content = re.sub(r"\\([$])", r"\1", content) # un-escape bash-escaped characters
"overwrite" if ">>" not in self[2].string else "append to", return "<div class='write-to'><div class='filename'>%s <span>(%s)</span></div><pre>%s</pre></div>\n" \
cgi.escape(content)) % (self[4].string,
"overwrite" if ">>" not in self[2].string else "append to",
cgi.escape(content))
class HideOutput(Grammar): class HideOutput(Grammar):
grammar = (L("hide_output "), REF("BashElement")) grammar = (L("hide_output "), REF("BashElement"))
def value(self):
return self[1].value() def value(self):
return self[1].value()
class EchoLine(Grammar): class EchoLine(Grammar):
grammar = (OPTIONAL(SPACE), L("echo "), REST_OF_LINE, EOL) grammar = (OPTIONAL(SPACE), L("echo "), REST_OF_LINE, EOL)
def value(self):
if "|" in self.string or ">" in self.string: def value(self):
return "<pre class='shell'><div>" + recode_bash(self.string.strip()) + "</div></pre>\n" if "|" in self.string or ">" in self.string:
return "" return "<pre class='shell'><div>" + recode_bash(self.string.strip()) + "</div></pre>\n"
return ""
class EditConf(Grammar): class EditConf(Grammar):
grammar = ( grammar = (
L('tools/editconf.py '), L('tools/editconf.py '),
FILENAME, FILENAME,
SPACE, SPACE,
OPTIONAL((LIST_OF( OPTIONAL((LIST_OF(
L("-w") | L("-s") | L("-c ;"), L("-w") | L("-s") | L("-c ;"),
sep=SPACE, sep=SPACE,
), SPACE)), ), SPACE)),
REST_OF_LINE, REST_OF_LINE,
OPTIONAL(SPACE), OPTIONAL(SPACE),
EOL EOL
) )
def value(self):
conffile = self[1] def value(self):
options = [] conffile = self[1]
eq = "=" options = []
if self[3] and "-s" in self[3].string: eq = " " eq = "="
for opt in re.split("\s+", self[4].string): if self[3] and "-s" in self[3].string:
k, v = opt.split("=", 1) eq = " "
v = re.sub(r"\n+", "", fixup_tokens(v)) # not sure why newlines are getting doubled for opt in re.split("\s+", self[4].string):
options.append("%s%s%s" % (k, eq, v)) k, v = opt.split("=", 1)
return "<div class='write-to'><div class='filename'>" + self[1].string + " <span>(change settings)</span></div><pre>" + "\n".join(cgi.escape(s) for s in options) + "</pre></div>\n" v = re.sub(r"\n+", "", fixup_tokens(v)) # not sure why newlines are getting doubled
options.append("%s%s%s" % (k, eq, v))
return "<div class='write-to'><div class='filename'>" + self[1].string +
" <span>(change settings)</span></div><pre>" +
"\n".join(cgi.escape(s) for s in options) + "</pre></div>\n"
class CaptureOutput(Grammar): class CaptureOutput(Grammar):
grammar = OPTIONAL(SPACE), WORD("A-Za-z_"), L('=$('), REST_OF_LINE, L(")"), OPTIONAL(L(';')), EOL grammar = OPTIONAL(SPACE), WORD("A-Za-z_"), L('=$('), REST_OF_LINE, L(")"), OPTIONAL(L(';')), EOL
def value(self):
cmd = self[3].string def value(self):
cmd = cmd.replace("; ", "\n") cmd = self[3].string
return "<div class='write-to'><div class='filename'>$" + self[1].string + "=</div><pre>" + cgi.escape(cmd) + "</pre></div>\n" cmd = cmd.replace("; ", "\n")
return "<div class='write-to'><div class='filename'>$" +
self[1].string + "=</div><pre>" + cgi.escape(cmd) + "</pre></div>\n"
class SedReplace(Grammar): class SedReplace(Grammar):
grammar = OPTIONAL(SPACE), L('sed -i "s/'), OPTIONAL(L('^')), ONE_OR_MORE(WORD("-A-Za-z0-9 #=\\{};.*$_!()")), L('/'), ONE_OR_MORE(WORD("-A-Za-z0-9 #=\\{};.*$_!()")), L('/"'), SPACE, FILENAME, EOL grammar = OPTIONAL(SPACE), L('sed -i "s/'), OPTIONAL(L('^')), \
def value(self): ONE_OR_MORE(WORD("-A-Za-z0-9 #=\\{};.*$_!()")), L('/'), \
return "<div class='write-to'><div class='filename'>edit<br>" + self[8].string + "</div><p>replace</p><pre>" + cgi.escape(self[3].string.replace(".*", ". . .")) + "</pre><p>with</p><pre>" + cgi.escape(self[5].string.replace("\\n", "\n").replace("\\t", "\t")) + "</pre></div>\n" ONE_OR_MORE(WORD("-A-Za-z0-9 #=\\{};.*$_!()")), L('/"'), SPACE, FILENAME, EOL
def value(self):
return "<div class='write-to'><div class='filename'>edit<br>" + self[8].string +
"</div><p>replace</p><pre>" + cgi.escape(self[3].string.replace(".*", ". . .")) +
"</pre><p>with</p><pre>" +
cgi.escape(self[5].string.replace("\\n", "\n").replace("\\t", "\t")) + "</pre></div>\n"
class EchoPipe(Grammar): class EchoPipe(Grammar):
grammar = OPTIONAL(SPACE), L("echo "), REST_OF_LINE, L(' | '), REST_OF_LINE, EOL grammar = OPTIONAL(SPACE), L("echo "), REST_OF_LINE, L(' | '), REST_OF_LINE, EOL
def value(self):
text = " ".join("\"%s\"" % s for s in self[2].string.split(" ")) def value(self):
return "<pre class='shell'><div>echo " + recode_bash(text) + " \<br> | " + recode_bash(self[4].string) + "</div></pre>\n" text = " ".join("\"%s\"" % s for s in self[2].string.split(" "))
return "<pre class='shell'><div>echo " + recode_bash(text) +
" \<br> | " + recode_bash(self[4].string) + "</div></pre>\n"
def shell_line(bash): def shell_line(bash):
return "<pre class='shell'><div>" + recode_bash(bash.strip()) + "</div></pre>\n" return "<pre class='shell'><div>" + recode_bash(bash.strip()) + "</div></pre>\n"
class AptGet(Grammar): class AptGet(Grammar):
grammar = (ZERO_OR_MORE(SPACE), L("apt_install "), REST_OF_LINE, EOL) grammar = (ZERO_OR_MORE(SPACE), L("apt_install "), REST_OF_LINE, EOL)
def value(self):
return shell_line("apt-get install -y " + re.sub(r"\s+", " ", self[2].string)) def value(self):
return shell_line("apt-get install -y " + re.sub(r"\s+", " ", self[2].string))
class UfwAllow(Grammar): class UfwAllow(Grammar):
grammar = (ZERO_OR_MORE(SPACE), L("ufw_allow "), REST_OF_LINE, EOL) grammar = (ZERO_OR_MORE(SPACE), L("ufw_allow "), REST_OF_LINE, EOL)
def value(self):
return shell_line("ufw allow " + self[2].string) def value(self):
return shell_line("ufw allow " + self[2].string)
class RestartService(Grammar): class RestartService(Grammar):
grammar = (ZERO_OR_MORE(SPACE), L("restart_service "), REST_OF_LINE, EOL) grammar = (ZERO_OR_MORE(SPACE), L("restart_service "), REST_OF_LINE, EOL)
def value(self):
return shell_line("service " + self[2].string + " restart") def value(self):
return shell_line("service " + self[2].string + " restart")
class OtherLine(Grammar): class OtherLine(Grammar):
grammar = (REST_OF_LINE, EOL) grammar = (REST_OF_LINE, EOL)
def value(self):
if self.string.strip() == "": return "" def value(self):
if "source setup/functions.sh" in self.string: return "" if self.string.strip() == "":
if "source /etc/mailinabox.conf" in self.string: return "" return ""
return "<pre class='shell'><div>" + recode_bash(self.string.strip()) + "</div></pre>\n" if "source setup/functions.sh" in self.string:
return ""
if "source /etc/mailinabox.conf" in self.string:
return ""
return "<pre class='shell'><div>" + recode_bash(self.string.strip()) + "</div></pre>\n"
class BashElement(Grammar): class BashElement(Grammar):
grammar = Comment | CatEOF | EchoPipe | EchoLine | HideOutput | EditConf | SedReplace | AptGet | UfwAllow | RestartService | OtherLine grammar = Comment | CatEOF | EchoPipe | EchoLine | HideOutput | EditConf | \
def value(self): SedReplace | AptGet | UfwAllow | RestartService | OtherLine
return self[0].value()
def value(self):
return self[0].value()
# Make some special characters to private use Unicode code points. # Make some special characters to private use Unicode code points.
bash_special_characters1 = { bash_special_characters1 = {
"\n": "\uE000", "\n": "\uE000",
" ": "\uE001", " ": "\uE001",
} }
bash_special_characters2 = { bash_special_characters2 = {
"$": "\uE010", "$": "\uE010",
} }
bash_escapes = { bash_escapes = {
"n": "\uE020", "n": "\uE020",
"t": "\uE021", "t": "\uE021",
} }
def quasitokenize(bashscript): def quasitokenize(bashscript):
# Make a parse of bash easier by making the tokenization easy. # Make a parse of bash easier by making the tokenization easy.
newscript = "" newscript = ""
quote_mode = None quote_mode = None
escape_next = False escape_next = False
line_comment = False line_comment = False
subshell = 0 subshell = 0
for c in bashscript: for c in bashscript:
if line_comment: if line_comment:
# We're in a comment until the end of the line. # We're in a comment until the end of the line.
newscript += c newscript += c
if c == '\n': if c == '\n':
line_comment = False line_comment = False
elif escape_next: elif escape_next:
# Previous character was a \. Normally the next character # Previous character was a \. Normally the next character
# comes through literally, but escaped newlines are line # comes through literally, but escaped newlines are line
# continuations and some escapes are for special characters # continuations and some escapes are for special characters
# which we'll recode and then turn back into escapes later. # which we'll recode and then turn back into escapes later.
if c == "\n": if c == "\n":
c = " " c = " "
elif c in bash_escapes: elif c in bash_escapes:
c = bash_escapes[c] c = bash_escapes[c]
newscript += c newscript += c
escape_next = False escape_next = False
elif c == "\\": elif c == "\\":
# Escaping next character. # Escaping next character.
escape_next = True escape_next = True
elif quote_mode is None and c in ('"', "'"): elif quote_mode is None and c in ('"', "'"):
# Starting a quoted word. # Starting a quoted word.
quote_mode = c quote_mode = c
elif c == quote_mode: elif c == quote_mode:
# Ending a quoted word. # Ending a quoted word.
quote_mode = None quote_mode = None
elif quote_mode is not None and quote_mode != "EOF" and c in bash_special_characters1: elif quote_mode is not None and quote_mode != "EOF" and c in bash_special_characters1:
# Replace special tokens within quoted words so that they # Replace special tokens within quoted words so that they
# don't interfere with tokenization later. # don't interfere with tokenization later.
newscript += bash_special_characters1[c] newscript += bash_special_characters1[c]
elif quote_mode is None and c == '#': elif quote_mode is None and c == '#':
# Start of a line comment. # Start of a line comment.
newscript += c newscript += c
line_comment = True line_comment = True
elif quote_mode is None and c == ';' and subshell == 0: elif quote_mode is None and c == ';' and subshell == 0:
# End of a statement. # End of a statement.
newscript += "\n" newscript += "\n"
elif quote_mode is None and c == '(': elif quote_mode is None and c == '(':
# Start of a subshell. # Start of a subshell.
newscript += c newscript += c
subshell += 1 subshell += 1
elif quote_mode is None and c == ')': elif quote_mode is None and c == ')':
# End of a subshell. # End of a subshell.
newscript += c newscript += c
subshell -= 1 subshell -= 1
elif quote_mode is None and c == '\t': elif quote_mode is None and c == '\t':
# Make these just spaces. # Make these just spaces.
if newscript[-1] != " ": if newscript[-1] != " ":
newscript += " " newscript += " "
elif quote_mode is None and c == ' ': elif quote_mode is None and c == ' ':
# Collapse consecutive spaces. # Collapse consecutive spaces.
if newscript[-1] != " ": if newscript[-1] != " ":
newscript += " " newscript += " "
elif c in bash_special_characters2: elif c in bash_special_characters2:
newscript += bash_special_characters2[c] newscript += bash_special_characters2[c]
else: else:
# All other characters. # All other characters.
newscript += c newscript += c
# "<< EOF" escaping. # "<< EOF" escaping.
if quote_mode is None and re.search("<<\s*EOF\n$", newscript): if quote_mode is None and re.search("<<\s*EOF\n$", newscript):
quote_mode = "EOF" quote_mode = "EOF"
elif quote_mode == "EOF" and re.search("\nEOF\n$", newscript): elif quote_mode == "EOF" and re.search("\nEOF\n$", newscript):
quote_mode = None quote_mode = None
return newscript
return newscript
def recode_bash(s): def recode_bash(s):
def requote(tok): def requote(tok):
tok = tok.replace("\\", "\\\\") tok = tok.replace("\\", "\\\\")
for c in bash_special_characters2: for c in bash_special_characters2:
tok = tok.replace(c, "\\" + c) tok = tok.replace(c, "\\" + c)
tok = fixup_tokens(tok) tok = fixup_tokens(tok)
if " " in tok or '"' in tok: if " " in tok or '"' in tok:
tok = tok.replace("\"", "\\\"") tok = tok.replace("\"", "\\\"")
tok = '"' + tok +'"' tok = '"' + tok + '"'
else: else:
tok = tok.replace("'", "\\'") tok = tok.replace("'", "\\'")
return tok return tok
return cgi.escape(" ".join(requote(tok) for tok in s.split(" "))) return cgi.escape(" ".join(requote(tok) for tok in s.split(" ")))
def fixup_tokens(s): def fixup_tokens(s):
for c, enc in bash_special_characters1.items(): for c, enc in bash_special_characters1.items():
s = s.replace(enc, c) s = s.replace(enc, c)
for c, enc in bash_special_characters2.items(): for c, enc in bash_special_characters2.items():
s = s.replace(enc, c) s = s.replace(enc, c)
for esc, c in bash_escapes.items(): for esc, c in bash_escapes.items():
s = s.replace(c, "\\" + esc) s = s.replace(c, "\\" + esc)
return s return s
class BashScript(Grammar): class BashScript(Grammar):
grammar = (OPTIONAL(HashBang), REPEAT(BashElement)) grammar = (OPTIONAL(HashBang), REPEAT(BashElement))
def value(self):
return [line.value() for line in self[1]]
@staticmethod def value(self):
def parse(fn): return [line.value() for line in self[1]]
if fn in ("setup/functions.sh", "/etc/mailinabox.conf"): return ""
string = open(fn).read()
# tokenize @staticmethod
string = re.sub(".* #NODOC\n", "", string) def parse(fn):
string = re.sub("\n\s*if .*then.*|\n\s*fi|\n\s*else|\n\s*elif .*", "", string) if fn in ("setup/functions.sh", "/etc/mailinabox.conf"):
string = quasitokenize(string) return ""
string = re.sub("hide_output ", "", string) string = open(fn).read()
parser = BashScript.parser() # tokenize
result = parser.parse_string(string) string = re.sub(".* #NODOC\n", "", string)
string = re.sub("\n\s*if .*then.*|\n\s*fi|\n\s*else|\n\s*elif .*", "", string)
string = quasitokenize(string)
string = re.sub("hide_output ", "", string)
v = "<div class='row'><div class='col-xs-12 sourcefile'>view the bash source for the following section at <a href=\"%s\">%s</a></div></div>\n" \ parser = BashScript.parser()
% ("https://github.com/mail-in-a-box/mailinabox/tree/master/" + fn, fn) result = parser.parse_string(string)
mode = 0 v = ("<div class='row'><div class='col-xs-12 sourcefile'>view the bash source for \
for item in result.value(): the following section at <a href=\"%s\">%s</a></div></div>\n") \
if item.strip() == "": % ("https://github.com/mail-in-a-box/mailinabox/tree/master/" + fn, fn)
pass
elif item.startswith("<p") and not item.startswith("<pre"):
clz = ""
if mode == 2:
v += "</div>\n" # col
v += "</div>\n" # row
mode = 0
clz = "contd"
if mode == 0:
v += "<div class='row %s'>\n" % clz
v += "<div class='col-md-6 prose'>\n"
v += item
mode = 1
elif item.startswith("<h"):
if mode != 0:
v += "</div>\n" # col
v += "</div>\n" # row
v += "<div class='row'>\n"
v += "<div class='col-md-6 header'>\n"
v += item
v += "</div>\n" # col
v += "<div class='col-md-6 terminal'> </div>\n"
v += "</div>\n" # row
mode = 0
else:
if mode == 0:
v += "<div class='row'>\n"
v += "<div class='col-md-offset-6 col-md-6 terminal'>\n"
elif mode == 1:
v += "</div>\n"
v += "<div class='col-md-6 terminal'>\n"
mode = 2
v += item
v += "</div>\n" # col mode = 0
v += "</div>\n" # row for item in result.value():
if item.strip() == "":
pass
elif item.startswith("<p") and not item.startswith("<pre"):
clz = ""
if mode == 2:
v += "</div>\n" # col
v += "</div>\n" # row
mode = 0
clz = "contd"
if mode == 0:
v += "<div class='row %s'>\n" % clz
v += "<div class='col-md-6 prose'>\n"
v += item
mode = 1
elif item.startswith("<h"):
if mode != 0:
v += "</div>\n" # col
v += "</div>\n" # row
v += "<div class='row'>\n"
v += "<div class='col-md-6 header'>\n"
v += item
v += "</div>\n" # col
v += "<div class='col-md-6 terminal'> </div>\n"
v += "</div>\n" # row
mode = 0
else:
if mode == 0:
v += "<div class='row'>\n"
v += "<div class='col-md-offset-6 col-md-6 terminal'>\n"
elif mode == 1:
v += "</div>\n"
v += "<div class='col-md-6 terminal'>\n"
mode = 2
v += item
v = fixup_tokens(v) v += "</div>\n" # col
v += "</div>\n" # row
v = v.replace("</pre>\n<pre class='shell'>", "") v = fixup_tokens(v)
v = re.sub("<pre>([\w\W]*?)</pre>", lambda m : "<pre>" + strip_indent(m.group(1)) + "</pre>", v)
v = re.sub(r"(\$?)PRIMARY_HOSTNAME", r"<b>box.yourdomain.com</b>", v) v = v.replace("</pre>\n<pre class='shell'>", "")
v = re.sub(r"\$STORAGE_ROOT", r"<b>$STORE</b>", v) v = re.sub("<pre>([\w\W]*?)</pre>", lambda m: "<pre>" + strip_indent(m.group(1)) + "</pre>", v)
v = v.replace("`pwd`", "<code><b>/path/to/mailinabox</b></code>")
v = re.sub(r"(\$?)PRIMARY_HOSTNAME", r"<b>box.yourdomain.com</b>", v)
v = re.sub(r"\$STORAGE_ROOT", r"<b>$STORE</b>", v)
v = v.replace("`pwd`", "<code><b>/path/to/mailinabox</b></code>")
return v
return v
def wrap_lines(text, cols=60): def wrap_lines(text, cols=60):
ret = "" ret = ""
words = re.split("(\s+)", text) words = re.split("(\s+)", text)
linelen = 0 linelen = 0
for w in words: for w in words:
if linelen + len(w) > cols-1: if linelen + len(w) > cols-1:
ret += " \\\n" ret += " \\\n"
ret += " " ret += " "
linelen = 0 linelen = 0
if linelen == 0 and w.strip() == "": continue if linelen == 0 and w.strip() == "":
ret += w continue
linelen += len(w) ret += w
return ret linelen += len(w)
return ret
if __name__ == '__main__': if __name__ == '__main__':
generate_documentation() generate_documentation()

21
tools/update-subresource-integrity.py
View File

@ -4,21 +4,26 @@
# after updating the Bootstrap and jQuery <link> and <script> to compute the # after updating the Bootstrap and jQuery <link> and <script> to compute the
# appropriate hash and insert it into the template. # appropriate hash and insert it into the template.
import re, urllib.request, hashlib, base64 import re
import urllib.request
import hashlib
import base64
fn = "management/templates/index.html" fn = "management/templates/index.html"
with open(fn, 'r') as f: with open(fn, 'r') as f:
content = f.read() content = f.read()
def make_integrity(url): def make_integrity(url):
resource = urllib.request.urlopen(url).read() resource = urllib.request.urlopen(url).read()
return "sha256-" + base64.b64encode(hashlib.sha256(resource).digest()).decode('ascii') return "sha256-" + base64.b64encode(hashlib.sha256(resource).digest()).decode('ascii')
content = re.sub( content = re.sub(
r'<(link rel="stylesheet" href|script src)="(.*?)" integrity="(.*?)"', r'<(link rel="stylesheet" href|script src)="(.*?)" integrity="(.*?)"',
lambda m : '<' + m.group(1) + '="' + m.group(2) + '" integrity="' + make_integrity(m.group(2)) + '"', lambda m: '<' + m.group(1) + '="' + m.group(2) + '" integrity="' + make_integrity(m.group(2)) + '"',
content) content)
with open(fn, 'w') as f: with open(fn, 'w') as f:
f.write(content) f.write(content)