From 8cea79de8badd77d713d70c2f0d4935bf4ad732a Mon Sep 17 00:00:00 2001 From: Wolfgang Steitz Date: Sun, 20 Nov 2016 13:23:46 +0100 Subject: [PATCH] pep8 cleanup of Python source files pep8 (https://www.python.org/dev/peps/pep-0008/) is the commonly accepted and widely adopted code style convention for Python. I used pycodestyle (https://pycodestyle.readthedocs.io/en/latest/) to check for pep8 compatibility. Especially the mix of tabs and spaces in the Python files makes it hard to work with. I switched to spaces, because that's what pep8 expects and the majority of Python programmers use. --- tools/editconf.py | 167 +++-- tools/mail.py | 183 ++--- tools/parse-nginx-log-bootstrap-accesses.py | 55 +- tools/readable_bash.py | 785 +++++++++++--------- tools/update-subresource-integrity.py | 21 +- 5 files changed, 650 insertions(+), 561 deletions(-) diff --git a/tools/editconf.py b/tools/editconf.py index d665f861..06688c0d 100755 --- a/tools/editconf.py +++ b/tools/editconf.py @@ -14,20 +14,21 @@ # # NAME VALUE # -# If the -c option is given, then the supplied character becomes the comment character +# If the -c option is given, then the supplied character becomes the commentcharacter # # If the -w option is given, then setting lines continue onto following # lines while the lines start with whitespace, e.g.: # # NAME VAL -# UE +# UE -import sys, re +import sys +import re # sanity check if len(sys.argv) < 3: - print("usage: python3 editconf.py /etc/file.conf [-s] [-w] [-c ] [-t] NAME=VAL [NAME=VAL ...]") - sys.exit(1) + print("usage: python3 editconf.py /etc/file.conf [-s] [-w] [-c ] [-t] NAME=VAL [NAME=VAL ...]") + sys.exit(1) # parse command line arguments filename = sys.argv[1] @@ -39,30 +40,30 @@ comment_char = "#" folded_lines = False testing = False while settings[0][0] == "-" and settings[0] != "--": - opt = settings.pop(0) - if opt == "-s": - # Space is the delimiter - delimiter = " " - delimiter_re = r"\s+" - elif opt == "-w": - # Line folding is possible in this file. - folded_lines = True - elif opt == "-c": - # Specifies a different comment character. - comment_char = settings.pop(0) - elif opt == "-t": - testing = True - else: - print("Invalid option.") - sys.exit(1) + opt = settings.pop(0) + if opt == "-s": + # Space is the delimiter + delimiter = " " + delimiter_re = r"\s+" + elif opt == "-w": + # Line folding is possible in this file. + folded_lines = True + elif opt == "-c": + # Specifies a different comment character. + comment_char = settings.pop(0) + elif opt == "-t": + testing = True + else: + print("Invalid option.") + sys.exit(1) # sanity check command line for setting in settings: - try: - name, value = setting.split("=", 1) - except: - import subprocess - print("Invalid command line: ", subprocess.list2cmdline(sys.argv)) + try: + name, value = setting.split("=", 1) + except: + import subprocess + print("Invalid command line: ", subprocess.list2cmdline(sys.argv)) # create the new config file in memory @@ -71,67 +72,69 @@ buf = "" input_lines = list(open(filename)) while len(input_lines) > 0: - line = input_lines.pop(0) + line = input_lines.pop(0) - # If this configuration file uses folded lines, append any folded lines - # into our input buffer. - if folded_lines and line[0] not in (comment_char, " ", ""): - while len(input_lines) > 0 and input_lines[0][0] in " \t": - line += input_lines.pop(0) + # If this configuration file uses folded lines, append any folded lines + # into our input buffer. + if folded_lines and line[0] not in (comment_char, " ", ""): + while len(input_lines) > 0 and input_lines[0][0] in " \t": + line += input_lines.pop(0) - # See if this line is for any settings passed on the command line. - for i in range(len(settings)): - # Check that this line contain this setting from the command-line arguments. - name, val = settings[i].split("=", 1) - m = re.match( - "(\s*)" - + "(" + re.escape(comment_char) + "\s*)?" - + re.escape(name) + delimiter_re + "(.*?)\s*$", - line, re.S) - if not m: continue - indent, is_comment, existing_val = m.groups() + # See if this line is for any settings passed on the command line. + for i in range(len(settings)): + # Check that this line contain this setting from the command-line arguments. + name, val = settings[i].split("=", 1) + m = re.match( + "(\s*)" + + "(" + re.escape(comment_char) + "\s*)?" + + re.escape(name) + delimiter_re + "(.*?)\s*$", + line, re.S) + if not m: + continue + indent, is_comment, existing_val = m.groups() + + # If this is already the setting, do nothing. + if is_comment is None and existing_val == val: + # It may be that we've already inserted this setting higher + # in the file so check for that first. + if i in found: + break + buf += line + found.add(i) + break + + # comment-out the existing line (also comment any folded lines) + if is_comment is None: + buf += comment_char + line.rstrip().replace("\n", "\n" + comment_char) + "\n" + else: + # the line is already commented, pass it through + buf += line + + # if this option oddly appears more than once, don't add the setting again + if i in found: + break + + # add the new setting + buf += indent + name + delimiter + val + "\n" + + # note that we've applied this option + found.add(i) + + break + else: + # If did not match any setting names, pass this line through. + buf += line - # If this is already the setting, do nothing. - if is_comment is None and existing_val == val: - # It may be that we've already inserted this setting higher - # in the file so check for that first. - if i in found: break - buf += line - found.add(i) - break - - # comment-out the existing line (also comment any folded lines) - if is_comment is None: - buf += comment_char + line.rstrip().replace("\n", "\n" + comment_char) + "\n" - else: - # the line is already commented, pass it through - buf += line - - # if this option oddly appears more than once, don't add the setting again - if i in found: - break - - # add the new setting - buf += indent + name + delimiter + val + "\n" - - # note that we've applied this option - found.add(i) - - break - else: - # If did not match any setting names, pass this line through. - buf += line - # Put any settings we didn't see at the end of the file. for i in range(len(settings)): - if i not in found: - name, val = settings[i].split("=", 1) - buf += name + delimiter + val + "\n" + if i not in found: + name, val = settings[i].split("=", 1) + buf += name + delimiter + val + "\n" if not testing: - # Write out the new file. - with open(filename, "w") as f: - f.write(buf) + # Write out the new file. + with open(filename, "w") as f: + f.write(buf) else: - # Just print the new file to stdout. - print(buf) + # Just print the new file to stdout. + print(buf) diff --git a/tools/mail.py b/tools/mail.py index d0e546ab..4c7c5a1e 100755 --- a/tools/mail.py +++ b/tools/mail.py @@ -1,31 +1,40 @@ #!/usr/bin/python3 -import sys, getpass, urllib.request, urllib.error, json, re +import sys +import getpass +import urllib.request +import urllib.error +import json +import re + def mgmt(cmd, data=None, is_json=False): - # The base URL for the management daemon. (Listens on IPv4 only.) - mgmt_uri = 'http://127.0.0.1:10222' + # The base URL for the management daemon. (Listens on IPv4 only.) + mgmt_uri = 'http://127.0.0.1:10222' - setup_key_auth(mgmt_uri) + setup_key_auth(mgmt_uri) + + req = urllib.request.Request(mgmt_uri + cmd, urllib.parse.urlencode(data).encode("utf8") if data else None) + try: + response = urllib.request.urlopen(req) + except urllib.error.HTTPError as e: + if e.code == 401: + try: + print(e.read().decode("utf8")) + except: + pass + print("The management daemon refused access. The API key file may be out of sync. \ + Try 'service mailinabox restart'.", file=sys.stderr) + elif hasattr(e, 'read'): + print(e.read().decode('utf8'), file=sys.stderr) + else: + print(e, file=sys.stderr) + sys.exit(1) + resp = response.read().decode('utf8') + if is_json: + resp = json.loads(resp) + return resp - req = urllib.request.Request(mgmt_uri + cmd, urllib.parse.urlencode(data).encode("utf8") if data else None) - try: - response = urllib.request.urlopen(req) - except urllib.error.HTTPError as e: - if e.code == 401: - try: - print(e.read().decode("utf8")) - except: - pass - print("The management daemon refused access. The API key file may be out of sync. Try 'service mailinabox restart'.", file=sys.stderr) - elif hasattr(e, 'read'): - print(e.read().decode('utf8'), file=sys.stderr) - else: - print(e, file=sys.stderr) - sys.exit(1) - resp = response.read().decode('utf8') - if is_json: resp = json.loads(resp) - return resp def read_password(): while True: @@ -43,89 +52,91 @@ def read_password(): break return first -def setup_key_auth(mgmt_uri): - key = open('/var/lib/mailinabox/api.key').read().strip() - auth_handler = urllib.request.HTTPBasicAuthHandler() - auth_handler.add_password( - realm='Mail-in-a-Box Management Server', - uri=mgmt_uri, - user=key, - passwd='') - opener = urllib.request.build_opener(auth_handler) - urllib.request.install_opener(opener) +def setup_key_auth(mgmt_uri): + key = open('/var/lib/mailinabox/api.key').read().strip() + + auth_handler = urllib.request.HTTPBasicAuthHandler() + auth_handler.add_password( + realm='Mail-in-a-Box Management Server', + uri=mgmt_uri, + user=key, + passwd='') + opener = urllib.request.build_opener(auth_handler) + urllib.request.install_opener(opener) + if len(sys.argv) < 2: - print("Usage: ") - print(" tools/mail.py user (lists users)") - print(" tools/mail.py user add user@domain.com [password]") - print(" tools/mail.py user password user@domain.com [password]") - print(" tools/mail.py user remove user@domain.com") - print(" tools/mail.py user make-admin user@domain.com") - print(" tools/mail.py user remove-admin user@domain.com") - print(" tools/mail.py user admins (lists admins)") - print(" tools/mail.py alias (lists aliases)") - print(" tools/mail.py alias add incoming.name@domain.com sent.to@other.domain.com") - print(" tools/mail.py alias add incoming.name@domain.com 'sent.to@other.domain.com, multiple.people@other.domain.com'") - print(" tools/mail.py alias remove incoming.name@domain.com") - print() - print("Removing a mail user does not delete their mail folders on disk. It only prevents IMAP/SMTP login.") - print() + print("Usage: ") + print(" tools/mail.py user (lists users)") + print(" tools/mail.py user add user@domain.com [password]") + print(" tools/mail.py user password user@domain.com [password]") + print(" tools/mail.py user remove user@domain.com") + print(" tools/mail.py user make-admin user@domain.com") + print(" tools/mail.py user remove-admin user@domain.com") + print(" tools/mail.py user admins (lists admins)") + print(" tools/mail.py alias (lists aliases)") + print(" tools/mail.py alias add incoming.name@domain.com sent.to@other.domain.com") + print(" tools/mail.py alias add incoming.name@domain.com 'sent.to@other.domain.com, multiple.people@other.domain.com'") + print(" tools/mail.py alias remove incoming.name@domain.com") + print() + print("Removing a mail user does not delete their mail folders on disk. It only prevents IMAP/SMTP login.") + print() elif sys.argv[1] == "user" and len(sys.argv) == 2: - # Dump a list of users, one per line. Mark admins with an asterisk. - users = mgmt("/mail/users?format=json", is_json=True) - for domain in users: - for user in domain["users"]: - if user['status'] == 'inactive': continue - print(user['email'], end='') - if "admin" in user['privileges']: - print("*", end='') - print() + # Dump a list of users, one per line. Mark admins with an asterisk. + users = mgmt("/mail/users?format=json", is_json=True) + for domain in users: + for user in domain["users"]: + if user['status'] == 'inactive': + continue + print(user['email'], end='') + if "admin" in user['privileges']: + print("*", end='') + print() elif sys.argv[1] == "user" and sys.argv[2] in ("add", "password"): - if len(sys.argv) < 5: - if len(sys.argv) < 4: - email = input("email: ") - else: - email = sys.argv[3] - pw = read_password() - else: - email, pw = sys.argv[3:5] + if len(sys.argv) < 5: + if len(sys.argv) < 4: + email = input("email: ") + else: + email = sys.argv[3] + pw = read_password() + else: + email, pw = sys.argv[3:5] - if sys.argv[2] == "add": - print(mgmt("/mail/users/add", { "email": email, "password": pw })) - elif sys.argv[2] == "password": - print(mgmt("/mail/users/password", { "email": email, "password": pw })) + if sys.argv[2] == "add": + print(mgmt("/mail/users/add", {"email": email, "password": pw})) + elif sys.argv[2] == "password": + print(mgmt("/mail/users/password", {"email": email, "password": pw})) elif sys.argv[1] == "user" and sys.argv[2] == "remove" and len(sys.argv) == 4: - print(mgmt("/mail/users/remove", { "email": sys.argv[3] })) + print(mgmt("/mail/users/remove", {"email": sys.argv[3]})) elif sys.argv[1] == "user" and sys.argv[2] in ("make-admin", "remove-admin") and len(sys.argv) == 4: - if sys.argv[2] == "make-admin": - action = "add" - else: - action = "remove" - print(mgmt("/mail/users/privileges/" + action, { "email": sys.argv[3], "privilege": "admin" })) + if sys.argv[2] == "make-admin": + action = "add" + else: + action = "remove" + print(mgmt("/mail/users/privileges/" + action, {"email": sys.argv[3], "privilege": "admin"})) elif sys.argv[1] == "user" and sys.argv[2] == "admins": - # Dump a list of admin users. - users = mgmt("/mail/users?format=json", is_json=True) - for domain in users: - for user in domain["users"]: - if "admin" in user['privileges']: - print(user['email']) + # Dump a list of admin users. + users = mgmt("/mail/users?format=json", is_json=True) + for domain in users: + for user in domain["users"]: + if "admin" in user['privileges']: + print(user['email']) elif sys.argv[1] == "alias" and len(sys.argv) == 2: - print(mgmt("/mail/aliases")) + print(mgmt("/mail/aliases")) elif sys.argv[1] == "alias" and sys.argv[2] == "add" and len(sys.argv) == 5: - print(mgmt("/mail/aliases/add", { "address": sys.argv[3], "forwards_to": sys.argv[4] })) + print(mgmt("/mail/aliases/add", {"address": sys.argv[3], "forwards_to": sys.argv[4]})) elif sys.argv[1] == "alias" and sys.argv[2] == "remove" and len(sys.argv) == 4: - print(mgmt("/mail/aliases/remove", { "address": sys.argv[3] })) + print(mgmt("/mail/aliases/remove", {"address": sys.argv[3]})) else: - print("Invalid command-line arguments.") - sys.exit(1) - + print("Invalid command-line arguments.") + sys.exit(1) diff --git a/tools/parse-nginx-log-bootstrap-accesses.py b/tools/parse-nginx-log-bootstrap-accesses.py index b08bc253..65ccad43 100755 --- a/tools/parse-nginx-log-bootstrap-accesses.py +++ b/tools/parse-nginx-log-bootstrap-accesses.py @@ -5,7 +5,11 @@ # looking at accesses to the bootstrap.sh script (which is currently at the URL # .../setup.sh). -import re, glob, gzip, os.path, json +import re +import glob +import gzip +import os.path +import json import dateutil.parser outfn = "/home/user-data/www/mailinabox.email/install-stats.json" @@ -16,37 +20,38 @@ accesses = set() # Scan the current and rotated access logs. for fn in glob.glob("/var/log/nginx/access.log*"): - # Gunzip if necessary. - if fn.endswith(".gz"): - f = gzip.open(fn) - else: - f = open(fn, "rb") + # Gunzip if necessary. + if fn.endswith(".gz"): + f = gzip.open(fn) + else: + f = open(fn, "rb") - # Loop through the lines in the access log. - with f: - for line in f: - # Find lines that are GETs on the bootstrap script by either curl or wget. - # (Note that we purposely skip ...?ping=1 requests which is the admin panel querying us for updates.) - # (Also, the URL changed in January 2016, but we'll accept both.) - m = re.match(rb"(?P\S+) - - \[(?P.*?)\] \"GET /(bootstrap.sh|setup.sh) HTTP/.*\" 200 \d+ .* \"(?:curl|wget)", line, re.I) - if m: - date, time = m.group("date").decode("ascii").split(":", 1) - date = dateutil.parser.parse(date).date().isoformat() - ip = m.group("ip").decode("ascii") - accesses.add( (date, ip) ) + # Loop through the lines in the access log. + with f: + for line in f: + # Find lines that are GETs on the bootstrap script by either curl or wget. + # (Note that we purposely skip ...?ping=1 requests which is the admin + # panel querying us for updates.) + # (Also, the URL changed in January 2016, but we'll accept both.) + m = re.match(rb"(?P\S+) - - \[(?P.*?)\] \"GET /(bootstrap.sh|setup.sh) HTTP/.*\" 200 \d+ .* \"(?:curl|wget)", line, re.I) + if m: + date, time = m.group("date").decode("ascii").split(":", 1) + date = dateutil.parser.parse(date).date().isoformat() + ip = m.group("ip").decode("ascii") + accesses.add((date, ip)) # Aggregate by date. -by_date = { } +by_date = {} for date, ip in accesses: - by_date[date] = by_date.get(date, 0) + 1 + by_date[date] = by_date.get(date, 0) + 1 # Since logs are rotated, store the statistics permanently in a JSON file. # Load in the stats from an existing file. if os.path.exists(outfn): - existing_data = json.load(open(outfn)) - for date, count in existing_data: - if date not in by_date: - by_date[date] = count + existing_data = json.load(open(outfn)) + for date, count in existing_data: + if date not in by_date: + by_date[date] = count # Turn into a list rather than a dict structure to make it ordered. by_date = sorted(by_date.items()) @@ -56,4 +61,4 @@ by_date.pop(-1) # Write out. with open(outfn, "w") as f: - json.dump(by_date, f, sort_keys=True, indent=True) + json.dump(by_date, f, sort_keys=True, indent=True) diff --git a/tools/readable_bash.py b/tools/readable_bash.py index 5207a78a..8f02b4f9 100644 --- a/tools/readable_bash.py +++ b/tools/readable_bash.py @@ -3,12 +3,14 @@ # Generate documentation for how this machine works by # parsing our bash scripts! -import cgi, re +import cgi +import re import markdown from modgrammar import * + def generate_documentation(): - print(""" + print(""" @@ -21,93 +23,93 @@ def generate_documentation(): @@ -123,358 +125,421 @@ def generate_documentation():
""") - parser = Source.parser() - for line in open("setup/start.sh"): - try: - fn = parser.parse_string(line).filename() - except: - continue - if fn in ("setup/start.sh", "setup/preflight.sh", "setup/questions.sh", "setup/firstuser.sh", "setup/management.sh"): - continue + parser = Source.parser() + for line in open("setup/start.sh"): + try: + fn = parser.parse_string(line).filename() + except: + continue + if fn in ("setup/start.sh", "setup/preflight.sh", "setup/questions.sh", + "setup/firstuser.sh", "setup/management.sh"): + continue - import sys - print(fn, file=sys.stderr) + import sys + print(fn, file=sys.stderr) - print(BashScript.parse(fn)) + print(BashScript.parse(fn)) - print(""" + print(""" + """) + class HashBang(Grammar): - grammar = (L('#!'), REST_OF_LINE, EOL) - def value(self): - return "" + grammar = (L('#!'), REST_OF_LINE, EOL) + + def value(self): + return "" + def strip_indent(s): - s = s.replace("\t", " ") - lines = s.split("\n") - try: - min_indent = min(len(re.match(r"\s*", line).group(0)) for line in lines if len(line) > 0) - except ValueError: - # No non-empty lines. - min_indent = 0 - lines = [line[min_indent:] for line in lines] - return "\n".join(lines) + s = s.replace("\t", " ") + lines = s.split("\n") + try: + min_indent = min(len(re.match(r"\s*", line).group(0)) for line in lines if len(line) > 0) + except ValueError: + # No non-empty lines. + min_indent = 0 + lines = [line[min_indent:] for line in lines] + return "\n".join(lines) + class Comment(Grammar): - grammar = ONE_OR_MORE(ZERO_OR_MORE(SPACE), L('#'), REST_OF_LINE, EOL) - def value(self): - if self.string.replace("#", "").strip() == "": - return "\n" - lines = [x[2].string for x in self[0]] - content = "\n".join(lines) - content = strip_indent(content) - return markdown.markdown(content, output_format="html4") + "\n\n" + grammar = ONE_OR_MORE(ZERO_OR_MORE(SPACE), L('#'), REST_OF_LINE, EOL) + + def value(self): + if self.string.replace("#", "").strip() == "": + return "\n" + lines = [x[2].string for x in self[0]] + content = "\n".join(lines) + content = strip_indent(content) + return markdown.markdown(content, output_format="html4") + "\n\n" + FILENAME = WORD('a-z0-9-/.') + class Source(Grammar): - grammar = ((L('.') | L('source')), L(' '), FILENAME, Comment | EOL) - def filename(self): - return self[2].string.strip() - def value(self): - return BashScript.parse(self.filename()) + grammar = ((L('.') | L('source')), L(' '), FILENAME, Comment | EOL) + + def filename(self): + return self[2].string.strip() + + def value(self): + return BashScript.parse(self.filename()) + class CatEOF(Grammar): - grammar = (ZERO_OR_MORE(SPACE), L('cat '), L('>') | L('>>'), L(' '), ANY_EXCEPT(WHITESPACE), L(" <<"), OPTIONAL(SPACE), L("EOF"), EOL, REPEAT(ANY, greedy=False), EOL, L("EOF"), EOL) - def value(self): - content = self[9].string - content = re.sub(r"\\([$])", r"\1", content) # un-escape bash-escaped characters - return "
%s (%s)
%s
\n" \ - % (self[4].string, - "overwrite" if ">>" not in self[2].string else "append to", - cgi.escape(content)) + grammar = (ZERO_OR_MORE(SPACE), L('cat '), L('>') | L('>>'), L(' '), ANY_EXCEPT(WHITESPACE), + L(" <<"), OPTIONAL(SPACE), L("EOF"), EOL, REPEAT(ANY, greedy=False), EOL, L("EOF"), EOL) + + def value(self): + content = self[9].string + content = re.sub(r"\\([$])", r"\1", content) # un-escape bash-escaped characters + return "
%s (%s)
%s
\n" \ + % (self[4].string, + "overwrite" if ">>" not in self[2].string else "append to", + cgi.escape(content)) + class HideOutput(Grammar): - grammar = (L("hide_output "), REF("BashElement")) - def value(self): - return self[1].value() + grammar = (L("hide_output "), REF("BashElement")) + + def value(self): + return self[1].value() + class EchoLine(Grammar): - grammar = (OPTIONAL(SPACE), L("echo "), REST_OF_LINE, EOL) - def value(self): - if "|" in self.string or ">" in self.string: - return "
" + recode_bash(self.string.strip()) + "
\n" - return "" + grammar = (OPTIONAL(SPACE), L("echo "), REST_OF_LINE, EOL) + + def value(self): + if "|" in self.string or ">" in self.string: + return "
" + recode_bash(self.string.strip()) + "
\n" + return "" + class EditConf(Grammar): - grammar = ( - L('tools/editconf.py '), - FILENAME, - SPACE, - OPTIONAL((LIST_OF( - L("-w") | L("-s") | L("-c ;"), - sep=SPACE, - ), SPACE)), - REST_OF_LINE, - OPTIONAL(SPACE), - EOL - ) - def value(self): - conffile = self[1] - options = [] - eq = "=" - if self[3] and "-s" in self[3].string: eq = " " - for opt in re.split("\s+", self[4].string): - k, v = opt.split("=", 1) - v = re.sub(r"\n+", "", fixup_tokens(v)) # not sure why newlines are getting doubled - options.append("%s%s%s" % (k, eq, v)) - return "
" + self[1].string + " (change settings)
" + "\n".join(cgi.escape(s) for s in options) + "
\n" + grammar = ( + L('tools/editconf.py '), + FILENAME, + SPACE, + OPTIONAL((LIST_OF( + L("-w") | L("-s") | L("-c ;"), + sep=SPACE, + ), SPACE)), + REST_OF_LINE, + OPTIONAL(SPACE), + EOL + ) + + def value(self): + conffile = self[1] + options = [] + eq = "=" + if self[3] and "-s" in self[3].string: + eq = " " + for opt in re.split("\s+", self[4].string): + k, v = opt.split("=", 1) + v = re.sub(r"\n+", "", fixup_tokens(v)) # not sure why newlines are getting doubled + options.append("%s%s%s" % (k, eq, v)) + return "
" + self[1].string + + " (change settings)
" +
+        "\n".join(cgi.escape(s) for s in options) + "
\n" + class CaptureOutput(Grammar): - grammar = OPTIONAL(SPACE), WORD("A-Za-z_"), L('=$('), REST_OF_LINE, L(")"), OPTIONAL(L(';')), EOL - def value(self): - cmd = self[3].string - cmd = cmd.replace("; ", "\n") - return "
$" + self[1].string + "=
" + cgi.escape(cmd) + "
\n" + grammar = OPTIONAL(SPACE), WORD("A-Za-z_"), L('=$('), REST_OF_LINE, L(")"), OPTIONAL(L(';')), EOL + + def value(self): + cmd = self[3].string + cmd = cmd.replace("; ", "\n") + return "
$" + + self[1].string + "=
" + cgi.escape(cmd) + "
\n" + class SedReplace(Grammar): - grammar = OPTIONAL(SPACE), L('sed -i "s/'), OPTIONAL(L('^')), ONE_OR_MORE(WORD("-A-Za-z0-9 #=\\{};.*$_!()")), L('/'), ONE_OR_MORE(WORD("-A-Za-z0-9 #=\\{};.*$_!()")), L('/"'), SPACE, FILENAME, EOL - def value(self): - return "
edit
" + self[8].string + "

replace

" + cgi.escape(self[3].string.replace(".*", ". . .")) + "

with

" + cgi.escape(self[5].string.replace("\\n", "\n").replace("\\t", "\t")) + "
\n" + grammar = OPTIONAL(SPACE), L('sed -i "s/'), OPTIONAL(L('^')), \ + ONE_OR_MORE(WORD("-A-Za-z0-9 #=\\{};.*$_!()")), L('/'), \ + ONE_OR_MORE(WORD("-A-Za-z0-9 #=\\{};.*$_!()")), L('/"'), SPACE, FILENAME, EOL + + def value(self): + return "
edit
" + self[8].string + + "

replace

" + cgi.escape(self[3].string.replace(".*", ". . .")) +
+        "

with

" +
+        cgi.escape(self[5].string.replace("\\n", "\n").replace("\\t", "\t")) + "
\n" + class EchoPipe(Grammar): - grammar = OPTIONAL(SPACE), L("echo "), REST_OF_LINE, L(' | '), REST_OF_LINE, EOL - def value(self): - text = " ".join("\"%s\"" % s for s in self[2].string.split(" ")) - return "
echo " + recode_bash(text) + " \
 | " + recode_bash(self[4].string) + "
\n" + grammar = OPTIONAL(SPACE), L("echo "), REST_OF_LINE, L(' | '), REST_OF_LINE, EOL + + def value(self): + text = " ".join("\"%s\"" % s for s in self[2].string.split(" ")) + return "
echo " + recode_bash(text) +
+        " \
 | " + recode_bash(self[4].string) + "
\n" + def shell_line(bash): - return "
" + recode_bash(bash.strip()) + "
\n" + return "
" + recode_bash(bash.strip()) + "
\n" + class AptGet(Grammar): - grammar = (ZERO_OR_MORE(SPACE), L("apt_install "), REST_OF_LINE, EOL) - def value(self): - return shell_line("apt-get install -y " + re.sub(r"\s+", " ", self[2].string)) + grammar = (ZERO_OR_MORE(SPACE), L("apt_install "), REST_OF_LINE, EOL) + + def value(self): + return shell_line("apt-get install -y " + re.sub(r"\s+", " ", self[2].string)) + + class UfwAllow(Grammar): - grammar = (ZERO_OR_MORE(SPACE), L("ufw_allow "), REST_OF_LINE, EOL) - def value(self): - return shell_line("ufw allow " + self[2].string) + grammar = (ZERO_OR_MORE(SPACE), L("ufw_allow "), REST_OF_LINE, EOL) + + def value(self): + return shell_line("ufw allow " + self[2].string) + + class RestartService(Grammar): - grammar = (ZERO_OR_MORE(SPACE), L("restart_service "), REST_OF_LINE, EOL) - def value(self): - return shell_line("service " + self[2].string + " restart") + grammar = (ZERO_OR_MORE(SPACE), L("restart_service "), REST_OF_LINE, EOL) + + def value(self): + return shell_line("service " + self[2].string + " restart") + class OtherLine(Grammar): - grammar = (REST_OF_LINE, EOL) - def value(self): - if self.string.strip() == "": return "" - if "source setup/functions.sh" in self.string: return "" - if "source /etc/mailinabox.conf" in self.string: return "" - return "
" + recode_bash(self.string.strip()) + "
\n" + grammar = (REST_OF_LINE, EOL) + + def value(self): + if self.string.strip() == "": + return "" + if "source setup/functions.sh" in self.string: + return "" + if "source /etc/mailinabox.conf" in self.string: + return "" + return "
" + recode_bash(self.string.strip()) + "
\n" + class BashElement(Grammar): - grammar = Comment | CatEOF | EchoPipe | EchoLine | HideOutput | EditConf | SedReplace | AptGet | UfwAllow | RestartService | OtherLine - def value(self): - return self[0].value() + grammar = Comment | CatEOF | EchoPipe | EchoLine | HideOutput | EditConf | \ + SedReplace | AptGet | UfwAllow | RestartService | OtherLine + + def value(self): + return self[0].value() + # Make some special characters to private use Unicode code points. bash_special_characters1 = { - "\n": "\uE000", - " ": "\uE001", + "\n": "\uE000", + " ": "\uE001", } bash_special_characters2 = { - "$": "\uE010", + "$": "\uE010", } bash_escapes = { - "n": "\uE020", - "t": "\uE021", + "n": "\uE020", + "t": "\uE021", } + def quasitokenize(bashscript): - # Make a parse of bash easier by making the tokenization easy. - newscript = "" - quote_mode = None - escape_next = False - line_comment = False - subshell = 0 - for c in bashscript: - if line_comment: - # We're in a comment until the end of the line. - newscript += c - if c == '\n': - line_comment = False - elif escape_next: - # Previous character was a \. Normally the next character - # comes through literally, but escaped newlines are line - # continuations and some escapes are for special characters - # which we'll recode and then turn back into escapes later. - if c == "\n": - c = " " - elif c in bash_escapes: - c = bash_escapes[c] - newscript += c - escape_next = False - elif c == "\\": - # Escaping next character. - escape_next = True - elif quote_mode is None and c in ('"', "'"): - # Starting a quoted word. - quote_mode = c - elif c == quote_mode: - # Ending a quoted word. - quote_mode = None - elif quote_mode is not None and quote_mode != "EOF" and c in bash_special_characters1: - # Replace special tokens within quoted words so that they - # don't interfere with tokenization later. - newscript += bash_special_characters1[c] - elif quote_mode is None and c == '#': - # Start of a line comment. - newscript += c - line_comment = True - elif quote_mode is None and c == ';' and subshell == 0: - # End of a statement. - newscript += "\n" - elif quote_mode is None and c == '(': - # Start of a subshell. - newscript += c - subshell += 1 - elif quote_mode is None and c == ')': - # End of a subshell. - newscript += c - subshell -= 1 - elif quote_mode is None and c == '\t': - # Make these just spaces. - if newscript[-1] != " ": - newscript += " " - elif quote_mode is None and c == ' ': - # Collapse consecutive spaces. - if newscript[-1] != " ": - newscript += " " - elif c in bash_special_characters2: - newscript += bash_special_characters2[c] - else: - # All other characters. - newscript += c + # Make a parse of bash easier by making the tokenization easy. + newscript = "" + quote_mode = None + escape_next = False + line_comment = False + subshell = 0 + for c in bashscript: + if line_comment: + # We're in a comment until the end of the line. + newscript += c + if c == '\n': + line_comment = False + elif escape_next: + # Previous character was a \. Normally the next character + # comes through literally, but escaped newlines are line + # continuations and some escapes are for special characters + # which we'll recode and then turn back into escapes later. + if c == "\n": + c = " " + elif c in bash_escapes: + c = bash_escapes[c] + newscript += c + escape_next = False + elif c == "\\": + # Escaping next character. + escape_next = True + elif quote_mode is None and c in ('"', "'"): + # Starting a quoted word. + quote_mode = c + elif c == quote_mode: + # Ending a quoted word. + quote_mode = None + elif quote_mode is not None and quote_mode != "EOF" and c in bash_special_characters1: + # Replace special tokens within quoted words so that they + # don't interfere with tokenization later. + newscript += bash_special_characters1[c] + elif quote_mode is None and c == '#': + # Start of a line comment. + newscript += c + line_comment = True + elif quote_mode is None and c == ';' and subshell == 0: + # End of a statement. + newscript += "\n" + elif quote_mode is None and c == '(': + # Start of a subshell. + newscript += c + subshell += 1 + elif quote_mode is None and c == ')': + # End of a subshell. + newscript += c + subshell -= 1 + elif quote_mode is None and c == '\t': + # Make these just spaces. + if newscript[-1] != " ": + newscript += " " + elif quote_mode is None and c == ' ': + # Collapse consecutive spaces. + if newscript[-1] != " ": + newscript += " " + elif c in bash_special_characters2: + newscript += bash_special_characters2[c] + else: + # All other characters. + newscript += c - # "<< EOF" escaping. - if quote_mode is None and re.search("<<\s*EOF\n$", newscript): - quote_mode = "EOF" - elif quote_mode == "EOF" and re.search("\nEOF\n$", newscript): - quote_mode = None + # "<< EOF" escaping. + if quote_mode is None and re.search("<<\s*EOF\n$", newscript): + quote_mode = "EOF" + elif quote_mode == "EOF" and re.search("\nEOF\n$", newscript): + quote_mode = None + + return newscript - return newscript def recode_bash(s): - def requote(tok): - tok = tok.replace("\\", "\\\\") - for c in bash_special_characters2: - tok = tok.replace(c, "\\" + c) - tok = fixup_tokens(tok) - if " " in tok or '"' in tok: - tok = tok.replace("\"", "\\\"") - tok = '"' + tok +'"' - else: - tok = tok.replace("'", "\\'") - return tok - return cgi.escape(" ".join(requote(tok) for tok in s.split(" "))) + def requote(tok): + tok = tok.replace("\\", "\\\\") + for c in bash_special_characters2: + tok = tok.replace(c, "\\" + c) + tok = fixup_tokens(tok) + if " " in tok or '"' in tok: + tok = tok.replace("\"", "\\\"") + tok = '"' + tok + '"' + else: + tok = tok.replace("'", "\\'") + return tok + return cgi.escape(" ".join(requote(tok) for tok in s.split(" "))) + def fixup_tokens(s): - for c, enc in bash_special_characters1.items(): - s = s.replace(enc, c) - for c, enc in bash_special_characters2.items(): - s = s.replace(enc, c) - for esc, c in bash_escapes.items(): - s = s.replace(c, "\\" + esc) - return s + for c, enc in bash_special_characters1.items(): + s = s.replace(enc, c) + for c, enc in bash_special_characters2.items(): + s = s.replace(enc, c) + for esc, c in bash_escapes.items(): + s = s.replace(c, "\\" + esc) + return s + class BashScript(Grammar): - grammar = (OPTIONAL(HashBang), REPEAT(BashElement)) - def value(self): - return [line.value() for line in self[1]] + grammar = (OPTIONAL(HashBang), REPEAT(BashElement)) - @staticmethod - def parse(fn): - if fn in ("setup/functions.sh", "/etc/mailinabox.conf"): return "" - string = open(fn).read() + def value(self): + return [line.value() for line in self[1]] - # tokenize - string = re.sub(".* #NODOC\n", "", string) - string = re.sub("\n\s*if .*then.*|\n\s*fi|\n\s*else|\n\s*elif .*", "", string) - string = quasitokenize(string) - string = re.sub("hide_output ", "", string) + @staticmethod + def parse(fn): + if fn in ("setup/functions.sh", "/etc/mailinabox.conf"): + return "" + string = open(fn).read() - parser = BashScript.parser() - result = parser.parse_string(string) + # tokenize + string = re.sub(".* #NODOC\n", "", string) + string = re.sub("\n\s*if .*then.*|\n\s*fi|\n\s*else|\n\s*elif .*", "", string) + string = quasitokenize(string) + string = re.sub("hide_output ", "", string) - v = "
view the bash source for the following section at %s
\n" \ - % ("https://github.com/mail-in-a-box/mailinabox/tree/master/" + fn, fn) + parser = BashScript.parser() + result = parser.parse_string(string) - mode = 0 - for item in result.value(): - if item.strip() == "": - pass - elif item.startswith("
view the bash source for \ + the following section at %s
\n") \ + % ("https://github.com/mail-in-a-box/mailinabox/tree/master/" + fn, fn) - v += "\n" # col - v += "\n" # row + mode = 0 + for item in result.value(): + if item.strip() == "": + pass + elif item.startswith("\n
", "")
-		v = re.sub("
([\w\W]*?)
", lambda m : "
" + strip_indent(m.group(1)) + "
", v)
+        v = fixup_tokens(v)
 
-		v = re.sub(r"(\$?)PRIMARY_HOSTNAME", r"box.yourdomain.com", v)
-		v = re.sub(r"\$STORAGE_ROOT", r"$STORE", v)
-		v = v.replace("`pwd`",  "/path/to/mailinabox")
+        v = v.replace("
\n
", "")
+        v = re.sub("
([\w\W]*?)
", lambda m: "
" + strip_indent(m.group(1)) + "
", v)
+
+        v = re.sub(r"(\$?)PRIMARY_HOSTNAME", r"box.yourdomain.com", v)
+        v = re.sub(r"\$STORAGE_ROOT", r"$STORE", v)
+        v = v.replace("`pwd`",  "/path/to/mailinabox")
+
+        return v
 
-		return v
 
 def wrap_lines(text, cols=60):
-	ret = ""
-	words = re.split("(\s+)", text)
-	linelen = 0
-	for w in words:
-		if linelen + len(w) > cols-1:
-			ret += " \\\n"
-			ret += "   "
-			linelen = 0
-		if linelen == 0 and w.strip() == "": continue
-		ret += w
-		linelen += len(w)
-	return ret
+    ret = ""
+    words = re.split("(\s+)", text)
+    linelen = 0
+    for w in words:
+        if linelen + len(w) > cols-1:
+            ret += " \\\n"
+            ret += "   "
+            linelen = 0
+        if linelen == 0 and w.strip() == "":
+            continue
+        ret += w
+        linelen += len(w)
+    return ret
+
 
 if __name__ == '__main__':
-	generate_documentation()
+    generate_documentation()
diff --git a/tools/update-subresource-integrity.py b/tools/update-subresource-integrity.py
index 0400c20a..79746cc7 100755
--- a/tools/update-subresource-integrity.py
+++ b/tools/update-subresource-integrity.py
@@ -4,21 +4,26 @@
 # after updating the Bootstrap and jQuery  and