Merge 8cea79de8b into ae73dc5d30

tools/editconf.py

@@ -22,7 +22,8 @@
 # NAME VAL
 #   UE
 
-import sys, re
+import sys
+import re
 
 # sanity check
 if len(sys.argv) < 3:
@@ -84,18 +85,20 @@ while len(input_lines) > 0:
         # Check that this line contain this setting from the command-line arguments.
         name, val = settings[i].split("=", 1)
         m = re.match(
-			   "(\s*)"
+               "(\s*)" +
-			 + "(" + re.escape(comment_char) + "\s*)?"
+               "(" + re.escape(comment_char) + "\s*)?" +
-			 + re.escape(name) + delimiter_re + "(.*?)\s*$",
+               re.escape(name) + delimiter_re + "(.*?)\s*$",
                line, re.S)
-		if not m: continue
+        if not m:
+            continue
         indent, is_comment, existing_val = m.groups()
 
         # If this is already the setting, do nothing.
         if is_comment is None and existing_val == val:
             # It may be that we've already inserted this setting higher
             # in the file so check for that first.
-			if i in found: break
+            if i in found:
+                break
             buf += line
             found.add(i)
             break

tools/mail.py

@@ -1,6 +1,12 @@
 #!/usr/bin/python3
 
-import sys, getpass, urllib.request, urllib.error, json, re
+import sys
+import getpass
+import urllib.request
+import urllib.error
+import json
+import re
+
 
 def mgmt(cmd, data=None, is_json=False):
     # The base URL for the management daemon. (Listens on IPv4 only.)
@@ -17,16 +23,19 @@ def mgmt(cmd, data=None, is_json=False):
                 print(e.read().decode("utf8"))
             except:
                 pass
-			print("The management daemon refused access. The API key file may be out of sync. Try 'service mailinabox restart'.", file=sys.stderr)
+            print("The management daemon refused access. The API key file may be out of sync. \
+                   Try 'service mailinabox restart'.", file=sys.stderr)
         elif hasattr(e, 'read'):
             print(e.read().decode('utf8'), file=sys.stderr)
         else:
             print(e, file=sys.stderr)
         sys.exit(1)
     resp = response.read().decode('utf8')
-	if is_json: resp = json.loads(resp)
+    if is_json:
+        resp = json.loads(resp)
     return resp
 
+
 def read_password():
     while True:
         first = getpass.getpass('password: ')
@@ -43,6 +52,7 @@ def read_password():
         break
     return first
 
+
 def setup_key_auth(mgmt_uri):
     key = open('/var/lib/mailinabox/api.key').read().strip()
 
@@ -55,6 +65,7 @@ def setup_key_auth(mgmt_uri):
     opener = urllib.request.build_opener(auth_handler)
     urllib.request.install_opener(opener)
 
+
 if len(sys.argv) < 2:
     print("Usage: ")
     print("  tools/mail.py user  (lists users)")
@@ -77,7 +88,8 @@ elif sys.argv[1] == "user" and len(sys.argv) == 2:
     users = mgmt("/mail/users?format=json", is_json=True)
     for domain in users:
         for user in domain["users"]:
-			if user['status'] == 'inactive': continue
+            if user['status'] == 'inactive':
+                continue
             print(user['email'], end='')
             if "admin" in user['privileges']:
                 print("*", end='')
@@ -128,4 +140,3 @@ elif sys.argv[1] == "alias" and sys.argv[2] == "remove" and len(sys.argv) == 4:
 else:
     print("Invalid command-line arguments.")
     sys.exit(1)
-

tools/parse-nginx-log-bootstrap-accesses.py

@@ -5,7 +5,11 @@
 # looking at accesses to the bootstrap.sh script (which is currently at the URL
 # .../setup.sh).
 
-import re, glob, gzip, os.path, json
+import re
+import glob
+import gzip
+import os.path
+import json
 import dateutil.parser
 
 outfn = "/home/user-data/www/mailinabox.email/install-stats.json"
@@ -26,7 +30,8 @@ for fn in glob.glob("/var/log/nginx/access.log*"):
     with f:
         for line in f:
             # Find lines that are GETs on the bootstrap script by either curl or wget.
-			# (Note that we purposely skip ...?ping=1 requests which is the admin panel querying us for updates.)
+            # (Note that we purposely skip ...?ping=1 requests which is the admin
+            # panel querying us for updates.)
             # (Also, the URL changed in January 2016, but we'll accept both.)
             m = re.match(rb"(?P<ip>\S+) - - \[(?P<date>.*?)\] \"GET /(bootstrap.sh|setup.sh) HTTP/.*\" 200 \d+ .* \"(?:curl|wget)", line, re.I)
             if m:

tools/readable_bash.py

@@ -3,10 +3,12 @@
 # Generate documentation for how this machine works by
 # parsing our bash scripts!
 
-import cgi, re
+import cgi
+import re
 import markdown
 from modgrammar import *
 
+
 def generate_documentation():
     print("""<!DOCTYPE html>
 <html>
@@ -129,7 +131,8 @@ def generate_documentation():
             fn = parser.parse_string(line).filename()
         except:
             continue
-		if fn in ("setup/start.sh", "setup/preflight.sh", "setup/questions.sh", "setup/firstuser.sh", "setup/management.sh"):
+        if fn in ("setup/start.sh", "setup/preflight.sh", "setup/questions.sh",
+                  "setup/firstuser.sh", "setup/management.sh"):
             continue
 
         import sys
@@ -151,11 +154,14 @@ def generate_documentation():
 </html>
 """)
 
+
 class HashBang(Grammar):
     grammar = (L('#!'), REST_OF_LINE, EOL)
+
     def value(self):
         return ""
 
+
 def strip_indent(s):
     s = s.replace("\t", "    ")
     lines = s.split("\n")
@@ -167,8 +173,10 @@ def strip_indent(s):
     lines = [line[min_indent:] for line in lines]
     return "\n".join(lines)
 
+
 class Comment(Grammar):
     grammar = ONE_OR_MORE(ZERO_OR_MORE(SPACE), L('#'), REST_OF_LINE, EOL)
+
     def value(self):
         if self.string.replace("#", "").strip() == "":
             return "\n"
@@ -177,17 +185,24 @@ class Comment(Grammar):
         content = strip_indent(content)
         return markdown.markdown(content, output_format="html4") + "\n\n"
 
+
 FILENAME = WORD('a-z0-9-/.')
 
+
 class Source(Grammar):
     grammar = ((L('.') | L('source')), L(' '), FILENAME, Comment | EOL)
+
     def filename(self):
         return self[2].string.strip()
+
     def value(self):
         return BashScript.parse(self.filename())
 
+
 class CatEOF(Grammar):
-	grammar = (ZERO_OR_MORE(SPACE), L('cat '), L('>') | L('>>'), L(' '), ANY_EXCEPT(WHITESPACE), L(" <<"), OPTIONAL(SPACE), L("EOF"), EOL, REPEAT(ANY, greedy=False), EOL, L("EOF"), EOL)
+    grammar = (ZERO_OR_MORE(SPACE), L('cat '), L('>') | L('>>'), L(' '), ANY_EXCEPT(WHITESPACE),
+               L(" <<"), OPTIONAL(SPACE), L("EOF"), EOL, REPEAT(ANY, greedy=False), EOL, L("EOF"), EOL)
+
     def value(self):
         content = self[9].string
         content = re.sub(r"\\([$])", r"\1", content)  # un-escape bash-escaped characters
@@ -196,18 +211,23 @@ class CatEOF(Grammar):
                "overwrite" if ">>" not in self[2].string else "append to",
                cgi.escape(content))
 
+
 class HideOutput(Grammar):
     grammar = (L("hide_output "), REF("BashElement"))
+
     def value(self):
         return self[1].value()
 
+
 class EchoLine(Grammar):
     grammar = (OPTIONAL(SPACE), L("echo "), REST_OF_LINE, EOL)
+
     def value(self):
         if "|" in self.string or ">" in self.string:
             return "<pre class='shell'><div>" + recode_bash(self.string.strip()) + "</div></pre>\n"
         return ""
 
+
 class EditConf(Grammar):
     grammar = (
         L('tools/editconf.py '),
@@ -221,64 +241,99 @@ class EditConf(Grammar):
         OPTIONAL(SPACE),
         EOL
         )
+
     def value(self):
         conffile = self[1]
         options = []
         eq = "="
-		if self[3] and "-s" in self[3].string: eq = " "
+        if self[3] and "-s" in self[3].string:
+            eq = " "
         for opt in re.split("\s+", self[4].string):
             k, v = opt.split("=", 1)
             v = re.sub(r"\n+", "", fixup_tokens(v))  # not sure why newlines are getting doubled
             options.append("%s%s%s" % (k, eq, v))
-		return "<div class='write-to'><div class='filename'>" + self[1].string + " <span>(change settings)</span></div><pre>" + "\n".join(cgi.escape(s) for s in options) + "</pre></div>\n"
+        return "<div class='write-to'><div class='filename'>" + self[1].string +
+        " <span>(change settings)</span></div><pre>" +
+        "\n".join(cgi.escape(s) for s in options) + "</pre></div>\n"
+
 
 class CaptureOutput(Grammar):
     grammar = OPTIONAL(SPACE), WORD("A-Za-z_"), L('=$('), REST_OF_LINE, L(")"), OPTIONAL(L(';')), EOL
+
     def value(self):
         cmd = self[3].string
         cmd = cmd.replace("; ", "\n")
-		return "<div class='write-to'><div class='filename'>$" + self[1].string + "=</div><pre>" + cgi.escape(cmd) + "</pre></div>\n"
+        return "<div class='write-to'><div class='filename'>$" +
+        self[1].string + "=</div><pre>" + cgi.escape(cmd) + "</pre></div>\n"
+
 
 class SedReplace(Grammar):
-	grammar = OPTIONAL(SPACE), L('sed -i "s/'), OPTIONAL(L('^')), ONE_OR_MORE(WORD("-A-Za-z0-9 #=\\{};.*$_!()")), L('/'), ONE_OR_MORE(WORD("-A-Za-z0-9 #=\\{};.*$_!()")), L('/"'), SPACE, FILENAME, EOL
+    grammar = OPTIONAL(SPACE), L('sed -i "s/'), OPTIONAL(L('^')), \
+        ONE_OR_MORE(WORD("-A-Za-z0-9 #=\\{};.*$_!()")), L('/'), \
+        ONE_OR_MORE(WORD("-A-Za-z0-9 #=\\{};.*$_!()")), L('/"'), SPACE, FILENAME, EOL
+
     def value(self):
-		return "<div class='write-to'><div class='filename'>edit<br>" + self[8].string + "</div><p>replace</p><pre>" + cgi.escape(self[3].string.replace(".*", ". . .")) + "</pre><p>with</p><pre>" + cgi.escape(self[5].string.replace("\\n", "\n").replace("\\t", "\t")) + "</pre></div>\n"
+        return "<div class='write-to'><div class='filename'>edit<br>" + self[8].string +
+        "</div><p>replace</p><pre>" + cgi.escape(self[3].string.replace(".*", ". . .")) +
+        "</pre><p>with</p><pre>" +
+        cgi.escape(self[5].string.replace("\\n", "\n").replace("\\t", "\t")) + "</pre></div>\n"
+
 
 class EchoPipe(Grammar):
     grammar = OPTIONAL(SPACE), L("echo "), REST_OF_LINE, L(' | '), REST_OF_LINE, EOL
+
     def value(self):
         text = " ".join("\"%s\"" % s for s in self[2].string.split(" "))
-		return "<pre class='shell'><div>echo " + recode_bash(text) + " \<br> | " + recode_bash(self[4].string) + "</div></pre>\n"
+        return "<pre class='shell'><div>echo " + recode_bash(text) +
+        " \<br> | " + recode_bash(self[4].string) + "</div></pre>\n"
+
 
 def shell_line(bash):
     return "<pre class='shell'><div>" + recode_bash(bash.strip()) + "</div></pre>\n"
 
+
 class AptGet(Grammar):
     grammar = (ZERO_OR_MORE(SPACE), L("apt_install "), REST_OF_LINE, EOL)
+
     def value(self):
         return shell_line("apt-get install -y " + re.sub(r"\s+", " ", self[2].string))
+
+
 class UfwAllow(Grammar):
     grammar = (ZERO_OR_MORE(SPACE), L("ufw_allow "), REST_OF_LINE, EOL)
+
     def value(self):
         return shell_line("ufw allow " + self[2].string)
+
+
 class RestartService(Grammar):
     grammar = (ZERO_OR_MORE(SPACE), L("restart_service "), REST_OF_LINE, EOL)
+
     def value(self):
         return shell_line("service " + self[2].string + " restart")
 
+
 class OtherLine(Grammar):
     grammar = (REST_OF_LINE, EOL)
+
     def value(self):
-		if self.string.strip() == "": return ""
+        if self.string.strip() == "":
-		if "source setup/functions.sh" in self.string: return ""
+            return ""
-		if "source /etc/mailinabox.conf" in self.string: return ""
+        if "source setup/functions.sh" in self.string:
+            return ""
+        if "source /etc/mailinabox.conf" in self.string:
+            return ""
         return "<pre class='shell'><div>" + recode_bash(self.string.strip()) + "</div></pre>\n"
 
+
 class BashElement(Grammar):
-	grammar = Comment | CatEOF | EchoPipe | EchoLine | HideOutput | EditConf | SedReplace | AptGet | UfwAllow | RestartService | OtherLine
+    grammar = Comment | CatEOF | EchoPipe | EchoLine | HideOutput | EditConf | \
+       SedReplace | AptGet | UfwAllow | RestartService | OtherLine
+
     def value(self):
         return self[0].value()
 
+
 # Make some special characters to private use Unicode code points.
 bash_special_characters1 = {
     "\n": "\uE000",
@@ -292,6 +347,7 @@ bash_escapes = {
     "t": "\uE021",
 }
 
+
 def quasitokenize(bashscript):
     # Make a parse of bash easier by making the tokenization easy.
     newscript = ""
@@ -366,6 +422,7 @@ def quasitokenize(bashscript):
 
     return newscript
 
+
 def recode_bash(s):
     def requote(tok):
         tok = tok.replace("\\", "\\\\")
@@ -380,6 +437,7 @@ def recode_bash(s):
         return tok
     return cgi.escape(" ".join(requote(tok) for tok in s.split(" ")))
 
+
 def fixup_tokens(s):
     for c, enc in bash_special_characters1.items():
         s = s.replace(enc, c)
@@ -389,14 +447,17 @@ def fixup_tokens(s):
         s = s.replace(c, "\\" + esc)
     return s
 
+
 class BashScript(Grammar):
     grammar = (OPTIONAL(HashBang), REPEAT(BashElement))
+
     def value(self):
         return [line.value() for line in self[1]]
 
     @staticmethod
     def parse(fn):
-		if fn in ("setup/functions.sh", "/etc/mailinabox.conf"): return ""
+        if fn in ("setup/functions.sh", "/etc/mailinabox.conf"):
+            return ""
         string = open(fn).read()
 
         # tokenize
@@ -408,7 +469,8 @@ class BashScript(Grammar):
         parser = BashScript.parser()
         result = parser.parse_string(string)
 
-		v = "<div class='row'><div class='col-xs-12 sourcefile'>view the bash source for the following section at <a href=\"%s\">%s</a></div></div>\n" \
+        v = ("<div class='row'><div class='col-xs-12 sourcefile'>view the bash source for \
+              the following section at <a href=\"%s\">%s</a></div></div>\n") \
             % ("https://github.com/mail-in-a-box/mailinabox/tree/master/" + fn, fn)
 
         mode = 0
@@ -462,6 +524,7 @@ class BashScript(Grammar):
 
         return v
 
+
 def wrap_lines(text, cols=60):
     ret = ""
     words = re.split("(\s+)", text)
@@ -471,10 +534,12 @@ def wrap_lines(text, cols=60):
             ret += " \\\n"
             ret += "   "
             linelen = 0
-		if linelen == 0 and w.strip() == "": continue
+        if linelen == 0 and w.strip() == "":
+            continue
         ret += w
         linelen += len(w)
     return ret
 
+
 if __name__ == '__main__':
     generate_documentation()

tools/update-subresource-integrity.py

@@ -4,17 +4,22 @@
 # after updating the Bootstrap and jQuery <link> and <script> to compute the
 # appropriate hash and insert it into the template.
 
-import re, urllib.request, hashlib, base64
+import re
+import urllib.request
+import hashlib
+import base64
 
 fn = "management/templates/index.html"
 
 with open(fn, 'r') as f:
     content = f.read()
 
+
 def make_integrity(url):
     resource = urllib.request.urlopen(url).read()
     return "sha256-" + base64.b64encode(hashlib.sha256(resource).digest()).decode('ascii')
 
+
 content = re.sub(
     r'<(link rel="stylesheet" href|script src)="(.*?)" integrity="(.*?)"',
     lambda m: '<' + m.group(1) + '="' + m.group(2) + '" integrity="' + make_integrity(m.group(2)) + '"',