pep8 cleanup of Python source files

pep8 (https://www.python.org/dev/peps/pep-0008/) is the commonly accepted and widely adopted code style convention for Python.
I used pycodestyle (https://pycodestyle.readthedocs.io/en/latest/) to check for pep8 compatibility.
Especially the mix of tabs and spaces in the Python files makes it hard to work with. I switched to spaces, because that's what
pep8 expects and the majority of Python programmers use.
This commit is contained in:
Wolfgang Steitz 2016-11-20 13:23:46 +01:00
parent c3605f6211
commit 8cea79de8b
5 changed files with 650 additions and 561 deletions

View File

@ -14,7 +14,7 @@
#
# NAME VALUE
#
# If the -c option is given, then the supplied character becomes the comment character
# If the -c option is given, then the supplied character becomes the commentcharacter
#
# If the -w option is given, then setting lines continue onto following
# lines while the lines start with whitespace, e.g.:
@ -22,7 +22,8 @@
# NAME VAL
# UE
import sys, re
import sys
import re
# sanity check
if len(sys.argv) < 3:
@ -84,18 +85,20 @@ while len(input_lines) > 0:
# Check that this line contain this setting from the command-line arguments.
name, val = settings[i].split("=", 1)
m = re.match(
"(\s*)"
+ "(" + re.escape(comment_char) + "\s*)?"
+ re.escape(name) + delimiter_re + "(.*?)\s*$",
"(\s*)" +
"(" + re.escape(comment_char) + "\s*)?" +
re.escape(name) + delimiter_re + "(.*?)\s*$",
line, re.S)
if not m: continue
if not m:
continue
indent, is_comment, existing_val = m.groups()
# If this is already the setting, do nothing.
if is_comment is None and existing_val == val:
# It may be that we've already inserted this setting higher
# in the file so check for that first.
if i in found: break
if i in found:
break
buf += line
found.add(i)
break

View File

@ -1,6 +1,12 @@
#!/usr/bin/python3
import sys, getpass, urllib.request, urllib.error, json, re
import sys
import getpass
import urllib.request
import urllib.error
import json
import re
def mgmt(cmd, data=None, is_json=False):
# The base URL for the management daemon. (Listens on IPv4 only.)
@ -17,16 +23,19 @@ def mgmt(cmd, data=None, is_json=False):
print(e.read().decode("utf8"))
except:
pass
print("The management daemon refused access. The API key file may be out of sync. Try 'service mailinabox restart'.", file=sys.stderr)
print("The management daemon refused access. The API key file may be out of sync. \
Try 'service mailinabox restart'.", file=sys.stderr)
elif hasattr(e, 'read'):
print(e.read().decode('utf8'), file=sys.stderr)
else:
print(e, file=sys.stderr)
sys.exit(1)
resp = response.read().decode('utf8')
if is_json: resp = json.loads(resp)
if is_json:
resp = json.loads(resp)
return resp
def read_password():
while True:
first = getpass.getpass('password: ')
@ -43,6 +52,7 @@ def read_password():
break
return first
def setup_key_auth(mgmt_uri):
key = open('/var/lib/mailinabox/api.key').read().strip()
@ -55,6 +65,7 @@ def setup_key_auth(mgmt_uri):
opener = urllib.request.build_opener(auth_handler)
urllib.request.install_opener(opener)
if len(sys.argv) < 2:
print("Usage: ")
print(" tools/mail.py user (lists users)")
@ -77,7 +88,8 @@ elif sys.argv[1] == "user" and len(sys.argv) == 2:
users = mgmt("/mail/users?format=json", is_json=True)
for domain in users:
for user in domain["users"]:
if user['status'] == 'inactive': continue
if user['status'] == 'inactive':
continue
print(user['email'], end='')
if "admin" in user['privileges']:
print("*", end='')
@ -94,19 +106,19 @@ elif sys.argv[1] == "user" and sys.argv[2] in ("add", "password"):
email, pw = sys.argv[3:5]
if sys.argv[2] == "add":
print(mgmt("/mail/users/add", { "email": email, "password": pw }))
print(mgmt("/mail/users/add", {"email": email, "password": pw}))
elif sys.argv[2] == "password":
print(mgmt("/mail/users/password", { "email": email, "password": pw }))
print(mgmt("/mail/users/password", {"email": email, "password": pw}))
elif sys.argv[1] == "user" and sys.argv[2] == "remove" and len(sys.argv) == 4:
print(mgmt("/mail/users/remove", { "email": sys.argv[3] }))
print(mgmt("/mail/users/remove", {"email": sys.argv[3]}))
elif sys.argv[1] == "user" and sys.argv[2] in ("make-admin", "remove-admin") and len(sys.argv) == 4:
if sys.argv[2] == "make-admin":
action = "add"
else:
action = "remove"
print(mgmt("/mail/users/privileges/" + action, { "email": sys.argv[3], "privilege": "admin" }))
print(mgmt("/mail/users/privileges/" + action, {"email": sys.argv[3], "privilege": "admin"}))
elif sys.argv[1] == "user" and sys.argv[2] == "admins":
# Dump a list of admin users.
@ -120,12 +132,11 @@ elif sys.argv[1] == "alias" and len(sys.argv) == 2:
print(mgmt("/mail/aliases"))
elif sys.argv[1] == "alias" and sys.argv[2] == "add" and len(sys.argv) == 5:
print(mgmt("/mail/aliases/add", { "address": sys.argv[3], "forwards_to": sys.argv[4] }))
print(mgmt("/mail/aliases/add", {"address": sys.argv[3], "forwards_to": sys.argv[4]}))
elif sys.argv[1] == "alias" and sys.argv[2] == "remove" and len(sys.argv) == 4:
print(mgmt("/mail/aliases/remove", { "address": sys.argv[3] }))
print(mgmt("/mail/aliases/remove", {"address": sys.argv[3]}))
else:
print("Invalid command-line arguments.")
sys.exit(1)

View File

@ -5,7 +5,11 @@
# looking at accesses to the bootstrap.sh script (which is currently at the URL
# .../setup.sh).
import re, glob, gzip, os.path, json
import re
import glob
import gzip
import os.path
import json
import dateutil.parser
outfn = "/home/user-data/www/mailinabox.email/install-stats.json"
@ -26,17 +30,18 @@ for fn in glob.glob("/var/log/nginx/access.log*"):
with f:
for line in f:
# Find lines that are GETs on the bootstrap script by either curl or wget.
# (Note that we purposely skip ...?ping=1 requests which is the admin panel querying us for updates.)
# (Note that we purposely skip ...?ping=1 requests which is the admin
# panel querying us for updates.)
# (Also, the URL changed in January 2016, but we'll accept both.)
m = re.match(rb"(?P<ip>\S+) - - \[(?P<date>.*?)\] \"GET /(bootstrap.sh|setup.sh) HTTP/.*\" 200 \d+ .* \"(?:curl|wget)", line, re.I)
if m:
date, time = m.group("date").decode("ascii").split(":", 1)
date = dateutil.parser.parse(date).date().isoformat()
ip = m.group("ip").decode("ascii")
accesses.add( (date, ip) )
accesses.add((date, ip))
# Aggregate by date.
by_date = { }
by_date = {}
for date, ip in accesses:
by_date[date] = by_date.get(date, 0) + 1

View File

@ -3,10 +3,12 @@
# Generate documentation for how this machine works by
# parsing our bash scripts!
import cgi, re
import cgi
import re
import markdown
from modgrammar import *
def generate_documentation():
print("""<!DOCTYPE html>
<html>
@ -129,7 +131,8 @@ def generate_documentation():
fn = parser.parse_string(line).filename()
except:
continue
if fn in ("setup/start.sh", "setup/preflight.sh", "setup/questions.sh", "setup/firstuser.sh", "setup/management.sh"):
if fn in ("setup/start.sh", "setup/preflight.sh", "setup/questions.sh",
"setup/firstuser.sh", "setup/management.sh"):
continue
import sys
@ -151,11 +154,14 @@ def generate_documentation():
</html>
""")
class HashBang(Grammar):
grammar = (L('#!'), REST_OF_LINE, EOL)
def value(self):
return ""
def strip_indent(s):
s = s.replace("\t", " ")
lines = s.split("\n")
@ -167,8 +173,10 @@ def strip_indent(s):
lines = [line[min_indent:] for line in lines]
return "\n".join(lines)
class Comment(Grammar):
grammar = ONE_OR_MORE(ZERO_OR_MORE(SPACE), L('#'), REST_OF_LINE, EOL)
def value(self):
if self.string.replace("#", "").strip() == "":
return "\n"
@ -177,17 +185,24 @@ class Comment(Grammar):
content = strip_indent(content)
return markdown.markdown(content, output_format="html4") + "\n\n"
FILENAME = WORD('a-z0-9-/.')
class Source(Grammar):
grammar = ((L('.') | L('source')), L(' '), FILENAME, Comment | EOL)
def filename(self):
return self[2].string.strip()
def value(self):
return BashScript.parse(self.filename())
class CatEOF(Grammar):
grammar = (ZERO_OR_MORE(SPACE), L('cat '), L('>') | L('>>'), L(' '), ANY_EXCEPT(WHITESPACE), L(" <<"), OPTIONAL(SPACE), L("EOF"), EOL, REPEAT(ANY, greedy=False), EOL, L("EOF"), EOL)
grammar = (ZERO_OR_MORE(SPACE), L('cat '), L('>') | L('>>'), L(' '), ANY_EXCEPT(WHITESPACE),
L(" <<"), OPTIONAL(SPACE), L("EOF"), EOL, REPEAT(ANY, greedy=False), EOL, L("EOF"), EOL)
def value(self):
content = self[9].string
content = re.sub(r"\\([$])", r"\1", content) # un-escape bash-escaped characters
@ -196,18 +211,23 @@ class CatEOF(Grammar):
"overwrite" if ">>" not in self[2].string else "append to",
cgi.escape(content))
class HideOutput(Grammar):
grammar = (L("hide_output "), REF("BashElement"))
def value(self):
return self[1].value()
class EchoLine(Grammar):
grammar = (OPTIONAL(SPACE), L("echo "), REST_OF_LINE, EOL)
def value(self):
if "|" in self.string or ">" in self.string:
return "<pre class='shell'><div>" + recode_bash(self.string.strip()) + "</div></pre>\n"
return ""
class EditConf(Grammar):
grammar = (
L('tools/editconf.py '),
@ -221,64 +241,99 @@ class EditConf(Grammar):
OPTIONAL(SPACE),
EOL
)
def value(self):
conffile = self[1]
options = []
eq = "="
if self[3] and "-s" in self[3].string: eq = " "
if self[3] and "-s" in self[3].string:
eq = " "
for opt in re.split("\s+", self[4].string):
k, v = opt.split("=", 1)
v = re.sub(r"\n+", "", fixup_tokens(v)) # not sure why newlines are getting doubled
options.append("%s%s%s" % (k, eq, v))
return "<div class='write-to'><div class='filename'>" + self[1].string + " <span>(change settings)</span></div><pre>" + "\n".join(cgi.escape(s) for s in options) + "</pre></div>\n"
return "<div class='write-to'><div class='filename'>" + self[1].string +
" <span>(change settings)</span></div><pre>" +
"\n".join(cgi.escape(s) for s in options) + "</pre></div>\n"
class CaptureOutput(Grammar):
grammar = OPTIONAL(SPACE), WORD("A-Za-z_"), L('=$('), REST_OF_LINE, L(")"), OPTIONAL(L(';')), EOL
def value(self):
cmd = self[3].string
cmd = cmd.replace("; ", "\n")
return "<div class='write-to'><div class='filename'>$" + self[1].string + "=</div><pre>" + cgi.escape(cmd) + "</pre></div>\n"
return "<div class='write-to'><div class='filename'>$" +
self[1].string + "=</div><pre>" + cgi.escape(cmd) + "</pre></div>\n"
class SedReplace(Grammar):
grammar = OPTIONAL(SPACE), L('sed -i "s/'), OPTIONAL(L('^')), ONE_OR_MORE(WORD("-A-Za-z0-9 #=\\{};.*$_!()")), L('/'), ONE_OR_MORE(WORD("-A-Za-z0-9 #=\\{};.*$_!()")), L('/"'), SPACE, FILENAME, EOL
grammar = OPTIONAL(SPACE), L('sed -i "s/'), OPTIONAL(L('^')), \
ONE_OR_MORE(WORD("-A-Za-z0-9 #=\\{};.*$_!()")), L('/'), \
ONE_OR_MORE(WORD("-A-Za-z0-9 #=\\{};.*$_!()")), L('/"'), SPACE, FILENAME, EOL
def value(self):
return "<div class='write-to'><div class='filename'>edit<br>" + self[8].string + "</div><p>replace</p><pre>" + cgi.escape(self[3].string.replace(".*", ". . .")) + "</pre><p>with</p><pre>" + cgi.escape(self[5].string.replace("\\n", "\n").replace("\\t", "\t")) + "</pre></div>\n"
return "<div class='write-to'><div class='filename'>edit<br>" + self[8].string +
"</div><p>replace</p><pre>" + cgi.escape(self[3].string.replace(".*", ". . .")) +
"</pre><p>with</p><pre>" +
cgi.escape(self[5].string.replace("\\n", "\n").replace("\\t", "\t")) + "</pre></div>\n"
class EchoPipe(Grammar):
grammar = OPTIONAL(SPACE), L("echo "), REST_OF_LINE, L(' | '), REST_OF_LINE, EOL
def value(self):
text = " ".join("\"%s\"" % s for s in self[2].string.split(" "))
return "<pre class='shell'><div>echo " + recode_bash(text) + " \<br> | " + recode_bash(self[4].string) + "</div></pre>\n"
return "<pre class='shell'><div>echo " + recode_bash(text) +
" \<br> | " + recode_bash(self[4].string) + "</div></pre>\n"
def shell_line(bash):
return "<pre class='shell'><div>" + recode_bash(bash.strip()) + "</div></pre>\n"
class AptGet(Grammar):
grammar = (ZERO_OR_MORE(SPACE), L("apt_install "), REST_OF_LINE, EOL)
def value(self):
return shell_line("apt-get install -y " + re.sub(r"\s+", " ", self[2].string))
class UfwAllow(Grammar):
grammar = (ZERO_OR_MORE(SPACE), L("ufw_allow "), REST_OF_LINE, EOL)
def value(self):
return shell_line("ufw allow " + self[2].string)
class RestartService(Grammar):
grammar = (ZERO_OR_MORE(SPACE), L("restart_service "), REST_OF_LINE, EOL)
def value(self):
return shell_line("service " + self[2].string + " restart")
class OtherLine(Grammar):
grammar = (REST_OF_LINE, EOL)
def value(self):
if self.string.strip() == "": return ""
if "source setup/functions.sh" in self.string: return ""
if "source /etc/mailinabox.conf" in self.string: return ""
if self.string.strip() == "":
return ""
if "source setup/functions.sh" in self.string:
return ""
if "source /etc/mailinabox.conf" in self.string:
return ""
return "<pre class='shell'><div>" + recode_bash(self.string.strip()) + "</div></pre>\n"
class BashElement(Grammar):
grammar = Comment | CatEOF | EchoPipe | EchoLine | HideOutput | EditConf | SedReplace | AptGet | UfwAllow | RestartService | OtherLine
grammar = Comment | CatEOF | EchoPipe | EchoLine | HideOutput | EditConf | \
SedReplace | AptGet | UfwAllow | RestartService | OtherLine
def value(self):
return self[0].value()
# Make some special characters to private use Unicode code points.
bash_special_characters1 = {
"\n": "\uE000",
@ -292,6 +347,7 @@ bash_escapes = {
"t": "\uE021",
}
def quasitokenize(bashscript):
# Make a parse of bash easier by making the tokenization easy.
newscript = ""
@ -366,6 +422,7 @@ def quasitokenize(bashscript):
return newscript
def recode_bash(s):
def requote(tok):
tok = tok.replace("\\", "\\\\")
@ -374,12 +431,13 @@ def recode_bash(s):
tok = fixup_tokens(tok)
if " " in tok or '"' in tok:
tok = tok.replace("\"", "\\\"")
tok = '"' + tok +'"'
tok = '"' + tok + '"'
else:
tok = tok.replace("'", "\\'")
return tok
return cgi.escape(" ".join(requote(tok) for tok in s.split(" ")))
def fixup_tokens(s):
for c, enc in bash_special_characters1.items():
s = s.replace(enc, c)
@ -389,14 +447,17 @@ def fixup_tokens(s):
s = s.replace(c, "\\" + esc)
return s
class BashScript(Grammar):
grammar = (OPTIONAL(HashBang), REPEAT(BashElement))
def value(self):
return [line.value() for line in self[1]]
@staticmethod
def parse(fn):
if fn in ("setup/functions.sh", "/etc/mailinabox.conf"): return ""
if fn in ("setup/functions.sh", "/etc/mailinabox.conf"):
return ""
string = open(fn).read()
# tokenize
@ -408,7 +469,8 @@ class BashScript(Grammar):
parser = BashScript.parser()
result = parser.parse_string(string)
v = "<div class='row'><div class='col-xs-12 sourcefile'>view the bash source for the following section at <a href=\"%s\">%s</a></div></div>\n" \
v = ("<div class='row'><div class='col-xs-12 sourcefile'>view the bash source for \
the following section at <a href=\"%s\">%s</a></div></div>\n") \
% ("https://github.com/mail-in-a-box/mailinabox/tree/master/" + fn, fn)
mode = 0
@ -454,7 +516,7 @@ class BashScript(Grammar):
v = fixup_tokens(v)
v = v.replace("</pre>\n<pre class='shell'>", "")
v = re.sub("<pre>([\w\W]*?)</pre>", lambda m : "<pre>" + strip_indent(m.group(1)) + "</pre>", v)
v = re.sub("<pre>([\w\W]*?)</pre>", lambda m: "<pre>" + strip_indent(m.group(1)) + "</pre>", v)
v = re.sub(r"(\$?)PRIMARY_HOSTNAME", r"<b>box.yourdomain.com</b>", v)
v = re.sub(r"\$STORAGE_ROOT", r"<b>$STORE</b>", v)
@ -462,6 +524,7 @@ class BashScript(Grammar):
return v
def wrap_lines(text, cols=60):
ret = ""
words = re.split("(\s+)", text)
@ -471,10 +534,12 @@ def wrap_lines(text, cols=60):
ret += " \\\n"
ret += " "
linelen = 0
if linelen == 0 and w.strip() == "": continue
if linelen == 0 and w.strip() == "":
continue
ret += w
linelen += len(w)
return ret
if __name__ == '__main__':
generate_documentation()

View File

@ -4,20 +4,25 @@
# after updating the Bootstrap and jQuery <link> and <script> to compute the
# appropriate hash and insert it into the template.
import re, urllib.request, hashlib, base64
import re
import urllib.request
import hashlib
import base64
fn = "management/templates/index.html"
with open(fn, 'r') as f:
content = f.read()
def make_integrity(url):
resource = urllib.request.urlopen(url).read()
return "sha256-" + base64.b64encode(hashlib.sha256(resource).digest()).decode('ascii')
content = re.sub(
r'<(link rel="stylesheet" href|script src)="(.*?)" integrity="(.*?)"',
lambda m : '<' + m.group(1) + '="' + m.group(2) + '" integrity="' + make_integrity(m.group(2)) + '"',
lambda m: '<' + m.group(1) + '="' + m.group(2) + '" integrity="' + make_integrity(m.group(2)) + '"',
content)
with open(fn, 'w') as f: