Merge 7c3c956228 into 3efd4257b5

tools/parse-nginx-log-bootstrap-accesses.py

@@ -32,7 +32,7 @@ for fn in glob.glob("/var/log/nginx/access.log*"):
 
 # Aggregate by date.
 by_date = { }
-for date, ip in accesses:
+for date, _ip in accesses:
 	by_date[date] = by_date.get(date, 0) + 1
 
 # Since logs are rotated, store the statistics permanently in a JSON file.

tools/readable_bash.py

@@ -124,14 +124,14 @@ def generate_documentation():
  """)
 
 	parser = Source.parser()
-	with open("setup/start.sh", "r") as start_file:
-                for line in start_file:
-                        try:
-                                fn = parser.parse_string(line).filename()
-                        except:
-                                continue
-                        if fn in ("setup/start.sh", "setup/preflight.sh", "setup/questions.sh", "setup/firstuser.sh", "setup/management.sh"):
-                                continue
+	with open("setup/start.sh", encoding="utf-8") as start_file:
+		for line in start_file:
+			try:
+				fn = parser.parse_string(line).filename()
+			except:
+				continue
+			if fn in {"setup/start.sh", "setup/preflight.sh", "setup/questions.sh", "setup/firstuser.sh", "setup/management.sh"}:
+				continue
 
 		import sys
 		print(fn, file=sys.stderr)
@@ -171,7 +171,7 @@ def strip_indent(s):
 class Comment(Grammar):
 	grammar = ONE_OR_MORE(ZERO_OR_MORE(SPACE), L('#'), REST_OF_LINE, EOL)
 	def value(self):
-		if self.string.replace("#", "").strip() == "":
+		if not self.string.replace("#", "").strip():
 			return "\n"
 		lines = [x[2].string for x in self[0]]
 		content = "\n".join(lines)
@@ -192,8 +192,7 @@ class CatEOF(Grammar):
 	def value(self):
 		content = self[9].string
 		content = re.sub(r"\\([$])", r"\1", content) # un-escape bash-escaped characters
-		return "<div class='write-to'><div class='filename'>%s <span>(%s)</span></div><pre>%s</pre></div>\n" \
-			% (self[4].string,
+		return "<div class='write-to'><div class='filename'>{} <span>({})</span></div><pre>{}</pre></div>\n".format(self[4].string,
 			   "overwrite" if ">>" not in self[2].string else "append to",
 			   cgi.escape(content))
 
@@ -223,14 +222,14 @@ class EditConf(Grammar):
 		EOL
 		)
 	def value(self):
-		conffile = self[1]
+		# conffile = self[1]
 		options = []
 		eq = "="
 		if self[3] and "-s" in self[3].string: eq = " "
-		for opt in re.split("\s+", self[4].string):
+		for opt in re.split(r"\s+", self[4].string):
 			k, v = opt.split("=", 1)
 			v = re.sub(r"\n+", "", fixup_tokens(v)) # not sure why newlines are getting doubled
-			options.append("%s%s%s" % (k, eq, v))
+			options.append(f"{k}{eq}{v}")
 		return "<div class='write-to'><div class='filename'>" + self[1].string + " <span>(change settings)</span></div><pre>" + "\n".join(cgi.escape(s) for s in options) + "</pre></div>\n"
 
 class CaptureOutput(Grammar):
@@ -248,8 +247,8 @@ class SedReplace(Grammar):
 class EchoPipe(Grammar):
 	grammar = OPTIONAL(SPACE), L("echo "), REST_OF_LINE, L(' | '), REST_OF_LINE, EOL
 	def value(self):
-		text = " ".join("\"%s\"" % s for s in self[2].string.split(" "))
-		return "<pre class='shell'><div>echo " + recode_bash(text) + " \<br> | " + recode_bash(self[4].string) + "</div></pre>\n"
+		text = " ".join(f'"{s}"' for s in self[2].string.split(" "))
+		return "<pre class='shell'><div>echo " + recode_bash(text) + r" \<br> | " + recode_bash(self[4].string) + "</div></pre>\n"
 
 def shell_line(bash):
 	return "<pre class='shell'><div>" + recode_bash(bash.strip()) + "</div></pre>\n"
@@ -274,7 +273,7 @@ class RestartService(Grammar):
 class OtherLine(Grammar):
 	grammar = (REST_OF_LINE, EOL)
 	def value(self):
-		if self.string.strip() == "": return ""
+		if not self.string.strip(): return ""
 		if "source setup/functions.sh" in self.string: return ""
 		if "source /etc/mailinabox.conf" in self.string: return ""
 		return "<pre class='shell'><div>" + recode_bash(self.string.strip()) + "</div></pre>\n"
@@ -324,7 +323,7 @@ def quasitokenize(bashscript):
 		elif c == "\\":
 			# Escaping next character.
 			escape_next = True
-		elif quote_mode is None and c in ('"', "'"):
+		elif quote_mode is None and c in {'"', "'"}:
 			# Starting a quoted word.
 			quote_mode = c
 		elif c == quote_mode:
@@ -364,9 +363,9 @@ def quasitokenize(bashscript):
 			newscript += c
 
 		# "<< EOF" escaping.
-		if quote_mode is None and re.search("<<\s*EOF\n$", newscript):
+		if quote_mode is None and re.search(r"<<\s*EOF\n$", newscript):
 			quote_mode = "EOF"
-		elif quote_mode == "EOF" and re.search("\nEOF\n$", newscript):
+		elif quote_mode == "EOF" and re.search(r"\nEOF\n$", newscript):
 			quote_mode = None
 
 	return newscript
@@ -378,7 +377,7 @@ def recode_bash(s):
 			tok = tok.replace(c, "\\" + c)
 		tok = fixup_tokens(tok)
 		if " " in tok or '"' in tok:
-			tok = tok.replace("\"", "\\\"")
+			tok = tok.replace('"', '\\"')
 			tok = '"' + tok +'"'
 		else:
 			tok = tok.replace("'", "\\'")
@@ -401,25 +400,24 @@ class BashScript(Grammar):
 
 	@staticmethod
 	def parse(fn):
-		if fn in ("setup/functions.sh", "/etc/mailinabox.conf"): return ""
-		with open(fn, "r") as f:
+		if fn in {"setup/functions.sh", "/etc/mailinabox.conf"}: return ""
+		with open(fn, encoding="utf-8") as f:
 			string = f.read()
 
 		# tokenize
-		string = re.sub(".* #NODOC\n", "", string)
-		string = re.sub("\n\s*if .*then.*|\n\s*fi|\n\s*else|\n\s*elif .*", "", string)
+		string = re.sub(r".* #NODOC\n", "", string)
+		string = re.sub(r"\n\s*if .*then.*|\n\s*fi|\n\s*else|\n\s*elif .*", "", string)
 		string = quasitokenize(string)
-		string = re.sub("hide_output ", "", string)
+		string = string.replace("hide_output ", "")
 
 		parser = BashScript.parser()
 		result = parser.parse_string(string)
 
-		v = "<div class='row'><div class='col-xs-12 sourcefile'>view the bash source for the following section at <a href=\"%s\">%s</a></div></div>\n" \
-			 % ("https://github.com/mail-in-a-box/mailinabox/tree/master/" + fn, fn)
+		v = "<div class='row'><div class='col-xs-12 sourcefile'>view the bash source for the following section at <a href=\"{}\">{}</a></div></div>\n".format("https://github.com/mail-in-a-box/mailinabox/tree/master/" + fn, fn)
 
 		mode = 0
 		for item in result.value():
-			if item.strip() == "":
+			if not item.strip():
 				pass
 			elif item.startswith("<p") and not item.startswith("<pre"):
 				clz = ""
@@ -429,7 +427,7 @@ class BashScript(Grammar):
 					mode = 0
 					clz = "contd"
 				if mode == 0:
-					v += "<div class='row %s'>\n" % clz
+					v += f"<div class='row {clz}'>\n"
 					v += "<div class='col-md-6 prose'>\n"
 				v += item
 				mode = 1
@@ -460,24 +458,23 @@ class BashScript(Grammar):
 		v = fixup_tokens(v)
 
 		v = v.replace("</pre>\n<pre class='shell'>", "")
-		v = re.sub("<pre>([\w\W]*?)</pre>", lambda m : "<pre>" + strip_indent(m.group(1)) + "</pre>", v)
+		v = re.sub(r"<pre>([\w\W]*?)</pre>", lambda m : "<pre>" + strip_indent(m.group(1)) + "</pre>", v)
 
 		v = re.sub(r"(\$?)PRIMARY_HOSTNAME", r"<b>box.yourdomain.com</b>", v)
 		v = re.sub(r"\$STORAGE_ROOT", r"<b>$STORE</b>", v)
-		v = v.replace("`pwd`",  "<code><b>/path/to/mailinabox</b></code>")
+		return v.replace("`pwd`",  "<code><b>/path/to/mailinabox</b></code>")
 
-		return v
 
 def wrap_lines(text, cols=60):
 	ret = ""
-	words = re.split("(\s+)", text)
+	words = re.split(r"(\s+)", text)
 	linelen = 0
 	for w in words:
 		if linelen + len(w) > cols-1:
 			ret += " \\\n"
 			ret += "   "
 			linelen = 0
-		if linelen == 0 and w.strip() == "": continue
+		if linelen == 0 and not w.strip(): continue
 		ret += w
 		linelen += len(w)
 	return ret