mirror of
https://github.com/mail-in-a-box/mailinabox.git
synced 2026-03-12 17:07:23 +01:00
Compare commits
10 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
0c0a079354 | ||
|
|
42e86610ba | ||
|
7c62f4b8e9 | ||
|
|
1eba7b0616 | ||
|
|
9c7820f422 | ||
|
|
87ec4e9f82 | ||
|
|
08becf7fa3 | ||
|
|
5eb4a53de1 | ||
|
|
598ade3f7a | ||
|
8f399df5bb |
@@ -1,6 +1,13 @@
|
||||
CHANGELOG
|
||||
=========
|
||||
|
||||
v0.27 (June 14, 2018)
|
||||
---------------------
|
||||
|
||||
* A report of box activity, including sent/received mail totals and logins by user, is now emailed to the box's administrator user each week.
|
||||
* Update Roundcube to version 1.3.6 and Z-Push to version 2.3.9.
|
||||
* The undocumented feature for proxying web requests to another server now sets X-Forwarded-For.
|
||||
|
||||
v0.26c (February 13, 2018)
|
||||
--------------------------
|
||||
|
||||
|
||||
@@ -59,7 +59,7 @@ by me:
|
||||
$ curl -s https://keybase.io/joshdata/key.asc | gpg --import
|
||||
gpg: key C10BDD81: public key "Joshua Tauberer <jt@occams.info>" imported
|
||||
|
||||
$ git verify-tag v0.26c
|
||||
$ git verify-tag v0.27
|
||||
gpg: Signature made ..... using RSA key ID C10BDD81
|
||||
gpg: Good signature from "Joshua Tauberer <jt@occams.info>"
|
||||
gpg: WARNING: This key is not certified with a trusted signature!
|
||||
@@ -72,7 +72,7 @@ and on my [personal homepage](https://razor.occams.info/). (Of course, if this r
|
||||
|
||||
Checkout the tag corresponding to the most recent release:
|
||||
|
||||
$ git checkout v0.26c
|
||||
$ git checkout v0.27
|
||||
|
||||
Begin the installation.
|
||||
|
||||
|
||||
@@ -9,6 +9,12 @@ export LC_ALL=en_US.UTF-8
|
||||
export LANG=en_US.UTF-8
|
||||
export LC_TYPE=en_US.UTF-8
|
||||
|
||||
# On Mondays, i.e. once a week, send the administrator a report of total emails
|
||||
# sent and received so the admin might notice server abuse.
|
||||
if [ `date "+%u"` -eq 1 ]; then
|
||||
management/mail_log.py -t week | management/email_administrator.py "Mail-in-a-Box Usage Report"
|
||||
fi
|
||||
|
||||
# Take a backup.
|
||||
management/backup.py | management/email_administrator.py "Backup Status"
|
||||
|
||||
|
||||
@@ -4,8 +4,14 @@
|
||||
|
||||
import sys
|
||||
|
||||
import html
|
||||
import smtplib
|
||||
from email.message import Message
|
||||
|
||||
from email.mime.multipart import MIMEMultipart
|
||||
from email.mime.text import MIMEText
|
||||
|
||||
# In Python 3.6:
|
||||
#from email.message import Message
|
||||
|
||||
from utils import load_environment
|
||||
|
||||
@@ -26,11 +32,23 @@ if content == "":
|
||||
sys.exit(0)
|
||||
|
||||
# create MIME message
|
||||
msg = Message()
|
||||
msg = MIMEMultipart('alternative')
|
||||
|
||||
# In Python 3.6:
|
||||
#msg = Message()
|
||||
|
||||
msg['From'] = "\"%s\" <%s>" % (env['PRIMARY_HOSTNAME'], admin_addr)
|
||||
msg['To'] = admin_addr
|
||||
msg['Subject'] = "[%s] %s" % (env['PRIMARY_HOSTNAME'], subject)
|
||||
msg.set_payload(content, "UTF-8")
|
||||
|
||||
content_html = "<html><body><pre>{}</pre></body></html>".format(html.escape(content))
|
||||
|
||||
msg.attach(MIMEText(content, 'plain'))
|
||||
msg.attach(MIMEText(content_html, 'html'))
|
||||
|
||||
# In Python 3.6:
|
||||
#msg.set_content(content)
|
||||
#msg.add_alternative(content_html, "html")
|
||||
|
||||
# send
|
||||
smtpclient = smtplib.SMTP('127.0.0.1', 25)
|
||||
|
||||
@@ -53,10 +53,10 @@ VERBOSE = False
|
||||
# List of strings to filter users with
|
||||
FILTERS = None
|
||||
|
||||
# What to show by default
|
||||
# What to show (with defaults)
|
||||
SCAN_OUT = True # Outgoing email
|
||||
SCAN_IN = True # Incoming email
|
||||
SCAN_CONN = False # IMAP and POP3 logins
|
||||
SCAN_DOVECOT_LOGIN = True # Dovecot Logins
|
||||
SCAN_GREY = False # Greylisted email
|
||||
SCAN_BLOCKED = False # Rejected email
|
||||
|
||||
@@ -76,7 +76,8 @@ def scan_files(collector):
|
||||
tmp_file = tempfile.NamedTemporaryFile()
|
||||
shutil.copyfileobj(gzip.open(fn), tmp_file)
|
||||
|
||||
print("Processing file", fn, "...")
|
||||
if VERBOSE:
|
||||
print("Processing file", fn, "...")
|
||||
fn = tmp_file.name if tmp_file else fn
|
||||
|
||||
for line in reverse_readline(fn):
|
||||
@@ -105,7 +106,7 @@ def scan_mail_log(env):
|
||||
"scan_time": time.time(), # The time in seconds the scan took
|
||||
"sent_mail": OrderedDict(), # Data about email sent by users
|
||||
"received_mail": OrderedDict(), # Data about email received by users
|
||||
"dovecot": OrderedDict(), # Data about Dovecot activity
|
||||
"logins": OrderedDict(), # Data about login activity
|
||||
"postgrey": {}, # Data about greylisting of email addresses
|
||||
"rejected": OrderedDict(), # Emails that were blocked
|
||||
"known_addresses": None, # Addresses handled by the Miab installation
|
||||
@@ -119,8 +120,8 @@ def scan_mail_log(env):
|
||||
except ImportError:
|
||||
pass
|
||||
|
||||
print("Scanning from {:%Y-%m-%d %H:%M:%S} back to {:%Y-%m-%d %H:%M:%S}".format(
|
||||
START_DATE, END_DATE)
|
||||
print("Scanning logs from {:%Y-%m-%d %H:%M:%S} to {:%Y-%m-%d %H:%M:%S}".format(
|
||||
END_DATE, START_DATE)
|
||||
)
|
||||
|
||||
# Scan the lines in the log files until the date goes out of range
|
||||
@@ -138,8 +139,8 @@ def scan_mail_log(env):
|
||||
# Print Sent Mail report
|
||||
|
||||
if collector["sent_mail"]:
|
||||
msg = "Sent email between {:%Y-%m-%d %H:%M:%S} and {:%Y-%m-%d %H:%M:%S}"
|
||||
print_header(msg.format(END_DATE, START_DATE))
|
||||
msg = "Sent email"
|
||||
print_header(msg)
|
||||
|
||||
data = OrderedDict(sorted(collector["sent_mail"].items(), key=email_sort))
|
||||
|
||||
@@ -173,8 +174,8 @@ def scan_mail_log(env):
|
||||
# Print Received Mail report
|
||||
|
||||
if collector["received_mail"]:
|
||||
msg = "Received email between {:%Y-%m-%d %H:%M:%S} and {:%Y-%m-%d %H:%M:%S}"
|
||||
print_header(msg.format(END_DATE, START_DATE))
|
||||
msg = "Received email"
|
||||
print_header(msg)
|
||||
|
||||
data = OrderedDict(sorted(collector["received_mail"].items(), key=email_sort))
|
||||
|
||||
@@ -199,43 +200,55 @@ def scan_mail_log(env):
|
||||
[accum]
|
||||
)
|
||||
|
||||
# Print Dovecot report
|
||||
# Print login report
|
||||
|
||||
if collector["dovecot"]:
|
||||
msg = "Email client logins between {:%Y-%m-%d %H:%M:%S} and {:%Y-%m-%d %H:%M:%S}"
|
||||
print_header(msg.format(END_DATE, START_DATE))
|
||||
if collector["logins"]:
|
||||
msg = "User logins per hour"
|
||||
print_header(msg)
|
||||
|
||||
data = OrderedDict(sorted(collector["dovecot"].items(), key=email_sort))
|
||||
data = OrderedDict(sorted(collector["logins"].items(), key=email_sort))
|
||||
|
||||
# Get a list of all of the protocols seen in the logs in reverse count order.
|
||||
all_protocols = defaultdict(int)
|
||||
for u in data.values():
|
||||
for protocol_name, count in u["totals_by_protocol"].items():
|
||||
all_protocols[protocol_name] += count
|
||||
all_protocols = [k for k, v in sorted(all_protocols.items(), key=lambda kv : -kv[1])]
|
||||
|
||||
print_user_table(
|
||||
data.keys(),
|
||||
data=[
|
||||
("imap", [u["imap"] for u in data.values()]),
|
||||
("pop3", [u["pop3"] for u in data.values()]),
|
||||
(protocol_name, [
|
||||
round(u["totals_by_protocol"][protocol_name] / (u["latest"]-u["earliest"]).total_seconds() * 60*60, 1)
|
||||
if (u["latest"]-u["earliest"]).total_seconds() > 0
|
||||
else 0 # prevent division by zero
|
||||
for u in data.values()])
|
||||
for protocol_name in all_protocols
|
||||
],
|
||||
sub_data=[
|
||||
("IMAP IP addresses", [[k + " (%d)" % v for k, v in u["imap-logins"].items()]
|
||||
for u in data.values()]),
|
||||
("POP3 IP addresses", [[k + " (%d)" % v for k, v in u["pop3-logins"].items()]
|
||||
for u in data.values()]),
|
||||
("Protocol and Source", [[
|
||||
"{} {}: {} times".format(protocol_name, host, count)
|
||||
for (protocol_name, host), count
|
||||
in sorted(u["totals_by_protocol_and_host"].items(), key=lambda kv:-kv[1])
|
||||
] for u in data.values()])
|
||||
],
|
||||
activity=[
|
||||
("imap", [u["activity-by-hour"]["imap"] for u in data.values()]),
|
||||
("pop3", [u["activity-by-hour"]["pop3"] for u in data.values()]),
|
||||
(protocol_name, [u["activity-by-hour"][protocol_name] for u in data.values()])
|
||||
for protocol_name in all_protocols
|
||||
],
|
||||
earliest=[u["earliest"] for u in data.values()],
|
||||
latest=[u["latest"] for u in data.values()],
|
||||
numstr=lambda n : str(round(n, 1)),
|
||||
)
|
||||
|
||||
accum = {"imap": defaultdict(int), "pop3": defaultdict(int), "both": defaultdict(int)}
|
||||
accum = { protocol_name: defaultdict(int) for protocol_name in all_protocols }
|
||||
for h in range(24):
|
||||
accum["imap"][h] = sum(d["activity-by-hour"]["imap"][h] for d in data.values())
|
||||
accum["pop3"][h] = sum(d["activity-by-hour"]["pop3"][h] for d in data.values())
|
||||
accum["both"][h] = accum["imap"][h] + accum["pop3"][h]
|
||||
for protocol_name in all_protocols:
|
||||
accum[protocol_name][h] = sum(d["activity-by-hour"][protocol_name][h] for d in data.values())
|
||||
|
||||
print_time_table(
|
||||
["imap", "pop3", " +"],
|
||||
[accum["imap"], accum["pop3"], accum["both"]]
|
||||
all_protocols,
|
||||
[accum[protocol_name] for protocol_name in all_protocols]
|
||||
)
|
||||
|
||||
if collector["postgrey"]:
|
||||
@@ -348,9 +361,9 @@ def scan_mail_log_line(line, collector):
|
||||
elif service == "postfix/lmtp":
|
||||
if SCAN_IN:
|
||||
scan_postfix_lmtp_line(date, log, collector)
|
||||
elif service in ("imap-login", "pop3-login"):
|
||||
if SCAN_CONN:
|
||||
scan_dovecot_line(date, log, collector, service[:4])
|
||||
elif service.endswith("-login"):
|
||||
if SCAN_DOVECOT_LOGIN:
|
||||
scan_dovecot_login_line(date, log, collector, service[:4])
|
||||
elif service == "postgrey":
|
||||
if SCAN_GREY:
|
||||
scan_postgrey_line(date, log, collector)
|
||||
@@ -448,44 +461,43 @@ def scan_postfix_smtpd_line(date, log, collector):
|
||||
collector["rejected"][user] = data
|
||||
|
||||
|
||||
def scan_dovecot_line(date, log, collector, prot):
|
||||
""" Scan a dovecot log line and extract interesting data """
|
||||
def scan_dovecot_login_line(date, log, collector, protocol_name):
|
||||
""" Scan a dovecot login log line and extract interesting data """
|
||||
|
||||
m = re.match("Info: Login: user=<(.*?)>, method=PLAIN, rip=(.*?),", log)
|
||||
|
||||
if m:
|
||||
# TODO: CHECK DIT
|
||||
user, rip = m.groups()
|
||||
user, host = m.groups()
|
||||
|
||||
if user_match(user):
|
||||
add_login(user, date, protocol_name, host, collector)
|
||||
|
||||
|
||||
def add_login(user, date, protocol_name, host, collector):
|
||||
# Get the user data, or create it if the user is new
|
||||
data = collector["dovecot"].get(
|
||||
data = collector["logins"].get(
|
||||
user,
|
||||
{
|
||||
"imap": 0,
|
||||
"pop3": 0,
|
||||
"earliest": None,
|
||||
"latest": None,
|
||||
"imap-logins": defaultdict(int),
|
||||
"pop3-logins": defaultdict(int),
|
||||
"activity-by-hour": {
|
||||
"imap": defaultdict(int),
|
||||
"pop3": defaultdict(int),
|
||||
},
|
||||
"totals_by_protocol": defaultdict(int),
|
||||
"totals_by_protocol_and_host": defaultdict(int),
|
||||
"activity-by-hour": defaultdict(lambda : defaultdict(int)),
|
||||
}
|
||||
)
|
||||
|
||||
data[prot] += 1
|
||||
data["activity-by-hour"][prot][date.hour] += 1
|
||||
|
||||
if data["latest"] is None:
|
||||
data["latest"] = date
|
||||
data["earliest"] = date
|
||||
|
||||
if rip not in ("127.0.0.1", "::1") or True:
|
||||
data["%s-logins" % prot][rip] += 1
|
||||
data["totals_by_protocol"][protocol_name] += 1
|
||||
data["totals_by_protocol_and_host"][(protocol_name, host)] += 1
|
||||
|
||||
collector["dovecot"][user] = data
|
||||
if host not in ("127.0.0.1", "::1") or True:
|
||||
data["activity-by-hour"][protocol_name][date.hour] += 1
|
||||
|
||||
collector["logins"][user] = data
|
||||
|
||||
|
||||
def scan_postfix_lmtp_line(date, log, collector):
|
||||
@@ -561,6 +573,8 @@ def scan_postfix_submission_line(date, log, collector):
|
||||
|
||||
collector["sent_mail"][user] = data
|
||||
|
||||
# Also log this as a login.
|
||||
add_login(user, date, "smtp", client, collector)
|
||||
|
||||
# Utility functions
|
||||
|
||||
@@ -640,7 +654,7 @@ def print_time_table(labels, data, do_print=True):
|
||||
for i, d in enumerate(data):
|
||||
lines[i] += base.format(d[h])
|
||||
|
||||
lines.insert(0, "┬")
|
||||
lines.insert(0, "┬ totals by time of day:")
|
||||
lines.append("└" + (len(lines[-1]) - 2) * "─")
|
||||
|
||||
if do_print:
|
||||
@@ -650,7 +664,7 @@ def print_time_table(labels, data, do_print=True):
|
||||
|
||||
|
||||
def print_user_table(users, data=None, sub_data=None, activity=None, latest=None, earliest=None,
|
||||
delimit=False):
|
||||
delimit=False, numstr=str):
|
||||
str_temp = "{:<32} "
|
||||
lines = []
|
||||
data = data or []
|
||||
@@ -764,7 +778,7 @@ def print_user_table(users, data=None, sub_data=None, activity=None, latest=None
|
||||
|
||||
# Print totals
|
||||
|
||||
data_accum = [str(a) for a in data_accum]
|
||||
data_accum = [numstr(a) for a in data_accum]
|
||||
footer = str_temp.format("Totals:" if do_accum else " ")
|
||||
for row, (l, _) in enumerate(data):
|
||||
temp = "{:>%d}" % max(5, len(l) + 1)
|
||||
@@ -818,7 +832,7 @@ if __name__ == "__main__":
|
||||
action="store_true")
|
||||
parser.add_argument("-s", "--sent", help="Scan for sent emails.",
|
||||
action="store_true")
|
||||
parser.add_argument("-l", "--logins", help="Scan for IMAP/POP logins.",
|
||||
parser.add_argument("-l", "--logins", help="Scan for user logins to IMAP/POP3.",
|
||||
action="store_true")
|
||||
parser.add_argument("-g", "--grey", help="Scan for greylisted emails.",
|
||||
action="store_true")
|
||||
@@ -863,8 +877,8 @@ if __name__ == "__main__":
|
||||
if not SCAN_OUT:
|
||||
print("Ignoring sent emails")
|
||||
|
||||
SCAN_CONN = args.logins
|
||||
if not SCAN_CONN:
|
||||
SCAN_DOVECOT_LOGIN = args.logins
|
||||
if not SCAN_DOVECOT_LOGIN:
|
||||
print("Ignoring logins")
|
||||
|
||||
SCAN_GREY = args.grey
|
||||
|
||||
@@ -149,7 +149,10 @@ def make_domain_config(domain, templates, ssl_certificates, env):
|
||||
|
||||
# any proxy or redirect here?
|
||||
for path, url in yaml.get("proxies", {}).items():
|
||||
nginx_conf_extra += "\tlocation %s {\n\t\tproxy_pass %s;\n\t}\n" % (path, url)
|
||||
nginx_conf_extra += "\tlocation %s {" % path
|
||||
nginx_conf_extra += "\n\t\tproxy_pass %s;" % url
|
||||
nginx_conf_extra += "\n\t\tproxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;"
|
||||
nginx_conf_extra += "\n\t}\n"
|
||||
for path, url in yaml.get("redirects", {}).items():
|
||||
nginx_conf_extra += "\trewrite %s %s permanent;\n" % (path, url)
|
||||
|
||||
|
||||
@@ -7,7 +7,7 @@
|
||||
#########################################################
|
||||
|
||||
if [ -z "$TAG" ]; then
|
||||
TAG=v0.26c
|
||||
TAG=v0.27
|
||||
fi
|
||||
|
||||
# Are we running as root?
|
||||
|
||||
@@ -35,8 +35,8 @@ apt-get purge -qq -y roundcube* #NODOC
|
||||
# Install Roundcube from source if it is not already present or if it is out of date.
|
||||
# Combine the Roundcube version number with the commit hash of plugins to track
|
||||
# whether we have the latest version of everything.
|
||||
VERSION=1.3.3
|
||||
HASH=903a4eb1bfc25e9a08d782a7f98502cddfa579de
|
||||
VERSION=1.3.6
|
||||
HASH=ece5cfc9c7af0cbe90c0065ef33e85ed42991830
|
||||
PERSISTENT_LOGIN_VERSION=dc5ca3d3f4415cc41edb2fde533c8a8628a94c76
|
||||
HTML5_NOTIFIER_VERSION=4b370e3cd60dabd2f428a26f45b677ad1b7118d5
|
||||
CARDDAV_VERSION=2.0.4
|
||||
|
||||
@@ -22,7 +22,7 @@ apt_install \
|
||||
phpenmod -v php7.0 imap
|
||||
|
||||
# Copy Z-Push into place.
|
||||
VERSION=2.3.8
|
||||
VERSION=2.3.9
|
||||
needs_update=0 #NODOC
|
||||
if [ ! -f /usr/local/lib/z-push/version ]; then
|
||||
needs_update=1 #NODOC
|
||||
|
||||
@@ -1,24 +0,0 @@
|
||||
#!/usr/bin/python3
|
||||
# Updates subresource integrity attributes in management/templates/index.html
|
||||
# to prevent CDN-hosted resources from being used as an attack vector. Run this
|
||||
# after updating the Bootstrap and jQuery <link> and <script> to compute the
|
||||
# appropriate hash and insert it into the template.
|
||||
|
||||
import re, urllib.request, hashlib, base64
|
||||
|
||||
fn = "management/templates/index.html"
|
||||
|
||||
with open(fn, 'r') as f:
|
||||
content = f.read()
|
||||
|
||||
def make_integrity(url):
|
||||
resource = urllib.request.urlopen(url).read()
|
||||
return "sha256-" + base64.b64encode(hashlib.sha256(resource).digest()).decode('ascii')
|
||||
|
||||
content = re.sub(
|
||||
r'<(link rel="stylesheet" href|script src)="(.*?)" integrity="(.*?)"',
|
||||
lambda m : '<' + m.group(1) + '="' + m.group(2) + '" integrity="' + make_integrity(m.group(2)) + '"',
|
||||
content)
|
||||
|
||||
with open(fn, 'w') as f:
|
||||
f.write(content)
|
||||
Reference in New Issue
Block a user