external
/
mailinabox
mirror of https://github.com/mail-in-a-box/mailinabox.git
1
0
Fork 0
Code Issues Releases Wiki Activity

mail_log.py: refactor the dovecot login collector

This commit is contained in:
Joshua Tauberer 2018-02-24 08:45:21 -05:00
parent 08becf7fa3
commit 87ec4e9f82
1 changed files with 54 additions and 47 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

101
management/mail_log.py
View File

@ -53,10 +53,10 @@ VERBOSE = False
# List of strings to filter users with # List of strings to filter users with
FILTERS = None FILTERS = None
# What to show by default # What to show (with defaults)
SCAN_OUT = True # Outgoing email SCAN_OUT = True # Outgoing email
SCAN_IN = True # Incoming email SCAN_IN = True # Incoming email
SCAN_CONN = False # IMAP and POP3 logins SCAN_DOVECOT_LOGIN = True # Dovecot Logins
SCAN_GREY = False # Greylisted email SCAN_GREY = False # Greylisted email
SCAN_BLOCKED = False # Rejected email SCAN_BLOCKED = False # Rejected email
@ -105,7 +105,7 @@ def scan_mail_log(env):
"scan_time": time.time(), # The time in seconds the scan took "scan_time": time.time(), # The time in seconds the scan took
"sent_mail": OrderedDict(), # Data about email sent by users "sent_mail": OrderedDict(), # Data about email sent by users
"received_mail": OrderedDict(), # Data about email received by users "received_mail": OrderedDict(), # Data about email received by users
"dovecot": OrderedDict(), # Data about Dovecot activity "logins": OrderedDict(), # Data about Dovecot login activity
"postgrey": {}, # Data about greylisting of email addresses "postgrey": {}, # Data about greylisting of email addresses
"rejected": OrderedDict(), # Emails that were blocked "rejected": OrderedDict(), # Emails that were blocked
"known_addresses": None, # Addresses handled by the Miab installation "known_addresses": None, # Addresses handled by the Miab installation
@ -199,43 +199,55 @@ def scan_mail_log(env):
[accum] [accum]
) )
# Print Dovecot report # Print login report
if collector["dovecot"]: if collector["logins"]:
msg = "Email client logins between {:%Y-%m-%d %H:%M:%S} and {:%Y-%m-%d %H:%M:%S}" msg = "User logins per hour between {:%Y-%m-%d %H:%M:%S} and {:%Y-%m-%d %H:%M:%S}"
print_header(msg.format(END_DATE, START_DATE)) print_header(msg.format(END_DATE, START_DATE))
data = OrderedDict(sorted(collector["dovecot"].items(), key=email_sort)) data = OrderedDict(sorted(collector["logins"].items(), key=email_sort))
# Get a list of all of the protocols seen in the logs in reverse count order.
all_protocols = defaultdict(int)
for u in data.values():
for protocol_name, count in u["totals_by_protocol"].items():
all_protocols[protocol_name] += count
all_protocols = [k for k, v in sorted(all_protocols.items(), key=lambda kv : -kv[1])]
print_user_table( print_user_table(
data.keys(), data.keys(),
data=[ data=[
("imap", [u["imap"] for u in data.values()]), (protocol_name, [
("pop3", [u["pop3"] for u in data.values()]), round(u["totals_by_protocol"][protocol_name] / (u["latest"]-u["earliest"]).total_seconds() * 60*60, 1)
if (u["latest"]-u["earliest"]).total_seconds() > 0
else 0 # prevent division by zero
for u in data.values()])
for protocol_name in all_protocols
], ],
sub_data=[ sub_data=[
("IMAP IP addresses", [[k + " (%d)" % v for k, v in u["imap-logins"].items()] ("Protocol and Source", [[
for u in data.values()]), "{} {}: {} times".format(protocol_name, host, count)
("POP3 IP addresses", [[k + " (%d)" % v for k, v in u["pop3-logins"].items()] for (protocol_name, host), count
for u in data.values()]), in sorted(u["totals_by_protocol_and_host"].items(), key=lambda kv:-kv[1])
] for u in data.values()])
], ],
activity=[ activity=[
("imap", [u["activity-by-hour"]["imap"] for u in data.values()]), (protocol_name, [u["activity-by-hour"][protocol_name] for u in data.values()])
("pop3", [u["activity-by-hour"]["pop3"] for u in data.values()]), for protocol_name in all_protocols
], ],
earliest=[u["earliest"] for u in data.values()], earliest=[u["earliest"] for u in data.values()],
latest=[u["latest"] for u in data.values()], latest=[u["latest"] for u in data.values()],
numstr=lambda n : str(round(n, 1)),
) )
accum = {"imap": defaultdict(int), "pop3": defaultdict(int), "both": defaultdict(int)} accum = { protocol_name: defaultdict(int) for protocol_name in all_protocols }
for h in range(24): for h in range(24):
accum["imap"][h] = sum(d["activity-by-hour"]["imap"][h] for d in data.values()) for protocol_name in all_protocols:
accum["pop3"][h] = sum(d["activity-by-hour"]["pop3"][h] for d in data.values()) accum[protocol_name][h] = sum(d["activity-by-hour"][protocol_name][h] for d in data.values())
accum["both"][h] = accum["imap"][h] + accum["pop3"][h]
print_time_table( print_time_table(
["imap", "pop3", " +"], all_protocols,
[accum["imap"], accum["pop3"], accum["both"]] [accum[protocol_name] for protocol_name in all_protocols]
) )
if collector["postgrey"]: if collector["postgrey"]:
@ -348,9 +360,9 @@ def scan_mail_log_line(line, collector):
elif service == "postfix/lmtp": elif service == "postfix/lmtp":
if SCAN_IN: if SCAN_IN:
scan_postfix_lmtp_line(date, log, collector) scan_postfix_lmtp_line(date, log, collector)
elif service in ("imap-login", "pop3-login"): elif service.endswith("-login"):
if SCAN_CONN: if SCAN_DOVECOT_LOGIN:
scan_dovecot_line(date, log, collector, service[:4]) scan_dovecot_login_line(date, log, collector, service[:4])
elif service == "postgrey": elif service == "postgrey":
if SCAN_GREY: if SCAN_GREY:
scan_postgrey_line(date, log, collector) scan_postgrey_line(date, log, collector)
@ -448,8 +460,8 @@ def scan_postfix_smtpd_line(date, log, collector):
collector["rejected"][user] = data collector["rejected"][user] = data
def scan_dovecot_line(date, log, collector, prot): def scan_dovecot_login_line(date, log, collector, protocol_name):
""" Scan a dovecot log line and extract interesting data """ """ Scan a dovecot login log line and extract interesting data """
m = re.match("Info: Login: user=<(.*?)>, method=PLAIN, rip=(.*?),", log) m = re.match("Info: Login: user=<(.*?)>, method=PLAIN, rip=(.*?),", log)
@ -459,33 +471,28 @@ def scan_dovecot_line(date, log, collector, prot):
if user_match(user): if user_match(user):
# Get the user data, or create it if the user is new # Get the user data, or create it if the user is new
data = collector["dovecot"].get( data = collector["logins"].get(
user, user,
{ {
"imap": 0,
"pop3": 0,
"earliest": None, "earliest": None,
"latest": None, "latest": None,
"imap-logins": defaultdict(int), "totals_by_protocol": defaultdict(int),
"pop3-logins": defaultdict(int), "totals_by_protocol_and_host": defaultdict(int),
"activity-by-hour": { "activity-by-hour": defaultdict(lambda : defaultdict(int)),
"imap": defaultdict(int),
"pop3": defaultdict(int),
},
} }
) )
data[prot] += 1
data["activity-by-hour"][prot][date.hour] += 1
if data["latest"] is None: if data["latest"] is None:
data["latest"] = date data["latest"] = date
data["earliest"] = date data["earliest"] = date
if rip not in ("127.0.0.1", "::1") or True: data["totals_by_protocol"][protocol_name] += 1
data["%s-logins" % prot][rip] += 1 data["totals_by_protocol_and_host"][(protocol_name, rip)] += 1
collector["dovecot"][user] = data if rip not in ("127.0.0.1", "::1") or True:
data["activity-by-hour"][protocol_name][date.hour] += 1
collector["logins"][user] = data
def scan_postfix_lmtp_line(date, log, collector): def scan_postfix_lmtp_line(date, log, collector):
@ -640,7 +647,7 @@ def print_time_table(labels, data, do_print=True):
for i, d in enumerate(data): for i, d in enumerate(data):
lines[i] += base.format(d[h]) lines[i] += base.format(d[h])
lines.insert(0, "┬") lines.insert(0, "┬ totals by time of day:")
lines.append("└" + (len(lines[-1]) - 2) * "─") lines.append("└" + (len(lines[-1]) - 2) * "─")
if do_print: if do_print:
@ -650,7 +657,7 @@ def print_time_table(labels, data, do_print=True):
def print_user_table(users, data=None, sub_data=None, activity=None, latest=None, earliest=None, def print_user_table(users, data=None, sub_data=None, activity=None, latest=None, earliest=None,
delimit=False): delimit=False, numstr=str):
str_temp = "{:<32} " str_temp = "{:<32} "
lines = [] lines = []
data = data or [] data = data or []
@ -764,7 +771,7 @@ def print_user_table(users, data=None, sub_data=None, activity=None, latest=None
# Print totals # Print totals
data_accum = [str(a) for a in data_accum] data_accum = [numstr(a) for a in data_accum]
footer = str_temp.format("Totals:" if do_accum else " ") footer = str_temp.format("Totals:" if do_accum else " ")
for row, (l, _) in enumerate(data): for row, (l, _) in enumerate(data):
temp = "{:>%d}" % max(5, len(l) + 1) temp = "{:>%d}" % max(5, len(l) + 1)
@ -818,7 +825,7 @@ if __name__ == "__main__":
action="store_true") action="store_true")
parser.add_argument("-s", "--sent", help="Scan for sent emails.", parser.add_argument("-s", "--sent", help="Scan for sent emails.",
action="store_true") action="store_true")
parser.add_argument("-l", "--logins", help="Scan for IMAP/POP logins.", parser.add_argument("-l", "--logins", help="Scan for user logins to IMAP/POP3.",
action="store_true") action="store_true")
parser.add_argument("-g", "--grey", help="Scan for greylisted emails.", parser.add_argument("-g", "--grey", help="Scan for greylisted emails.",
action="store_true") action="store_true")
@ -863,8 +870,8 @@ if __name__ == "__main__":
if not SCAN_OUT: if not SCAN_OUT:
print("Ignoring sent emails") print("Ignoring sent emails")
SCAN_CONN = args.logins SCAN_DOVECOT_LOGIN = args.logins
if not SCAN_CONN: if not SCAN_DOVECOT_LOGIN:
print("Ignoring logins") print("Ignoring logins")
SCAN_GREY = args.grey SCAN_GREY = args.grey