mirror of
https://github.com/mail-in-a-box/mailinabox.git
synced 2025-04-03 00:07:05 +00:00
468 lines
12 KiB
Python
468 lines
12 KiB
Python
# -*- indent-tabs-mode: t; tab-width: 4; python-indent-offset: 4; -*-
|
|
#####
|
|
##### This file is part of Mail-in-a-Box-LDAP which is released under the
|
|
##### terms of the GNU Affero General Public License as published by the
|
|
##### Free Software Foundation, either version 3 of the License, or (at
|
|
##### your option) any later version. See file LICENSE or go to
|
|
##### https://github.com/downtownallday/mailinabox-ldap for full license
|
|
##### details.
|
|
#####
|
|
|
|
|
|
import sqlite3
|
|
import os, stat
|
|
import logging
|
|
import json
|
|
import datetime
|
|
from .EventStore import EventStore
|
|
|
|
log = logging.getLogger(__name__)
|
|
|
|
#
|
|
# schema
|
|
#
|
|
mta_conn_fields = [
|
|
'service',
|
|
'service_tid',
|
|
'connect_time',
|
|
'disconnect_time',
|
|
'remote_host',
|
|
'remote_ip',
|
|
'sasl_method',
|
|
'sasl_username',
|
|
'remote_auth_success',
|
|
'remote_auth_attempts',
|
|
'remote_used_starttls',
|
|
'remote_used_tls',
|
|
'tls_version',
|
|
'tls_cipher',
|
|
'disposition',
|
|
]
|
|
|
|
mta_accept_fields = [
|
|
'mta_conn_id',
|
|
'queue_time',
|
|
'queue_remove_time',
|
|
'subsystems',
|
|
# 'spf_tid',
|
|
'spf_result',
|
|
'spf_reason',
|
|
'postfix_msg_id',
|
|
'message_id',
|
|
'dkim_result',
|
|
'dkim_reason',
|
|
'dmarc_result',
|
|
'dmarc_reason',
|
|
'envelope_from',
|
|
'message_size',
|
|
'message_nrcpt',
|
|
'accept_status',
|
|
'failure_info',
|
|
'failure_category',
|
|
]
|
|
|
|
mta_delivery_fields = [
|
|
'mta_accept_id',
|
|
'service',
|
|
# 'service_tid',
|
|
'rcpt_to',
|
|
'orig_to',
|
|
# 'postgrey_tid',
|
|
'postgrey_result',
|
|
'postgrey_reason',
|
|
'postgrey_delay',
|
|
# 'spam_tid',
|
|
'spam_result',
|
|
'spam_score',
|
|
'relay',
|
|
'status',
|
|
'delay',
|
|
'delivery_connection',
|
|
'delivery_connection_info',
|
|
'delivery_info',
|
|
'failure_category',
|
|
]
|
|
|
|
imap_conn_fields = [
|
|
'service',
|
|
'service_tid',
|
|
'connect_time',
|
|
'disconnect_time',
|
|
'disconnect_reason',
|
|
'remote_host',
|
|
'remote_ip',
|
|
'sasl_method',
|
|
'sasl_username',
|
|
'remote_auth_success',
|
|
'remote_auth_attempts',
|
|
'connection_security',
|
|
'in_bytes',
|
|
'out_bytes',
|
|
'disposition'
|
|
]
|
|
|
|
db_info_create_table_stmt = "CREATE TABLE IF NOT EXISTS db_info(id INTEGER PRIMARY KEY AUTOINCREMENT, key TEXT NOT NULL, value TEXT NOT NULL)"
|
|
|
|
schema_updates = [
|
|
# update 0
|
|
[
|
|
# three "mta" tables having a one-to-many-to-many relationship:
|
|
# mta_connection(1) -> mta_accept(0:N) -> mta_delivery(0:N)
|
|
#
|
|
|
|
"CREATE TABLE mta_connection(\
|
|
mta_conn_id INTEGER PRIMARY KEY AUTOINCREMENT,\
|
|
service TEXT NOT NULL, /* 'smtpd', 'submission' or 'pickup' */\
|
|
service_tid TEXT NOT NULL,\
|
|
connect_time TEXT NOT NULL,\
|
|
disconnect_time TEXT,\
|
|
remote_host TEXT COLLATE NOCASE,\
|
|
remote_ip TEXT COLLATE NOCASE,\
|
|
sasl_method TEXT, /* sasl: submission service only */\
|
|
sasl_username TEXT COLLATE NOCASE,\
|
|
remote_auth_success INTEGER, /* count of successes */\
|
|
remote_auth_attempts INTEGER, /* count of attempts */\
|
|
remote_used_starttls INTEGER, /* 1 if STARTTLS used */\
|
|
disposition TEXT /* 'normal','scanner','login_attempt',etc */\
|
|
)",
|
|
|
|
"CREATE INDEX idx_mta_connection_connect_time ON mta_connection(connect_time, sasl_username COLLATE NOCASE)",
|
|
|
|
|
|
"CREATE TABLE mta_accept(\
|
|
mta_accept_id INTEGER PRIMARY KEY AUTOINCREMENT,\
|
|
mta_conn_id INTEGER,\
|
|
queue_time TEXT,\
|
|
queue_remove_time TEXT,\
|
|
subsystems TEXT,\
|
|
/*spf_tid TEXT,*/\
|
|
spf_result TEXT,\
|
|
spf_reason TEXT,\
|
|
postfix_msg_id TEXT,\
|
|
message_id TEXT,\
|
|
dkim_result TEXT,\
|
|
dkim_reason TEXT,\
|
|
dmarc_result TEXT,\
|
|
dmarc_reason TEXT,\
|
|
envelope_from TEXT COLLATE NOCASE,\
|
|
message_size INTEGER,\
|
|
message_nrcpt INTEGER,\
|
|
accept_status TEXT, /* 'accept','greylist','spf-reject',others... */\
|
|
failure_info TEXT, /* details from mta or subsystems */\
|
|
failure_category TEXT,\
|
|
FOREIGN KEY(mta_conn_id) REFERENCES mta_connection(mta_conn_id) ON DELETE RESTRICT\
|
|
)",
|
|
|
|
"CREATE TABLE mta_delivery(\
|
|
mta_delivery_id INTEGER PRIMARY KEY AUTOINCREMENT,\
|
|
mta_accept_id INTEGER,\
|
|
service TEXT, /* 'lmtp' or 'smtp' */\
|
|
/*service_tid TEXT,*/\
|
|
rcpt_to TEXT COLLATE NOCASE, /* email addr */\
|
|
/*postgrey_tid TEXT,*/\
|
|
postgrey_result TEXT,\
|
|
postgrey_reason TEXT,\
|
|
postgrey_delay NUMBER,\
|
|
/*spam_tid TEXT,*/ /* spam: lmtp only */\
|
|
spam_result TEXT, /* 'clean' or 'spam' */\
|
|
spam_score NUMBER, /* eg: 2.10 */\
|
|
relay TEXT, /* hostname[IP]:port */\
|
|
status TEXT, /* 'sent', 'bounce', 'reject', etc */\
|
|
delay NUMBER, /* fractional seconds, 'sent' status only */\
|
|
delivery_connection TEXT, /* 'trusted' or 'untrusted' */\
|
|
delivery_connection_info TEXT, /* details on TLS connection */\
|
|
delivery_info TEXT, /* details from the remote mta */\
|
|
failure_category TEXT,\
|
|
FOREIGN KEY(mta_accept_id) REFERENCES mta_accept(mta_accept_id) ON DELETE RESTRICT\
|
|
)",
|
|
|
|
"CREATE INDEX idx_mta_delivery_rcpt_to ON mta_delivery(rcpt_to COLLATE NOCASE)",
|
|
|
|
"CREATE TABLE state_cache(\
|
|
state_cache_id INTEGER PRIMARY KEY AUTOINCREMENT,\
|
|
owner_id INTEGER NOT NULL,\
|
|
state TEXT\
|
|
)",
|
|
|
|
"INSERT INTO db_info (key,value) VALUES ('schema_version', '0')"
|
|
],
|
|
|
|
# update 1
|
|
[
|
|
"ALTER TABLE mta_delivery ADD COLUMN orig_to TEXT COLLATE NOCASE",
|
|
"UPDATE db_info SET value='1' WHERE key='schema_version'"
|
|
],
|
|
|
|
# update 2
|
|
[
|
|
"CREATE TABLE imap_connection(\
|
|
imap_conn_id INTEGER PRIMARY KEY AUTOINCREMENT,\
|
|
service TEXT NOT NULL, /* 'imap', 'imap-login', 'pop3-login' */\
|
|
service_tid TEXT,\
|
|
connect_time TEXT NOT NULL,\
|
|
disconnect_time TEXT,\
|
|
disconnect_reason TEXT,\
|
|
remote_host TEXT COLLATE NOCASE,\
|
|
remote_ip TEXT COLLATE NOCASE,\
|
|
sasl_method TEXT, /* eg. 'PLAIN' */\
|
|
sasl_username TEXT COLLATE NOCASE,\
|
|
remote_auth_success INTEGER, /* count of successes */\
|
|
remote_auth_attempts INTEGER, /* count of attempts */\
|
|
connection_security TEXT, /* eg 'TLS' */\
|
|
in_bytes INTEGER,\
|
|
out_bytes INTEGER,\
|
|
disposition TEXT /* 'ok','failed_login_attempt',etc */\
|
|
)",
|
|
|
|
"CREATE INDEX idx_imap_connection_connect_time ON imap_connection(connect_time, sasl_username COLLATE NOCASE)",
|
|
|
|
"UPDATE db_info SET value='2' WHERE key='schema_version'"
|
|
],
|
|
|
|
# update 3
|
|
[
|
|
"ALTER TABLE mta_connection ADD COLUMN remote_used_tls INTEGER DEFAULT 0",
|
|
"ALTER TABLE mta_connection ADD COLUMN tls_version TEXT DEFAULT NULL COLLATE NOCASE",
|
|
"ALTER TABLE mta_connection ADD COLUMN tls_cipher TEXT DEFAULT NULL COLLATE NOCASE",
|
|
|
|
"UPDATE db_info SET value='3' WHERE key='schema_version'"
|
|
],
|
|
|
|
]
|
|
|
|
|
|
|
|
class SqliteEventStore(EventStore):
|
|
|
|
def __init__(self, db_conn_factory):
|
|
super(SqliteEventStore, self).__init__(db_conn_factory)
|
|
self.update_schema()
|
|
|
|
def update_schema(self):
|
|
''' update the schema to the latest version
|
|
|
|
'''
|
|
c = None
|
|
conn = None
|
|
try:
|
|
conn = self.connect()
|
|
c = conn.cursor()
|
|
c.execute(db_info_create_table_stmt)
|
|
conn.commit()
|
|
c.execute("SELECT value from db_info WHERE key='schema_version'")
|
|
v = c.fetchone()
|
|
if v is None:
|
|
v = -1
|
|
else:
|
|
v = int(v[0])
|
|
for idx in range(v+1, len(schema_updates)):
|
|
log.info('updating database to v%s', idx)
|
|
for stmt in schema_updates[idx]:
|
|
try:
|
|
c.execute(stmt)
|
|
except Exception as e:
|
|
log.error('problem with sql statement at version=%s error="%s" stmt="%s"' % (idx, e, stmt))
|
|
raise e
|
|
|
|
conn.commit()
|
|
|
|
finally:
|
|
if c: c.close(); c=None
|
|
if conn: self.close(conn); conn=None
|
|
|
|
|
|
|
|
def write_rec(self, conn, type, rec):
|
|
if type=='mta_mail':
|
|
self.write_mta_mail(conn, rec)
|
|
|
|
elif type=='imap_mail':
|
|
self.write_imap_mail(conn, rec)
|
|
|
|
elif type=='state':
|
|
''' rec: {
|
|
owner_id: int,
|
|
state: list
|
|
}
|
|
'''
|
|
self.write_state(conn, rec)
|
|
else:
|
|
raise ValueError('type "%s" not implemented' % type)
|
|
|
|
|
|
def _insert(self, table, fields):
|
|
insert = 'INSERT INTO ' + table + ' (' + \
|
|
",".join(fields) + \
|
|
') VALUES (' + \
|
|
"?,"*(len(fields)-1) + \
|
|
'?)'
|
|
return insert
|
|
|
|
def _values(self, fields, data_dict):
|
|
values = []
|
|
for field in fields:
|
|
if field in data_dict:
|
|
values.append(data_dict[field])
|
|
data_dict.pop(field)
|
|
else:
|
|
values.append(None)
|
|
|
|
for field in data_dict:
|
|
if type(data_dict[field]) != list and not field.startswith('_') and not field.endswith('_tid'):
|
|
log.warning('unused field: %s', field)
|
|
return values
|
|
|
|
|
|
def write_mta_mail(self, conn, rec):
|
|
c = None
|
|
try:
|
|
c = conn.cursor()
|
|
|
|
# mta_connection
|
|
insert = self._insert('mta_connection', mta_conn_fields)
|
|
values = self._values(mta_conn_fields, rec)
|
|
#log.debug('INSERT: %s VALUES: %s REC=%s', insert, values, rec)
|
|
c.execute(insert, values)
|
|
conn_id = c.lastrowid
|
|
|
|
accept_insert = self._insert('mta_accept', mta_accept_fields)
|
|
delivery_insert = self._insert('mta_delivery', mta_delivery_fields)
|
|
for accept in rec.get('mta_accept', []):
|
|
accept['mta_conn_id'] = conn_id
|
|
values = self._values(mta_accept_fields, accept)
|
|
c.execute(accept_insert, values)
|
|
accept_id = c.lastrowid
|
|
|
|
for delivery in accept.get('mta_delivery', []):
|
|
delivery['mta_accept_id'] = accept_id
|
|
values = self._values(mta_delivery_fields, delivery)
|
|
c.execute(delivery_insert, values)
|
|
|
|
conn.commit()
|
|
|
|
except sqlite3.Error as e:
|
|
conn.rollback()
|
|
raise e
|
|
|
|
finally:
|
|
if c: c.close(); c=None
|
|
|
|
|
|
|
|
def write_imap_mail(self, conn, rec):
|
|
c = None
|
|
try:
|
|
c = conn.cursor()
|
|
|
|
# imap_connection
|
|
insert = self._insert('imap_connection', imap_conn_fields)
|
|
values = self._values(imap_conn_fields, rec)
|
|
#log.debug('INSERT: %s VALUES: %s REC=%s', insert, values, rec)
|
|
c.execute(insert, values)
|
|
conn_id = c.lastrowid
|
|
|
|
conn.commit()
|
|
|
|
except sqlite3.Error as e:
|
|
conn.rollback()
|
|
raise e
|
|
|
|
finally:
|
|
if c: c.close(); c=None
|
|
|
|
|
|
def write_state(self, conn, rec):
|
|
c = None
|
|
try:
|
|
c = conn.cursor()
|
|
|
|
owner_id = rec['owner_id']
|
|
insert = 'INSERT INTO state_cache (owner_id, state) VALUES (?, ?)'
|
|
for item in rec['state']:
|
|
item_json = json.dumps(item)
|
|
c.execute(insert, (owner_id, item_json))
|
|
|
|
conn.commit()
|
|
|
|
except sqlite3.Error as e:
|
|
conn.rollback()
|
|
raise e
|
|
|
|
finally:
|
|
if c: c.close(); c=None
|
|
|
|
|
|
def read_rec(self, conn, type, args):
|
|
if type=='state':
|
|
return self.read_state(
|
|
conn,
|
|
args['owner_id'],
|
|
args.get('clear',False)
|
|
)
|
|
else:
|
|
raise ValueError('type "%s" not implemented' % type)
|
|
|
|
def read_state(self, conn, owner_id, clear):
|
|
c = None
|
|
state = []
|
|
try:
|
|
c = conn.cursor()
|
|
select = 'SELECT state FROM state_cache WHERE owner_id=? ORDER BY state_cache_id'
|
|
for row in c.execute(select, (owner_id,)):
|
|
state.append(json.loads(row[0]))
|
|
|
|
if clear:
|
|
delete = 'DELETE FROM state_cache WHERE owner_id=?'
|
|
c.execute(delete, (owner_id,))
|
|
conn.commit()
|
|
|
|
finally:
|
|
if c: c.close(); c=None
|
|
|
|
return state
|
|
|
|
def prune(self, conn, policy):
|
|
older_than_days = datetime.timedelta(days=policy['older_than_days'])
|
|
if older_than_days.days <= 0:
|
|
return
|
|
now = datetime.datetime.now(datetime.timezone.utc)
|
|
d = (now - older_than_days)
|
|
dstr = d.isoformat(sep=' ', timespec='seconds')
|
|
|
|
c = None
|
|
try:
|
|
c = conn.cursor()
|
|
deletes = [
|
|
'DELETE FROM mta_delivery WHERE mta_accept_id IN (\
|
|
SELECT mta_accept.mta_accept_id FROM mta_accept\
|
|
JOIN mta_connection ON mta_connection.mta_conn_id = mta_accept.mta_conn_id\
|
|
WHERE connect_time < ?)',
|
|
|
|
'DELETE FROM mta_accept WHERE mta_accept_id IN (\
|
|
SELECT mta_accept.mta_accept_id FROM mta_accept\
|
|
JOIN mta_connection ON mta_connection.mta_conn_id = mta_accept.mta_conn_id\
|
|
WHERE connect_time < ?)',
|
|
|
|
'DELETE FROM mta_connection WHERE connect_time < ?',
|
|
|
|
'DELETE FROM imap_connection WHERE connect_time < ?',
|
|
]
|
|
|
|
counts = []
|
|
for delete in deletes:
|
|
c.execute(delete, (dstr,))
|
|
counts.append(str(c.rowcount))
|
|
conn.commit()
|
|
counts.reverse()
|
|
log.info("pruned %s rows", "/".join(counts))
|
|
|
|
except sqlite3.Error as e:
|
|
conn.rollback()
|
|
raise e
|
|
|
|
finally:
|
|
if c: c.close()
|
|
|
|
|
|
|