external
/
mailinabox
mirror of https://github.com/mail-in-a-box/mailinabox.git
1
0
Fork 0
Code Issues Releases Wiki Activity

First big pass on PEP8'ing all the things. 

All PEP8 errors (except line length) have been fixed except one.  That
one will require a little bit of refactoring.
This commit is contained in:
Jack Twilley 2015-02-22 13:20:02 -08:00
parent 7ec662c83f
commit 86a31cd978
17 changed files with 3672 additions and 3337 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

203
management/auth.py
View File

@ -1,129 +1,134 @@
import base64, os, os.path, hmac
import base64
import os
import os.path
import hmac
from flask import make_response
import utils
from mailconfig import get_mail_password, get_mail_user_privileges
DEFAULT_KEY_PATH = '/var/lib/mailinabox/api.key'
DEFAULT_KEY_PATH = '/var/lib/mailinabox/api.key'
DEFAULT_AUTH_REALM = 'Mail-in-a-Box Management Server'
class KeyAuthService:
"""Generate an API key for authenticating clients
"""Generate an API key for authenticating clients
Clients must read the key from the key file and send the key with all HTTP
requests. The key is passed as the username field in the standard HTTP
Basic Auth header.
"""
def __init__(self):
self.auth_realm = DEFAULT_AUTH_REALM
self.key = self._generate_key()
self.key_path = DEFAULT_KEY_PATH
Clients must read the key from the key file and send the key with all HTTP
requests. The key is passed as the username field in the standard HTTP
Basic Auth header.
"""
def __init__(self):
self.auth_realm = DEFAULT_AUTH_REALM
self.key = self._generate_key()
self.key_path = DEFAULT_KEY_PATH
def write_key(self):
"""Write key to file so authorized clients can get the key
def write_key(self):
"""Write key to file so authorized clients can get the key
The key file is created with mode 0640 so that additional users can be
authorized to access the API by granting group/ACL read permissions on
the key file.
"""
def create_file_with_mode(path, mode):
# Based on answer by A-B-B: http://stackoverflow.com/a/15015748
old_umask = os.umask(0)
try:
return os.fdopen(os.open(path, os.O_WRONLY | os.O_CREAT, mode), 'w')
finally:
os.umask(old_umask)
The key file is created with mode 0640 so that additional users can be
authorized to access the API by granting group/ACL read permissions on
the key file.
"""
def create_file_with_mode(path, mode):
# Based on answer by A-B-B: http://stackoverflow.com/a/15015748
old_umask = os.umask(0)
try:
return os.fdopen(os.open(path, os.O_WRONLY | os.O_CREAT, mode), 'w')
finally:
os.umask(old_umask)
os.makedirs(os.path.dirname(self.key_path), exist_ok=True)
os.makedirs(os.path.dirname(self.key_path), exist_ok=True)
with create_file_with_mode(self.key_path, 0o640) as key_file:
key_file.write(self.key + '\n')
with create_file_with_mode(self.key_path, 0o640) as key_file:
key_file.write(self.key + '\n')
def authenticate(self, request, env):
"""Test if the client key passed in HTTP Authorization header matches the service key
or if the or username/password passed in the header matches an administrator user.
Returns a tuple of the user's email address and list of user privileges (e.g.
('my@email', []) or ('my@email', ['admin']); raises a ValueError on login failure.
If the user used an API key, the user's email is returned as None."""
def authenticate(self, request, env):
"""Test if the client key passed in HTTP Authorization header matches the service key
or if the or username/password passed in the header matches an administrator user.
Returns a tuple of the user's email address and list of user privileges (e.g.
('my@email', []) or ('my@email', ['admin']); raises a ValueError on login failure.
If the user used an API key, the user's email is returned as None."""
def decode(s):
return base64.b64decode(s.encode('ascii')).decode('ascii')
def decode(s):
return base64.b64decode(s.encode('ascii')).decode('ascii')
def parse_basic_auth(header):
if " " not in header:
return None, None
scheme, credentials = header.split(maxsplit=1)
if scheme != 'Basic':
return None, None
def parse_basic_auth(header):
if " " not in header:
return None, None
scheme, credentials = header.split(maxsplit=1)
if scheme != 'Basic':
return None, None
credentials = decode(credentials)
if ":" not in credentials:
return None, None
username, password = credentials.split(':', maxsplit=1)
return username, password
credentials = decode(credentials)
if ":" not in credentials:
return None, None
username, password = credentials.split(':', maxsplit=1)
return username, password
header = request.headers.get('Authorization')
if not header:
raise ValueError("No authorization header provided.")
header = request.headers.get('Authorization')
if not header:
raise ValueError("No authorization header provided.")
username, password = parse_basic_auth(header)
username, password = parse_basic_auth(header)
if username in (None, ""):
raise ValueError("Authorization header invalid.")
elif username == self.key:
# The user passed the API key which grants administrative privs.
return (None, ["admin"])
else:
# The user is trying to log in with a username and user-specific
# API key or password. Raises or returns privs.
return (username, self.get_user_credentials(username, password, env))
if username in (None, ""):
raise ValueError("Authorization header invalid.")
elif username == self.key:
# The user passed the API key which grants administrative privs.
return (None, ["admin"])
else:
# The user is trying to log in with a username and user-specific
# API key or password. Raises or returns privs.
return (username, self.get_user_credentials(username, password, env))
def get_user_credentials(self, email, pw, env):
# Validate a user's credentials. On success returns a list of
# privileges (e.g. [] or ['admin']). On failure raises a ValueError
# with a login error message.
def get_user_credentials(self, email, pw, env):
# Validate a user's credentials. On success returns a list of
# privileges (e.g. [] or ['admin']). On failure raises a ValueError
# with a login error message.
# Sanity check.
if email == "" or pw == "":
raise ValueError("Enter an email address and password.")
# Sanity check.
if email == "" or pw == "":
raise ValueError("Enter an email address and password.")
# The password might be a user-specific API key.
if hmac.compare_digest(self.create_user_key(email), pw):
# OK.
pass
else:
# Get the hashed password of the user. Raise a ValueError if the
# email address does not correspond to a user.
pw_hash = get_mail_password(email, env)
# The password might be a user-specific API key.
if hmac.compare_digest(self.create_user_key(email), pw):
# OK.
pass
else:
# Get the hashed password of the user. Raise a ValueError if the
# email address does not correspond to a user.
pw_hash = get_mail_password(email, env)
# Authenticate.
try:
# Use 'doveadm pw' to check credentials. doveadm will return
# a non-zero exit status if the credentials are no good,
# and check_call will raise an exception in that case.
utils.shell('check_call', [
"/usr/bin/doveadm", "pw",
"-p", pw,
"-t", pw_hash,
])
except:
# Login failed.
raise ValueError("Invalid password.")
# Authenticate.
try:
# Use 'doveadm pw' to check credentials. doveadm will return
# a non-zero exit status if the credentials are no good,
# and check_call will raise an exception in that case.
utils.shell('check_call', [
"/usr/bin/doveadm", "pw",
"-p", pw,
"-t", pw_hash,
])
except:
# Login failed.
raise ValueError("Invalid password.")
# Get privileges for authorization.
# Get privileges for authorization.
# (This call should never fail on a valid user. But if it did fail, it would
# return a tuple of an error message and an HTTP status code.)
privs = get_mail_user_privileges(email, env)
if isinstance(privs, tuple): raise Exception("Error getting privileges.")
# (This call should never fail on a valid user. But if it did fail, it would
# return a tuple of an error message and an HTTP status code.)
privs = get_mail_user_privileges(email, env)
if isinstance(privs, tuple):
raise Exception("Error getting privileges.")
# Return a list of privileges.
return privs
# Return a list of privileges.
return privs
def create_user_key(self, email):
return hmac.new(self.key.encode('ascii'), b"AUTH:" + email.encode("utf8"), digestmod="sha1").hexdigest()
def create_user_key(self, email):
return hmac.new(self.key.encode('ascii'), b"AUTH:" + email.encode("utf8"), digestmod="sha1").hexdigest()
def _generate_key(self):
raw_key = os.urandom(32)
return base64.b64encode(raw_key).decode('ascii')
def _generate_key(self):
raw_key = os.urandom(32)
return base64.b64encode(raw_key).decode('ascii')

355
management/backup.py
View File

@ -9,8 +9,15 @@
# backup/secret_key.txt) to STORAGE_ROOT/backup/encrypted.
# 5) STORAGE_ROOT/backup/after-backup is executd if it exists.
import os, os.path, shutil, glob, re, datetime
import dateutil.parser, dateutil.relativedelta, dateutil.tz
import os
import os.path
import shutil
import glob
import re
import datetime
import dateutil.parser
import dateutil.relativedelta
import dateutil.tz
from utils import exclusive_process, load_environment, shell
@ -18,195 +25,209 @@ from utils import exclusive_process, load_environment, shell
# that depends on it is this many days old.
keep_backups_for_days = 3
def backup_status(env):
# What is the current status of backups?
# Loop through all of the files in STORAGE_ROOT/backup/duplicity to
# get a list of all of the backups taken and sum up file sizes to
# see how large the storage is.
# What is the current status of backups?
# Loop through all of the files in STORAGE_ROOT/backup/duplicity to
# get a list of all of the backups taken and sum up file sizes to
# see how large the storage is.
now = datetime.datetime.now(dateutil.tz.tzlocal())
def reldate(date, ref, clip):
if ref < date: return clip
rd = dateutil.relativedelta.relativedelta(ref, date)
if rd.months > 1: return "%d months, %d days" % (rd.months, rd.days)
if rd.months == 1: return "%d month, %d days" % (rd.months, rd.days)
if rd.days >= 7: return "%d days" % rd.days
if rd.days > 1: return "%d days, %d hours" % (rd.days, rd.hours)
if rd.days == 1: return "%d day, %d hours" % (rd.days, rd.hours)
return "%d hours, %d minutes" % (rd.hours, rd.minutes)
now = datetime.datetime.now(dateutil.tz.tzlocal())
backups = { }
basedir = os.path.join(env['STORAGE_ROOT'], 'backup/duplicity/')
encdir = os.path.join(env['STORAGE_ROOT'], 'backup/encrypted/')
os.makedirs(basedir, exist_ok=True) # os.listdir fails if directory does not exist
for fn in os.listdir(basedir):
m = re.match(r"duplicity-(full|full-signatures|(inc|new-signatures)\.(?P<incbase>\d+T\d+Z)\.to)\.(?P<date>\d+T\d+Z)\.", fn)
if not m: raise ValueError(fn)
def reldate(date, ref, clip):
if ref < date:
return clip
rd = dateutil.relativedelta.relativedelta(ref, date)
if rd.months > 1:
return "%d months, %d days" % (rd.months, rd.days)
if rd.months == 1:
return "%d month, %d days" % (rd.months, rd.days)
if rd.days >= 7:
return "%d days" % rd.days
if rd.days > 1:
return "%d days, %d hours" % (rd.days, rd.hours)
if rd.days == 1:
return "%d day, %d hours" % (rd.days, rd.hours)
return "%d hours, %d minutes" % (rd.hours, rd.minutes)
key = m.group("date")
if key not in backups:
date = dateutil.parser.parse(m.group("date"))
backups[key] = {
"date": m.group("date"),
"date_str": date.strftime("%x %X"),
"date_delta": reldate(date, now, "the future?"),
"full": m.group("incbase") is None,
"previous": m.group("incbase"),
"size": 0,
"encsize": 0,
}
backups = {}
basedir = os.path.join(env['STORAGE_ROOT'], 'backup/duplicity/')
encdir = os.path.join(env['STORAGE_ROOT'], 'backup/encrypted/')
# os.listdir fails if directory does not exist
os.makedirs(basedir, exist_ok=True)
for fn in os.listdir(basedir):
m = re.match(r"duplicity-(full|full-signatures|(inc|new-signatures)\.(?P<incbase>\d+T\d+Z)\.to)\.(?P<date>\d+T\d+Z)\.", fn)
if not m:
raise ValueError(fn)
backups[key]["size"] += os.path.getsize(os.path.join(basedir, fn))
key = m.group("date")
if key not in backups:
date = dateutil.parser.parse(m.group("date"))
backups[key] = {
"date": m.group("date"),
"date_str": date.strftime("%x %X"),
"date_delta": reldate(date, now, "the future?"),
"full": m.group("incbase") is None,
"previous": m.group("incbase"),
"size": 0,
"encsize": 0,
}
# Also check encrypted size.
encfn = os.path.join(encdir, fn + ".enc")
if os.path.exists(encfn):
backups[key]["encsize"] += os.path.getsize(encfn)
backups[key]["size"] += os.path.getsize(os.path.join(basedir, fn))
# Ensure the rows are sorted reverse chronologically.
# This is relied on by should_force_full() and the next step.
backups = sorted(backups.values(), key = lambda b : b["date"], reverse=True)
# Also check encrypted size.
encfn = os.path.join(encdir, fn + ".enc")
if os.path.exists(encfn):
backups[key]["encsize"] += os.path.getsize(encfn)
# When will a backup be deleted?
saw_full = False
deleted_in = None
days_ago = now - datetime.timedelta(days=keep_backups_for_days)
for bak in backups:
if deleted_in:
# Subsequent backups are deleted when the most recent increment
# in the chain would be deleted.
bak["deleted_in"] = deleted_in
if bak["full"]:
# Reset when we get to a full backup. A new chain start next.
saw_full = True
deleted_in = None
elif saw_full and not deleted_in:
# Mark deleted_in only on the first increment after a full backup.
deleted_in = reldate(days_ago, dateutil.parser.parse(bak["date"]), "on next daily backup")
bak["deleted_in"] = deleted_in
# Ensure the rows are sorted reverse chronologically.
# This is relied on by should_force_full() and the next step.
backups = sorted(backups.values(), key=lambda b: b["date"], reverse=True)
# When will a backup be deleted?
saw_full = False
deleted_in = None
days_ago = now - datetime.timedelta(days=keep_backups_for_days)
for bak in backups:
if deleted_in:
# Subsequent backups are deleted when the most recent increment
# in the chain would be deleted.
bak["deleted_in"] = deleted_in
if bak["full"]:
# Reset when we get to a full backup. A new chain start next.
saw_full = True
deleted_in = None
elif saw_full and not deleted_in:
# Mark deleted_in only on the first increment after a full backup.
deleted_in = reldate(days_ago, dateutil.parser.parse(bak["date"]), "on next daily backup")
bak["deleted_in"] = deleted_in
return {
"directory": basedir,
"encpwfile": os.path.join(env['STORAGE_ROOT'], 'backup/secret_key.txt'),
"encdirectory": encdir,
"tz": now.tzname(),
"backups": backups,
}
return {
"directory": basedir,
"encpwfile": os.path.join(env['STORAGE_ROOT'], 'backup/secret_key.txt'),
"encdirectory": encdir,
"tz": now.tzname(),
"backups": backups,
}
def should_force_full(env):
# Force a full backup when the total size of the increments
# since the last full backup is greater than half the size
# of that full backup.
inc_size = 0
for bak in backup_status(env)["backups"]:
if not bak["full"]:
# Scan through the incremental backups cumulating
# size...
inc_size += bak["size"]
else:
# ...until we reach the most recent full backup.
# Return if we should to a full backup.
return inc_size > .5*bak["size"]
else:
# If we got here there are no (full) backups, so make one.
# (I love for/else blocks. Here it's just to show off.)
return True
# Force a full backup when the total size of the increments
# since the last full backup is greater than half the size
# of that full backup.
inc_size = 0
for bak in backup_status(env)["backups"]:
if not bak["full"]:
# Scan through the incremental backups cumulating
# size...
inc_size += bak["size"]
else:
# ...until we reach the most recent full backup.
# Return if we should to a full backup.
return inc_size > .5*bak["size"]
else:
# If we got here there are no (full) backups, so make one.
# (I love for/else blocks. Here it's just to show off.)
return True
def perform_backup(full_backup):
env = load_environment()
env = load_environment()
exclusive_process("backup")
exclusive_process("backup")
# Ensure the backup directory exists.
backup_dir = os.path.join(env["STORAGE_ROOT"], 'backup')
backup_duplicity_dir = os.path.join(backup_dir, 'duplicity')
os.makedirs(backup_duplicity_dir, exist_ok=True)
# Ensure the backup directory exists.
backup_dir = os.path.join(env["STORAGE_ROOT"], 'backup')
backup_duplicity_dir = os.path.join(backup_dir, 'duplicity')
os.makedirs(backup_duplicity_dir, exist_ok=True)
# On the first run, always do a full backup. Incremental
# will fail. Otherwise do a full backup when the size of
# the increments since the most recent full backup are
# large.
full_backup = full_backup or should_force_full(env)
# On the first run, always do a full backup. Incremental
# will fail. Otherwise do a full backup when the size of
# the increments since the most recent full backup are
# large.
full_backup = full_backup or should_force_full(env)
# Stop services.
shell('check_call', ["/usr/sbin/service", "dovecot", "stop"])
shell('check_call', ["/usr/sbin/service", "postfix", "stop"])
# Stop services.
shell('check_call', ["/usr/sbin/service", "dovecot", "stop"])
shell('check_call', ["/usr/sbin/service", "postfix", "stop"])
# Update the backup mirror directory which mirrors the current
# STORAGE_ROOT (but excluding the backups themselves!).
try:
shell('check_call', [
"/usr/bin/duplicity",
"full" if full_backup else "incr",
"--no-encryption",
"--archive-dir", "/tmp/duplicity-archive-dir",
"--name", "mailinabox",
"--exclude", backup_dir,
"--volsize", "100",
"--verbosity", "warning",
env["STORAGE_ROOT"],
"file://" + backup_duplicity_dir
])
finally:
# Start services again.
shell('check_call', ["/usr/sbin/service", "dovecot", "start"])
shell('check_call', ["/usr/sbin/service", "postfix", "start"])
# Update the backup mirror directory which mirrors the current
# STORAGE_ROOT (but excluding the backups themselves!).
try:
shell('check_call', [
"/usr/bin/duplicity",
"full" if full_backup else "incr",
"--no-encryption",
"--archive-dir", "/tmp/duplicity-archive-dir",
"--name", "mailinabox",
"--exclude", backup_dir,
"--volsize", "100",
"--verbosity", "warning",
env["STORAGE_ROOT"],
"file://" + backup_duplicity_dir
])
finally:
# Start services again.
shell('check_call', ["/usr/sbin/service", "dovecot", "start"])
shell('check_call', ["/usr/sbin/service", "postfix", "start"])
# Remove old backups. This deletes all backup data no longer needed
# from more than 31 days ago. Must do this before destroying the
# cache directory or else this command will re-create it.
shell('check_call', [
"/usr/bin/duplicity",
"remove-older-than",
"%dD" % keep_backups_for_days,
"--archive-dir", "/tmp/duplicity-archive-dir",
"--name", "mailinabox",
"--force",
"--verbosity", "warning",
"file://" + backup_duplicity_dir
])
# Remove old backups. This deletes all backup data no longer needed
# from more than 31 days ago. Must do this before destroying the
# cache directory or else this command will re-create it.
shell('check_call', [
"/usr/bin/duplicity",
"remove-older-than",
"%dD" % keep_backups_for_days,
"--archive-dir", "/tmp/duplicity-archive-dir",
"--name", "mailinabox",
"--force",
"--verbosity", "warning",
"file://" + backup_duplicity_dir
])
# Remove duplicity's cache directory because it's redundant with our backup directory.
shutil.rmtree("/tmp/duplicity-archive-dir")
# Remove duplicity's cache directory because it's redundant with our backup directory.
shutil.rmtree("/tmp/duplicity-archive-dir")
# Encrypt all of the new files.
backup_encrypted_dir = os.path.join(backup_dir, 'encrypted')
os.makedirs(backup_encrypted_dir, exist_ok=True)
for fn in os.listdir(backup_duplicity_dir):
fn2 = os.path.join(backup_encrypted_dir, fn) + ".enc"
if os.path.exists(fn2): continue
# Encrypt all of the new files.
backup_encrypted_dir = os.path.join(backup_dir, 'encrypted')
os.makedirs(backup_encrypted_dir, exist_ok=True)
for fn in os.listdir(backup_duplicity_dir):
fn2 = os.path.join(backup_encrypted_dir, fn) + ".enc"
if os.path.exists(fn2):
continue
# Encrypt the backup using the backup private key.
shell('check_call', [
"/usr/bin/openssl",
"enc",
"-aes-256-cbc",
"-a",
"-salt",
"-in", os.path.join(backup_duplicity_dir, fn),
"-out", fn2,
"-pass", "file:%s" % os.path.join(backup_dir, "secret_key.txt"),
])
# Encrypt the backup using the backup private key.
shell('check_call', [
"/usr/bin/openssl",
"enc",
"-aes-256-cbc",
"-a",
"-salt",
"-in", os.path.join(backup_duplicity_dir, fn),
"-out", fn2,
"-pass", "file:%s" % os.path.join(backup_dir, "secret_key.txt"),
])
# The backup can be decrypted with:
# openssl enc -d -aes-256-cbc -a -in latest.tgz.enc -out /dev/stdout -pass file:secret_key.txt | tar -z
# The backup can be decrypted with:
# openssl enc -d -aes-256-cbc -a -in latest.tgz.enc -out /dev/stdout -pass file:secret_key.txt | tar -z
# Remove encrypted backups that are no longer needed.
for fn in os.listdir(backup_encrypted_dir):
fn2 = os.path.join(backup_duplicity_dir, fn.replace(".enc", ""))
if os.path.exists(fn2): continue
os.unlink(os.path.join(backup_encrypted_dir, fn))
# Remove encrypted backups that are no longer needed.
for fn in os.listdir(backup_encrypted_dir):
fn2 = os.path.join(backup_duplicity_dir, fn.replace(".enc", ""))
if os.path.exists(fn2):
continue
os.unlink(os.path.join(backup_encrypted_dir, fn))
# Execute a post-backup script that does the copying to a remote server.
# Run as the STORAGE_USER user, not as root. Pass our settings in
# environment variables so the script has access to STORAGE_ROOT.
post_script = os.path.join(backup_dir, 'after-backup')
if os.path.exists(post_script):
shell('check_call',
['su', env['STORAGE_USER'], '-c', post_script],
env=env)
# Execute a post-backup script that does the copying to a remote server.
# Run as the STORAGE_USER user, not as root. Pass our settings in
# environment variables so the script has access to STORAGE_ROOT.
post_script = os.path.join(backup_dir, 'after-backup')
if os.path.exists(post_script):
shell('check_call',
['su', env['STORAGE_USER'], '-c', post_script],
env=env)
if __name__ == "__main__":
import sys
full_backup = "--full" in sys.argv
perform_backup(full_backup)
import sys
full_backup = "--full" in sys.argv
perform_backup(full_backup)

429
management/daemon.py
View File

@ -1,12 +1,16 @@
#!/usr/bin/python3
import os, os.path, re, json
import os
import os.path
import re
import json
from functools import wraps
from flask import Flask, request, render_template, abort, Response
import auth, utils
import auth
import utils
from mailconfig import get_mail_users, get_mail_users_ex, get_admins, add_mail_user, set_mail_password, remove_mail_user
from mailconfig import get_mail_user_privileges, add_remove_mail_user_privilege
from mailconfig import get_mail_aliases, get_mail_aliases_ex, get_mail_domains, add_mail_alias, remove_mail_alias
@ -24,175 +28,192 @@ auth_service = auth.KeyAuthService()
# We may deploy via a symbolic link, which confuses flask's template finding.
me = __file__
try:
me = os.readlink(__file__)
me = os.readlink(__file__)
except OSError:
pass
pass
app = Flask(__name__, template_folder=os.path.abspath(os.path.join(os.path.dirname(me), "templates")))
# Decorator to protect views that require a user with 'admin' privileges.
def authorized_personnel_only(viewfunc):
@wraps(viewfunc)
def newview(*args, **kwargs):
# Authenticate the passed credentials, which is either the API key or a username:password pair.
error = None
try:
email, privs = auth_service.authenticate(request, env)
except ValueError as e:
# Authentication failed.
privs = []
error = str(e)
@wraps(viewfunc)
def newview(*args, **kwargs):
# Authenticate the passed credentials, which is either the API key or a username:password pair.
error = None
try:
email, privs = auth_service.authenticate(request, env)
except ValueError as e:
# Authentication failed.
privs = []
error = str(e)
# Authorized to access an API view?
if "admin" in privs:
# Call view func.
return viewfunc(*args, **kwargs)
elif not error:
error = "You are not an administrator."
# Authorized to access an API view?
if "admin" in privs:
# Call view func.
return viewfunc(*args, **kwargs)
elif not error:
error = "You are not an administrator."
# Not authorized. Return a 401 (send auth) and a prompt to authorize by default.
status = 401
headers = {
'WWW-Authenticate': 'Basic realm="{0}"'.format(auth_service.auth_realm),
'X-Reason': error,
}
# Not authorized. Return a 401 (send auth) and a prompt to authorize by default.
status = 401
headers = {
'WWW-Authenticate': 'Basic realm="{0}"'.format(auth_service.auth_realm),
'X-Reason': error,
}
if request.headers.get('X-Requested-With') == 'XMLHttpRequest':
# Don't issue a 401 to an AJAX request because the user will
# be prompted for credentials, which is not helpful.
status = 403
headers = None
if request.headers.get('X-Requested-With') == 'XMLHttpRequest':
# Don't issue a 401 to an AJAX request because the user will
# be prompted for credentials, which is not helpful.
status = 403
headers = None
if request.headers.get('Accept') in (None, "", "*/*"):
# Return plain text output.
return Response(error+"\n", status=status, mimetype='text/plain', headers=headers)
else:
# Return JSON output.
return Response(json.dumps({
"status": "error",
"reason": error,
})+"\n", status=status, mimetype='application/json', headers=headers)
if request.headers.get('Accept') in (None, "", "*/*"):
# Return plain text output.
return Response(error+"\n", status=status, mimetype='text/plain', headers=headers)
else:
# Return JSON output.
return Response(json.dumps({
"status": "error",
"reason": error,
})+"\n", status=status, mimetype='application/json', headers=headers)
return newview
return newview
@app.errorhandler(401)
def unauthorized(error):
return auth_service.make_unauthorized_response()
return auth_service.make_unauthorized_response()
def json_response(data):
return Response(json.dumps(data), status=200, mimetype='application/json')
return Response(json.dumps(data), status=200, mimetype='application/json')
###################################
# Control Panel (unauthenticated views)
@app.route('/')
def index():
# Render the control panel. This route does not require user authentication
# so it must be safe!
no_admins_exist = (len(get_admins(env)) == 0)
return render_template('index.html',
hostname=env['PRIMARY_HOSTNAME'],
storage_root=env['STORAGE_ROOT'],
no_admins_exist=no_admins_exist,
)
# Render the control panel. This route does not require user authentication
# so it must be safe!
no_admins_exist = (len(get_admins(env)) == 0)
return render_template(
'index.html',
hostname=env['PRIMARY_HOSTNAME'],
storage_root=env['STORAGE_ROOT'],
no_admins_exist=no_admins_exist,
)
@app.route('/me')
def me():
# Is the caller authorized?
try:
email, privs = auth_service.authenticate(request, env)
except ValueError as e:
return json_response({
"status": "invalid",
"reason": str(e),
})
# Is the caller authorized?
try:
email, privs = auth_service.authenticate(request, env)
except ValueError as e:
return json_response({
"status": "invalid",
"reason": str(e),
})
resp = {
"status": "ok",
"email": email,
"privileges": privs,
}
resp = {
"status": "ok",
"email": email,
"privileges": privs,
}
# Is authorized as admin? Return an API key for future use.
if "admin" in privs:
resp["api_key"] = auth_service.create_user_key(email)
# Is authorized as admin? Return an API key for future use.
if "admin" in privs:
resp["api_key"] = auth_service.create_user_key(email)
# Return.
return json_response(resp)
# Return.
return json_response(resp)
# MAIL
@app.route('/mail/users')
@authorized_personnel_only
def mail_users():
if request.args.get("format", "") == "json":
return json_response(get_mail_users_ex(env, with_archived=True, with_slow_info=True))
else:
return "".join(x+"\n" for x in get_mail_users(env))
if request.args.get("format", "") == "json":
return json_response(get_mail_users_ex(env, with_archived=True, with_slow_info=True))
else:
return "".join(x+"\n" for x in get_mail_users(env))
@app.route('/mail/users/add', methods=['POST'])
@authorized_personnel_only
def mail_users_add():
try:
return add_mail_user(request.form.get('email', ''), request.form.get('password', ''), request.form.get('privileges', ''), env)
except ValueError as e:
return (str(e), 400)
try:
return add_mail_user(request.form.get('email', ''), request.form.get('password', ''), request.form.get('privileges', ''), env)
except ValueError as e:
return (str(e), 400)
@app.route('/mail/users/password', methods=['POST'])
@authorized_personnel_only
def mail_users_password():
try:
return set_mail_password(request.form.get('email', ''), request.form.get('password', ''), env)
except ValueError as e:
return (str(e), 400)
try:
return set_mail_password(request.form.get('email', ''), request.form.get('password', ''), env)
except ValueError as e:
return (str(e), 400)
@app.route('/mail/users/remove', methods=['POST'])
@authorized_personnel_only
def mail_users_remove():
return remove_mail_user(request.form.get('email', ''), env)
return remove_mail_user(request.form.get('email', ''), env)
@app.route('/mail/users/privileges')
@authorized_personnel_only
def mail_user_privs():
privs = get_mail_user_privileges(request.args.get('email', ''), env)
if isinstance(privs, tuple): return privs # error
return "\n".join(privs)
privs = get_mail_user_privileges(request.args.get('email', ''), env)
# error
if isinstance(privs, tuple):
return privs
return "\n".join(privs)
@app.route('/mail/users/privileges/add', methods=['POST'])
@authorized_personnel_only
def mail_user_privs_add():
return add_remove_mail_user_privilege(request.form.get('email', ''), request.form.get('privilege', ''), "add", env)
return add_remove_mail_user_privilege(request.form.get('email', ''), request.form.get('privilege', ''), "add", env)
@app.route('/mail/users/privileges/remove', methods=['POST'])
@authorized_personnel_only
def mail_user_privs_remove():
return add_remove_mail_user_privilege(request.form.get('email', ''), request.form.get('privilege', ''), "remove", env)
return add_remove_mail_user_privilege(request.form.get('email', ''), request.form.get('privilege', ''), "remove", env)
@app.route('/mail/aliases')
@authorized_personnel_only
def mail_aliases():
if request.args.get("format", "") == "json":
return json_response(get_mail_aliases_ex(env))
else:
return "".join(x+"\t"+y+"\n" for x, y in get_mail_aliases(env))
if request.args.get("format", "") == "json":
return json_response(get_mail_aliases_ex(env))
else:
return "".join(x+"\t"+y+"\n" for x, y in get_mail_aliases(env))
@app.route('/mail/aliases/add', methods=['POST'])
@authorized_personnel_only
def mail_aliases_add():
return add_mail_alias(
request.form.get('source', ''),
request.form.get('destination', ''),
env,
update_if_exists=(request.form.get('update_if_exists', '') == '1')
)
return add_mail_alias(
request.form.get('source', ''),
request.form.get('destination', ''),
env,
update_if_exists=(request.form.get('update_if_exists', '') == '1')
)
@app.route('/mail/aliases/remove', methods=['POST'])
@authorized_personnel_only
def mail_aliases_remove():
return remove_mail_alias(request.form.get('source', ''), env)
return remove_mail_alias(request.form.get('source', ''), env)
@app.route('/mail/domains')
@authorized_personnel_only
@ -201,172 +222,196 @@ def mail_domains():
# DNS
@app.route('/dns/zones')
@authorized_personnel_only
def dns_zones():
from dns_update import get_dns_zones
return json_response([z[0] for z in get_dns_zones(env)])
from dns_update import get_dns_zones
return json_response([z[0] for z in get_dns_zones(env)])
@app.route('/dns/update', methods=['POST'])
@authorized_personnel_only
def dns_update():
from dns_update import do_dns_update
try:
return do_dns_update(env, force=request.form.get('force', '') == '1')
except Exception as e:
return (str(e), 500)
from dns_update import do_dns_update
try:
return do_dns_update(env, force=request.form.get('force', '') == '1')
except Exception as e:
return (str(e), 500)
@app.route('/dns/secondary-nameserver')
@authorized_personnel_only
def dns_get_secondary_nameserver():
from dns_update import get_custom_dns_config
return json_response({ "hostname": get_custom_dns_config(env).get("_secondary_nameserver") })
from dns_update import get_custom_dns_config
return json_response({"hostname": get_custom_dns_config(env).get("_secondary_nameserver")})
@app.route('/dns/secondary-nameserver', methods=['POST'])
@authorized_personnel_only
def dns_set_secondary_nameserver():
from dns_update import set_secondary_dns
try:
return set_secondary_dns(request.form.get('hostname'), env)
except ValueError as e:
return (str(e), 400)
from dns_update import set_secondary_dns
try:
return set_secondary_dns(request.form.get('hostname'), env)
except ValueError as e:
return (str(e), 400)
@app.route('/dns/set')
@authorized_personnel_only
def dns_get_records():
from dns_update import get_custom_dns_config, get_custom_records
additional_records = get_custom_dns_config(env)
records = get_custom_records(None, additional_records, env)
return json_response([{
"qname": r[0],
"rtype": r[1],
"value": r[2],
} for r in records])
from dns_update import get_custom_dns_config, get_custom_records
additional_records = get_custom_dns_config(env)
records = get_custom_records(None, additional_records, env)
return json_response([{
"qname": r[0],
"rtype": r[1],
"value": r[2],
} for r in records])
@app.route('/dns/set/<qname>', methods=['POST'])
@app.route('/dns/set/<qname>/<rtype>', methods=['POST'])
@app.route('/dns/set/<qname>/<rtype>/<value>', methods=['POST'])
@authorized_personnel_only
def dns_set_record(qname, rtype="A", value=None):
from dns_update import do_dns_update, set_custom_dns_record
try:
# Get the value from the URL, then the POST parameters, or if it is not set then
# use the remote IP address of the request --- makes dynamic DNS easy. To clear a
# value, '' must be explicitly passed.
if value is None:
value = request.form.get("value")
if value is None:
value = request.environ.get("HTTP_X_FORWARDED_FOR") # normally REMOTE_ADDR but we're behind nginx as a reverse proxy
if value == '' or value == '__delete__':
# request deletion
value = None
if set_custom_dns_record(qname, rtype, value, env):
return do_dns_update(env)
return "OK"
except ValueError as e:
return (str(e), 400)
from dns_update import do_dns_update, set_custom_dns_record
try:
# Get the value from the URL, then the POST parameters, or if it is not set then
# use the remote IP address of the request --- makes dynamic DNS easy. To clear a
# value, '' must be explicitly passed.
if value is None:
value = request.form.get("value")
if value is None:
# normally REMOTE_ADDR but we're behind nginx as a reverse proxy
value = request.environ.get("HTTP_X_FORWARDED_FOR")
if value == '' or value == '__delete__':
# request deletion
value = None
if set_custom_dns_record(qname, rtype, value, env):
return do_dns_update(env)
return "OK"
except ValueError as e:
return (str(e), 400)
@app.route('/dns/dump')
@authorized_personnel_only
def dns_get_dump():
from dns_update import build_recommended_dns
return json_response(build_recommended_dns(env))
from dns_update import build_recommended_dns
return json_response(build_recommended_dns(env))
# SSL
@app.route('/ssl/csr/<domain>', methods=['POST'])
@authorized_personnel_only
def ssl_get_csr(domain):
from web_update import get_domain_ssl_files, create_csr
ssl_key, ssl_certificate, ssl_via = get_domain_ssl_files(domain, env)
return create_csr(domain, ssl_key, env)
from web_update import get_domain_ssl_files, create_csr
ssl_key, ssl_certificate, ssl_via = get_domain_ssl_files(domain, env)
return create_csr(domain, ssl_key, env)
@app.route('/ssl/install', methods=['POST'])
@authorized_personnel_only
def ssl_install_cert():
from web_update import install_cert
domain = request.form.get('domain')
ssl_cert = request.form.get('cert')
ssl_chain = request.form.get('chain')
return install_cert(domain, ssl_cert, ssl_chain, env)
from web_update import install_cert
domain = request.form.get('domain')
ssl_cert = request.form.get('cert')
ssl_chain = request.form.get('chain')
return install_cert(domain, ssl_cert, ssl_chain, env)
# WEB
@app.route('/web/domains')
@authorized_personnel_only
def web_get_domains():
from web_update import get_web_domains_info
return json_response(get_web_domains_info(env))
from web_update import get_web_domains_info
return json_response(get_web_domains_info(env))
@app.route('/web/update', methods=['POST'])
@authorized_personnel_only
def web_update():
from web_update import do_web_update
return do_web_update(env)
from web_update import do_web_update
return do_web_update(env)
# System
@app.route('/system/status', methods=["POST"])
@authorized_personnel_only
def system_status():
from status_checks import run_checks
class WebOutput:
def __init__(self):
self.items = []
def add_heading(self, heading):
self.items.append({ "type": "heading", "text": heading, "extra": [] })
def print_ok(self, message):
self.items.append({ "type": "ok", "text": message, "extra": [] })
def print_error(self, message):
self.items.append({ "type": "error", "text": message, "extra": [] })
def print_warning(self, message):
self.items.append({ "type": "warning", "text": message, "extra": [] })
def print_line(self, message, monospace=False):
self.items[-1]["extra"].append({ "text": message, "monospace": monospace })
output = WebOutput()
run_checks(env, output, pool)
return json_response(output.items)
from status_checks import run_checks
class WebOutput:
def __init__(self):
self.items = []
def add_heading(self, heading):
self.items.append({"type": "heading", "text": heading, "extra": []})
def print_ok(self, message):
self.items.append({"type": "ok", "text": message, "extra": []})
def print_error(self, message):
self.items.append({"type": "error", "text": message, "extra": []})
def print_warning(self, message):
self.items.append({"type": "warning", "text": message, "extra": []})
def print_line(self, message, monospace=False):
self.items[-1]["extra"].append({"text": message, "monospace": monospace})
output = WebOutput()
run_checks(env, output, pool)
return json_response(output.items)
@app.route('/system/updates')
@authorized_personnel_only
def show_updates():
from status_checks import list_apt_updates
return "".join(
"%s (%s)\n"
% (p["package"], p["version"])
for p in list_apt_updates())
from status_checks import list_apt_updates
return "".join(
"%s (%s)\n"
% (p["package"], p["version"])
for p in list_apt_updates())
@app.route('/system/update-packages', methods=["POST"])
@authorized_personnel_only
def do_updates():
utils.shell("check_call", ["/usr/bin/apt-get", "-qq", "update"])
return utils.shell("check_output", ["/usr/bin/apt-get", "-y", "upgrade"], env={
"DEBIAN_FRONTEND": "noninteractive"
})
utils.shell("check_call", ["/usr/bin/apt-get", "-qq", "update"])
return utils.shell("check_output", ["/usr/bin/apt-get", "-y", "upgrade"], env={
"DEBIAN_FRONTEND": "noninteractive"
})
@app.route('/system/backup/status')
@authorized_personnel_only
def backup_status():
from backup import backup_status
return json_response(backup_status(env))
from backup import backup_status
return json_response(backup_status(env))
# APP
if __name__ == '__main__':
if "DEBUG" in os.environ: app.debug = True
if "APIKEY" in os.environ: auth_service.key = os.environ["APIKEY"]
if "DEBUG" in os.environ:
app.debug = True
if "APIKEY" in os.environ:
auth_service.key = os.environ["APIKEY"]
if not app.debug:
app.logger.addHandler(utils.create_syslog_handler())
if not app.debug:
app.logger.addHandler(utils.create_syslog_handler())
# For testing on the command line, you can use `curl` like so:
# curl --user $(</var/lib/mailinabox/api.key): http://localhost:10222/mail/users
auth_service.write_key()
# For testing on the command line, you can use `curl` like so:
# curl --user $(</var/lib/mailinabox/api.key): http://localhost:10222/mail/users
auth_service.write_key()
# For testing in the browser, you can copy the API key that's output to the
# debug console and enter that as the username
app.logger.info('API key: ' + auth_service.key)
# Start the application server. Listens on 127.0.0.1 (IPv4 only).
app.run(port=10222)
# For testing in the browser, you can copy the API key that's output to the
# debug console and enter that as the username
app.logger.info('API key: ' + auth_service.key)
# Start the application server. Listens on 127.0.0.1 (IPv4 only).
app.run(port=10222)

1396
management/dns_update.py
View File

File diff suppressed because it is too large Load Diff

183
management/mail_log.py
View File

@ -1,121 +1,130 @@
#!/usr/bin/python3
import re, os.path
import re
import os.path
import dateutil.parser
import mailconfig
import utils
def scan_mail_log(logger, env):
collector = {
"other-services": set(),
"imap-logins": { },
"postgrey": { },
"rejected-mail": { },
}
collector = {
"other-services": set(),
"imap-logins": {},
"postgrey": {},
"rejected-mail": {},
}
collector["real_mail_addresses"] = set(mailconfig.get_mail_users(env)) | set(alias[0] for alias in mailconfig.get_mail_aliases(env))
collector["real_mail_addresses"] = set(mailconfig.get_mail_users(env)) | set(alias[0] for alias in mailconfig.get_mail_aliases(env))
for fn in ('/var/log/mail.log.1', '/var/log/mail.log'):
if not os.path.exists(fn): continue
with open(fn, 'rb') as log:
for line in log:
line = line.decode("utf8", errors='replace')
scan_mail_log_line(line.strip(), collector)
for fn in ('/var/log/mail.log.1', '/var/log/mail.log'):
if not os.path.exists(fn):
continue
with open(fn, 'rb') as log:
for line in log:
line = line.decode("utf8", errors='replace')
scan_mail_log_line(line.strip(), collector)
if collector["imap-logins"]:
logger.add_heading("Recent IMAP Logins")
logger.print_block("The most recent login from each remote IP adddress is show.")
for k in utils.sort_email_addresses(collector["imap-logins"], env):
for ip, date in sorted(collector["imap-logins"][k].items(), key = lambda kv : kv[1]):
logger.print_line(k + "\t" + str(date) + "\t" + ip)
if collector["imap-logins"]:
logger.add_heading("Recent IMAP Logins")
logger.print_block("The most recent login from each remote IP adddress is show.")
for k in utils.sort_email_addresses(collector["imap-logins"], env):
for ip, date in sorted(collector["imap-logins"][k].items(), key=lambda kv: kv[1]):
logger.print_line(k + "\t" + str(date) + "\t" + ip)
if collector["postgrey"]:
logger.add_heading("Greylisted Mail")
logger.print_block("The following mail was greylisted, meaning the emails were temporarily rejected. Legitimate senders will try again within ten minutes.")
logger.print_line("recipient" + "\t" + "received" + "\t" + "sender" + "\t" + "delivered")
for recipient in utils.sort_email_addresses(collector["postgrey"], env):
for (client_address, sender), (first_date, delivered_date) in sorted(collector["postgrey"][recipient].items(), key = lambda kv : kv[1][0]):
logger.print_line(recipient + "\t" + str(first_date) + "\t" + sender + "\t" + (("delivered " + str(delivered_date)) if delivered_date else "no retry yet"))
if collector["postgrey"]:
logger.add_heading("Greylisted Mail")
logger.print_block("The following mail was greylisted, meaning the emails were temporarily rejected. Legitimate senders will try again within ten minutes.")
logger.print_line("recipient" + "\t" + "received" + "\t" + "sender" + "\t" + "delivered")
for recipient in utils.sort_email_addresses(collector["postgrey"], env):
for (client_address, sender), (first_date, delivered_date) in sorted(collector["postgrey"][recipient].items(), key=lambda kv: kv[1][0]):
logger.print_line(recipient + "\t" + str(first_date) + "\t" + sender + "\t" + (("delivered " + str(delivered_date)) if delivered_date else "no retry yet"))
if collector["rejected-mail"]:
logger.add_heading("Rejected Mail")
logger.print_block("The following incoming mail was rejected.")
for k in utils.sort_email_addresses(collector["rejected-mail"], env):
for date, sender, message in collector["rejected-mail"][k]:
logger.print_line(k + "\t" + str(date) + "\t" + sender + "\t" + message)
if collector["rejected-mail"]:
logger.add_heading("Rejected Mail")
logger.print_block("The following incoming mail was rejected.")
for k in utils.sort_email_addresses(collector["rejected-mail"], env):
for date, sender, message in collector["rejected-mail"][k]:
logger.print_line(k + "\t" + str(date) + "\t" + sender + "\t" + message)
if len(collector["other-services"]) > 0:
logger.add_heading("Other")
logger.print_block("Unrecognized services in the log: " + ", ".join(collector["other-services"]))
if len(collector["other-services"]) > 0:
logger.add_heading("Other")
logger.print_block("Unrecognized services in the log: " + ", ".join(collector["other-services"]))
def scan_mail_log_line(line, collector):
m = re.match(r"(\S+ \d+ \d+:\d+:\d+) (\S+) (\S+?)(\[\d+\])?: (.*)", line)
if not m: return
m = re.match(r"(\S+ \d+ \d+:\d+:\d+) (\S+) (\S+?)(\[\d+\])?: (.*)", line)
if not m:
return
date, system, service, pid, log = m.groups()
date = dateutil.parser.parse(date)
if service == "dovecot":
scan_dovecot_line(date, log, collector)
date, system, service, pid, log = m.groups()
date = dateutil.parser.parse(date)
elif service == "postgrey":
scan_postgrey_line(date, log, collector)
if service == "dovecot":
scan_dovecot_line(date, log, collector)
elif service == "postfix/smtpd":
scan_postfix_smtpd_line(date, log, collector)
elif service == "postgrey":
scan_postgrey_line(date, log, collector)
elif service in ("postfix/qmgr", "postfix/pickup", "postfix/cleanup",
"postfix/scache", "spampd", "postfix/anvil", "postfix/master",
"opendkim", "postfix/lmtp", "postfix/tlsmgr"):
# nothing to look at
pass
elif service == "postfix/smtpd":
scan_postfix_smtpd_line(date, log, collector)
elif service in ("postfix/qmgr", "postfix/pickup", "postfix/cleanup",
"postfix/scache", "spampd", "postfix/anvil",
"postfix/master", "opendkim", "postfix/lmtp",
"postfix/tlsmgr"):
# nothing to look at
pass
else:
collector["other-services"].add(service)
else:
collector["other-services"].add(service)
def scan_dovecot_line(date, log, collector):
m = re.match("imap-login: Login: user=<(.*?)>, method=PLAIN, rip=(.*?),", log)
if m:
login, ip = m.group(1), m.group(2)
if ip != "127.0.0.1": # local login from webmail/zpush
collector["imap-logins"].setdefault(login, {})[ip] = date
m = re.match("imap-login: Login: user=<(.*?)>, method=PLAIN, rip=(.*?),", log)
if m:
login, ip = m.group(1), m.group(2)
if ip != "127.0.0.1": # local login from webmail/zpush
collector["imap-logins"].setdefault(login, {})[ip] = date
def scan_postgrey_line(date, log, collector):
m = re.match("action=(greylist|pass), reason=(.*?), (?:delay=\d+, )?client_name=(.*), client_address=(.*), sender=(.*), recipient=(.*)", log)
if m:
action, reason, client_name, client_address, sender, recipient = m.groups()
key = (client_address, sender)
if action == "greylist" and reason == "new":
collector["postgrey"].setdefault(recipient, {})[key] = (date, None)
elif action == "pass" and reason == "triplet found" and key in collector["postgrey"].get(recipient, {}):
collector["postgrey"][recipient][key] = (collector["postgrey"][recipient][key][0], date)
m = re.match("action=(greylist|pass), reason=(.*?), (?:delay=\d+, )?client_name=(.*), client_address=(.*), sender=(.*), recipient=(.*)", log)
if m:
action, reason, client_name, client_address, sender, recipient = m.groups()
key = (client_address, sender)
if action == "greylist" and reason == "new":
collector["postgrey"].setdefault(recipient, {})[key] = (date, None)
elif action == "pass" and reason == "triplet found" and key in collector["postgrey"].get(recipient, {}):
collector["postgrey"][recipient][key] = (collector["postgrey"][recipient][key][0], date)
def scan_postfix_smtpd_line(date, log, collector):
m = re.match("NOQUEUE: reject: RCPT from .*?: (.*?); from=<(.*?)> to=<(.*?)>", log)
if m:
message, sender, recipient = m.groups()
if recipient in collector["real_mail_addresses"]:
# only log mail to real recipients
m = re.match("NOQUEUE: reject: RCPT from .*?: (.*?); from=<(.*?)> to=<(.*?)>", log)
if m:
message, sender, recipient = m.groups()
if recipient in collector["real_mail_addresses"]:
# only log mail to real recipients
# skip this, is reported in the greylisting report
if "Recipient address rejected: Greylisted" in message:
return
# skip this, is reported in the greylisting report
if "Recipient address rejected: Greylisted" in message:
return
# simplify this one
m = re.search(r"Client host \[(.*?)\] blocked using zen.spamhaus.org; (.*)", message)
if m:
message = "ip blocked: " + m.group(2)
# simplify this one
m = re.search(r"Client host \[(.*?)\] blocked using zen.spamhaus.org; (.*)", message)
if m:
message = "ip blocked: " + m.group(2)
# simplify this one too
m = re.search(r"Sender address \[.*@(.*)\] blocked using dbl.spamhaus.org; (.*)", message)
if m:
message = "domain blocked: " + m.group(2)
# simplify this one too
m = re.search(r"Sender address \[.*@(.*)\] blocked using dbl.spamhaus.org; (.*)", message)
if m:
message = "domain blocked: " + m.group(2)
collector["rejected-mail"].setdefault(recipient, []).append( (date, sender, message) )
collector["rejected-mail"].setdefault(recipient, []).append((date, sender, message))
if __name__ == "__main__":
from status_checks import ConsoleOutput
env = utils.load_environment()
scan_mail_log(ConsoleOutput(), env)
from status_checks import ConsoleOutput
env = utils.load_environment()
scan_mail_log(ConsoleOutput(), env)

949
management/mailconfig.py
View File

File diff suppressed because it is too large Load Diff

1298
management/status_checks.py
View File

File diff suppressed because it is too large Load Diff

53
management/utils.py
View File

@ -2,33 +2,39 @@ import os.path
CONF_DIR = os.path.join(os.path.dirname(__file__), "../conf")
def load_environment():
# Load settings from /etc/mailinabox.conf.
return load_env_vars_from_file("/etc/mailinabox.conf")
def load_env_vars_from_file(fn):
# Load settings from a KEY=VALUE file.
import collections
env = collections.OrderedDict()
for line in open(fn): env.setdefault(*line.strip().split("=", 1))
for line in open(fn):
env.setdefault(*line.strip().split("=", 1))
return env
def save_environment(env):
with open("/etc/mailinabox.conf", "w") as f:
for k, v in env.items():
f.write("%s=%s\n" % (k, v))
def safe_domain_name(name):
# Sanitize a domain name so it is safe to use as a file name on disk.
import urllib.parse
return urllib.parse.quote(name, safe='')
def sort_domains(domain_names, env):
# Put domain names in a nice sorted order. For web_update, PRIMARY_HOSTNAME
# must appear first so it becomes the nginx default server.
# First group PRIMARY_HOSTNAME and its subdomains, then parent domains of PRIMARY_HOSTNAME, then other domains.
groups = ( [], [], [] )
groups = ([], [], [])
for d in domain_names:
if d == env['PRIMARY_HOSTNAME'] or d.endswith("." + env['PRIMARY_HOSTNAME']):
groups[0].append(d)
@ -44,13 +50,14 @@ def sort_domains(domain_names, env):
ret = []
for d in top_domains:
ret.append(d)
ret.extend( sort_group([s for s in group if s.endswith("." + d)]) )
ret.extend(sort_group([s for s in group if s.endswith("." + d)]))
return ret
groups = [sort_group(g) for g in groups]
return groups[0] + groups[1] + groups[2]
def sort_email_addresses(email_addresses, env):
email_addresses = set(email_addresses)
domains = set(email.split("@", 1)[1] for email in email_addresses if "@" in email)
@ -59,13 +66,17 @@ def sort_email_addresses(email_addresses, env):
domain_emails = set(email for email in email_addresses if email.endswith("@" + domain))
ret.extend(sorted(domain_emails))
email_addresses -= domain_emails
ret.extend(sorted(email_addresses)) # whatever is left
# whatever is left
ret.extend(sorted(email_addresses))
return ret
def exclusive_process(name):
# Ensure that a process named `name` does not execute multiple
# times concurrently.
import os, sys, atexit
import os
import sys
import atexit
pidfile = '/var/run/mailinabox-%s.pid' % name
mypid = os.getpid()
@ -95,7 +106,8 @@ def exclusive_process(name):
try:
existing_pid = int(f.read().strip())
except ValueError:
pass # No valid integer in the file.
# No valid integer in the file.
pass
# Check if the pid in it is valid.
if existing_pid:
@ -108,7 +120,7 @@ def exclusive_process(name):
f.write(str(mypid))
f.truncate()
atexit.register(clear_my_pid, pidfile)
def clear_my_pid(pidfile):
import os
@ -118,26 +130,32 @@ def clear_my_pid(pidfile):
def is_pid_valid(pid):
"""Checks whether a pid is a valid process ID of a currently running process."""
# adapted from http://stackoverflow.com/questions/568271/how-to-check-if-there-exists-a-process-with-a-given-pid
import os, errno
if pid <= 0: raise ValueError('Invalid PID.')
import os
import errno
if pid <= 0:
raise ValueError('Invalid PID.')
try:
os.kill(pid, 0)
except OSError as err:
if err.errno == errno.ESRCH: # No such process
# No such process
if err.errno == errno.ESRCH:
return False
elif err.errno == errno.EPERM: # Not permitted to send signal
# Not permitted to send signal
elif err.errno == errno.EPERM:
return True
else: # EINVAL
# EINVAL
else:
raise
else:
return True
def shell(method, cmd_args, env={}, capture_stderr=False, return_bytes=False, trap=False, input=None):
# A safe way to execute processes.
# Some processes like apt-get require being given a sane PATH.
import subprocess
env.update({ "PATH": "/sbin:/bin:/usr/sbin:/usr/bin" })
env.update({"PATH": "/sbin:/bin:/usr/sbin:/usr/bin"})
kwargs = {
'env': env,
'stderr': None if not capture_stderr else subprocess.STDOUT,
@ -154,18 +172,21 @@ def shell(method, cmd_args, env={}, capture_stderr=False, return_bytes=False, tr
except subprocess.CalledProcessError as e:
ret = e.output
code = e.returncode
if not return_bytes and isinstance(ret, bytes): ret = ret.decode("utf8")
if not return_bytes and isinstance(ret, bytes):
ret = ret.decode("utf8")
if not trap:
return ret
else:
return code, ret
def create_syslog_handler():
import logging.handlers
handler = logging.handlers.SysLogHandler(address='/dev/log')
handler.setLevel(logging.WARNING)
return handler
def du(path):
# Computes the size of all files in the path, like the `du` command.
# Based on http://stackoverflow.com/a/17936789. Takes into account

480
management/web_update.py
View File

@ -2,297 +2,313 @@
# domains for which a mail account has been set up.
########################################################################
import os, os.path, shutil, re, tempfile, rtyaml
import os
import os.path
import shutil
import re
import tempfile
import rtyaml
from mailconfig import get_mail_domains
from dns_update import get_custom_dns_config, do_dns_update
from utils import shell, safe_domain_name, sort_domains
def get_web_domains(env):
# What domains should we serve websites for?
domains = set()
# What domains should we serve websites for?
domains = set()
# At the least it's the PRIMARY_HOSTNAME so we can serve webmail
# as well as Z-Push for Exchange ActiveSync.
domains.add(env['PRIMARY_HOSTNAME'])
# At the least it's the PRIMARY_HOSTNAME so we can serve webmail
# as well as Z-Push for Exchange ActiveSync.
domains.add(env['PRIMARY_HOSTNAME'])
# Also serve web for all mail domains so that we might at least
# provide auto-discover of email settings, and also a static website
# if the user wants to make one. These will require an SSL cert.
domains |= get_mail_domains(env)
# Also serve web for all mail domains so that we might at least
# provide auto-discover of email settings, and also a static website
# if the user wants to make one. These will require an SSL cert.
domains |= get_mail_domains(env)
# ...Unless the domain has an A/AAAA record that maps it to a different
# IP address than this box. Remove those domains from our list.
dns = get_custom_dns_config(env)
for domain, value in dns.items():
if domain not in domains: continue
if (isinstance(value, str) and (value != "local")) \
or (isinstance(value, dict) and ("CNAME" in value)) \
or (isinstance(value, dict) and ("A" in value) and (value["A"] != "local")) \
or (isinstance(value, dict) and ("AAAA" in value) and (value["AAAA"] != "local")):
domains.remove(domain)
# ...Unless the domain has an A/AAAA record that maps it to a different
# IP address than this box. Remove those domains from our list.
dns = get_custom_dns_config(env)
for domain, value in dns.items():
if domain not in domains:
continue
if (isinstance(value, str) and (value != "local")) or (isinstance(value, dict) and ("CNAME" in value)) or (isinstance(value, dict) and ("A" in value) and (value["A"] != "local")) or (isinstance(value, dict) and ("AAAA" in value) and (value["AAAA"] != "local")):
domains.remove(domain)
# Sort the list. Put PRIMARY_HOSTNAME first so it becomes the
# default server (nginx's default_server).
domains = sort_domains(domains, env)
return domains
# Sort the list. Put PRIMARY_HOSTNAME first so it becomes the
# default server (nginx's default_server).
domains = sort_domains(domains, env)
return domains
def do_web_update(env, ok_status="web updated\n"):
# Build an nginx configuration file.
nginx_conf = open(os.path.join(os.path.dirname(__file__), "../conf/nginx-top.conf")).read()
# Build an nginx configuration file.
nginx_conf = open(os.path.join(os.path.dirname(__file__), "../conf/nginx-top.conf")).read()
# Add configuration for each web domain.
template1 = open(os.path.join(os.path.dirname(__file__), "../conf/nginx.conf")).read()
template2 = open(os.path.join(os.path.dirname(__file__), "../conf/nginx-primaryonly.conf")).read()
for domain in get_web_domains(env):
nginx_conf += make_domain_config(domain, template1, template2, env)
# Add configuration for each web domain.
template1 = open(os.path.join(os.path.dirname(__file__), "../conf/nginx.conf")).read()
template2 = open(os.path.join(os.path.dirname(__file__), "../conf/nginx-primaryonly.conf")).read()
for domain in get_web_domains(env):
nginx_conf += make_domain_config(domain, template1, template2, env)
# Did the file change? If not, don't bother writing & restarting nginx.
nginx_conf_fn = "/etc/nginx/conf.d/local.conf"
if os.path.exists(nginx_conf_fn):
with open(nginx_conf_fn) as f:
if f.read() == nginx_conf:
return ""
# Did the file change? If not, don't bother writing & restarting nginx.
nginx_conf_fn = "/etc/nginx/conf.d/local.conf"
if os.path.exists(nginx_conf_fn):
with open(nginx_conf_fn) as f:
if f.read() == nginx_conf:
return ""
# Save the file.
with open(nginx_conf_fn, "w") as f:
f.write(nginx_conf)
# Save the file.
with open(nginx_conf_fn, "w") as f:
f.write(nginx_conf)
# Kick nginx. Since this might be called from the web admin
# don't do a 'restart'. That would kill the connection before
# the API returns its response. A 'reload' should be good
# enough and doesn't break any open connections.
shell('check_call', ["/usr/sbin/service", "nginx", "reload"])
# Kick nginx. Since this might be called from the web admin
# don't do a 'restart'. That would kill the connection before
# the API returns its response. A 'reload' should be good
# enough and doesn't break any open connections.
shell('check_call', ["/usr/sbin/service", "nginx", "reload"])
return ok_status
return ok_status
def make_domain_config(domain, template, template_for_primaryhost, env):
# How will we configure this domain.
# How will we configure this domain.
# Where will its root directory be for static files?
# Where will its root directory be for static files?
root = get_web_root(domain, env)
root = get_web_root(domain, env)
# What private key and SSL certificate will we use for this domain?
ssl_key, ssl_certificate, ssl_via = get_domain_ssl_files(domain, env)
# What private key and SSL certificate will we use for this domain?
ssl_key, ssl_certificate, ssl_via = get_domain_ssl_files(domain, env)
# For hostnames created after the initial setup, ensure we have an SSL certificate
# available. Make a self-signed one now if one doesn't exist.
ensure_ssl_certificate_exists(domain, ssl_key, ssl_certificate, env)
# For hostnames created after the initial setup, ensure we have an SSL certificate
# available. Make a self-signed one now if one doesn't exist.
ensure_ssl_certificate_exists(domain, ssl_key, ssl_certificate, env)
# Put pieces together.
nginx_conf_parts = re.split("\s*# ADDITIONAL DIRECTIVES HERE\s*", template)
nginx_conf = nginx_conf_parts[0] + "\n"
if domain == env['PRIMARY_HOSTNAME']:
nginx_conf += template_for_primaryhost + "\n"
# Put pieces together.
nginx_conf_parts = re.split("\s*# ADDITIONAL DIRECTIVES HERE\s*", template)
nginx_conf = nginx_conf_parts[0] + "\n"
if domain == env['PRIMARY_HOSTNAME']:
nginx_conf += template_for_primaryhost + "\n"
# Replace substitution strings in the template & return.
nginx_conf = nginx_conf.replace("$STORAGE_ROOT", env['STORAGE_ROOT'])
nginx_conf = nginx_conf.replace("$HOSTNAME", domain.encode("idna").decode("ascii"))
nginx_conf = nginx_conf.replace("$ROOT", root)
nginx_conf = nginx_conf.replace("$SSL_KEY", ssl_key)
nginx_conf = nginx_conf.replace("$SSL_CERTIFICATE", ssl_certificate)
# Replace substitution strings in the template & return.
nginx_conf = nginx_conf.replace("$STORAGE_ROOT", env['STORAGE_ROOT'])
nginx_conf = nginx_conf.replace("$HOSTNAME", domain.encode("idna").decode("ascii"))
nginx_conf = nginx_conf.replace("$ROOT", root)
nginx_conf = nginx_conf.replace("$SSL_KEY", ssl_key)
nginx_conf = nginx_conf.replace("$SSL_CERTIFICATE", ssl_certificate)
# Because the certificate may change, we should recognize this so we
# can trigger an nginx update.
def hashfile(filepath):
import hashlib
sha1 = hashlib.sha1()
f = open(filepath, 'rb')
try:
sha1.update(f.read())
finally:
f.close()
return sha1.hexdigest()
nginx_conf += "# ssl files sha1: %s / %s\n" % (hashfile(ssl_key), hashfile(ssl_certificate))
# Because the certificate may change, we should recognize this so we
# can trigger an nginx update.
def hashfile(filepath):
import hashlib
sha1 = hashlib.sha1()
f = open(filepath, 'rb')
try:
sha1.update(f.read())
finally:
f.close()
return sha1.hexdigest()
nginx_conf += "# ssl files sha1: %s / %s\n" % (hashfile(ssl_key), hashfile(ssl_certificate))
# Add in any user customizations in YAML format.
nginx_conf_custom_fn = os.path.join(env["STORAGE_ROOT"], "www/custom.yaml")
if os.path.exists(nginx_conf_custom_fn):
yaml = rtyaml.load(open(nginx_conf_custom_fn))
if domain in yaml:
yaml = yaml[domain]
for path, url in yaml.get("proxies", {}).items():
nginx_conf += "\tlocation %s {\n\t\tproxy_pass %s;\n\t}\n" % (path, url)
for path, url in yaml.get("redirects", {}).items():
nginx_conf += "\trewrite %s %s permanent;\n" % (path, url)
# Add in any user customizations in YAML format.
nginx_conf_custom_fn = os.path.join(env["STORAGE_ROOT"], "www/custom.yaml")
if os.path.exists(nginx_conf_custom_fn):
yaml = rtyaml.load(open(nginx_conf_custom_fn))
if domain in yaml:
yaml = yaml[domain]
for path, url in yaml.get("proxies", {}).items():
nginx_conf += "\tlocation %s {\n\t\tproxy_pass %s;\n\t}\n" % (path, url)
for path, url in yaml.get("redirects", {}).items():
nginx_conf += "\trewrite %s %s permanent;\n" % (path, url)
# Add in any user customizations in the includes/ folder.
nginx_conf_custom_include = os.path.join(env["STORAGE_ROOT"], "www", safe_domain_name(domain) + ".conf")
if os.path.exists(nginx_conf_custom_include):
nginx_conf += "\tinclude %s;\n" % (nginx_conf_custom_include)
# Add in any user customizations in the includes/ folder.
nginx_conf_custom_include = os.path.join(env["STORAGE_ROOT"], "www", safe_domain_name(domain) + ".conf")
if os.path.exists(nginx_conf_custom_include):
nginx_conf += "\tinclude %s;\n" % (nginx_conf_custom_include)
# Ending.
nginx_conf += nginx_conf_parts[1]
# Ending.
nginx_conf += nginx_conf_parts[1]
return nginx_conf
return nginx_conf
def get_web_root(domain, env, test_exists=True):
# Try STORAGE_ROOT/web/domain_name if it exists, but fall back to STORAGE_ROOT/web/default.
for test_domain in (domain, 'default'):
root = os.path.join(env["STORAGE_ROOT"], "www", safe_domain_name(test_domain))
if os.path.exists(root) or not test_exists: break
return root
# Try STORAGE_ROOT/web/domain_name if it exists, but fall back to STORAGE_ROOT/web/default.
for test_domain in (domain, 'default'):
root = os.path.join(env["STORAGE_ROOT"], "www", safe_domain_name(test_domain))
if os.path.exists(root) or not test_exists:
break
return root
def get_domain_ssl_files(domain, env, allow_shared_cert=True):
# What SSL private key will we use? Allow the user to override this, but
# in many cases using the same private key for all domains would be fine.
# Don't allow the user to override the key for PRIMARY_HOSTNAME because
# that's what's in the main file.
ssl_key = os.path.join(env["STORAGE_ROOT"], 'ssl/ssl_private_key.pem')
ssl_key_is_alt = False
alt_key = os.path.join(env["STORAGE_ROOT"], 'ssl/%s/private_key.pem' % safe_domain_name(domain))
if domain != env['PRIMARY_HOSTNAME'] and os.path.exists(alt_key):
ssl_key = alt_key
ssl_key_is_alt = True
# What SSL private key will we use? Allow the user to override this, but
# in many cases using the same private key for all domains would be fine.
# Don't allow the user to override the key for PRIMARY_HOSTNAME because
# that's what's in the main file.
ssl_key = os.path.join(env["STORAGE_ROOT"], 'ssl/ssl_private_key.pem')
ssl_key_is_alt = False
alt_key = os.path.join(env["STORAGE_ROOT"], 'ssl/%s/private_key.pem' % safe_domain_name(domain))
if domain != env['PRIMARY_HOSTNAME'] and os.path.exists(alt_key):
ssl_key = alt_key
ssl_key_is_alt = True
# What SSL certificate will we use?
ssl_certificate_primary = os.path.join(env["STORAGE_ROOT"], 'ssl/ssl_certificate.pem')
ssl_via = None
if domain == env['PRIMARY_HOSTNAME']:
# For PRIMARY_HOSTNAME, use the one we generated at set-up time.
ssl_certificate = ssl_certificate_primary
else:
# For other domains, we'll probably use a certificate in a different path.
ssl_certificate = os.path.join(env["STORAGE_ROOT"], 'ssl/%s/ssl_certificate.pem' % safe_domain_name(domain))
# What SSL certificate will we use?
ssl_certificate_primary = os.path.join(env["STORAGE_ROOT"], 'ssl/ssl_certificate.pem')
ssl_via = None
if domain == env['PRIMARY_HOSTNAME']:
# For PRIMARY_HOSTNAME, use the one we generated at set-up time.
ssl_certificate = ssl_certificate_primary
else:
# For other domains, we'll probably use a certificate in a different path.
ssl_certificate = os.path.join(env["STORAGE_ROOT"], 'ssl/%s/ssl_certificate.pem' % safe_domain_name(domain))
# But we can be smart and reuse the main SSL certificate if is has
# a Subject Alternative Name matching this domain. Don't do this if
# the user has uploaded a different private key for this domain.
if not ssl_key_is_alt and allow_shared_cert:
from status_checks import check_certificate
if check_certificate(domain, ssl_certificate_primary, None)[0] == "OK":
ssl_certificate = ssl_certificate_primary
ssl_via = "Using multi/wildcard certificate of %s." % env['PRIMARY_HOSTNAME']
# But we can be smart and reuse the main SSL certificate if is has
# a Subject Alternative Name matching this domain. Don't do this if
# the user has uploaded a different private key for this domain.
if not ssl_key_is_alt and allow_shared_cert:
from status_checks import check_certificate
if check_certificate(domain, ssl_certificate_primary, None)[0] == "OK":
ssl_certificate = ssl_certificate_primary
ssl_via = "Using multi/wildcard certificate of %s." % env['PRIMARY_HOSTNAME']
# For a 'www.' domain, see if we can reuse the cert of the parent.
elif domain.startswith('www.'):
ssl_certificate_parent = os.path.join(env["STORAGE_ROOT"], 'ssl/%s/ssl_certificate.pem' % safe_domain_name(domain[4:]))
if os.path.exists(ssl_certificate_parent) and check_certificate(domain, ssl_certificate_parent, None)[0] == "OK":
ssl_certificate = ssl_certificate_parent
ssl_via = "Using multi/wildcard certificate of %s." % domain[4:]
# For a 'www.' domain, see if we can reuse the cert of the parent.
elif domain.startswith('www.'):
ssl_certificate_parent = os.path.join(env["STORAGE_ROOT"], 'ssl/%s/ssl_certificate.pem' % safe_domain_name(domain[4:]))
if os.path.exists(ssl_certificate_parent) and check_certificate(domain, ssl_certificate_parent, None)[0] == "OK":
ssl_certificate = ssl_certificate_parent
ssl_via = "Using multi/wildcard certificate of %s." % domain[4:]
return ssl_key, ssl_certificate, ssl_via
return ssl_key, ssl_certificate, ssl_via
def ensure_ssl_certificate_exists(domain, ssl_key, ssl_certificate, env):
# For domains besides PRIMARY_HOSTNAME, generate a self-signed certificate if
# a certificate doesn't already exist. See setup/mail.sh for documentation.
# For domains besides PRIMARY_HOSTNAME, generate a self-signed certificate if
# a certificate doesn't already exist. See setup/mail.sh for documentation.
if domain == env['PRIMARY_HOSTNAME']:
return
if domain == env['PRIMARY_HOSTNAME']:
return
# Sanity check. Shouldn't happen. A non-primary domain might use this
# certificate (see above), but then the certificate should exist anyway.
if ssl_certificate == os.path.join(env["STORAGE_ROOT"], 'ssl/ssl_certificate.pem'):
return
# Sanity check. Shouldn't happen. A non-primary domain might use this
# certificate (see above), but then the certificate should exist anyway.
if ssl_certificate == os.path.join(env["STORAGE_ROOT"], 'ssl/ssl_certificate.pem'):
return
if os.path.exists(ssl_certificate):
return
if os.path.exists(ssl_certificate):
return
os.makedirs(os.path.dirname(ssl_certificate), exist_ok=True)
os.makedirs(os.path.dirname(ssl_certificate), exist_ok=True)
# Generate a new self-signed certificate using the same private key that we already have.
# Generate a new self-signed certificate using the same private key that we already have.
# Start with a CSR written to a temporary file.
with tempfile.NamedTemporaryFile(mode="w") as csr_fp:
csr_fp.write(create_csr(domain, ssl_key, env))
csr_fp.flush() # since we won't close until after running 'openssl x509', since close triggers delete.
# Start with a CSR written to a temporary file.
with tempfile.NamedTemporaryFile(mode="w") as csr_fp:
csr_fp.write(create_csr(domain, ssl_key, env))
# since we won't close until after running 'openssl x509', since close triggers delete.
csr_fp.flush()
# And then make the certificate.
shell("check_call", [
"openssl", "x509", "-req",
"-days", "365",
"-in", csr_fp.name,
"-signkey", ssl_key,
"-out", ssl_certificate])
# And then make the certificate.
shell("check_call", [
"openssl", "x509", "-req",
"-days", "365",
"-in", csr_fp.name,
"-signkey", ssl_key,
"-out", ssl_certificate])
def create_csr(domain, ssl_key, env):
return shell("check_output", [
"openssl", "req", "-new",
"-key", ssl_key,
"-out", "/dev/stdout",
"-sha256",
"-subj", "/C=%s/ST=/L=/O=/CN=%s" % (env["CSR_COUNTRY"], domain.encode("idna").decode("ascii"))])
return shell("check_output", [
"openssl", "req", "-new",
"-key", ssl_key,
"-out", "/dev/stdout",
"-sha256",
"-subj", "/C=%s/ST=/L=/O=/CN=%s" % (env["CSR_COUNTRY"], domain.encode("idna").decode("ascii"))])
def install_cert(domain, ssl_cert, ssl_chain, env):
if domain not in get_web_domains(env):
return "Invalid domain name."
if domain not in get_web_domains(env):
return "Invalid domain name."
# Write the combined cert+chain to a temporary path and validate that it is OK.
# The certificate always goes above the chain.
import tempfile, os
fd, fn = tempfile.mkstemp('.pem')
os.write(fd, (ssl_cert + '\n' + ssl_chain).encode("ascii"))
os.close(fd)
# Write the combined cert+chain to a temporary path and validate that it is OK.
# The certificate always goes above the chain.
import tempfile
import os
fd, fn = tempfile.mkstemp('.pem')
os.write(fd, (ssl_cert + '\n' + ssl_chain).encode("ascii"))
os.close(fd)
# Do validation on the certificate before installing it.
from status_checks import check_certificate
ssl_key, ssl_certificate, ssl_via = get_domain_ssl_files(domain, env, allow_shared_cert=False)
cert_status, cert_status_details = check_certificate(domain, fn, ssl_key)
if cert_status != "OK":
if cert_status == "SELF-SIGNED":
cert_status = "This is a self-signed certificate. I can't install that."
os.unlink(fn)
if cert_status_details is not None:
cert_status += " " + cert_status_details
return cert_status
# Do validation on the certificate before installing it.
from status_checks import check_certificate
ssl_key, ssl_certificate, ssl_via = get_domain_ssl_files(domain, env, allow_shared_cert=False)
cert_status, cert_status_details = check_certificate(domain, fn, ssl_key)
if cert_status != "OK":
if cert_status == "SELF-SIGNED":
cert_status = "This is a self-signed certificate. I can't install that."
os.unlink(fn)
if cert_status_details is not None:
cert_status += " " + cert_status_details
return cert_status
# Copy the certificate to its expected location.
os.makedirs(os.path.dirname(ssl_certificate), exist_ok=True)
shutil.move(fn, ssl_certificate)
# Copy the certificate to its expected location.
os.makedirs(os.path.dirname(ssl_certificate), exist_ok=True)
shutil.move(fn, ssl_certificate)
ret = []
ret = []
# When updating the cert for PRIMARY_HOSTNAME, also update DNS because it is
# used in the DANE TLSA record and restart postfix and dovecot which use
# that certificate.
if domain == env['PRIMARY_HOSTNAME']:
ret.append( do_dns_update(env) )
# When updating the cert for PRIMARY_HOSTNAME, also update DNS because it is
# used in the DANE TLSA record and restart postfix and dovecot which use
# that certificate.
if domain == env['PRIMARY_HOSTNAME']:
ret.append(do_dns_update(env))
shell('check_call', ["/usr/sbin/service", "postfix", "restart"])
shell('check_call', ["/usr/sbin/service", "dovecot", "restart"])
ret.append("mail services restarted")
shell('check_call', ["/usr/sbin/service", "postfix", "restart"])
shell('check_call', ["/usr/sbin/service", "dovecot", "restart"])
ret.append("mail services restarted")
# Kick nginx so it sees the cert.
ret.append(do_web_update(env, ok_status=""))
return "\n".join(r for r in ret if r.strip() != "")
# Kick nginx so it sees the cert.
ret.append( do_web_update(env, ok_status="") )
return "\n".join(r for r in ret if r.strip() != "")
def get_web_domains_info(env):
# load custom settings so we can tell what domains have a redirect or proxy set up on '/',
# which means static hosting is not happening
custom_settings = { }
nginx_conf_custom_fn = os.path.join(env["STORAGE_ROOT"], "www/custom.yaml")
if os.path.exists(nginx_conf_custom_fn):
custom_settings = rtyaml.load(open(nginx_conf_custom_fn))
def has_root_proxy_or_redirect(domain):
return custom_settings.get(domain, {}).get('redirects', {}).get('/') or custom_settings.get(domain, {}).get('proxies', {}).get('/')
# load custom settings so we can tell what domains have a redirect or proxy set up on '/',
# which means static hosting is not happening
custom_settings = {}
nginx_conf_custom_fn = os.path.join(env["STORAGE_ROOT"], "www/custom.yaml")
if os.path.exists(nginx_conf_custom_fn):
custom_settings = rtyaml.load(open(nginx_conf_custom_fn))
# for the SSL config panel, get cert status
def check_cert(domain):
from status_checks import check_certificate
ssl_key, ssl_certificate, ssl_via = get_domain_ssl_files(domain, env)
if not os.path.exists(ssl_certificate):
return ("danger", "No Certificate Installed")
cert_status, cert_status_details = check_certificate(domain, ssl_certificate, ssl_key)
if cert_status == "OK":
if not ssl_via:
return ("success", "Signed & valid. " + cert_status_details)
else:
# This is an alternate domain but using the same cert as the primary domain.
return ("success", "Signed & valid. " + ssl_via)
elif cert_status == "SELF-SIGNED":
return ("warning", "Self-signed. Get a signed certificate to stop warnings.")
else:
return ("danger", "Certificate has a problem: " + cert_status)
def has_root_proxy_or_redirect(domain):
return custom_settings.get(domain, {}).get('redirects', {}).get('/') or custom_settings.get(domain, {}).get('proxies', {}).get('/')
return [
{
"domain": domain,
"root": get_web_root(domain, env),
"custom_root": get_web_root(domain, env, test_exists=False),
"ssl_certificate": check_cert(domain),
"static_enabled": not has_root_proxy_or_redirect(domain),
}
for domain in get_web_domains(env)
]
# for the SSL config panel, get cert status
def check_cert(domain):
from status_checks import check_certificate
ssl_key, ssl_certificate, ssl_via = get_domain_ssl_files(domain, env)
if not os.path.exists(ssl_certificate):
return ("danger", "No Certificate Installed")
cert_status, cert_status_details = check_certificate(domain, ssl_certificate, ssl_key)
if cert_status == "OK":
if not ssl_via:
return ("success", "Signed & valid. " + cert_status_details)
else:
# This is an alternate domain but using the same cert as the primary domain.
return ("success", "Signed & valid. " + ssl_via)
elif cert_status == "SELF-SIGNED":
return ("warning", "Self-signed. Get a signed certificate to stop warnings.")
else:
return ("danger", "Certificate has a problem: " + cert_status)
return [
{
"domain": domain,
"root": get_web_root(domain, env),
"custom_root": get_web_root(domain, env, test_exists=False),
"ssl_certificate": check_cert(domain),
"static_enabled": not has_root_proxy_or_redirect(domain),
}
for domain in get_web_domains(env)
]

210
setup/migrate.py
View File

@ -5,134 +5,150 @@
# We have to be careful here that any dependencies are already installed in the previous
# version since this script runs before all other aspects of the setup script.
import sys, os, os.path, glob, re, shutil
import sys
import os
import os.path
import glob
import re
import shutil
sys.path.insert(0, 'management')
from utils import load_environment, save_environment, shell
def migration_1(env):
# Re-arrange where we store SSL certificates. There was a typo also.
# Re-arrange where we store SSL certificates. There was a typo also.
def move_file(fn, domain_name_escaped, filename):
# Moves an SSL-related file into the right place.
fn1 = os.path.join( env["STORAGE_ROOT"], 'ssl', domain_name_escaped, file_type)
os.makedirs(os.path.dirname(fn1), exist_ok=True)
shutil.move(fn, fn1)
def move_file(fn, domain_name_escaped, filename):
# Moves an SSL-related file into the right place.
fn1 = os.path.join(env["STORAGE_ROOT"], 'ssl', domain_name_escaped, file_type)
os.makedirs(os.path.dirname(fn1), exist_ok=True)
shutil.move(fn, fn1)
# Migrate the 'domains' directory.
for sslfn in glob.glob(os.path.join( env["STORAGE_ROOT"], 'ssl/domains/*' )):
fn = os.path.basename(sslfn)
m = re.match("(.*)_(certifiate.pem|cert_sign_req.csr|private_key.pem)$", fn)
if m:
# get the new name for the file
domain_name, file_type = m.groups()
if file_type == "certifiate.pem": file_type = "ssl_certificate.pem" # typo
if file_type == "cert_sign_req.csr": file_type = "certificate_signing_request.csr" # nicer
move_file(sslfn, domain_name, file_type)
# Migrate the 'domains' directory.
for sslfn in glob.glob(os.path.join(env["STORAGE_ROOT"], 'ssl/domains/*')):
fn = os.path.basename(sslfn)
m = re.match("(.*)_(certifiate.pem|cert_sign_req.csr|private_key.pem)$", fn)
if m:
# get the new name for the file
domain_name, file_type = m.groups()
# typo
if file_type == "certifiate.pem":
file_type = "ssl_certificate.pem"
# nicer
if file_type == "cert_sign_req.csr":
file_type = "certificate_signing_request.csr"
move_file(sslfn, domain_name, file_type)
# Move the old domains directory if it is now empty.
try:
os.rmdir(os.path.join(env["STORAGE_ROOT"], 'ssl/domains'))
except:
pass
# Move the old domains directory if it is now empty.
try:
os.rmdir(os.path.join( env["STORAGE_ROOT"], 'ssl/domains'))
except:
pass
def migration_2(env):
# Delete the .dovecot_sieve script everywhere. This was formerly a copy of our spam -> Spam
# script. We now install it as a global script, and we use managesieve, so the old file is
# irrelevant. Also delete the compiled binary form.
for fn in glob.glob(os.path.join(env["STORAGE_ROOT"], 'mail/mailboxes/*/*/.dovecot.sieve')):
os.unlink(fn)
for fn in glob.glob(os.path.join(env["STORAGE_ROOT"], 'mail/mailboxes/*/*/.dovecot.svbin')):
os.unlink(fn)
# Delete the .dovecot_sieve script everywhere. This was formerly a copy of our spam -> Spam
# script. We now install it as a global script, and we use managesieve, so the old file is
# irrelevant. Also delete the compiled binary form.
for fn in glob.glob(os.path.join(env["STORAGE_ROOT"], 'mail/mailboxes/*/*/.dovecot.sieve')):
os.unlink(fn)
for fn in glob.glob(os.path.join(env["STORAGE_ROOT"], 'mail/mailboxes/*/*/.dovecot.svbin')):
os.unlink(fn)
def migration_3(env):
# Move the migration ID from /etc/mailinabox.conf to $STORAGE_ROOT/mailinabox.version
# so that the ID stays with the data files that it describes the format of. The writing
# of the file will be handled by the main function.
pass
# Move the migration ID from /etc/mailinabox.conf to $STORAGE_ROOT/mailinabox.version
# so that the ID stays with the data files that it describes the format of. The writing
# of the file will be handled by the main function.
pass
def migration_4(env):
# Add a new column to the mail users table where we can store administrative privileges.
db = os.path.join(env["STORAGE_ROOT"], 'mail/users.sqlite')
shell("check_call", ["sqlite3", db, "ALTER TABLE users ADD privileges TEXT NOT NULL DEFAULT ''"])
# Add a new column to the mail users table where we can store administrative privileges.
db = os.path.join(env["STORAGE_ROOT"], 'mail/users.sqlite')
shell("check_call", ["sqlite3", db, "ALTER TABLE users ADD privileges TEXT NOT NULL DEFAULT ''"])
def migration_5(env):
# The secret key for encrypting backups was world readable. Fix here.
os.chmod(os.path.join(env["STORAGE_ROOT"], 'backup/secret_key.txt'), 0o600)
# The secret key for encrypting backups was world readable. Fix here.
os.chmod(os.path.join(env["STORAGE_ROOT"], 'backup/secret_key.txt'), 0o600)
def migration_6(env):
# We now will generate multiple DNSSEC keys for different algorithms, since TLDs may
# not support them all. .email only supports RSA/SHA-256. Rename the keys.conf file
# to be algorithm-specific.
basepath = os.path.join(env["STORAGE_ROOT"], 'dns/dnssec')
shutil.move(os.path.join(basepath, 'keys.conf'), os.path.join(basepath, 'RSASHA1-NSEC3-SHA1.conf'))
# We now will generate multiple DNSSEC keys for different algorithms, since TLDs may
# not support them all. .email only supports RSA/SHA-256. Rename the keys.conf file
# to be algorithm-specific.
basepath = os.path.join(env["STORAGE_ROOT"], 'dns/dnssec')
shutil.move(os.path.join(basepath, 'keys.conf'), os.path.join(basepath, 'RSASHA1-NSEC3-SHA1.conf'))
def get_current_migration():
ver = 0
while True:
next_ver = (ver + 1)
migration_func = globals().get("migration_%d" % next_ver)
if not migration_func:
return ver
ver = next_ver
ver = 0
while True:
next_ver = (ver + 1)
migration_func = globals().get("migration_%d" % next_ver)
if not migration_func:
return ver
ver = next_ver
def run_migrations():
if not os.access("/etc/mailinabox.conf", os.W_OK, effective_ids=True):
print("This script must be run as root.", file=sys.stderr)
sys.exit(1)
if not os.access("/etc/mailinabox.conf", os.W_OK, effective_ids=True):
print("This script must be run as root.", file=sys.stderr)
sys.exit(1)
env = load_environment()
env = load_environment()
migration_id_file = os.path.join(env['STORAGE_ROOT'], 'mailinabox.version')
if os.path.exists(migration_id_file):
with open(migration_id_file) as f:
ourver = int(f.read().strip())
else:
# Load the legacy location of the migration ID. We'll drop support
# for this eventually.
ourver = int(env.get("MIGRATIONID", "0"))
migration_id_file = os.path.join(env['STORAGE_ROOT'], 'mailinabox.version')
if os.path.exists(migration_id_file):
with open(migration_id_file) as f:
ourver = int(f.read().strip())
else:
# Load the legacy location of the migration ID. We'll drop support
# for this eventually.
ourver = int(env.get("MIGRATIONID", "0"))
while True:
next_ver = (ourver + 1)
migration_func = globals().get("migration_%d" % next_ver)
while True:
next_ver = (ourver + 1)
migration_func = globals().get("migration_%d" % next_ver)
if not migration_func:
# No more migrations to run.
break
if not migration_func:
# No more migrations to run.
break
print()
print("Running migration to Mail-in-a-Box #%d..." % next_ver)
print()
print("Running migration to Mail-in-a-Box #%d..." % next_ver)
try:
migration_func(env)
except Exception as e:
print()
print("Error running the migration script:")
print()
print(e)
print()
print("Your system may be in an inconsistent state now. We're terribly sorry. A re-install from a backup might be the best way to continue.")
sys.exit(1)
try:
migration_func(env)
except Exception as e:
print()
print("Error running the migration script:")
print()
print(e)
print()
print("Your system may be in an inconsistent state now. We're terribly sorry. A re-install from a backup might be the best way to continue.")
sys.exit(1)
ourver = next_ver
ourver = next_ver
# Write out our current version now. Do this sooner rather than later
# in case of any problems.
with open(migration_id_file, "w") as f:
f.write(str(ourver) + "\n")
# Write out our current version now. Do this sooner rather than later
# in case of any problems.
with open(migration_id_file, "w") as f:
f.write(str(ourver) + "\n")
# Delete the legacy location of this field.
if "MIGRATIONID" in env:
del env["MIGRATIONID"]
save_environment(env)
# Delete the legacy location of this field.
if "MIGRATIONID" in env:
del env["MIGRATIONID"]
save_environment(env)
# iterate and try next version...
# iterate and try next version...
if __name__ == "__main__":
if sys.argv[-1] == "--current":
# Return the number of the highest migration.
print(str(get_current_migration()))
elif sys.argv[-1] == "--migrate":
# Perform migrations.
run_migrations()
if sys.argv[-1] == "--current":
# Return the number of the highest migration.
print(str(get_current_migration()))
elif sys.argv[-1] == "--migrate":
# Perform migrations.
run_migrations()

162
tests/test_dns.py
View File

@ -7,100 +7,110 @@
# where ipaddr is the IP address of your Mail-in-a-Box
# and hostname is the domain name to check the DNS for.
import sys, re, difflib
import dns.reversename, dns.resolver
import sys
import re
import difflib
import dns.reversename
import dns.resolver
if len(sys.argv) < 3:
print("Usage: tests/dns.py ipaddress hostname [primary hostname]")
sys.exit(1)
print("Usage: tests/dns.py ipaddress hostname [primary hostname]")
sys.exit(1)
ipaddr, hostname = sys.argv[1:3]
primary_hostname = hostname
if len(sys.argv) == 4:
primary_hostname = sys.argv[3]
primary_hostname = sys.argv[3]
def test(server, description):
tests = [
(hostname, "A", ipaddr),
#(hostname, "NS", "ns1.%s.;ns2.%s." % (primary_hostname, primary_hostname)),
("ns1." + primary_hostname, "A", ipaddr),
("ns2." + primary_hostname, "A", ipaddr),
("www." + hostname, "A", ipaddr),
(hostname, "MX", "10 " + primary_hostname + "."),
(hostname, "TXT", "\"v=spf1 mx -all\""),
("mail._domainkey." + hostname, "TXT", "\"v=DKIM1; k=rsa; s=email; \" \"p=__KEY__\""),
#("_adsp._domainkey." + hostname, "TXT", "\"dkim=all\""),
("_dmarc." + hostname, "TXT", "\"v=DMARC1; p=quarantine\""),
]
return test2(tests, server, description)
tests = [
(hostname, "A", ipaddr),
#(hostname, "NS", "ns1.%s.;ns2.%s." % (primary_hostname, primary_hostname)),
("ns1." + primary_hostname, "A", ipaddr),
("ns2." + primary_hostname, "A", ipaddr),
("www." + hostname, "A", ipaddr),
(hostname, "MX", "10 " + primary_hostname + "."),
(hostname, "TXT", "\"v=spf1 mx -all\""),
("mail._domainkey." + hostname, "TXT", "\"v=DKIM1; k=rsa; s=email; \" \"p=__KEY__\""),
#("_adsp._domainkey." + hostname, "TXT", "\"dkim=all\""),
("_dmarc." + hostname, "TXT", "\"v=DMARC1; p=quarantine\""),
]
return test2(tests, server, description)
def test_ptr(server, description):
ipaddr_rev = dns.reversename.from_address(ipaddr)
tests = [
(ipaddr_rev, "PTR", hostname+'.'),
]
return test2(tests, server, description)
ipaddr_rev = dns.reversename.from_address(ipaddr)
tests = [
(ipaddr_rev, "PTR", hostname+'.'),
]
return test2(tests, server, description)
def test2(tests, server, description):
first = True
resolver = dns.resolver.get_default_resolver()
resolver.nameservers = [server]
for qname, rtype, expected_answer in tests:
# do the query and format the result as a string
try:
response = dns.resolver.query(qname, rtype)
except dns.resolver.NoNameservers:
# host did not have an answer for this query
print("Could not connect to %s for DNS query." % server)
sys.exit(1)
except (dns.resolver.NXDOMAIN, dns.resolver.NoAnswer):
# host did not have an answer for this query; not sure what the
# difference is between the two exceptions
response = ["[no value]"]
response = ";".join(str(r) for r in response)
response = re.sub(r"(\"p=).*(\")", r"\1__KEY__\2", response) # normalize DKIM key
response = response.replace("\"\" ", "") # normalize TXT records (DNSSEC signing inserts empty text string components)
first = True
resolver = dns.resolver.get_default_resolver()
resolver.nameservers = [server]
for qname, rtype, expected_answer in tests:
# do the query and format the result as a string
try:
response = dns.resolver.query(qname, rtype)
except dns.resolver.NoNameservers:
# host did not have an answer for this query
print("Could not connect to %s for DNS query." % server)
sys.exit(1)
except (dns.resolver.NXDOMAIN, dns.resolver.NoAnswer):
# host did not have an answer for this query; not sure what the
# difference is between the two exceptions
response = ["[no value]"]
response = ";".join(str(r) for r in response)
# normalize DKIM key
response = re.sub(r"(\"p=).*(\")", r"\1__KEY__\2", response)
# normalize TXT records (DNSSEC signing inserts empty text
# string components)
response = response.replace("\"\" ", "")
# is it right?
if response == expected_answer:
#print(server, ":", qname, rtype, "?", response)
continue
# is it right?
if response == expected_answer:
#print(server, ":", qname, rtype, "?", response)
continue
# show prolem
if first:
print("Incorrect DNS Response from", description)
print()
print("QUERY ", "RESPONSE ", "CORRECT VALUE", sep='\t')
first = False
# show problem
if first:
print("Incorrect DNS Response from", description)
print()
print("QUERY ", "RESPONSE ", "CORRECT VALUE", sep='\t')
first = False
print((qname + "/" + rtype).ljust(20), response.ljust(12), expected_answer, sep='\t')
return first # success
print((qname + "/" + rtype).ljust(20), response.ljust(12), expected_answer, sep='\t')
# success
return first
# Test the response from the machine itself.
if not test(ipaddr, "Mail-in-a-Box"):
print ()
print ("Please run the Mail-in-a-Box setup script on %s again." % hostname)
sys.exit(1)
print ()
print ("Please run the Mail-in-a-Box setup script on %s again." % hostname)
sys.exit(1)
else:
print ("The Mail-in-a-Box provided correct DNS answers.")
print ()
print ("The Mail-in-a-Box provided correct DNS answers.")
print ()
# If those settings are OK, also test Google's Public DNS
# to see if the machine is hooked up to recursive DNS properly.
if not test("8.8.8.8", "Google Public DNS"):
print ()
print ("Check that the nameserver settings for %s are correct at your domain registrar. It may take a few hours for Google Public DNS to update after changes on your Mail-in-a-Box." % hostname)
sys.exit(1)
else:
print ("Your domain registrar or DNS host appears to be configured correctly as well. Public DNS provides the same answers.")
print ()
# If those settings are OK, also test Google's Public DNS
# to see if the machine is hooked up to recursive DNS properly.
if not test("8.8.8.8", "Google Public DNS"):
print ()
print ("Check that the nameserver settings for %s are correct at your domain registrar. It may take a few hours for Google Public DNS to update after changes on your Mail-in-a-Box." % hostname)
sys.exit(1)
else:
print ("Your domain registrar or DNS host appears to be configured correctly as well. Public DNS provides the same answers.")
print ()
# And if that's OK, also check reverse DNS (the PTR record).
if not test_ptr("8.8.8.8", "Google Public DNS (Reverse DNS)"):
print ()
print ("The reverse DNS for %s is not correct. Consult your ISP for how to set the reverse DNS (also called the PTR record) for %s to %s." % (hostname, hostname, ipaddr))
sys.exit(1)
else:
print ("And the reverse DNS for the domain is correct.")
print ()
print ("DNS is OK.")
# And if that's OK, also check reverse DNS (the PTR record).
if not test_ptr("8.8.8.8", "Google Public DNS (Reverse DNS)"):
print ()
print ("The reverse DNS for %s is not correct. Consult your ISP for how to set the reverse DNS (also called the PTR record) for %s to %s." % (hostname, hostname, ipaddr))
sys.exit(1)
else:
print ("And the reverse DNS for the domain is correct.")
print ()
print ("DNS is OK.")

117
tests/test_mail.py
View File

@ -1,28 +1,34 @@
#!/usr/bin/env python3
# Tests sending and receiving mail by sending a test message to yourself.
import sys, imaplib, smtplib, uuid, time
import socket, dns.reversename, dns.resolver
import sys
import imaplib
import smtplib
import uuid
import time
import socket
import dns.reversename
import dns.resolver
if len(sys.argv) < 3:
print("Usage: tests/mail.py hostname emailaddress password")
sys.exit(1)
print("Usage: tests/mail.py hostname emailaddress password")
sys.exit(1)
host, emailaddress, pw = sys.argv[1:4]
# Attempt to login with IMAP. Our setup uses email addresses
# as IMAP/SMTP usernames.
try:
M = imaplib.IMAP4_SSL(host)
M.login(emailaddress, pw)
M = imaplib.IMAP4_SSL(host)
M.login(emailaddress, pw)
except OSError as e:
print("Connection error:", e)
sys.exit(1)
print("Connection error:", e)
sys.exit(1)
except imaplib.IMAP4.error as e:
# any sort of login error
e = ", ".join(a.decode("utf8") for a in e.args)
print("IMAP error:", e)
sys.exit(1)
# any sort of login error
e = ", ".join(a.decode("utf8") for a in e.args)
print("IMAP error:", e)
sys.exit(1)
M.select()
print("IMAP login is OK.")
@ -35,10 +41,10 @@ To: {emailto}
Subject: {subject}
This is a test message. It should be automatically deleted by the test script.""".format(
emailaddress=emailaddress,
emailto=emailto,
subject=mailsubject,
)
emailaddress=emailaddress,
emailto=emailto,
subject=mailsubject,
)
# Connect to the server on the SMTP submission TLS port.
server = smtplib.SMTP(host, 587)
@ -46,20 +52,21 @@ server = smtplib.SMTP(host, 587)
server.starttls()
# Verify that the EHLO name matches the server's reverse DNS.
ipaddr = socket.gethostbyname(host) # IPv4 only!
reverse_ip = dns.reversename.from_address(ipaddr) # e.g. "1.0.0.127.in-addr.arpa."
ipaddr = socket.gethostbyname(host) # IPv4 only!
reverse_ip = dns.reversename.from_address(ipaddr) # e.g. "1.0.0.127.in-addr.arpa."
try:
reverse_dns = dns.resolver.query(reverse_ip, 'PTR')[0].target.to_text(omit_final_dot=True) # => hostname
reverse_dns = dns.resolver.query(reverse_ip, 'PTR')[0].target.to_text(omit_final_dot=True) # => hostname
except dns.resolver.NXDOMAIN:
print("Reverse DNS lookup failed for %s. SMTP EHLO name check skipped." % ipaddr)
reverse_dns = None
print("Reverse DNS lookup failed for %s. SMTP EHLO name check skipped." % ipaddr)
reverse_dns = None
if reverse_dns is not None:
server.ehlo_or_helo_if_needed() # must send EHLO before getting the server's EHLO name
helo_name = server.ehlo_resp.decode("utf8").split("\n")[0] # first line is the EHLO name
if helo_name != reverse_dns:
print("The server's EHLO name does not match its reverse hostname. Check DNS settings.")
else:
print("SMTP EHLO name (%s) is OK." % helo_name)
server.ehlo_or_helo_if_needed() # must send EHLO before getting the server's EHLO name
helo_name = server.ehlo_resp.decode("utf8").split("\n")[0] # first line is the EHLO name
if helo_name != reverse_dns:
print("The server's EHLO name does not match its reverse hostname. Check DNS settings.")
else:
print("SMTP EHLO name (%s) is OK." % helo_name)
# Login and send a test email.
server.login(emailaddress, pw)
@ -68,40 +75,40 @@ server.quit()
print("SMTP submission is OK.")
while True:
# Wait so the message can propagate to the inbox.
time.sleep(10)
# Wait so the message can propagate to the inbox.
time.sleep(10)
# Read the subject lines of all of the emails in the inbox
# to find our test message, and then delete it.
found = False
typ, data = M.search(None, 'ALL')
for num in data[0].split():
typ, data = M.fetch(num, '(BODY[HEADER.FIELDS (SUBJECT)])')
imapsubjectline = data[0][1].strip().decode("utf8")
if imapsubjectline == "Subject: " + mailsubject:
# We found our test message.
found = True
# Read the subject lines of all of the emails in the inbox
# to find our test message, and then delete it.
found = False
typ, data = M.search(None, 'ALL')
for num in data[0].split():
typ, data = M.fetch(num, '(BODY[HEADER.FIELDS (SUBJECT)])')
imapsubjectline = data[0][1].strip().decode("utf8")
if imapsubjectline == "Subject: " + mailsubject:
# We found our test message.
found = True
# To test DKIM, download the whole mssage body. Unfortunately,
# pydkim doesn't actually work.
# You must 'sudo apt-get install python3-dkim python3-dnspython' first.
#typ, msgdata = M.fetch(num, '(RFC822)')
#msg = msgdata[0][1]
#if dkim.verify(msg):
# print("DKIM signature on the test message is OK (verified).")
#else:
# print("DKIM signature on the test message failed verification.")
# To test DKIM, download the whole mssage body. Unfortunately,
# pydkim doesn't actually work.
# You must 'sudo apt-get install python3-dkim python3-dnspython' first.
#typ, msgdata = M.fetch(num, '(RFC822)')
#msg = msgdata[0][1]
#if dkim.verify(msg):
# print("DKIM signature on the test message is OK (verified).")
#else:
# print("DKIM signature on the test message failed verification.")
# Delete the test message.
M.store(num, '+FLAGS', '\\Deleted')
M.expunge()
# Delete the test message.
M.store(num, '+FLAGS', '\\Deleted')
M.expunge()
break
break
if found:
break
if found:
break
print("Test message not present in the inbox yet...")
print("Test message not present in the inbox yet...")
M.close()
M.logout()

4
tests/test_smtp_server.py
View File

@ -1,5 +1,6 @@
#!/usr/bin/env python3
import smtplib, sys
import smtplib
import sys
if len(sys.argv) < 3:
print("Usage: tests/smtp_server.py host email.to email.from")
@ -16,4 +17,3 @@ server = smtplib.SMTP(host, 25)
server.set_debuglevel(1)
server.sendmail(fromaddr, [toaddr], msg)
server.quit()

155
tools/editconf.py
View File

@ -18,14 +18,15 @@
# lines while the lines start with whitespace, e.g.:
#
# NAME VAL
# UE
# UE
import sys, re
import sys
import re
# sanity check
if len(sys.argv) < 3:
print("usage: python3 editconf.py /etc/file.conf [-s] [-w] [-t] NAME=VAL [NAME=VAL ...]")
sys.exit(1)
print("usage: python3 editconf.py /etc/file.conf [-s] [-w] [-t] NAME=VAL [NAME=VAL ...]")
sys.exit(1)
# parse command line arguments
filename = sys.argv[1]
@ -37,22 +38,22 @@ comment_char = "#"
folded_lines = False
testing = False
while settings[0][0] == "-" and settings[0] != "--":
opt = settings.pop(0)
if opt == "-s":
# Space is the delimiter
delimiter = " "
delimiter_re = r"\s+"
elif opt == "-w":
# Line folding is possible in this file.
folded_lines = True
elif opt == "-c":
# Specifies a different comment character.
comment_char = settings.pop(0)
elif opt == "-t":
testing = True
else:
print("Invalid option.")
sys.exit(1)
opt = settings.pop(0)
if opt == "-s":
# Space is the delimiter
delimiter = " "
delimiter_re = r"\s+"
elif opt == "-w":
# Line folding is possible in this file.
folded_lines = True
elif opt == "-c":
# Specifies a different comment character.
comment_char = settings.pop(0)
elif opt == "-t":
testing = True
else:
print("Invalid option.")
sys.exit(1)
# create the new config file in memory
@ -61,67 +62,69 @@ buf = ""
input_lines = list(open(filename))
while len(input_lines) > 0:
line = input_lines.pop(0)
line = input_lines.pop(0)
# If this configuration file uses folded lines, append any folded lines
# into our input buffer.
if folded_lines and line[0] not in (comment_char, " ", ""):
while len(input_lines) > 0 and input_lines[0][0] in " \t":
line += input_lines.pop(0)
# If this configuration file uses folded lines, append any folded lines
# into our input buffer.
if folded_lines and line[0] not in (comment_char, " ", ""):
while len(input_lines) > 0 and input_lines[0][0] in " \t":
line += input_lines.pop(0)
# See if this line is for any settings passed on the command line.
for i in range(len(settings)):
# Check that this line contain this setting from the command-line arguments.
name, val = settings[i].split("=", 1)
m = re.match(
"(\s*)"
+ "(" + re.escape(comment_char) + "\s*)?"
+ re.escape(name) + delimiter_re + "(.*?)\s*$",
line, re.S)
if not m: continue
indent, is_comment, existing_val = m.groups()
# See if this line is for any settings passed on the command line.
for i in range(len(settings)):
# Check that this line contain this setting from the command-line arguments.
name, val = settings[i].split("=", 1)
m = re.match(
"(\s*)" +
"(" + re.escape(comment_char) + "\s*)?" +
re.escape(name) + delimiter_re + "(.*?)\s*$",
line, re.S)
if not m:
continue
indent, is_comment, existing_val = m.groups()
# If this is already the setting, do nothing.
if is_comment is None and existing_val == val:
# It may be that we've already inserted this setting higher
# in the file so check for that first.
if i in found:
break
buf += line
found.add(i)
break
# comment-out the existing line (also comment any folded lines)
if is_comment is None:
buf += comment_char + line.rstrip().replace("\n", "\n" + comment_char) + "\n"
else:
# the line is already commented, pass it through
buf += line
# if this option oddly appears more than once, don't add the setting again
if i in found:
break
# add the new setting
buf += indent + name + delimiter + val + "\n"
# note that we've applied this option
found.add(i)
break
else:
# If did not match any setting names, pass this line through.
buf += line
# If this is already the setting, do nothing.
if is_comment is None and existing_val == val:
# It may be that we've already inserted this setting higher
# in the file so check for that first.
if i in found: break
buf += line
found.add(i)
break
# comment-out the existing line (also comment any folded lines)
if is_comment is None:
buf += comment_char + line.rstrip().replace("\n", "\n" + comment_char) + "\n"
else:
# the line is already commented, pass it through
buf += line
# if this option oddly appears more than once, don't add the setting again
if i in found:
break
# add the new setting
buf += indent + name + delimiter + val + "\n"
# note that we've applied this option
found.add(i)
break
else:
# If did not match any setting names, pass this line through.
buf += line
# Put any settings we didn't see at the end of the file.
for i in range(len(settings)):
if i not in found:
name, val = settings[i].split("=", 1)
buf += name + delimiter + val + "\n"
if i not in found:
name, val = settings[i].split("=", 1)
buf += name + delimiter + val + "\n"
if not testing:
# Write out the new file.
with open(filename, "w") as f:
f.write(buf)
# Write out the new file.
with open(filename, "w") as f:
f.write(buf)
else:
# Just print the new file to stdout.
print(buf)
# Just print the new file to stdout.
print(buf)

192
tools/mail.py
View File

@ -1,124 +1,132 @@
#!/usr/bin/python3
import sys, getpass, urllib.request, urllib.error, json
import sys
import getpass
import urllib.request
import urllib.error
import json
def mgmt(cmd, data=None, is_json=False):
# The base URL for the management daemon. (Listens on IPv4 only.)
mgmt_uri = 'http://127.0.0.1:10222'
# The base URL for the management daemon. (Listens on IPv4 only.)
mgmt_uri = 'http://127.0.0.1:10222'
setup_key_auth(mgmt_uri)
setup_key_auth(mgmt_uri)
req = urllib.request.Request(mgmt_uri + cmd, urllib.parse.urlencode(data).encode("utf8") if data else None)
try:
response = urllib.request.urlopen(req)
except urllib.error.HTTPError as e:
if e.code == 401:
try:
print(e.read().decode("utf8"))
except:
pass
print("The management daemon refused access. The API key file may be out of sync. Try 'service mailinabox restart'.", file=sys.stderr)
elif hasattr(e, 'read'):
print(e.read().decode('utf8'), file=sys.stderr)
else:
print(e, file=sys.stderr)
sys.exit(1)
resp = response.read().decode('utf8')
if is_json:
resp = json.loads(resp)
return resp
req = urllib.request.Request(mgmt_uri + cmd, urllib.parse.urlencode(data).encode("utf8") if data else None)
try:
response = urllib.request.urlopen(req)
except urllib.error.HTTPError as e:
if e.code == 401:
try:
print(e.read().decode("utf8"))
except:
pass
print("The management daemon refused access. The API key file may be out of sync. Try 'service mailinabox restart'.", file=sys.stderr)
elif hasattr(e, 'read'):
print(e.read().decode('utf8'), file=sys.stderr)
else:
print(e, file=sys.stderr)
sys.exit(1)
resp = response.read().decode('utf8')
if is_json: resp = json.loads(resp)
return resp
def read_password():
first = getpass.getpass('password: ')
second = getpass.getpass(' (again): ')
while first != second:
print('Passwords not the same. Try again.')
first = getpass.getpass('password: ')
second = getpass.getpass(' (again): ')
return first
first = getpass.getpass('password: ')
second = getpass.getpass(' (again): ')
while first != second:
print('Passwords not the same. Try again.')
first = getpass.getpass('password: ')
second = getpass.getpass(' (again): ')
return first
def setup_key_auth(mgmt_uri):
key = open('/var/lib/mailinabox/api.key').read().strip()
key = open('/var/lib/mailinabox/api.key').read().strip()
auth_handler = urllib.request.HTTPBasicAuthHandler()
auth_handler.add_password(
realm='Mail-in-a-Box Management Server',
uri=mgmt_uri,
user=key,
passwd='')
opener = urllib.request.build_opener(auth_handler)
urllib.request.install_opener(opener)
auth_handler = urllib.request.HTTPBasicAuthHandler()
auth_handler.add_password(
realm='Mail-in-a-Box Management Server',
uri=mgmt_uri,
user=key,
passwd='')
opener = urllib.request.build_opener(auth_handler)
urllib.request.install_opener(opener)
if len(sys.argv) < 2:
print("Usage: ")
print(" tools/mail.py user (lists users)")
print(" tools/mail.py user add user@domain.com [password]")
print(" tools/mail.py user password user@domain.com [password]")
print(" tools/mail.py user remove user@domain.com")
print(" tools/mail.py user make-admin user@domain.com")
print(" tools/mail.py user remove-admin user@domain.com")
print(" tools/mail.py user admins (lists admins)")
print(" tools/mail.py alias (lists aliases)")
print(" tools/mail.py alias add incoming.name@domain.com sent.to@other.domain.com")
print(" tools/mail.py alias add incoming.name@domain.com 'sent.to@other.domain.com, multiple.people@other.domain.com'")
print(" tools/mail.py alias remove incoming.name@domain.com")
print()
print("Removing a mail user does not delete their mail folders on disk. It only prevents IMAP/SMTP login.")
print()
print("Usage: ")
print(" tools/mail.py user (lists users)")
print(" tools/mail.py user add user@domain.com [password]")
print(" tools/mail.py user password user@domain.com [password]")
print(" tools/mail.py user remove user@domain.com")
print(" tools/mail.py user make-admin user@domain.com")
print(" tools/mail.py user remove-admin user@domain.com")
print(" tools/mail.py user admins (lists admins)")
print(" tools/mail.py alias (lists aliases)")
print(" tools/mail.py alias add incoming.name@domain.com sent.to@other.domain.com")
print(" tools/mail.py alias add incoming.name@domain.com 'sent.to@other.domain.com, multiple.people@other.domain.com'")
print(" tools/mail.py alias remove incoming.name@domain.com")
print()
print("Removing a mail user does not delete their mail folders on disk. It only prevents IMAP/SMTP login.")
print()
elif sys.argv[1] == "user" and len(sys.argv) == 2:
# Dump a list of users, one per line. Mark admins with an asterisk.
users = mgmt("/mail/users?format=json", is_json=True)
for domain in users:
for user in domain["users"]:
if user['status'] == 'inactive': continue
print(user['email'], end='')
if "admin" in user['privileges']:
print("*", end='')
print()
# Dump a list of users, one per line. Mark admins with an asterisk.
users = mgmt("/mail/users?format=json", is_json=True)
for domain in users:
for user in domain["users"]:
if user['status'] == 'inactive':
continue
print(user['email'], end='')
if "admin" in user['privileges']:
print("*", end='')
print()
elif sys.argv[1] == "user" and sys.argv[2] in ("add", "password"):
if len(sys.argv) < 5:
if len(sys.argv) < 4:
email = input("email: ")
else:
email = sys.argv[3]
pw = read_password()
else:
email, pw = sys.argv[3:5]
if len(sys.argv) < 5:
if len(sys.argv) < 4:
email = input("email: ")
else:
email = sys.argv[3]
pw = read_password()
else:
email, pw = sys.argv[3:5]
if sys.argv[2] == "add":
print(mgmt("/mail/users/add", { "email": email, "password": pw }))
elif sys.argv[2] == "password":
print(mgmt("/mail/users/password", { "email": email, "password": pw }))
if sys.argv[2] == "add":
print(mgmt("/mail/users/add", {"email": email, "password": pw}))
elif sys.argv[2] == "password":
print(mgmt("/mail/users/password", {"email": email, "password": pw}))
elif sys.argv[1] == "user" and sys.argv[2] == "remove" and len(sys.argv) == 4:
print(mgmt("/mail/users/remove", { "email": sys.argv[3] }))
print(mgmt("/mail/users/remove", {"email": sys.argv[3]}))
elif sys.argv[1] == "user" and sys.argv[2] in ("make-admin", "remove-admin") and len(sys.argv) == 4:
if sys.argv[2] == "make-admin":
action = "add"
else:
action = "remove"
print(mgmt("/mail/users/privileges/" + action, { "email": sys.argv[3], "privilege": "admin" }))
if sys.argv[2] == "make-admin":
action = "add"
else:
action = "remove"
print(mgmt("/mail/users/privileges/" + action, {"email": sys.argv[3], "privilege": "admin"}))
elif sys.argv[1] == "user" and sys.argv[2] == "admins":
# Dump a list of admin users.
users = mgmt("/mail/users?format=json", is_json=True)
for domain in users:
for user in domain["users"]:
if "admin" in user['privileges']:
print(user['email'])
# Dump a list of admin users.
users = mgmt("/mail/users?format=json", is_json=True)
for domain in users:
for user in domain["users"]:
if "admin" in user['privileges']:
print(user['email'])
elif sys.argv[1] == "alias" and len(sys.argv) == 2:
print(mgmt("/mail/aliases"))
print(mgmt("/mail/aliases"))
elif sys.argv[1] == "alias" and sys.argv[2] == "add" and len(sys.argv) == 5:
print(mgmt("/mail/aliases/add", { "source": sys.argv[3], "destination": sys.argv[4] }))
print(mgmt("/mail/aliases/add", {"source": sys.argv[3], "destination": sys.argv[4]}))
elif sys.argv[1] == "alias" and sys.argv[2] == "remove" and len(sys.argv) == 4:
print(mgmt("/mail/aliases/remove", { "source": sys.argv[3] }))
print(mgmt("/mail/aliases/remove", {"source": sys.argv[3]}))
else:
print("Invalid command-line arguments.")
sys.exit(1)
print("Invalid command-line arguments.")
sys.exit(1)

50
tools/parse-nginx-log-bootstrap-accesses.py
View File

@ -4,7 +4,11 @@
# access log to see how many people are installing Mail-in-a-Box each day, by
# looking at accesses to the bootstrap.sh script.
import re, glob, gzip, os.path, json
import re
import glob
import gzip
import os.path
import json
import dateutil.parser
outfn = "/home/user-data/www/mailinabox.email/install-stats.json"
@ -15,35 +19,35 @@ accesses = set()
# Scan the current and rotated access logs.
for fn in glob.glob("/var/log/nginx/access.log*"):
# Gunzip if necessary.
if fn.endswith(".gz"):
f = gzip.open(fn)
else:
f = open(fn, "rb")
# Gunzip if necessary.
if fn.endswith(".gz"):
f = gzip.open(fn)
else:
f = open(fn, "rb")
# Loop through the lines in the access log.
with f:
for line in f:
# Find lines that are GETs on /bootstrap.sh by either curl or wget.
m = re.match(rb"(?P<ip>\S+) - - \[(?P<date>.*?)\] \"GET /bootstrap.sh HTTP/.*\" 200 \d+ .* \"(?:curl|wget)", line, re.I)
if m:
date, time = m.group("date").decode("ascii").split(":", 1)
date = dateutil.parser.parse(date).date().isoformat()
ip = m.group("ip").decode("ascii")
accesses.add( (date, ip) )
# Loop through the lines in the access log.
with f:
for line in f:
# Find lines that are GETs on /bootstrap.sh by either curl or wget.
m = re.match(rb"(?P<ip>\S+) - - \[(?P<date>.*?)\] \"GET /bootstrap.sh HTTP/.*\" 200 \d+ .* \"(?:curl|wget)", line, re.I)
if m:
date, time = m.group("date").decode("ascii").split(":", 1)
date = dateutil.parser.parse(date).date().isoformat()
ip = m.group("ip").decode("ascii")
accesses.add((date, ip))
# Aggregate by date.
by_date = { }
by_date = {}
for date, ip in accesses:
by_date[date] = by_date.get(date, 0) + 1
by_date[date] = by_date.get(date, 0) + 1
# Since logs are rotated, store the statistics permanently in a JSON file.
# Load in the stats from an existing file.
if os.path.exists(outfn):
existing_data = json.load(open(outfn))
for date, count in existing_data:
if date not in by_date:
by_date[date] = count
existing_data = json.load(open(outfn))
for date, count in existing_data:
if date not in by_date:
by_date[date] = count
# Turn into a list rather than a dict structure to make it ordered.
by_date = sorted(by_date.items())
@ -53,4 +57,4 @@ by_date.pop(-1)
# Write out.
with open(outfn, "w") as f:
json.dump(by_date, f, sort_keys=True, indent=True)
json.dump(by_date, f, sort_keys=True, indent=True)

773
tools/readable_bash.py
View File

@ -3,12 +3,14 @@
# Generate documentation for how this machine works by
# parsing our bash scripts!
import cgi, re
import cgi
import re
import markdown
from modgrammar import *
def generate_documentation():
print("""<!DOCTYPE html>
print("""<!DOCTYPE html>
<html>
<head>
<meta charset="utf-8">
@ -21,93 +23,93 @@ def generate_documentation():
<link rel="stylesheet" href="https://maxcdn.bootstrapcdn.com/bootstrap/3.2.0/css/bootstrap-theme.min.css">
<style>
@import url(https://fonts.googleapis.com/css?family=Iceland);
@import url(https://fonts.googleapis.com/css?family=Raleway:400,700);
@import url(https://fonts.googleapis.com/css?family=Ubuntu:300,500);
body {
font-family: Raleway, sans-serif;
font-size: 16px;
color: #555;
}
h2, h3 {
margin-top: .25em;
margin-bottom: .75em;
}
p {
margin-bottom: 1em;
}
.intro p {
margin: 1.5em 0;
}
li {
margin-bottom: .33em;
}
@import url(https://fonts.googleapis.com/css?family=Iceland);
@import url(https://fonts.googleapis.com/css?family=Raleway:400,700);
@import url(https://fonts.googleapis.com/css?family=Ubuntu:300,500);
body {
font-family: Raleway, sans-serif;
font-size: 16px;
color: #555;
}
h2, h3 {
margin-top: .25em;
margin-bottom: .75em;
}
p {
margin-bottom: 1em;
}
.intro p {
margin: 1.5em 0;
}
li {
margin-bottom: .33em;
}
.sourcefile {
padding-top: 1.5em;
padding-bottom: 1em;
font-size: 90%;
text-align: right;
}
.sourcefile a {
color: red;
}
.sourcefile {
padding-top: 1.5em;
padding-bottom: 1em;
font-size: 90%;
text-align: right;
}
.sourcefile a {
color: red;
}
.instructions .row.contd {
border-top: 1px solid #E0E0E0;
}
.instructions .row.contd {
border-top: 1px solid #E0E0E0;
}
.prose {
padding-top: 1em;
padding-bottom: 1em;
}
.terminal {
background-color: #EEE;
padding-top: 1em;
padding-bottom: 1em;
}
.prose {
padding-top: 1em;
padding-bottom: 1em;
}
.terminal {
background-color: #EEE;
padding-top: 1em;
padding-bottom: 1em;
}
ul {
padding-left: 1.25em;
}
ul {
padding-left: 1.25em;
}
pre {
color: black;
border: 0;
background: none;
font-size: 100%;
}
pre {
color: black;
border: 0;
background: none;
font-size: 100%;
}
div.write-to {
margin: 0 0 1em .5em;
}
div.write-to p {
padding: .5em;
margin: 0;
}
div.write-to .filename {
padding: .25em .5em;
background-color: #666;
color: white;
font-family: monospace;
font-weight: bold;
}
div.write-to .filename span {
font-family: sans-serif;
font-weight: normal;
}
div.write-to pre {
margin: 0;
padding: .5em;
border: 1px solid #999;
border-radius: 0;
font-size: 90%;
}
div.write-to {
margin: 0 0 1em .5em;
}
div.write-to p {
padding: .5em;
margin: 0;
}
div.write-to .filename {
padding: .25em .5em;
background-color: #666;
color: white;
font-family: monospace;
font-weight: bold;
}
div.write-to .filename span {
font-family: sans-serif;
font-weight: normal;
}
div.write-to pre {
margin: 0;
padding: .5em;
border: 1px solid #999;
border-radius: 0;
font-size: 90%;
}
pre.shell > div:before {
content: "$ ";
color: #666;
}
pre.shell > div:before {
content: "$ ";
color: #666;
}
</style>
</head>
<body>
@ -123,359 +125,408 @@ def generate_documentation():
<div class="container instructions">
""")
parser = Source.parser()
for line in open("setup/start.sh"):
try:
fn = parser.parse_string(line).filename()
except:
continue
if fn in ("setup/start.sh", "setup/preflight.sh", "setup/questions.sh", "setup/firstuser.sh", "setup/management.sh"):
continue
parser = Source.parser()
for line in open("setup/start.sh"):
try:
fn = parser.parse_string(line).filename()
except:
continue
if fn in ("setup/start.sh", "setup/preflight.sh", "setup/questions.sh", "setup/firstuser.sh", "setup/management.sh"):
continue
import sys
print(fn, file=sys.stderr)
import sys
print(fn, file=sys.stderr)
print(BashScript.parse(fn))
print(BashScript.parse(fn))
print("""
print("""
<script src="https://ajax.googleapis.com/ajax/libs/jquery/1.10.1/jquery.min.js"></script>
<script src="https://maxcdn.bootstrapcdn.com/bootstrap/3.2.0/js/bootstrap.min.js"></script>
<script>
$(function() {
$('.terminal').each(function() {
$(this).outerHeight( $(this).parent().innerHeight() );
});
$('.terminal').each(function() {
$(this).outerHeight( $(this).parent().innerHeight() );
});
})
</script>
</script>
</body>
</html>
""")
class HashBang(Grammar):
grammar = (L('#!'), REST_OF_LINE, EOL)
def value(self):
return ""
grammar = (L('#!'), REST_OF_LINE, EOL)
def value(self):
return ""
def strip_indent(s):
s = s.replace("\t", " ")
lines = s.split("\n")
try:
min_indent = min(len(re.match(r"\s*", line).group(0)) for line in lines if len(line) > 0)
except ValueError:
# No non-empty lines.
min_indent = 0
lines = [line[min_indent:] for line in lines]
return "\n".join(lines)
s = s.replace("\t", " ")
lines = s.split("\n")
try:
min_indent = min(len(re.match(r"\s*", line).group(0)) for line in lines if len(line) > 0)
except ValueError:
# No non-empty lines.
min_indent = 0
lines = [line[min_indent:] for line in lines]
return "\n".join(lines)
class Comment(Grammar):
grammar = ONE_OR_MORE(ZERO_OR_MORE(SPACE), L('#'), REST_OF_LINE, EOL)
def value(self):
if self.string.replace("#", "").strip() == "":
return "\n"
lines = [x[2].string for x in self[0]]
content = "\n".join(lines)
content = strip_indent(content)
return markdown.markdown(content, output_format="html4") + "\n\n"
grammar = ONE_OR_MORE(ZERO_OR_MORE(SPACE), L('#'), REST_OF_LINE, EOL)
def value(self):
if self.string.replace("#", "").strip() == "":
return "\n"
lines = [x[2].string for x in self[0]]
content = "\n".join(lines)
content = strip_indent(content)
return markdown.markdown(content, output_format="html4") + "\n\n"
FILENAME = WORD('a-z0-9-/.')
class Source(Grammar):
grammar = ((L('.') | L('source')), L(' '), FILENAME, Comment | EOL)
def filename(self):
return self[2].string.strip()
def value(self):
return BashScript.parse(self.filename())
grammar = ((L('.') | L('source')), L(' '), FILENAME, Comment | EOL)
def filename(self):
return self[2].string.strip()
def value(self):
return BashScript.parse(self.filename())
class CatEOF(Grammar):
grammar = (ZERO_OR_MORE(SPACE), L('cat '), L('>') | L('>>'), L(' '), ANY_EXCEPT(WHITESPACE), L(" <<"), OPTIONAL(SPACE), L("EOF"), EOL, REPEAT(ANY, greedy=False), EOL, L("EOF"), EOL)
def value(self):
content = self[9].string
content = re.sub(r"\\([$])", r"\1", content) # un-escape bash-escaped characters
return "<div class='write-to'><div class='filename'>%s <span>(%s)</span></div><pre>%s</pre></div>\n" \
% (self[4].string,
"overwrite" if ">>" not in self[2].string else "append to",
cgi.escape(content))
grammar = (ZERO_OR_MORE(SPACE), L('cat '), L('>') | L('>>'), L(' '), ANY_EXCEPT(WHITESPACE), L(" <<"), OPTIONAL(SPACE), L("EOF"), EOL, REPEAT(ANY, greedy=False), EOL, L("EOF"), EOL)
def value(self):
content = self[9].string
# un-escape bash-escaped characters
content = re.sub(r"\\([$])", r"\1", content)
return "<div class='write-to'><div class='filename'>%s <span>(%s)</span></div><pre>%s</pre></div>\n" \
% (self[4].string,
"overwrite" if ">>" not in self[2].string else "append to",
cgi.escape(content))
class HideOutput(Grammar):
grammar = (L("hide_output "), REF("BashElement"))
def value(self):
return self[1].value()
grammar = (L("hide_output "), REF("BashElement"))
def value(self):
return self[1].value()
class EchoLine(Grammar):
grammar = (OPTIONAL(SPACE), L("echo "), REST_OF_LINE, EOL)
def value(self):
if "|" in self.string or ">" in self.string:
return "<pre class='shell'><div>" + recode_bash(self.string.strip()) + "</div></pre>\n"
return ""
grammar = (OPTIONAL(SPACE), L("echo "), REST_OF_LINE, EOL)
def value(self):
if "|" in self.string or ">" in self.string:
return "<pre class='shell'><div>" + recode_bash(self.string.strip()) + "</div></pre>\n"
return ""
class EditConf(Grammar):
grammar = (
L('tools/editconf.py '),
FILENAME,
SPACE,
OPTIONAL((LIST_OF(
L("-w") | L("-s") | L("-c ;"),
sep=SPACE,
), SPACE)),
REST_OF_LINE,
OPTIONAL(SPACE),
EOL
)
def value(self):
conffile = self[1]
options = []
eq = "="
if self[3] and "-s" in self[3].string: eq = " "
for opt in re.split("\s+", self[4].string):
k, v = opt.split("=", 1)
v = re.sub(r"\n+", "", fixup_tokens(v)) # not sure why newlines are getting doubled
options.append("%s%s%s" % (k, eq, v))
return "<div class='write-to'><div class='filename'>" + self[1].string + " <span>(change settings)</span></div><pre>" + "\n".join(cgi.escape(s) for s in options) + "</pre></div>\n"
grammar = (
L('tools/editconf.py '),
FILENAME,
SPACE,
OPTIONAL((LIST_OF(
L("-w") | L("-s") | L("-c ;"),
sep=SPACE,
), SPACE)),
REST_OF_LINE,
OPTIONAL(SPACE),
EOL
)
def value(self):
conffile = self[1]
options = []
eq = "="
if self[3] and "-s" in self[3].string:
eq = " "
for opt in re.split("\s+", self[4].string):
k, v = opt.split("=", 1)
# not sure why newlines are getting doubled
v = re.sub(r"\n+", "", fixup_tokens(v))
options.append("%s%s%s" % (k, eq, v))
return "<div class='write-to'><div class='filename'>" + self[1].string + " <span>(change settings)</span></div><pre>" + "\n".join(cgi.escape(s) for s in options) + "</pre></div>\n"
class CaptureOutput(Grammar):
grammar = OPTIONAL(SPACE), WORD("A-Za-z_"), L('=$('), REST_OF_LINE, L(")"), OPTIONAL(L(';')), EOL
def value(self):
cmd = self[3].string
cmd = cmd.replace("; ", "\n")
return "<div class='write-to'><div class='filename'>$" + self[1].string + "=</div><pre>" + cgi.escape(cmd) + "</pre></div>\n"
grammar = OPTIONAL(SPACE), WORD("A-Za-z_"), L('=$('), REST_OF_LINE, L(")"), OPTIONAL(L(';')), EOL
def value(self):
cmd = self[3].string
cmd = cmd.replace("; ", "\n")
return "<div class='write-to'><div class='filename'>$" + self[1].string + "=</div><pre>" + cgi.escape(cmd) + "</pre></div>\n"
class SedReplace(Grammar):
grammar = OPTIONAL(SPACE), L('sed -i "s/'), OPTIONAL(L('^')), ONE_OR_MORE(WORD("-A-Za-z0-9 #=\\{};.*$_!()")), L('/'), ONE_OR_MORE(WORD("-A-Za-z0-9 #=\\{};.*$_!()")), L('/"'), SPACE, FILENAME, EOL
def value(self):
return "<div class='write-to'><div class='filename'>edit<br>" + self[8].string + "</div><p>replace</p><pre>" + cgi.escape(self[3].string.replace(".*", ". . .")) + "</pre><p>with</p><pre>" + cgi.escape(self[5].string.replace("\\n", "\n").replace("\\t", "\t")) + "</pre></div>\n"
grammar = OPTIONAL(SPACE), L('sed -i "s/'), OPTIONAL(L('^')), ONE_OR_MORE(WORD("-A-Za-z0-9 #=\\{};.*$_!()")), L('/'), ONE_OR_MORE(WORD("-A-Za-z0-9 #=\\{};.*$_!()")), L('/"'), SPACE, FILENAME, EOL
def value(self):
return "<div class='write-to'><div class='filename'>edit<br>" + self[8].string + "</div><p>replace</p><pre>" + cgi.escape(self[3].string.replace(".*", ". . .")) + "</pre><p>with</p><pre>" + cgi.escape(self[5].string.replace("\\n", "\n").replace("\\t", "\t")) + "</pre></div>\n"
class EchoPipe(Grammar):
grammar = OPTIONAL(SPACE), L("echo "), REST_OF_LINE, L(' | '), REST_OF_LINE, EOL
def value(self):
text = " ".join("\"%s\"" % s for s in self[2].string.split(" "))
return "<pre class='shell'><div>echo " + recode_bash(text) + " \<br> | " + recode_bash(self[4].string) + "</div></pre>\n"
grammar = OPTIONAL(SPACE), L("echo "), REST_OF_LINE, L(' | '), REST_OF_LINE, EOL
def value(self):
text = " ".join("\"%s\"" % s for s in self[2].string.split(" "))
return "<pre class='shell'><div>echo " + recode_bash(text) + " \<br> | " + recode_bash(self[4].string) + "</div></pre>\n"
def shell_line(bash):
return "<pre class='shell'><div>" + recode_bash(bash.strip()) + "</div></pre>\n"
return "<pre class='shell'><div>" + recode_bash(bash.strip()) + "</div></pre>\n"
class AptGet(Grammar):
grammar = (ZERO_OR_MORE(SPACE), L("apt_install "), REST_OF_LINE, EOL)
def value(self):
return shell_line("apt-get install -y " + re.sub(r"\s+", " ", self[2].string))
grammar = (ZERO_OR_MORE(SPACE), L("apt_install "), REST_OF_LINE, EOL)
def value(self):
return shell_line("apt-get install -y " + re.sub(r"\s+", " ", self[2].string))
class UfwAllow(Grammar):
grammar = (ZERO_OR_MORE(SPACE), L("ufw_allow "), REST_OF_LINE, EOL)
def value(self):
return shell_line("ufw allow " + self[2].string)
grammar = (ZERO_OR_MORE(SPACE), L("ufw_allow "), REST_OF_LINE, EOL)
def value(self):
return shell_line("ufw allow " + self[2].string)
class RestartService(Grammar):
grammar = (ZERO_OR_MORE(SPACE), L("restart_service "), REST_OF_LINE, EOL)
def value(self):
return shell_line("service " + self[2].string + " restart")
grammar = (ZERO_OR_MORE(SPACE), L("restart_service "), REST_OF_LINE, EOL)
def value(self):
return shell_line("service " + self[2].string + " restart")
class OtherLine(Grammar):
grammar = (REST_OF_LINE, EOL)
def value(self):
if self.string.strip() == "": return ""
if "source setup/functions.sh" in self.string: return ""
if "source /etc/mailinabox.conf" in self.string: return ""
return "<pre class='shell'><div>" + recode_bash(self.string.strip()) + "</div></pre>\n"
grammar = (REST_OF_LINE, EOL)
def value(self):
if self.string.strip() == "":
return ""
if "source setup/functions.sh" in self.string:
return ""
if "source /etc/mailinabox.conf" in self.string:
return ""
return "<pre class='shell'><div>" + recode_bash(self.string.strip()) + "</div></pre>\n"
class BashElement(Grammar):
grammar = Comment | CatEOF | EchoPipe | EchoLine | HideOutput | EditConf | SedReplace | AptGet | UfwAllow | RestartService | OtherLine
def value(self):
return self[0].value()
grammar = Comment | CatEOF | EchoPipe | EchoLine | HideOutput | EditConf | SedReplace | AptGet | UfwAllow | RestartService | OtherLine
def value(self):
return self[0].value()
# Make some special characters to private use Unicode code points.
bash_special_characters1 = {
"\n": "\uE000",
" ": "\uE001",
"\n": "\uE000",
" ": "\uE001",
}
bash_special_characters2 = {
"$": "\uE010",
"$": "\uE010",
}
bash_escapes = {
"n": "\uE020",
"t": "\uE021",
"n": "\uE020",
"t": "\uE021",
}
def quasitokenize(bashscript):
# Make a parse of bash easier by making the tokenization easy.
newscript = ""
quote_mode = None
escape_next = False
line_comment = False
subshell = 0
for c in bashscript:
if line_comment:
# We're in a comment until the end of the line.
newscript += c
if c == '\n':
line_comment = False
elif escape_next:
# Previous character was a \. Normally the next character
# comes through literally, but escaped newlines are line
# continuations and some escapes are for special characters
# which we'll recode and then turn back into escapes later.
if c == "\n":
c = " "
elif c in bash_escapes:
c = bash_escapes[c]
newscript += c
escape_next = False
elif c == "\\":
# Escaping next character.
escape_next = True
elif quote_mode is None and c in ('"', "'"):
# Starting a quoted word.
quote_mode = c
elif c == quote_mode:
# Ending a quoted word.
quote_mode = None
elif quote_mode is not None and quote_mode != "EOF" and c in bash_special_characters1:
# Replace special tokens within quoted words so that they
# don't interfere with tokenization later.
newscript += bash_special_characters1[c]
elif quote_mode is None and c == '#':
# Start of a line comment.
newscript += c
line_comment = True
elif quote_mode is None and c == ';' and subshell == 0:
# End of a statement.
newscript += "\n"
elif quote_mode is None and c == '(':
# Start of a subshell.
newscript += c
subshell += 1
elif quote_mode is None and c == ')':
# End of a subshell.
newscript += c
subshell -= 1
elif quote_mode is None and c == '\t':
# Make these just spaces.
if newscript[-1] != " ":
newscript += " "
elif quote_mode is None and c == ' ':
# Collapse consecutive spaces.
if newscript[-1] != " ":
newscript += " "
elif c in bash_special_characters2:
newscript += bash_special_characters2[c]
else:
# All other characters.
newscript += c
# Make a parse of bash easier by making the tokenization easy.
newscript = ""
quote_mode = None
escape_next = False
line_comment = False
subshell = 0
for c in bashscript:
if line_comment:
# We're in a comment until the end of the line.
newscript += c
if c == '\n':
line_comment = False
elif escape_next:
# Previous character was a \. Normally the next character
# comes through literally, but escaped newlines are line
# continuations and some escapes are for special characters
# which we'll recode and then turn back into escapes later.
if c == "\n":
c = " "
elif c in bash_escapes:
c = bash_escapes[c]
newscript += c
escape_next = False
elif c == "\\":
# Escaping next character.
escape_next = True
elif quote_mode is None and c in ('"', "'"):
# Starting a quoted word.
quote_mode = c
elif c == quote_mode:
# Ending a quoted word.
quote_mode = None
elif quote_mode is not None and quote_mode != "EOF" and c in bash_special_characters1:
# Replace special tokens within quoted words so that they
# don't interfere with tokenization later.
newscript += bash_special_characters1[c]
elif quote_mode is None and c == '#':
# Start of a line comment.
newscript += c
line_comment = True
elif quote_mode is None and c == ';' and subshell == 0:
# End of a statement.
newscript += "\n"
elif quote_mode is None and c == '(':
# Start of a subshell.
newscript += c
subshell += 1
elif quote_mode is None and c == ')':
# End of a subshell.
newscript += c
subshell -= 1
elif quote_mode is None and c == '\t':
# Make these just spaces.
if newscript[-1] != " ":
newscript += " "
elif quote_mode is None and c == ' ':
# Collapse consecutive spaces.
if newscript[-1] != " ":
newscript += " "
elif c in bash_special_characters2:
newscript += bash_special_characters2[c]
else:
# All other characters.
newscript += c
# "<< EOF" escaping.
if quote_mode is None and re.search("<<\s*EOF\n$", newscript):
quote_mode = "EOF"
elif quote_mode == "EOF" and re.search("\nEOF\n$", newscript):
quote_mode = None
# "<< EOF" escaping.
if quote_mode is None and re.search("<<\s*EOF\n$", newscript):
quote_mode = "EOF"
elif quote_mode == "EOF" and re.search("\nEOF\n$", newscript):
quote_mode = None
return newscript
return newscript
def recode_bash(s):
def requote(tok):
tok = tok.replace("\\", "\\\\")
for c in bash_special_characters2:
tok = tok.replace(c, "\\" + c)
tok = fixup_tokens(tok)
if " " in tok or '"' in tok:
tok = tok.replace("\"", "\\\"")
tok = '"' + tok +'"'
else:
tok = tok.replace("'", "\\'")
return tok
return cgi.escape(" ".join(requote(tok) for tok in s.split(" ")))
def requote(tok):
tok = tok.replace("\\", "\\\\")
for c in bash_special_characters2:
tok = tok.replace(c, "\\" + c)
tok = fixup_tokens(tok)
if " " in tok or '"' in tok:
tok = tok.replace("\"", "\\\"")
tok = '"' + tok + '"'
else:
tok = tok.replace("'", "\\'")
return tok
return cgi.escape(" ".join(requote(tok) for tok in s.split(" ")))
def fixup_tokens(s):
for c, enc in bash_special_characters1.items():
s = s.replace(enc, c)
for c, enc in bash_special_characters2.items():
s = s.replace(enc, c)
for esc, c in bash_escapes.items():
s = s.replace(c, "\\" + esc)
return s
for c, enc in bash_special_characters1.items():
s = s.replace(enc, c)
for c, enc in bash_special_characters2.items():
s = s.replace(enc, c)
for esc, c in bash_escapes.items():
s = s.replace(c, "\\" + esc)
return s
class BashScript(Grammar):
grammar = (OPTIONAL(HashBang), REPEAT(BashElement))
def value(self):
return [line.value() for line in self[1]]
grammar = (OPTIONAL(HashBang), REPEAT(BashElement))
@staticmethod
def parse(fn):
if fn in ("setup/functions.sh", "/etc/mailinabox.conf"): return ""
string = open(fn).read()
def value(self):
return [line.value() for line in self[1]]
# tokenize
string = re.sub(".* #NODOC\n", "", string)
string = re.sub("\n\s*if .*then.*|\n\s*fi|\n\s*else|\n\s*elif .*", "", string)
string = quasitokenize(string)
string = re.sub("hide_output ", "", string)
@staticmethod
def parse(fn):
if fn in ("setup/functions.sh", "/etc/mailinabox.conf"):
return ""
string = open(fn).read()
parser = BashScript.parser()
result = parser.parse_string(string)
# tokenize
string = re.sub(".* #NODOC\n", "", string)
string = re.sub("\n\s*if .*then.*|\n\s*fi|\n\s*else|\n\s*elif .*", "", string)
string = quasitokenize(string)
string = re.sub("hide_output ", "", string)
v = "<div class='row'><div class='col-xs-12 sourcefile'>view the bash source for the following section at <a href=\"%s\">%s</a></div></div>\n" \
% ("https://github.com/mail-in-a-box/mailinabox/tree/master/" + fn, fn)
parser = BashScript.parser()
result = parser.parse_string(string)
mode = 0
for item in result.value():
if item.strip() == "":
pass
elif item.startswith("<p") and not item.startswith("<pre"):
clz = ""
if mode == 2:
v += "</div>\n" # col
v += "</div>\n" # row
mode = 0
clz = "contd"
if mode == 0:
v += "<div class='row %s'>\n" % clz
v += "<div class='col-md-6 prose'>\n"
v += item
mode = 1
elif item.startswith("<h"):
if mode != 0:
v += "</div>\n" # col
v += "</div>\n" # row
v += "<div class='row'>\n"
v += "<div class='col-md-6 header'>\n"
v += item
v += "</div>\n" # col
v += "<div class='col-md-6 terminal'> </div>\n"
v += "</div>\n" # row
mode = 0
else:
if mode == 0:
v += "<div class='row'>\n"
v += "<div class='col-md-offset-6 col-md-6 terminal'>\n"
elif mode == 1:
v += "</div>\n"
v += "<div class='col-md-6 terminal'>\n"
mode = 2
v += item
v = "<div class='row'><div class='col-xs-12 sourcefile'>view the bash source for the following section at <a href=\"%s\">%s</a></div></div>\n" \
% ("https://github.com/mail-in-a-box/mailinabox/tree/master/" + fn, fn)
v += "</div>\n" # col
v += "</div>\n" # row
mode = 0
for item in result.value():
if item.strip() == "":
pass
elif item.startswith("<p") and not item.startswith("<pre"):
clz = ""
if mode == 2:
v += "</div>\n" # col
v += "</div>\n" # row
mode = 0
clz = "contd"
if mode == 0:
v += "<div class='row %s'>\n" % clz
v += "<div class='col-md-6 prose'>\n"
v += item
mode = 1
elif item.startswith("<h"):
if mode != 0:
v += "</div>\n" # col
v += "</div>\n" # row
v += "<div class='row'>\n"
v += "<div class='col-md-6 header'>\n"
v += item
v += "</div>\n" # col
v += "<div class='col-md-6 terminal'> </div>\n"
v += "</div>\n" # row
mode = 0
else:
if mode == 0:
v += "<div class='row'>\n"
v += "<div class='col-md-offset-6 col-md-6 terminal'>\n"
elif mode == 1:
v += "</div>\n"
v += "<div class='col-md-6 terminal'>\n"
mode = 2
v += item
v = fixup_tokens(v)
v += "</div>\n" # col
v += "</div>\n" # row
v = v.replace("</pre>\n<pre class='shell'>", "")
v = re.sub("<pre>([\w\W]*?)</pre>", lambda m : "<pre>" + strip_indent(m.group(1)) + "</pre>", v)
v = fixup_tokens(v)
v = re.sub(r"(\$?)PRIMARY_HOSTNAME", r"<b>box.yourdomain.com</b>", v)
v = re.sub(r"\$STORAGE_ROOT", r"<b>$STORE</b>", v)
v = re.sub(r"\$CSR_COUNTRY", r"<b>US</b>", v)
v = v.replace("`pwd`", "<code><b>/path/to/mailinabox</b></code>")
v = v.replace("</pre>\n<pre class='shell'>", "")
v = re.sub("<pre>([\w\W]*?)</pre>", lambda m: "<pre>" + strip_indent(m.group(1)) + "</pre>", v)
v = re.sub(r"(\$?)PRIMARY_HOSTNAME", r"<b>box.yourdomain.com</b>", v)
v = re.sub(r"\$STORAGE_ROOT", r"<b>$STORE</b>", v)
v = re.sub(r"\$CSR_COUNTRY", r"<b>US</b>", v)
v = v.replace("`pwd`", "<code><b>/path/to/mailinabox</b></code>")
return v
return v
def wrap_lines(text, cols=60):
ret = ""
words = re.split("(\s+)", text)
linelen = 0
for w in words:
if linelen + len(w) > cols-1:
ret += " \\\n"
ret += " "
linelen = 0
if linelen == 0 and w.strip() == "": continue
ret += w
linelen += len(w)
return ret
ret = ""
words = re.split("(\s+)", text)
linelen = 0
for w in words:
if linelen + len(w) > cols-1:
ret += " \\\n"
ret += " "
linelen = 0
if linelen == 0 and w.strip() == "":
continue
ret += w
linelen += len(w)
return ret
if __name__ == '__main__':
generate_documentation()
generate_documentation()