mirror of
https://github.com/mail-in-a-box/mailinabox.git
synced 2025-04-01 23:57:05 +00:00
Merge remote-tracking branch 'upstream/main' into merge-upstream
# Conflicts: # .gitignore # management/auth.py # management/daemon.py # management/mail_log.py # management/mailconfig.py # management/mfa.py # management/ssl_certificates.py # management/status_checks.py # management/utils.py # management/web_update.py # setup/mail-postfix.sh # setup/migrate.py # setup/preflight.sh # setup/webmail.sh # tests/test_mail.py # tools/editconf.py
This commit is contained in:
commit
d349150dd0
5
.github/workflows/commit-tests.yml
vendored
5
.github/workflows/commit-tests.yml
vendored
@ -22,13 +22,14 @@ jobs:
|
||||
path: tests/out/**/screenshot.png
|
||||
if-no-files-found: ignore
|
||||
retention-days: 5
|
||||
|
||||
|
||||
# install upstream miab, then migrate to miabldap
|
||||
upgrade-from-upstream:
|
||||
runs-on: ubuntu-22.04
|
||||
env:
|
||||
PRIMARY_HOSTNAME: box2.abc.com
|
||||
UPSTREAM_TAG: main
|
||||
# TODO: change UPSTREAM_TAG to 'main' once upstream is installable
|
||||
UPSTREAM_TAG: v67
|
||||
steps:
|
||||
- uses: actions/checkout@v2
|
||||
- name: setup
|
||||
|
1
.gitignore
vendored
1
.gitignore
vendored
@ -6,3 +6,4 @@ externals/
|
||||
api/docs/api-docs.html
|
||||
downloads/
|
||||
MAINTAIN/
|
||||
*.code-workspace
|
||||
|
@ -3,5 +3,5 @@
|
||||
before = common.conf
|
||||
|
||||
[Definition]
|
||||
failregex=<HOST> - .*GET /admin/munin/.* HTTP/1.1\" 401.*
|
||||
failregex=<HOST> - .*GET /admin/munin/.* HTTP/\d+\.\d+\" 401.*
|
||||
ignoreregex =
|
||||
|
@ -8,7 +8,7 @@
|
||||
##### details.
|
||||
#####
|
||||
|
||||
import base64, os, os.path, hmac, json, secrets
|
||||
import base64, hmac, json, secrets
|
||||
from datetime import timedelta
|
||||
|
||||
from expiringdict import ExpiringDict
|
||||
@ -32,7 +32,7 @@ class AuthService:
|
||||
def init_system_api_key(self):
|
||||
"""Write an API key to a local file so local processes can use the API"""
|
||||
|
||||
with open(self.key_path, 'r') as file:
|
||||
with open(self.key_path, encoding='utf-8') as file:
|
||||
self.key = file.read()
|
||||
|
||||
def authenticate(self, request, env, login_only=False, logout=False):
|
||||
@ -58,11 +58,13 @@ class AuthService:
|
||||
return username, password
|
||||
|
||||
username, password = parse_http_authorization_basic(request.headers.get('Authorization', ''))
|
||||
if username in (None, ""):
|
||||
raise ValueError("Authorization header invalid.")
|
||||
if username in {None, ""}:
|
||||
msg = "Authorization header invalid."
|
||||
raise ValueError(msg)
|
||||
|
||||
if username.strip() == "" and password.strip() == "":
|
||||
raise ValueError("No email address, password, session key, or API key provided.")
|
||||
msg = "No email address, password, session key, or API key provided."
|
||||
raise ValueError(msg)
|
||||
|
||||
# If user passed the system API key, grant administrative privs. This key
|
||||
# is not associated with a user.
|
||||
@ -82,7 +84,8 @@ class AuthService:
|
||||
|
||||
# If no password was given, but a username was given, we're missing some information.
|
||||
elif password.strip() == "":
|
||||
raise ValueError("Enter a password.")
|
||||
msg = "Enter a password."
|
||||
raise ValueError(msg)
|
||||
|
||||
else:
|
||||
# The user is trying to log in with a username and a password
|
||||
@ -109,7 +112,8 @@ class AuthService:
|
||||
# Authenticate.
|
||||
if not validate_login(email, pw, env):
|
||||
# Login failed.
|
||||
raise ValueError("Incorrect email address or password.")
|
||||
msg = "Incorrect email address or password."
|
||||
raise ValueError(msg)
|
||||
|
||||
# If MFA is enabled, check that MFA passes.
|
||||
status, hints = validate_auth_mfa(email, request, env)
|
||||
|
@ -17,7 +17,7 @@
|
||||
# 4) The stopped services are restarted.
|
||||
# 5) STORAGE_ROOT/backup/after-backup is executed if it exists.
|
||||
|
||||
import os, os.path, shutil, glob, re, datetime, sys
|
||||
import os, os.path, re, datetime, sys
|
||||
import dateutil.parser, dateutil.relativedelta, dateutil.tz
|
||||
import rtyaml
|
||||
from exclusiveprocess import Lock
|
||||
@ -69,7 +69,7 @@ def backup_status(env):
|
||||
"--archive-dir", backup_cache_dir,
|
||||
"--gpg-options", "'--cipher-algo=AES256'",
|
||||
"--log-fd", "1",
|
||||
] + get_duplicity_additional_args(env) + [
|
||||
*get_duplicity_additional_args(env),
|
||||
get_duplicity_target_url(config)
|
||||
],
|
||||
get_duplicity_env_vars(env),
|
||||
@ -79,7 +79,7 @@ def backup_status(env):
|
||||
# destination for the backups or the last backup job terminated unexpectedly.
|
||||
raise Exception("Something is wrong with the backup: " + collection_status)
|
||||
for line in collection_status.split('\n'):
|
||||
if line.startswith(" full") or line.startswith(" inc"):
|
||||
if line.startswith((" full", " inc")):
|
||||
backup = parse_line(line)
|
||||
backups[backup["date"]] = backup
|
||||
|
||||
@ -195,7 +195,7 @@ def get_passphrase(env):
|
||||
# only needs to be 43 base64-characters to match AES256's key
|
||||
# length of 32 bytes.
|
||||
backup_root = os.path.join(env["STORAGE_ROOT"], 'backup')
|
||||
with open(os.path.join(backup_root, 'secret_key.txt')) as f:
|
||||
with open(os.path.join(backup_root, 'secret_key.txt'), encoding="utf-8") as f:
|
||||
passphrase = f.readline().strip()
|
||||
if len(passphrase) < 43: raise Exception("secret_key.txt's first line is too short!")
|
||||
|
||||
@ -236,7 +236,7 @@ def get_duplicity_additional_args(env):
|
||||
port = 22
|
||||
if port is None:
|
||||
port = 22
|
||||
|
||||
|
||||
return [
|
||||
f"--ssh-options='-i /root/.ssh/id_rsa_miab -p {port}'",
|
||||
f"--rsync-options='-e \"/usr/bin/ssh -oStrictHostKeyChecking=no -oBatchMode=yes -p {port} -i /root/.ssh/id_rsa_miab\"'",
|
||||
@ -267,8 +267,7 @@ def get_duplicity_env_vars(env):
|
||||
return env
|
||||
|
||||
def get_target_type(config):
|
||||
protocol = config["target"].split(":")[0]
|
||||
return protocol
|
||||
return config["target"].split(":")[0]
|
||||
|
||||
def nuke_local_files(backup_dir, backup_cache_dir, config, env):
|
||||
# the files must be removed manually, duplicity won't do
|
||||
@ -351,8 +350,8 @@ def perform_backup(full_backup):
|
||||
"--exclude", backup_root,
|
||||
"--volsize", "250",
|
||||
"--gpg-options", "'--cipher-algo=AES256'",
|
||||
"--allow-source-mismatch"
|
||||
] + get_duplicity_additional_args(env) + [
|
||||
"--allow-source-mismatch",
|
||||
*get_duplicity_additional_args(env),
|
||||
env["STORAGE_ROOT"],
|
||||
get_duplicity_target_url(config),
|
||||
],
|
||||
@ -375,7 +374,7 @@ def perform_backup(full_backup):
|
||||
"--verbosity", "error",
|
||||
"--archive-dir", backup_cache_dir,
|
||||
"--force",
|
||||
] + get_duplicity_additional_args(env) + [
|
||||
*get_duplicity_additional_args(env),
|
||||
get_duplicity_target_url(config)
|
||||
],
|
||||
get_duplicity_env_vars(env))
|
||||
@ -391,7 +390,7 @@ def perform_backup(full_backup):
|
||||
"--verbosity", "error",
|
||||
"--archive-dir", backup_cache_dir,
|
||||
"--force",
|
||||
] + get_duplicity_additional_args(env) + [
|
||||
*get_duplicity_additional_args(env),
|
||||
get_duplicity_target_url(config)
|
||||
],
|
||||
get_duplicity_env_vars(env))
|
||||
@ -430,7 +429,7 @@ def run_duplicity_verification():
|
||||
"--compare-data",
|
||||
"--archive-dir", backup_cache_dir,
|
||||
"--exclude", backup_root,
|
||||
] + get_duplicity_additional_args(env) + [
|
||||
*get_duplicity_additional_args(env),
|
||||
get_duplicity_target_url(config),
|
||||
env["STORAGE_ROOT"],
|
||||
], get_duplicity_env_vars(env))
|
||||
@ -443,9 +442,9 @@ def run_duplicity_restore(args):
|
||||
"/usr/bin/duplicity",
|
||||
"restore",
|
||||
"--archive-dir", backup_cache_dir,
|
||||
] + get_duplicity_additional_args(env) + [
|
||||
get_duplicity_target_url(config)
|
||||
] + args,
|
||||
*get_duplicity_additional_args(env),
|
||||
get_duplicity_target_url(config),
|
||||
*args],
|
||||
get_duplicity_env_vars(env))
|
||||
|
||||
def print_duplicity_command():
|
||||
@ -457,7 +456,7 @@ def print_duplicity_command():
|
||||
print(f"export {k}={shlex.quote(v)}")
|
||||
print("duplicity", "{command}", shlex.join([
|
||||
"--archive-dir", backup_cache_dir,
|
||||
] + get_duplicity_additional_args(env) + [
|
||||
*get_duplicity_additional_args(env),
|
||||
get_duplicity_target_url(config)
|
||||
]))
|
||||
|
||||
@ -513,21 +512,22 @@ def list_target_files(config):
|
||||
if 'Permission denied (publickey).' in listing:
|
||||
reason = "Invalid user or check you correctly copied the SSH key."
|
||||
elif 'No such file or directory' in listing:
|
||||
reason = "Provided path {} is invalid.".format(target_path)
|
||||
reason = f"Provided path {target_path} is invalid."
|
||||
elif 'Network is unreachable' in listing:
|
||||
reason = "The IP address {} is unreachable.".format(target.hostname)
|
||||
reason = f"The IP address {target.hostname} is unreachable."
|
||||
elif 'Could not resolve hostname' in listing:
|
||||
reason = "The hostname {} cannot be resolved.".format(target.hostname)
|
||||
reason = f"The hostname {target.hostname} cannot be resolved."
|
||||
else:
|
||||
reason = "Unknown error." \
|
||||
"Please check running 'management/backup.py --verify'" \
|
||||
"from mailinabox sources to debug the issue."
|
||||
raise ValueError("Connection to rsync host failed: {}".format(reason))
|
||||
reason = ("Unknown error."
|
||||
"Please check running 'management/backup.py --verify'"
|
||||
"from mailinabox sources to debug the issue.")
|
||||
msg = f"Connection to rsync host failed: {reason}"
|
||||
raise ValueError(msg)
|
||||
|
||||
elif target.scheme == "s3":
|
||||
import boto3.s3
|
||||
from botocore.exceptions import ClientError
|
||||
|
||||
|
||||
# separate bucket from path in target
|
||||
bucket = target.path[1:].split('/')[0]
|
||||
path = '/'.join(target.path[1:].split('/')[1:]) + '/'
|
||||
@ -537,7 +537,8 @@ def list_target_files(config):
|
||||
path = ''
|
||||
|
||||
if bucket == "":
|
||||
raise ValueError("Enter an S3 bucket name.")
|
||||
msg = "Enter an S3 bucket name."
|
||||
raise ValueError(msg)
|
||||
|
||||
# connect to the region & bucket
|
||||
try:
|
||||
@ -555,7 +556,7 @@ def list_target_files(config):
|
||||
from b2sdk.v1.exception import NonExistentBucket
|
||||
info = InMemoryAccountInfo()
|
||||
b2_api = B2Api(info)
|
||||
|
||||
|
||||
# Extract information from target
|
||||
b2_application_keyid = target.netloc[:target.netloc.index(':')]
|
||||
b2_application_key = urllib.parse.unquote(target.netloc[target.netloc.index(':')+1:target.netloc.index('@')])
|
||||
@ -564,8 +565,9 @@ def list_target_files(config):
|
||||
try:
|
||||
b2_api.authorize_account("production", b2_application_keyid, b2_application_key)
|
||||
bucket = b2_api.get_bucket_by_name(b2_bucket)
|
||||
except NonExistentBucket as e:
|
||||
raise ValueError("B2 Bucket does not exist. Please double check your information!")
|
||||
except NonExistentBucket:
|
||||
msg = "B2 Bucket does not exist. Please double check your information!"
|
||||
raise ValueError(msg)
|
||||
return [(key.file_name, key.size) for key, _ in bucket.ls()]
|
||||
|
||||
else:
|
||||
@ -586,7 +588,7 @@ def backup_set_custom(env, target, target_user, target_pass, min_age):
|
||||
|
||||
# Validate.
|
||||
try:
|
||||
if config["target"] not in ("off", "local"):
|
||||
if config["target"] not in {"off", "local"}:
|
||||
# these aren't supported by the following function, which expects a full url in the target key,
|
||||
# which is what is there except when loading the config prior to saving
|
||||
list_target_files(config)
|
||||
@ -608,9 +610,9 @@ def get_backup_config(env, for_save=False, for_ui=False):
|
||||
|
||||
# Merge in anything written to custom.yaml.
|
||||
try:
|
||||
with open(os.path.join(backup_root, 'custom.yaml'), 'r') as f:
|
||||
with open(os.path.join(backup_root, 'custom.yaml'), encoding="utf-8") as f:
|
||||
custom_config = rtyaml.load(f)
|
||||
if not isinstance(custom_config, dict): raise ValueError() # caught below
|
||||
if not isinstance(custom_config, dict): raise ValueError # caught below
|
||||
config.update(custom_config)
|
||||
except:
|
||||
pass
|
||||
@ -634,18 +636,17 @@ def get_backup_config(env, for_save=False, for_ui=False):
|
||||
config["target"] = "file://" + config["file_target_directory"]
|
||||
ssh_pub_key = os.path.join('/root', '.ssh', 'id_rsa_miab.pub')
|
||||
if os.path.exists(ssh_pub_key):
|
||||
with open(ssh_pub_key, 'r') as f:
|
||||
with open(ssh_pub_key, encoding="utf-8") as f:
|
||||
config["ssh_pub_key"] = f.read()
|
||||
|
||||
return config
|
||||
|
||||
def write_backup_config(env, newconfig):
|
||||
backup_root = os.path.join(env["STORAGE_ROOT"], 'backup')
|
||||
with open(os.path.join(backup_root, 'custom.yaml'), "w") as f:
|
||||
with open(os.path.join(backup_root, 'custom.yaml'), "w", encoding="utf-8") as f:
|
||||
f.write(rtyaml.dump(newconfig))
|
||||
|
||||
if __name__ == "__main__":
|
||||
import sys
|
||||
if sys.argv[-1] == "--verify":
|
||||
# Run duplicity's verification command to check a) the backup files
|
||||
# are readable, and b) report if they are up to date.
|
||||
@ -654,7 +655,7 @@ if __name__ == "__main__":
|
||||
elif sys.argv[-1] == "--list":
|
||||
# List the saved backup files.
|
||||
for fn, size in list_target_files(get_backup_config(load_environment())):
|
||||
print("{}\t{}".format(fn, size))
|
||||
print(f"{fn}\t{size}")
|
||||
|
||||
elif sys.argv[-1] == "--status":
|
||||
# Show backup status.
|
||||
|
@ -15,7 +15,8 @@
|
||||
# root API key. This file is readable only by root, so this
|
||||
# tool can only be used as root.
|
||||
|
||||
import sys, getpass, urllib.request, urllib.error, json, re, csv
|
||||
import sys, getpass, urllib.request, urllib.error, json, csv
|
||||
import contextlib
|
||||
|
||||
def mgmt(cmd, data=None, is_json=False):
|
||||
# The base URL for the management daemon. (Listens on IPv4 only.)
|
||||
@ -28,10 +29,8 @@ def mgmt(cmd, data=None, is_json=False):
|
||||
response = urllib.request.urlopen(req)
|
||||
except urllib.error.HTTPError as e:
|
||||
if e.code == 401:
|
||||
try:
|
||||
with contextlib.suppress(Exception):
|
||||
print(e.read().decode("utf8"))
|
||||
except:
|
||||
pass
|
||||
print("The management daemon refused access. The API key file may be out of sync. Try 'service mailinabox restart'.", file=sys.stderr)
|
||||
elif hasattr(e, 'read'):
|
||||
print(e.read().decode('utf8'), file=sys.stderr)
|
||||
@ -56,7 +55,7 @@ def read_password():
|
||||
return first
|
||||
|
||||
def setup_key_auth(mgmt_uri):
|
||||
with open('/var/lib/mailinabox/api.key', 'r') as f:
|
||||
with open('/var/lib/mailinabox/api.key', encoding='utf-8') as f:
|
||||
key = f.read().strip()
|
||||
|
||||
auth_handler = urllib.request.HTTPBasicAuthHandler()
|
||||
@ -100,12 +99,9 @@ elif sys.argv[1] == "user" and len(sys.argv) == 2:
|
||||
print("*", end='')
|
||||
print()
|
||||
|
||||
elif sys.argv[1] == "user" and sys.argv[2] in ("add", "password"):
|
||||
elif sys.argv[1] == "user" and sys.argv[2] in {"add", "password"}:
|
||||
if len(sys.argv) < 5:
|
||||
if len(sys.argv) < 4:
|
||||
email = input("email: ")
|
||||
else:
|
||||
email = sys.argv[3]
|
||||
email = input('email: ') if len(sys.argv) < 4 else sys.argv[3]
|
||||
pw = read_password()
|
||||
else:
|
||||
email, pw = sys.argv[3:5]
|
||||
@ -118,11 +114,8 @@ elif sys.argv[1] == "user" and sys.argv[2] in ("add", "password"):
|
||||
elif sys.argv[1] == "user" and sys.argv[2] == "remove" and len(sys.argv) == 4:
|
||||
print(mgmt("/mail/users/remove", { "email": sys.argv[3] }))
|
||||
|
||||
elif sys.argv[1] == "user" and sys.argv[2] in ("make-admin", "remove-admin") and len(sys.argv) == 4:
|
||||
if sys.argv[2] == "make-admin":
|
||||
action = "add"
|
||||
else:
|
||||
action = "remove"
|
||||
elif sys.argv[1] == "user" and sys.argv[2] in {"make-admin", "remove-admin"} and len(sys.argv) == 4:
|
||||
action = 'add' if sys.argv[2] == 'make-admin' else 'remove'
|
||||
print(mgmt("/mail/users/privileges/" + action, { "email": sys.argv[3], "privilege": "admin" }))
|
||||
|
||||
elif sys.argv[1] == "user" and sys.argv[2] == "admins":
|
||||
@ -141,7 +134,7 @@ elif sys.argv[1] == "user" and len(sys.argv) == 5 and sys.argv[2:4] == ["mfa", "
|
||||
for mfa in status["enabled_mfa"]:
|
||||
W.writerow([mfa["id"], mfa["type"], mfa["label"]])
|
||||
|
||||
elif sys.argv[1] == "user" and len(sys.argv) in (5, 6) and sys.argv[2:4] == ["mfa", "disable"]:
|
||||
elif sys.argv[1] == "user" and len(sys.argv) in {5, 6} and sys.argv[2:4] == ["mfa", "disable"]:
|
||||
# Disable MFA (all or a particular device) for a user.
|
||||
print(mgmt("/mfa/disable", { "user": sys.argv[4], "mfa-id": sys.argv[5] if len(sys.argv) == 6 else None }))
|
||||
|
||||
|
@ -20,11 +20,11 @@
|
||||
# service mailinabox start # when done debugging, start it up again
|
||||
|
||||
import os, os.path, re, json, time
|
||||
import multiprocessing.pool, subprocess
|
||||
import multiprocessing.pool
|
||||
|
||||
from functools import wraps
|
||||
|
||||
from flask import Flask, request, render_template, abort, Response, send_from_directory, make_response
|
||||
from flask import Flask, request, render_template, Response, send_from_directory, make_response
|
||||
|
||||
import auth, utils
|
||||
from mailconfig import get_mail_users, get_mail_users_ex, get_admins, add_mail_user, set_mail_password, set_mail_display_name, remove_mail_user
|
||||
@ -32,6 +32,7 @@ from mailconfig import get_mail_user_privileges, add_remove_mail_user_privilege
|
||||
from mailconfig import get_mail_aliases, get_mail_aliases_ex, get_mail_domains, add_mail_alias, remove_mail_alias
|
||||
from mfa import get_public_mfa_state, enable_mfa, disable_mfa
|
||||
import mfa_totp
|
||||
import contextlib
|
||||
|
||||
env = utils.load_environment()
|
||||
|
||||
@ -39,14 +40,12 @@ auth_service = auth.AuthService()
|
||||
|
||||
# We may deploy via a symbolic link, which confuses flask's template finding.
|
||||
me = __file__
|
||||
try:
|
||||
with contextlib.suppress(OSError):
|
||||
me = os.readlink(__file__)
|
||||
except OSError:
|
||||
pass
|
||||
|
||||
# for generating CSRs we need a list of country codes
|
||||
csr_country_codes = []
|
||||
with open(os.path.join(os.path.dirname(me), "csr_country_codes.tsv")) as f:
|
||||
with open(os.path.join(os.path.dirname(me), "csr_country_codes.tsv"), encoding="utf-8") as f:
|
||||
for line in f:
|
||||
if line.strip() == "" or line.startswith("#"): continue
|
||||
code, name = line.strip().split("\t")[0:2]
|
||||
@ -90,7 +89,7 @@ def authorized_personnel_only(viewfunc):
|
||||
# Not authorized. Return a 401 (send auth) and a prompt to authorize by default.
|
||||
status = 401
|
||||
headers = {
|
||||
'WWW-Authenticate': 'Basic realm="{0}"'.format(auth_service.auth_realm),
|
||||
'WWW-Authenticate': f'Basic realm="{auth_service.auth_realm}"',
|
||||
'X-Reason': error,
|
||||
}
|
||||
|
||||
@ -100,7 +99,7 @@ def authorized_personnel_only(viewfunc):
|
||||
status = 403
|
||||
headers = None
|
||||
|
||||
if request.headers.get('Accept') in (None, "", "*/*"):
|
||||
if request.headers.get('Accept') in {None, "", "*/*"}:
|
||||
# Return plain text output.
|
||||
return Response(error+"\n", status=status, mimetype='text/plain', headers=headers)
|
||||
else:
|
||||
@ -174,7 +173,7 @@ def login():
|
||||
"api_key": auth_service.create_session_key(email, env, type='login'),
|
||||
}
|
||||
|
||||
app.logger.info("New login session created for {}".format(email))
|
||||
app.logger.info(f"New login session created for {email}")
|
||||
|
||||
# Return.
|
||||
return json_response(resp)
|
||||
@ -183,8 +182,8 @@ def login():
|
||||
def logout():
|
||||
try:
|
||||
email, _ = auth_service.authenticate(request, env, logout=True)
|
||||
app.logger.info("{} logged out".format(email))
|
||||
except ValueError as e:
|
||||
app.logger.info(f"{email} logged out")
|
||||
except ValueError:
|
||||
pass
|
||||
finally:
|
||||
return json_response({ "status": "ok" })
|
||||
@ -374,9 +373,9 @@ def dns_set_record(qname, rtype="A"):
|
||||
# Get the existing records matching the qname and rtype.
|
||||
return dns_get_records(qname, rtype)
|
||||
|
||||
elif request.method in ("POST", "PUT"):
|
||||
elif request.method in {"POST", "PUT"}:
|
||||
# There is a default value for A/AAAA records.
|
||||
if rtype in ("A", "AAAA") and value == "":
|
||||
if rtype in {"A", "AAAA"} and value == "":
|
||||
value = request.environ.get("HTTP_X_FORWARDED_FOR") # normally REMOTE_ADDR but we're behind nginx as a reverse proxy
|
||||
|
||||
# Cannot add empty records.
|
||||
@ -438,7 +437,7 @@ def ssl_get_status():
|
||||
{
|
||||
"domain": d["domain"],
|
||||
"status": d["ssl_certificate"][0],
|
||||
"text": d["ssl_certificate"][1] + ((" " + cant_provision[d["domain"]] if d["domain"] in cant_provision else ""))
|
||||
"text": d["ssl_certificate"][1] + (" " + cant_provision[d["domain"]] if d["domain"] in cant_provision else "")
|
||||
} for d in domains_status ]
|
||||
|
||||
# Warn the user about domain names not hosted here because of other settings.
|
||||
@ -510,7 +509,7 @@ def totp_post_enable():
|
||||
secret = request.form.get('secret')
|
||||
token = request.form.get('token')
|
||||
label = request.form.get('label')
|
||||
if type(token) != str:
|
||||
if not isinstance(token, str):
|
||||
return ("Bad Input", 400)
|
||||
try:
|
||||
mfa_totp.validate_secret(secret)
|
||||
@ -606,8 +605,7 @@ def system_status():
|
||||
def show_updates():
|
||||
from status_checks import list_apt_updates
|
||||
return "".join(
|
||||
"%s (%s)\n"
|
||||
% (p["package"], p["version"])
|
||||
"{} ({})\n".format(p["package"], p["version"])
|
||||
for p in list_apt_updates())
|
||||
|
||||
@app.route('/system/update-packages', methods=["POST"])
|
||||
@ -806,14 +804,11 @@ def log_failed_login(request):
|
||||
# During setup we call the management interface directly to determine the user
|
||||
# status. So we can't always use X-Forwarded-For because during setup that header
|
||||
# will not be present.
|
||||
if request.headers.getlist("X-Forwarded-For"):
|
||||
ip = request.headers.getlist("X-Forwarded-For")[0]
|
||||
else:
|
||||
ip = request.remote_addr
|
||||
ip = request.headers.getlist("X-Forwarded-For")[0] if request.headers.getlist("X-Forwarded-For") else request.remote_addr
|
||||
|
||||
# We need to add a timestamp to the log message, otherwise /dev/log will eat the "duplicate"
|
||||
# message.
|
||||
app.logger.warning( "Mail-in-a-Box Management Daemon: Failed login attempt from ip %s - timestamp %s" % (ip, time.time()))
|
||||
app.logger.warning( f"Mail-in-a-Box Management Daemon: Failed login attempt from ip {ip} - timestamp {time.time()}")
|
||||
|
||||
|
||||
# APP
|
||||
|
@ -13,7 +13,7 @@
|
||||
# and mail aliases and restarts nsd.
|
||||
########################################################################
|
||||
|
||||
import sys, os, os.path, urllib.parse, datetime, re, hashlib, base64
|
||||
import sys, os, os.path, datetime, re, hashlib, base64
|
||||
import ipaddress
|
||||
import rtyaml
|
||||
import dns.resolver
|
||||
@ -21,12 +21,13 @@ import hooks
|
||||
|
||||
from utils import shell, load_env_vars_from_file, safe_domain_name, sort_domains
|
||||
from ssl_certificates import get_ssl_certificates, check_certificate
|
||||
import contextlib
|
||||
|
||||
# From https://stackoverflow.com/questions/3026957/how-to-validate-a-domain-name-using-regex-php/16491074#16491074
|
||||
# This regular expression matches domain names according to RFCs, it also accepts fqdn with an leading dot,
|
||||
# underscores, as well as asteriks which are allowed in domain names but not hostnames (i.e. allowed in
|
||||
# DNS but not in URLs), which are common in certain record types like for DKIM.
|
||||
DOMAIN_RE = "^(?!\-)(?:[*][.])?(?:[a-zA-Z\d\-_]{0,62}[a-zA-Z\d_]\.){1,126}(?!\d+)[a-zA-Z\d_]{1,63}(\.?)$"
|
||||
DOMAIN_RE = r"^(?!\-)(?:[*][.])?(?:[a-zA-Z\d\-_]{0,62}[a-zA-Z\d_]\.){1,126}(?!\d+)[a-zA-Z\d_]{1,63}(\.?)$"
|
||||
|
||||
def get_dns_domains(env):
|
||||
# Add all domain names in use by email users and mail aliases, any
|
||||
@ -48,7 +49,7 @@ def get_dns_zones(env):
|
||||
# Exclude domains that are subdomains of other domains we know. Proceed
|
||||
# by looking at shorter domains first.
|
||||
zone_domains = set()
|
||||
for domain in sorted(domains, key=lambda d : len(d)):
|
||||
for domain in sorted(domains, key=len):
|
||||
for d in zone_domains:
|
||||
if domain.endswith("." + d):
|
||||
# We found a parent domain already in the list.
|
||||
@ -58,9 +59,7 @@ def get_dns_zones(env):
|
||||
zone_domains.add(domain)
|
||||
|
||||
# Make a nice and safe filename for each domain.
|
||||
zonefiles = []
|
||||
for domain in zone_domains:
|
||||
zonefiles.append([domain, safe_domain_name(domain) + ".txt"])
|
||||
zonefiles = [[domain, safe_domain_name(domain) + ".txt"] for domain in zone_domains]
|
||||
|
||||
# Sort the list so that the order is nice and so that nsd.conf has a
|
||||
# stable order so we don't rewrite the file & restart the service
|
||||
@ -204,8 +203,7 @@ def build_zone(domain, domain_properties, additional_records, env, is_zone=True)
|
||||
# User may provide one or more additional nameservers
|
||||
secondary_ns_list = get_secondary_dns(additional_records, mode="NS") \
|
||||
or ["ns2." + env["PRIMARY_HOSTNAME"]]
|
||||
for secondary_ns in secondary_ns_list:
|
||||
records.append((None, "NS", secondary_ns+'.', False))
|
||||
records.extend((None, "NS", secondary_ns+'.', False) for secondary_ns in secondary_ns_list)
|
||||
|
||||
|
||||
# In PRIMARY_HOSTNAME...
|
||||
@ -222,8 +220,7 @@ def build_zone(domain, domain_properties, additional_records, env, is_zone=True)
|
||||
records.append(("_443._tcp", "TLSA", build_tlsa_record(env), "Optional. When DNSSEC is enabled, provides out-of-band HTTPS certificate validation for a few web clients that support it."))
|
||||
|
||||
# Add a SSHFP records to help SSH key validation. One per available SSH key on this system.
|
||||
for value in build_sshfp_records():
|
||||
records.append((None, "SSHFP", value, "Optional. Provides an out-of-band method for verifying an SSH key before connecting. Use 'VerifyHostKeyDNS yes' (or 'VerifyHostKeyDNS ask') when connecting with ssh."))
|
||||
records.extend((None, "SSHFP", value, "Optional. Provides an out-of-band method for verifying an SSH key before connecting. Use 'VerifyHostKeyDNS yes' (or 'VerifyHostKeyDNS ask') when connecting with ssh.") for value in build_sshfp_records())
|
||||
|
||||
# Add DNS records for any subdomains of this domain. We should not have a zone for
|
||||
# both a domain and one of its subdomains.
|
||||
@ -233,7 +230,7 @@ def build_zone(domain, domain_properties, additional_records, env, is_zone=True)
|
||||
subdomain_qname = subdomain[0:-len("." + domain)]
|
||||
subzone = build_zone(subdomain, domain_properties, additional_records, env, is_zone=False)
|
||||
for child_qname, child_rtype, child_value, child_explanation in subzone:
|
||||
if child_qname == None:
|
||||
if child_qname is None:
|
||||
child_qname = subdomain_qname
|
||||
else:
|
||||
child_qname += "." + subdomain_qname
|
||||
@ -241,10 +238,7 @@ def build_zone(domain, domain_properties, additional_records, env, is_zone=True)
|
||||
|
||||
has_rec_base = list(records) # clone current state
|
||||
def has_rec(qname, rtype, prefix=None):
|
||||
for rec in has_rec_base:
|
||||
if rec[0] == qname and rec[1] == rtype and (prefix is None or rec[2].startswith(prefix)):
|
||||
return True
|
||||
return False
|
||||
return any(rec[0] == qname and rec[1] == rtype and (prefix is None or rec[2].startswith(prefix)) for rec in has_rec_base)
|
||||
|
||||
# The user may set other records that don't conflict with our settings.
|
||||
# Don't put any TXT records above this line, or it'll prevent any custom TXT records.
|
||||
@ -272,7 +266,7 @@ def build_zone(domain, domain_properties, additional_records, env, is_zone=True)
|
||||
has_rec_base = list(records)
|
||||
a_expl = "Required. May have a different value. Sets the IP address that %s resolves to for web hosting and other services besides mail. The A record must be present but its value does not affect mail delivery." % domain
|
||||
if domain_properties[domain]["auto"]:
|
||||
if domain.startswith("ns1.") or domain.startswith("ns2."): a_expl = False # omit from 'External DNS' page since this only applies if box is its own DNS server
|
||||
if domain.startswith(("ns1.", "ns2.")): a_expl = False # omit from 'External DNS' page since this only applies if box is its own DNS server
|
||||
if domain.startswith("www."): a_expl = "Optional. Sets the IP address that %s resolves to so that the box can provide a redirect to the parent domain." % domain
|
||||
if domain.startswith("mta-sts."): a_expl = "Optional. MTA-STS Policy Host serving /.well-known/mta-sts.txt."
|
||||
if domain.startswith("autoconfig."): a_expl = "Provides email configuration autodiscovery support for Thunderbird Autoconfig."
|
||||
@ -308,7 +302,7 @@ def build_zone(domain, domain_properties, additional_records, env, is_zone=True)
|
||||
# Append the DKIM TXT record to the zone as generated by OpenDKIM.
|
||||
# Skip if the user has set a DKIM record already.
|
||||
opendkim_record_file = os.path.join(env['STORAGE_ROOT'], 'mail/dkim/mail.txt')
|
||||
with open(opendkim_record_file) as orf:
|
||||
with open(opendkim_record_file, encoding="utf-8") as orf:
|
||||
m = re.match(r'(\S+)\s+IN\s+TXT\s+\( ((?:"[^"]+"\s+)+)\)', orf.read(), re.S)
|
||||
val = "".join(re.findall(r'"([^"]+)"', m.group(2)))
|
||||
if not has_rec(m.group(1), "TXT", prefix="v=DKIM1; "):
|
||||
@ -374,8 +368,8 @@ def build_zone(domain, domain_properties, additional_records, env, is_zone=True)
|
||||
# non-mail domain and also may include qnames from custom DNS records.
|
||||
# Do this once at the end of generating a zone.
|
||||
if is_zone:
|
||||
qnames_with_a = set(qname for (qname, rtype, value, explanation) in records if rtype in ("A", "AAAA"))
|
||||
qnames_with_mx = set(qname for (qname, rtype, value, explanation) in records if rtype == "MX")
|
||||
qnames_with_a = {qname for (qname, rtype, value, explanation) in records if rtype in {"A", "AAAA"}}
|
||||
qnames_with_mx = {qname for (qname, rtype, value, explanation) in records if rtype == "MX"}
|
||||
for qname in qnames_with_a - qnames_with_mx:
|
||||
# Mark this domain as not sending mail with hard-fail SPF and DMARC records.
|
||||
d = (qname+"." if qname else "") + domain
|
||||
@ -471,14 +465,12 @@ def build_sshfp_records():
|
||||
# specify that port to sshkeyscan.
|
||||
|
||||
port = 22
|
||||
with open('/etc/ssh/sshd_config', 'r') as f:
|
||||
with open('/etc/ssh/sshd_config', encoding="utf-8") as f:
|
||||
for line in f:
|
||||
s = line.rstrip().split()
|
||||
if len(s) == 2 and s[0] == 'Port':
|
||||
try:
|
||||
with contextlib.suppress(ValueError):
|
||||
port = int(s[1])
|
||||
except ValueError:
|
||||
pass
|
||||
break
|
||||
|
||||
keys = shell("check_output", ["ssh-keyscan", "-4", "-t", "rsa,dsa,ecdsa,ed25519", "-p", str(port), "localhost"])
|
||||
@ -487,7 +479,7 @@ def build_sshfp_records():
|
||||
for key in keys:
|
||||
if key.strip() == "" or key[0] == "#": continue
|
||||
try:
|
||||
host, keytype, pubkey = key.split(" ")
|
||||
_host, keytype, pubkey = key.split(" ")
|
||||
yield "%d %d ( %s )" % (
|
||||
algorithm_number[keytype],
|
||||
2, # specifies we are using SHA-256 on next line
|
||||
@ -532,7 +524,7 @@ $TTL 86400 ; default time to live
|
||||
zone = zone.format(domain=domain, primary_domain=env["PRIMARY_HOSTNAME"])
|
||||
|
||||
# Add records.
|
||||
for subdomain, querytype, value, explanation in records:
|
||||
for subdomain, querytype, value, _explanation in records:
|
||||
if subdomain:
|
||||
zone += subdomain
|
||||
zone += "\tIN\t" + querytype + "\t"
|
||||
@ -550,7 +542,7 @@ $TTL 86400 ; default time to live
|
||||
zone += value + "\n"
|
||||
|
||||
# Append a stable hash of DNSSEC signing keys in a comment.
|
||||
zone += "\n; DNSSEC signing keys hash: {}\n".format(hash_dnssec_keys(domain, env))
|
||||
zone += f"\n; DNSSEC signing keys hash: {hash_dnssec_keys(domain, env)}\n"
|
||||
|
||||
# DNSSEC requires re-signing a zone periodically. That requires
|
||||
# bumping the serial number even if no other records have changed.
|
||||
@ -566,7 +558,7 @@ $TTL 86400 ; default time to live
|
||||
# We've signed the domain. Check if we are close to the expiration
|
||||
# time of the signature. If so, we'll force a bump of the serial
|
||||
# number so we can re-sign it.
|
||||
with open(zonefile + ".signed") as f:
|
||||
with open(zonefile + ".signed", encoding="utf-8") as f:
|
||||
signed_zone = f.read()
|
||||
expiration_times = re.findall(r"\sRRSIG\s+SOA\s+\d+\s+\d+\s\d+\s+(\d{14})", signed_zone)
|
||||
if len(expiration_times) == 0:
|
||||
@ -585,7 +577,7 @@ $TTL 86400 ; default time to live
|
||||
if os.path.exists(zonefile):
|
||||
# If the zone already exists, is different, and has a later serial number,
|
||||
# increment the number.
|
||||
with open(zonefile) as f:
|
||||
with open(zonefile, encoding="utf-8") as f:
|
||||
existing_zone = f.read()
|
||||
m = re.search(r"(\d+)\s*;\s*serial number", existing_zone)
|
||||
if m:
|
||||
@ -609,7 +601,7 @@ $TTL 86400 ; default time to live
|
||||
zone = zone.replace("__SERIAL__", serial)
|
||||
|
||||
# Write the zone file.
|
||||
with open(zonefile, "w") as f:
|
||||
with open(zonefile, "w", encoding="utf-8") as f:
|
||||
f.write(zone)
|
||||
|
||||
return True # file is updated
|
||||
@ -622,7 +614,7 @@ def get_dns_zonefile(zone, env):
|
||||
raise ValueError("%s is not a domain name that corresponds to a zone." % zone)
|
||||
|
||||
nsd_zonefile = "/etc/nsd/zones/" + fn
|
||||
with open(nsd_zonefile, "r") as f:
|
||||
with open(nsd_zonefile, encoding="utf-8") as f:
|
||||
return f.read()
|
||||
|
||||
########################################################################
|
||||
@ -634,11 +626,11 @@ def write_nsd_conf(zonefiles, additional_records, env):
|
||||
|
||||
# Append the zones.
|
||||
for domain, zonefile in zonefiles:
|
||||
nsdconf += """
|
||||
nsdconf += f"""
|
||||
zone:
|
||||
name: %s
|
||||
zonefile: %s
|
||||
""" % (domain, zonefile)
|
||||
name: {domain}
|
||||
zonefile: {zonefile}
|
||||
"""
|
||||
|
||||
# If custom secondary nameservers have been set, allow zone transfers
|
||||
# and, if not a subnet, notifies to them.
|
||||
@ -650,13 +642,13 @@ zone:
|
||||
# Check if the file is changing. If it isn't changing,
|
||||
# return False to flag that no change was made.
|
||||
if os.path.exists(nsd_conf_file):
|
||||
with open(nsd_conf_file) as f:
|
||||
with open(nsd_conf_file, encoding="utf-8") as f:
|
||||
if f.read() == nsdconf:
|
||||
return False
|
||||
|
||||
# Write out new contents and return True to signal that
|
||||
# configuration changed.
|
||||
with open(nsd_conf_file, "w") as f:
|
||||
with open(nsd_conf_file, "w", encoding="utf-8") as f:
|
||||
f.write(nsdconf)
|
||||
return True
|
||||
|
||||
@ -690,9 +682,8 @@ def hash_dnssec_keys(domain, env):
|
||||
keydata = []
|
||||
for keytype, keyfn in sorted(find_dnssec_signing_keys(domain, env)):
|
||||
oldkeyfn = os.path.join(env['STORAGE_ROOT'], 'dns/dnssec', keyfn + ".private")
|
||||
keydata.append(keytype)
|
||||
keydata.append(keyfn)
|
||||
with open(oldkeyfn, "r") as fr:
|
||||
keydata.extend((keytype, keyfn))
|
||||
with open(oldkeyfn, encoding="utf-8") as fr:
|
||||
keydata.append( fr.read() )
|
||||
keydata = "".join(keydata).encode("utf8")
|
||||
return hashlib.sha1(keydata).hexdigest()
|
||||
@ -720,12 +711,12 @@ def sign_zone(domain, zonefile, env):
|
||||
# Use os.umask and open().write() to securely create a copy that only
|
||||
# we (root) can read.
|
||||
oldkeyfn = os.path.join(env['STORAGE_ROOT'], 'dns/dnssec', keyfn + ext)
|
||||
with open(oldkeyfn, "r") as fr:
|
||||
with open(oldkeyfn, encoding="utf-8") as fr:
|
||||
keydata = fr.read()
|
||||
keydata = keydata.replace("_domain_", domain)
|
||||
prev_umask = os.umask(0o77) # ensure written file is not world-readable
|
||||
try:
|
||||
with open(newkeyfn + ext, "w") as fw:
|
||||
with open(newkeyfn + ext, "w", encoding="utf-8") as fw:
|
||||
fw.write(keydata)
|
||||
finally:
|
||||
os.umask(prev_umask) # other files we write should be world-readable
|
||||
@ -759,7 +750,7 @@ def sign_zone(domain, zonefile, env):
|
||||
# be used, so we'll pre-generate all for each key. One DS record per line. Only one
|
||||
# needs to actually be deployed at the registrar. We'll select the preferred one
|
||||
# in the status checks.
|
||||
with open("/etc/nsd/zones/" + zonefile + ".ds", "w") as f:
|
||||
with open("/etc/nsd/zones/" + zonefile + ".ds", "w", encoding="utf-8") as f:
|
||||
for key in ksk_keys:
|
||||
for digest_type in ('1', '2', '4'):
|
||||
rr_ds = shell('check_output', ["/usr/bin/ldns-key2ds",
|
||||
@ -796,7 +787,7 @@ def write_opendkim_tables(domains, env):
|
||||
# So we must have a separate KeyTable entry for each domain.
|
||||
"SigningTable":
|
||||
"".join(
|
||||
"*@{domain} {domain}\n".format(domain=domain)
|
||||
f"*@{domain} {domain}\n"
|
||||
for domain in domains
|
||||
),
|
||||
|
||||
@ -805,7 +796,7 @@ def write_opendkim_tables(domains, env):
|
||||
# signing domain must match the sender's From: domain.
|
||||
"KeyTable":
|
||||
"".join(
|
||||
"{domain} {domain}:mail:{key_file}\n".format(domain=domain, key_file=opendkim_key_file)
|
||||
f"{domain} {domain}:mail:{opendkim_key_file}\n"
|
||||
for domain in domains
|
||||
),
|
||||
}
|
||||
@ -814,12 +805,12 @@ def write_opendkim_tables(domains, env):
|
||||
for filename, content in config.items():
|
||||
# Don't write the file if it doesn't need an update.
|
||||
if os.path.exists("/etc/opendkim/" + filename):
|
||||
with open("/etc/opendkim/" + filename) as f:
|
||||
with open("/etc/opendkim/" + filename, encoding="utf-8") as f:
|
||||
if f.read() == content:
|
||||
continue
|
||||
|
||||
# The contents needs to change.
|
||||
with open("/etc/opendkim/" + filename, "w") as f:
|
||||
with open("/etc/opendkim/" + filename, "w", encoding="utf-8") as f:
|
||||
f.write(content)
|
||||
did_update = True
|
||||
|
||||
@ -831,9 +822,9 @@ def write_opendkim_tables(domains, env):
|
||||
|
||||
def get_custom_dns_config(env, only_real_records=False):
|
||||
try:
|
||||
with open(os.path.join(env['STORAGE_ROOT'], 'dns/custom.yaml'), 'r') as f:
|
||||
with open(os.path.join(env['STORAGE_ROOT'], 'dns/custom.yaml'), encoding="utf-8") as f:
|
||||
custom_dns = rtyaml.load(f)
|
||||
if not isinstance(custom_dns, dict): raise ValueError() # caught below
|
||||
if not isinstance(custom_dns, dict): raise ValueError # caught below
|
||||
except:
|
||||
return [ ]
|
||||
|
||||
@ -851,7 +842,7 @@ def get_custom_dns_config(env, only_real_records=False):
|
||||
|
||||
# No other type of data is allowed.
|
||||
else:
|
||||
raise ValueError()
|
||||
raise ValueError
|
||||
|
||||
for rtype, value2 in values:
|
||||
if isinstance(value2, str):
|
||||
@ -861,7 +852,7 @@ def get_custom_dns_config(env, only_real_records=False):
|
||||
yield (qname, rtype, value3)
|
||||
# No other type of data is allowed.
|
||||
else:
|
||||
raise ValueError()
|
||||
raise ValueError
|
||||
|
||||
def filter_custom_records(domain, custom_dns_iter):
|
||||
for qname, rtype, value in custom_dns_iter:
|
||||
@ -877,10 +868,7 @@ def filter_custom_records(domain, custom_dns_iter):
|
||||
# our short form (None => domain, or a relative QNAME) if
|
||||
# domain is not None.
|
||||
if domain is not None:
|
||||
if qname == domain:
|
||||
qname = None
|
||||
else:
|
||||
qname = qname[0:len(qname)-len("." + domain)]
|
||||
qname = None if qname == domain else qname[0:len(qname) - len("." + domain)]
|
||||
|
||||
yield (qname, rtype, value)
|
||||
|
||||
@ -916,12 +904,12 @@ def write_custom_dns_config(config, env):
|
||||
|
||||
# Write.
|
||||
config_yaml = rtyaml.dump(dns)
|
||||
with open(os.path.join(env['STORAGE_ROOT'], 'dns/custom.yaml'), "w") as f:
|
||||
with open(os.path.join(env['STORAGE_ROOT'], 'dns/custom.yaml'), "w", encoding="utf-8") as f:
|
||||
f.write(config_yaml)
|
||||
|
||||
def set_custom_dns_record(qname, rtype, value, action, env):
|
||||
# validate qname
|
||||
for zone, fn in get_dns_zones(env):
|
||||
for zone, _fn in get_dns_zones(env):
|
||||
# It must match a zone apex or be a subdomain of a zone
|
||||
# that we are otherwise hosting.
|
||||
if qname == zone or qname.endswith("."+zone):
|
||||
@ -935,24 +923,27 @@ def set_custom_dns_record(qname, rtype, value, action, env):
|
||||
rtype = rtype.upper()
|
||||
if value is not None and qname != "_secondary_nameserver":
|
||||
if not re.search(DOMAIN_RE, qname):
|
||||
raise ValueError("Invalid name.")
|
||||
msg = "Invalid name."
|
||||
raise ValueError(msg)
|
||||
|
||||
if rtype in ("A", "AAAA"):
|
||||
if rtype in {"A", "AAAA"}:
|
||||
if value != "local": # "local" is a special flag for us
|
||||
v = ipaddress.ip_address(value) # raises a ValueError if there's a problem
|
||||
if rtype == "A" and not isinstance(v, ipaddress.IPv4Address): raise ValueError("That's an IPv6 address.")
|
||||
if rtype == "AAAA" and not isinstance(v, ipaddress.IPv6Address): raise ValueError("That's an IPv4 address.")
|
||||
elif rtype in ("CNAME", "NS"):
|
||||
elif rtype in {"CNAME", "NS"}:
|
||||
if rtype == "NS" and qname == zone:
|
||||
raise ValueError("NS records can only be set for subdomains.")
|
||||
msg = "NS records can only be set for subdomains."
|
||||
raise ValueError(msg)
|
||||
|
||||
# ensure value has a trailing dot
|
||||
if not value.endswith("."):
|
||||
value = value + "."
|
||||
|
||||
if not re.search(DOMAIN_RE, value):
|
||||
raise ValueError("Invalid value.")
|
||||
elif rtype in ("CNAME", "TXT", "SRV", "MX", "SSHFP", "CAA"):
|
||||
msg = "Invalid value."
|
||||
raise ValueError(msg)
|
||||
elif rtype in {"CNAME", "TXT", "SRV", "MX", "SSHFP", "CAA"}:
|
||||
# anything goes
|
||||
pass
|
||||
else:
|
||||
@ -985,7 +976,7 @@ def set_custom_dns_record(qname, rtype, value, action, env):
|
||||
# Drop this record.
|
||||
made_change = True
|
||||
continue
|
||||
if value == None and (_qname, _rtype) == (qname, rtype):
|
||||
if value is None and (_qname, _rtype) == (qname, rtype):
|
||||
# Drop all qname-rtype records.
|
||||
made_change = True
|
||||
continue
|
||||
@ -995,7 +986,7 @@ def set_custom_dns_record(qname, rtype, value, action, env):
|
||||
# Preserve this record.
|
||||
newconfig.append((_qname, _rtype, _value))
|
||||
|
||||
if action in ("add", "set") and needs_add and value is not None:
|
||||
if action in {"add", "set"} and needs_add and value is not None:
|
||||
newconfig.append((qname, rtype, value))
|
||||
made_change = True
|
||||
|
||||
@ -1012,11 +1003,11 @@ def get_secondary_dns(custom_dns, mode=None):
|
||||
resolver.lifetime = 10
|
||||
|
||||
values = []
|
||||
for qname, rtype, value in custom_dns:
|
||||
for qname, _rtype, value in custom_dns:
|
||||
if qname != '_secondary_nameserver': continue
|
||||
for hostname in value.split(" "):
|
||||
hostname = hostname.strip()
|
||||
if mode == None:
|
||||
if mode is None:
|
||||
# Just return the setting.
|
||||
values.append(hostname)
|
||||
continue
|
||||
@ -1057,24 +1048,24 @@ def set_secondary_dns(hostnames, env):
|
||||
resolver = dns.resolver.get_default_resolver()
|
||||
resolver.timeout = 5
|
||||
resolver.lifetime = 5
|
||||
|
||||
|
||||
for item in hostnames:
|
||||
if not item.startswith("xfr:"):
|
||||
# Resolve hostname.
|
||||
try:
|
||||
response = resolver.resolve(item, "A")
|
||||
resolver.resolve(item, "A")
|
||||
except (dns.resolver.NoNameservers, dns.resolver.NXDOMAIN, dns.resolver.NoAnswer, dns.resolver.Timeout):
|
||||
try:
|
||||
response = resolver.resolve(item, "AAAA")
|
||||
resolver.resolve(item, "AAAA")
|
||||
except (dns.resolver.NoNameservers, dns.resolver.NXDOMAIN, dns.resolver.NoAnswer, dns.resolver.Timeout):
|
||||
raise ValueError("Could not resolve the IP address of %s." % item)
|
||||
else:
|
||||
# Validate IP address.
|
||||
try:
|
||||
if "/" in item[4:]:
|
||||
v = ipaddress.ip_network(item[4:]) # raises a ValueError if there's a problem
|
||||
ipaddress.ip_network(item[4:]) # raises a ValueError if there's a problem
|
||||
else:
|
||||
v = ipaddress.ip_address(item[4:]) # raises a ValueError if there's a problem
|
||||
ipaddress.ip_address(item[4:]) # raises a ValueError if there's a problem
|
||||
except ValueError:
|
||||
raise ValueError("'%s' is not an IPv4 or IPv6 address or subnet." % item[4:])
|
||||
|
||||
@ -1092,13 +1083,12 @@ def get_custom_dns_records(custom_dns, qname, rtype):
|
||||
for qname1, rtype1, value in custom_dns:
|
||||
if qname1 == qname and rtype1 == rtype:
|
||||
yield value
|
||||
return None
|
||||
|
||||
########################################################################
|
||||
|
||||
def build_recommended_dns(env):
|
||||
ret = []
|
||||
for (domain, zonefile, records) in build_zones(env):
|
||||
for (domain, _zonefile, records) in build_zones(env):
|
||||
# remove records that we don't display
|
||||
records = [r for r in records if r[3] is not False]
|
||||
|
||||
@ -1107,10 +1097,7 @@ def build_recommended_dns(env):
|
||||
|
||||
# expand qnames
|
||||
for i in range(len(records)):
|
||||
if records[i][0] == None:
|
||||
qname = domain
|
||||
else:
|
||||
qname = records[i][0] + "." + domain
|
||||
qname = domain if records[i][0] is None else records[i][0] + "." + domain
|
||||
|
||||
records[i] = {
|
||||
"qname": qname,
|
||||
@ -1129,7 +1116,7 @@ if __name__ == "__main__":
|
||||
if sys.argv[-1] == "--lint":
|
||||
write_custom_dns_config(get_custom_dns_config(env), env)
|
||||
else:
|
||||
for zone, records in build_recommended_dns(env):
|
||||
for _zone, records in build_recommended_dns(env):
|
||||
for record in records:
|
||||
print("; " + record['explanation'])
|
||||
print(record['qname'], record['rtype'], record['value'], sep="\t")
|
||||
|
@ -46,11 +46,11 @@ msg = MIMEMultipart('alternative')
|
||||
# In Python 3.6:
|
||||
#msg = Message()
|
||||
|
||||
msg['From'] = "\"%s\" <%s>" % (env['PRIMARY_HOSTNAME'], admin_addr)
|
||||
msg['From'] = '"{}" <{}>'.format(env['PRIMARY_HOSTNAME'], admin_addr)
|
||||
msg['To'] = admin_addr
|
||||
msg['Subject'] = "[%s] %s" % (env['PRIMARY_HOSTNAME'], subject)
|
||||
msg['Subject'] = "[{}] {}".format(env['PRIMARY_HOSTNAME'], subject)
|
||||
|
||||
content_html = '<html><body><pre style="overflow-x: scroll; white-space: pre;">{}</pre></body></html>'.format(html.escape(content))
|
||||
content_html = f'<html><body><pre style="overflow-x: scroll; white-space: pre;">{html.escape(content)}</pre></body></html>'
|
||||
|
||||
msg.attach(MIMEText(content, 'plain'))
|
||||
msg.attach(MIMEText(content_html, 'html'))
|
||||
|
@ -131,8 +131,7 @@ def scan_mail_log(env):
|
||||
except ImportError:
|
||||
pass
|
||||
|
||||
print("Scanning logs from {:%Y-%m-%d %H:%M:%S} to {:%Y-%m-%d %H:%M:%S}".format(
|
||||
START_DATE, END_DATE)
|
||||
print(f"Scanning logs from {START_DATE:%Y-%m-%d %H:%M:%S} to {END_DATE:%Y-%m-%d %H:%M:%S}"
|
||||
)
|
||||
|
||||
# Scan the lines in the log files until the date goes out of range
|
||||
@ -238,7 +237,7 @@ def scan_mail_log(env):
|
||||
],
|
||||
sub_data=[
|
||||
("Protocol and Source", [[
|
||||
"{} {}: {} times".format(protocol_name, host, count)
|
||||
f"{protocol_name} {host}: {count} times"
|
||||
for (protocol_name, host), count
|
||||
in sorted(u["totals_by_protocol_and_host"].items(), key=lambda kv:-kv[1])
|
||||
] for u in data.values()])
|
||||
@ -314,8 +313,7 @@ def scan_mail_log(env):
|
||||
for date, sender, message in user_data["blocked"]:
|
||||
if len(sender) > 64:
|
||||
sender = sender[:32] + "…" + sender[-32:]
|
||||
user_rejects.append("%s - %s " % (date, sender))
|
||||
user_rejects.append(" %s" % message)
|
||||
user_rejects.extend((f'{date} - {sender} ', ' %s' % message))
|
||||
rejects.append(user_rejects)
|
||||
|
||||
print_user_table(
|
||||
@ -333,7 +331,7 @@ def scan_mail_log(env):
|
||||
if collector["other-services"] and VERBOSE and False:
|
||||
print_header("Other services")
|
||||
print("The following unkown services were found in the log file.")
|
||||
print(" ", *sorted(list(collector["other-services"])), sep='\n│ ')
|
||||
print(" ", *sorted(collector["other-services"]), sep='\n│ ')
|
||||
|
||||
|
||||
def scan_mail_log_line(line, collector):
|
||||
@ -344,7 +342,7 @@ def scan_mail_log_line(line, collector):
|
||||
if not m:
|
||||
return True
|
||||
|
||||
date, system, service, log = m.groups()
|
||||
date, _system, service, log = m.groups()
|
||||
collector["scan_count"] += 1
|
||||
|
||||
# print()
|
||||
@ -355,7 +353,7 @@ def scan_mail_log_line(line, collector):
|
||||
|
||||
# Replaced the dateutil parser for a less clever way of parser that is roughly 4 times faster.
|
||||
# date = dateutil.parser.parse(date)
|
||||
|
||||
|
||||
# strptime fails on Feb 29 with ValueError: day is out of range for month if correct year is not provided.
|
||||
# See https://bugs.python.org/issue26460
|
||||
date = datetime.datetime.strptime(str(NOW.year) + ' ' + date, '%Y %b %d %H:%M:%S')
|
||||
@ -387,9 +385,9 @@ def scan_mail_log_line(line, collector):
|
||||
elif service == "postfix/smtpd":
|
||||
if SCAN_BLOCKED:
|
||||
scan_postfix_smtpd_line(date, log, collector)
|
||||
elif service in ("postfix/qmgr", "postfix/pickup", "postfix/cleanup", "postfix/scache",
|
||||
elif service in {"postfix/qmgr", "postfix/pickup", "postfix/cleanup", "postfix/scache",
|
||||
"spampd", "postfix/anvil", "postfix/master", "opendkim", "postfix/lmtp",
|
||||
"postfix/tlsmgr", "anvil"):
|
||||
"postfix/tlsmgr", "anvil"}:
|
||||
# nothing to look at
|
||||
return True
|
||||
else:
|
||||
@ -403,7 +401,7 @@ def scan_mail_log_line(line, collector):
|
||||
def scan_postgrey_line(date, log, collector):
|
||||
""" Scan a postgrey log line and extract interesting data """
|
||||
|
||||
m = re.match("action=(greylist|pass), reason=(.*?), (?:delay=\d+, )?client_name=(.*), "
|
||||
m = re.match(r"action=(greylist|pass), reason=(.*?), (?:delay=\d+, )?client_name=(.*), "
|
||||
"client_address=(.*), sender=(.*), recipient=(.*)",
|
||||
log)
|
||||
|
||||
@ -446,36 +444,35 @@ def scan_postfix_smtpd_line(date, log, collector):
|
||||
return
|
||||
|
||||
# only log mail to known recipients
|
||||
if user_match(user):
|
||||
if collector["known_addresses"] is None or user in collector["known_addresses"]:
|
||||
data = collector["rejected"].get(
|
||||
user,
|
||||
{
|
||||
"blocked": [],
|
||||
"earliest": None,
|
||||
"latest": None,
|
||||
}
|
||||
)
|
||||
# simplify this one
|
||||
if user_match(user) and (collector["known_addresses"] is None or user in collector["known_addresses"]):
|
||||
data = collector["rejected"].get(
|
||||
user,
|
||||
{
|
||||
"blocked": [],
|
||||
"earliest": None,
|
||||
"latest": None,
|
||||
}
|
||||
)
|
||||
# simplify this one
|
||||
m = re.search(
|
||||
r"Client host \[(.*?)\] blocked using zen.spamhaus.org; (.*)", message
|
||||
)
|
||||
if m:
|
||||
message = "ip blocked: " + m.group(2)
|
||||
else:
|
||||
# simplify this one too
|
||||
m = re.search(
|
||||
r"Client host \[(.*?)\] blocked using zen.spamhaus.org; (.*)", message
|
||||
r"Sender address \[.*@(.*)\] blocked using dbl.spamhaus.org; (.*)", message
|
||||
)
|
||||
if m:
|
||||
message = "ip blocked: " + m.group(2)
|
||||
else:
|
||||
# simplify this one too
|
||||
m = re.search(
|
||||
r"Sender address \[.*@(.*)\] blocked using dbl.spamhaus.org; (.*)", message
|
||||
)
|
||||
if m:
|
||||
message = "domain blocked: " + m.group(2)
|
||||
message = "domain blocked: " + m.group(2)
|
||||
|
||||
if data["earliest"] is None:
|
||||
data["earliest"] = date
|
||||
data["latest"] = date
|
||||
data["blocked"].append((date, sender, message))
|
||||
if data["earliest"] is None:
|
||||
data["earliest"] = date
|
||||
data["latest"] = date
|
||||
data["blocked"].append((date, sender, message))
|
||||
|
||||
collector["rejected"][user] = data
|
||||
collector["rejected"][user] = data
|
||||
|
||||
|
||||
def scan_dovecot_login_line(date, log, collector, protocol_name):
|
||||
@ -511,7 +508,7 @@ def add_login(user, date, protocol_name, host, collector):
|
||||
data["totals_by_protocol"][protocol_name] += 1
|
||||
data["totals_by_protocol_and_host"][(protocol_name, host)] += 1
|
||||
|
||||
if host not in ("127.0.0.1", "::1") or True:
|
||||
if host not in {"127.0.0.1", "::1"} or True:
|
||||
data["activity-by-hour"][protocol_name][date.hour] += 1
|
||||
|
||||
collector["logins"][user] = data
|
||||
@ -525,7 +522,7 @@ def scan_postfix_lmtp_line(date, log, collector):
|
||||
|
||||
"""
|
||||
|
||||
m = re.match("([A-Z0-9]+): to=<(\S+)>, .* Saved", log)
|
||||
m = re.match(r"([A-Z0-9]+): to=<(\S+)>, .* Saved", log)
|
||||
|
||||
if m:
|
||||
_, user = m.groups()
|
||||
@ -561,12 +558,12 @@ def scan_postfix_submission_line(date, log, collector):
|
||||
"""
|
||||
|
||||
# Match both the 'plain' and 'login' sasl methods, since both authentication methods are
|
||||
# allowed by Dovecot. Exclude trailing comma after the username when additional fields
|
||||
# allowed by Dovecot. Exclude trailing comma after the username when additional fields
|
||||
# follow after.
|
||||
m = re.match("([A-Z0-9]+): client=(\S+), sasl_method=(PLAIN|LOGIN), sasl_username=(\S+)(?<!,)", log)
|
||||
m = re.match(r"([A-Z0-9]+): client=(\S+), sasl_method=(PLAIN|LOGIN), sasl_username=(\S+)(?<!,)", log)
|
||||
|
||||
if m:
|
||||
_, client, method, user = m.groups()
|
||||
_, client, _method, user = m.groups()
|
||||
|
||||
if user_match(user):
|
||||
# Get the user data, or create it if the user is new
|
||||
@ -599,7 +596,7 @@ def scan_postfix_submission_line(date, log, collector):
|
||||
def readline(filename):
|
||||
""" A generator that returns the lines of a file
|
||||
"""
|
||||
with open(filename, errors='replace') as file:
|
||||
with open(filename, errors='replace', encoding='utf-8') as file:
|
||||
while True:
|
||||
line = file.readline()
|
||||
if not line:
|
||||
@ -633,10 +630,7 @@ def print_time_table(labels, data, do_print=True):
|
||||
data.insert(0, [str(h) for h in range(24)])
|
||||
|
||||
temp = "│ {:<%d} " % max(len(l) for l in labels)
|
||||
lines = []
|
||||
|
||||
for label in labels:
|
||||
lines.append(temp.format(label))
|
||||
lines = [temp.format(label) for label in labels]
|
||||
|
||||
for h in range(24):
|
||||
max_len = max(len(str(d[h])) for d in data)
|
||||
@ -650,6 +644,7 @@ def print_time_table(labels, data, do_print=True):
|
||||
|
||||
if do_print:
|
||||
print("\n".join(lines))
|
||||
return None
|
||||
else:
|
||||
return lines
|
||||
|
||||
@ -683,7 +678,7 @@ def print_user_table(users, data=None, sub_data=None, activity=None, latest=None
|
||||
col_str = str_temp.format(d[row][:31] + "…" if len(d[row]) > 32 else d[row])
|
||||
col_left[col] = True
|
||||
elif isinstance(d[row], datetime.datetime):
|
||||
col_str = "{:<20}".format(str(d[row]))
|
||||
col_str = f"{d[row]!s:<20}"
|
||||
col_left[col] = True
|
||||
else:
|
||||
temp = "{:>%s}" % max(5, len(l) + 1, len(str(d[row])) + 1)
|
||||
@ -695,7 +690,7 @@ def print_user_table(users, data=None, sub_data=None, activity=None, latest=None
|
||||
data_accum[col] += d[row]
|
||||
|
||||
try:
|
||||
if None not in [latest, earliest]:
|
||||
if None not in {latest, earliest}:
|
||||
vert_pos = len(line)
|
||||
e = earliest[row]
|
||||
l = latest[row]
|
||||
@ -723,10 +718,7 @@ def print_user_table(users, data=None, sub_data=None, activity=None, latest=None
|
||||
if sub_data is not None:
|
||||
for l, d in sub_data:
|
||||
if d[row]:
|
||||
lines.append("┬")
|
||||
lines.append("│ %s" % l)
|
||||
lines.append("├─%s─" % (len(l) * "─"))
|
||||
lines.append("│")
|
||||
lines.extend(('┬', '│ %s' % l, '├─%s─' % (len(l) * '─'), '│'))
|
||||
max_len = 0
|
||||
for v in list(d[row]):
|
||||
lines.append("│ %s" % v)
|
||||
@ -751,7 +743,7 @@ def print_user_table(users, data=None, sub_data=None, activity=None, latest=None
|
||||
else:
|
||||
header += l.rjust(max(5, len(l) + 1, col_widths[col]))
|
||||
|
||||
if None not in (latest, earliest):
|
||||
if None not in {latest, earliest}:
|
||||
header += " │ timespan "
|
||||
|
||||
lines.insert(0, header.rstrip())
|
||||
@ -776,7 +768,7 @@ def print_user_table(users, data=None, sub_data=None, activity=None, latest=None
|
||||
footer += temp.format(data_accum[row])
|
||||
|
||||
try:
|
||||
if None not in [latest, earliest]:
|
||||
if None not in {latest, earliest}:
|
||||
max_l = max(latest)
|
||||
min_e = min(earliest)
|
||||
timespan = relativedelta(max_l, min_e)
|
||||
@ -855,7 +847,7 @@ if __name__ == "__main__":
|
||||
END_DATE = args.enddate
|
||||
if args.timespan == 'today':
|
||||
args.timespan = 'day'
|
||||
print("Setting end date to {}".format(END_DATE))
|
||||
print(f"Setting end date to {END_DATE}")
|
||||
|
||||
START_DATE = END_DATE - TIME_DELTAS[args.timespan]
|
||||
|
||||
|
@ -97,7 +97,7 @@ def validate_email(email, mode=None):
|
||||
email_domain = get_domain(email)
|
||||
except IndexError:
|
||||
raise EmailNotValidError(email)
|
||||
|
||||
|
||||
test_env = (
|
||||
email_domain.endswith(".local") and
|
||||
email_domain == socket.getfqdn()
|
||||
@ -161,10 +161,7 @@ def prettify_idn_email_address(email):
|
||||
|
||||
def is_dcv_address(email):
|
||||
email = email.lower()
|
||||
for localpart in ("admin", "administrator", "postmaster", "hostmaster", "webmaster", "abuse"):
|
||||
if email.startswith(localpart+"@") or email.startswith(localpart+"+"):
|
||||
return True
|
||||
return False
|
||||
return any(email.startswith((localpart + "@", localpart + "+")) for localpart in ("admin", "administrator", "postmaster", "hostmaster", "webmaster", "abuse"))
|
||||
|
||||
def utf8_from_idna(domain_idna):
|
||||
try:
|
||||
@ -183,7 +180,7 @@ def find_mail_user(env, email, attributes=None, conn=None):
|
||||
# email is the users email address
|
||||
# attributes are a list of attributes to return eg ["mail","maildrop"]
|
||||
# conn is a ldap database connection, if not specified a new one
|
||||
# is established
|
||||
# is established
|
||||
#
|
||||
# The ldap record for the user is returned or None if not found.
|
||||
if not conn: conn = open_database(env)
|
||||
@ -196,7 +193,7 @@ def find_mail_user(env, email, attributes=None, conn=None):
|
||||
raise LookupError("Detected more than one user with the same email address (%s): %s" % (email, ";".join(dns)))
|
||||
else:
|
||||
return response.next()
|
||||
|
||||
|
||||
def find_mail_alias(env, email_idna, attributes=None, conn=None, auto=None):
|
||||
# Find the alias with the given address and return the ldap
|
||||
# records for it and the associated permitted senders (if one).
|
||||
@ -216,7 +213,7 @@ def find_mail_alias(env, email_idna, attributes=None, conn=None, auto=None):
|
||||
# A tuple having the two ldap records for the alias and it's
|
||||
# permitted senders (alias, permitted_senders) is returned. If
|
||||
# either is not found, the corresponding tuple value will be None.
|
||||
#
|
||||
#
|
||||
if not conn: conn = open_database(env)
|
||||
# get alias
|
||||
q = [
|
||||
@ -245,7 +242,7 @@ def find_mail_alias(env, email_idna, attributes=None, conn=None, auto=None):
|
||||
raise LookupError("Detected more than one permitted senders group with the same email address (%s): %s" % (email_idna, ";".join(dns)))
|
||||
permitted_senders = response.next()
|
||||
return (alias, permitted_senders)
|
||||
|
||||
|
||||
|
||||
def primary_address(mail):
|
||||
# return the first IDNA-encoded email address
|
||||
@ -285,7 +282,7 @@ def get_mail_users(env, as_map=False, map_by="maildrop"):
|
||||
users = [ primary_address(rec['mail']).lower() for rec in pager ]
|
||||
return utils.sort_email_addresses(users, env)
|
||||
|
||||
|
||||
|
||||
def get_mail_users_ex(env, with_archived=False):
|
||||
# Returns a complex data structure of all user accounts, optionally
|
||||
# including archived (status="inactive") accounts.
|
||||
@ -395,7 +392,7 @@ def get_mail_aliases(env, as_map=False, map_by="primary_address"):
|
||||
c = open_database(env)
|
||||
# get all permitted senders
|
||||
pager = c.paged_search(env.LDAP_PERMITTED_SENDERS_BASE, "(objectClass=mailGroup)", attributes=["mail", "member"])
|
||||
|
||||
|
||||
# make a dict of permitted senders, key=mail(lowercase) value=members
|
||||
permitted_senders = { }
|
||||
for rec in pager:
|
||||
@ -409,7 +406,7 @@ def get_mail_aliases(env, as_map=False, map_by="primary_address"):
|
||||
attributes=[
|
||||
'mail','member','mailMember','description','namedProperty'
|
||||
])
|
||||
|
||||
|
||||
# make a dict of aliases
|
||||
# key=email(lowercase), value=(email, forward-tos, permitted-senders, auto).
|
||||
aliases = {}
|
||||
@ -421,7 +418,7 @@ def get_mail_aliases(env, as_map=False, map_by="primary_address"):
|
||||
|
||||
for fwd_to in alias['mailMember']:
|
||||
forward_tos.append(fwd_to)
|
||||
|
||||
|
||||
# chase down permitted senders' email addresses
|
||||
allowed_senders = []
|
||||
primary_email_lc = primary_address(alias['mail']).lower()
|
||||
@ -457,10 +454,10 @@ def get_mail_aliases(env, as_map=False, map_by="primary_address"):
|
||||
xas = ",".join(alias["permitted_senders"])
|
||||
list.append( (address, xft, None if xas == "" else xas, alias["auto"]) )
|
||||
return list
|
||||
|
||||
|
||||
else:
|
||||
return aliases
|
||||
|
||||
|
||||
|
||||
def get_mail_aliases_ex(env):
|
||||
# Returns a complex data structure of all mail aliases, similar
|
||||
@ -486,24 +483,24 @@ def get_mail_aliases_ex(env):
|
||||
|
||||
aliases=get_mail_aliases(env, as_map=True, map_by="primary_address")
|
||||
domains = {}
|
||||
|
||||
|
||||
for mail in aliases:
|
||||
alias=aliases[mail]
|
||||
address=primary_address(alias['mail']).lower()
|
||||
|
||||
|
||||
# get alias info
|
||||
forwards_to=alias["forward_tos"]
|
||||
permitted_senders=alias["permitted_senders"]
|
||||
description=alias["description"]
|
||||
auto=alias["auto"]
|
||||
|
||||
|
||||
# skip auto domain maps since these are not informative in the control panel's aliases list
|
||||
if auto and address.startswith("@"): continue
|
||||
|
||||
|
||||
domain = get_domain(address)
|
||||
|
||||
|
||||
# add to list
|
||||
if not domain in domains:
|
||||
if domain not in domains:
|
||||
domains[domain] = {
|
||||
"domain": domain,
|
||||
"aliases": [],
|
||||
@ -608,7 +605,7 @@ def get_mail_domains(env, as_map=False, category=None, users_only=False):
|
||||
del domains[domain_idna]
|
||||
|
||||
return domains
|
||||
|
||||
|
||||
|
||||
|
||||
def add_mail_domain(env, domain_idna, validate=True):
|
||||
@ -634,9 +631,9 @@ def add_mail_domain(env, domain_idna, validate=True):
|
||||
if conn.wait(id).count() == 0:
|
||||
# no mail users are using that domain!
|
||||
return False
|
||||
|
||||
|
||||
dn = 'dc=%s,%s' % (domain_idna, env.LDAP_DOMAINS_BASE)
|
||||
domain_utf8 = utf8_from_idna(domain_idna)
|
||||
domain_utf8 = utf8_from_idna(domain_idna)
|
||||
try:
|
||||
response = conn.wait( conn.add(dn, [ 'domain', 'mailDomain' ], {
|
||||
"dcIntl": domain_utf8,
|
||||
@ -654,7 +651,7 @@ def add_mail_domain(env, domain_idna, validate=True):
|
||||
pass
|
||||
return False
|
||||
|
||||
|
||||
|
||||
def remove_mail_domain(env, domain_idna, validate=True):
|
||||
# Remove the specified domain from the list of domains that we
|
||||
# handle mail for. The domain must be IDNA encoded.
|
||||
@ -674,11 +671,11 @@ def remove_mail_domain(env, domain_idna, validate=True):
|
||||
# there is one or more user or alias with that domain
|
||||
log.debug("remove_mail_domain: %s: has users and/or aliases", domain_idna)
|
||||
return False
|
||||
|
||||
|
||||
id = conn.search(env.LDAP_DOMAINS_BASE,
|
||||
"(&(objectClass=domain)(dc=%s))" % domain_idna,
|
||||
attributes=['businessCategory'])
|
||||
|
||||
|
||||
existing = conn.wait(id).next()
|
||||
if existing is None:
|
||||
# the domain doesn't exist!
|
||||
@ -713,7 +710,7 @@ def add_mail_user(email, pw, privs, display_name, env):
|
||||
# http-status).
|
||||
#
|
||||
# If successful, the string "OK" is returned.
|
||||
|
||||
|
||||
# validate email
|
||||
if email.strip() == "":
|
||||
return ("No email address provided.", 400)
|
||||
@ -750,7 +747,7 @@ def add_mail_user(email, pw, privs, display_name, env):
|
||||
id=conn.search(env.LDAP_ALIASES_BASE, "(&(objectClass=mailGroup)(mail=%s))" % email)
|
||||
if conn.wait(id).count() > 0:
|
||||
return ("An alias exists with that address.", 400)
|
||||
|
||||
|
||||
## Generate a unique id for uid
|
||||
#uid = '%s' % uuid.uuid4()
|
||||
# use a sha-1 hash of maildrop for uid
|
||||
@ -768,7 +765,7 @@ def add_mail_user(email, pw, privs, display_name, env):
|
||||
|
||||
# get the utf8 version if an idna domain was given
|
||||
email_utf8 = email_name + "@" + get_domain(email, as_unicode=True)
|
||||
|
||||
|
||||
# compile user's attributes
|
||||
# for historical reasons, make the email address lowercase
|
||||
attrs = {
|
||||
@ -790,7 +787,7 @@ def add_mail_user(email, pw, privs, display_name, env):
|
||||
|
||||
# set the password - the ldap server will hash it
|
||||
conn.extend.standard.modify_password(user=dn, new_password=pw)
|
||||
|
||||
|
||||
# tell postfix the domain is local, if needed
|
||||
return_status = "mail user added"
|
||||
domain_idna = get_domain(email, as_unicode=False)
|
||||
@ -802,12 +799,12 @@ def add_mail_user(email, pw, privs, display_name, env):
|
||||
if isinstance(result, tuple):
|
||||
# error occurred
|
||||
return result
|
||||
elif result != '':
|
||||
elif result:
|
||||
return_status += "\n" + result
|
||||
|
||||
|
||||
# convert alias's mailMember to member
|
||||
convert_mailMember(env, conn, dn, email)
|
||||
|
||||
|
||||
# Update things in case any new domains are added.
|
||||
if domain_added:
|
||||
return kick(env, return_status)
|
||||
@ -818,7 +815,7 @@ def set_mail_password(email, pw, env):
|
||||
# validate that the password is acceptable
|
||||
validate_password(pw)
|
||||
|
||||
# find the user
|
||||
# find the user
|
||||
conn = open_database(env)
|
||||
user = find_mail_user(env, email, ['shadowLastChange'], conn)
|
||||
if user is None:
|
||||
@ -836,7 +833,7 @@ def set_mail_display_name(email, display_name, env):
|
||||
# validate arguments
|
||||
if not display_name or display_name.strip() == "":
|
||||
return ("Display name may not be empty!", 400)
|
||||
|
||||
|
||||
# find the user
|
||||
conn = open_database(env)
|
||||
user = find_mail_user(env, email, ['cn', 'sn'], conn)
|
||||
@ -846,7 +843,7 @@ def set_mail_display_name(email, display_name, env):
|
||||
# update cn and sn
|
||||
sn = display_name[display_name.strip().find(' ')+1:]
|
||||
conn.modify_record(user, {'cn': display_name.strip(), 'sn': sn})
|
||||
|
||||
|
||||
return "OK"
|
||||
|
||||
def validate_login(email, pw, env):
|
||||
@ -889,12 +886,12 @@ def remove_mail_user(email_idna, env):
|
||||
#
|
||||
# If successful, the string "OK" is returned.
|
||||
conn = open_database(env)
|
||||
|
||||
|
||||
# find the user
|
||||
user = find_mail_user(env, email_idna, conn=conn)
|
||||
if user is None:
|
||||
return ("That's not a user (%s)." % email_idna, 400)
|
||||
|
||||
|
||||
# delete the user
|
||||
conn.wait( conn.delete(user['dn']) )
|
||||
|
||||
@ -909,7 +906,7 @@ def remove_mail_user(email_idna, env):
|
||||
if isinstance(result, tuple):
|
||||
# error occurred
|
||||
return result
|
||||
elif result != '':
|
||||
elif result:
|
||||
return_status += "\n" + result
|
||||
|
||||
# Update things in case any domains are removed.
|
||||
@ -929,11 +926,11 @@ def get_mail_user_privileges(email, env, empty_on_error=False):
|
||||
except LookupError as e:
|
||||
if empty_on_error: return []
|
||||
raise e
|
||||
|
||||
|
||||
if user is None:
|
||||
if empty_on_error: return []
|
||||
return ("That's not a user (%s)." % email, 400)
|
||||
|
||||
|
||||
return user['mailaccess']
|
||||
|
||||
def validate_privilege(priv):
|
||||
@ -951,7 +948,7 @@ def add_remove_mail_user_privilege(email, priv, action, env):
|
||||
# http-status).
|
||||
#
|
||||
# If successful, the string "OK" is returned.
|
||||
|
||||
|
||||
# validate
|
||||
validation = validate_privilege(priv)
|
||||
if validation: return validation
|
||||
@ -960,7 +957,7 @@ def add_remove_mail_user_privilege(email, priv, action, env):
|
||||
user = find_mail_user(env, email, attributes=['mailaccess'])
|
||||
if user is None:
|
||||
return ("That's not a user (%s)." % email, 400)
|
||||
|
||||
|
||||
privs = user['mailaccess'].copy()
|
||||
|
||||
# update privs set
|
||||
@ -969,7 +966,7 @@ def add_remove_mail_user_privilege(email, priv, action, env):
|
||||
if priv not in privs:
|
||||
privs.append(priv)
|
||||
changed = True
|
||||
|
||||
|
||||
elif action == "remove":
|
||||
if priv in privs:
|
||||
privs.remove(priv)
|
||||
@ -981,7 +978,7 @@ def add_remove_mail_user_privilege(email, priv, action, env):
|
||||
if changed:
|
||||
conn = open_database(env)
|
||||
conn.modify_record( user, {'mailaccess': privs} )
|
||||
|
||||
|
||||
return "OK"
|
||||
|
||||
|
||||
@ -1010,7 +1007,7 @@ def add_required_aliases(env, conn, domain_idna):
|
||||
verbose_result=True
|
||||
))
|
||||
log.debug("add_required_alias: %s: %r", email_utf8, results[-1])
|
||||
|
||||
|
||||
return results
|
||||
|
||||
def remove_required_aliases(env, conn, domain_idna):
|
||||
@ -1027,7 +1024,7 @@ def remove_required_aliases(env, conn, domain_idna):
|
||||
ignore_if_not_exists=True
|
||||
))
|
||||
log.debug("remove_required_alias: %s: %r", email_utf8, results[-1])
|
||||
|
||||
|
||||
return results
|
||||
|
||||
|
||||
@ -1056,7 +1053,7 @@ def convert_mailMember(env, conn, dn, mail):
|
||||
except ldap3.core.exceptions.LDAPAttributeOrValueExistsResult:
|
||||
pass
|
||||
|
||||
|
||||
|
||||
def add_mail_alias(address_utf8, description, forwards_to, permitted_senders, env, auto=False, update_if_exists=False, do_kick=True, verbose_result=False):
|
||||
# Add a new alias group with permitted senders.
|
||||
#
|
||||
@ -1170,7 +1167,7 @@ def add_mail_alias(address_utf8, description, forwards_to, permitted_senders, en
|
||||
vfwd_tos_local.append(dn)
|
||||
else:
|
||||
vfwd_tos_remote.append(fwd_to["email_idna"])
|
||||
|
||||
|
||||
# save to db
|
||||
|
||||
conn = open_database(env)
|
||||
@ -1187,7 +1184,7 @@ def add_mail_alias(address_utf8, description, forwards_to, permitted_senders, en
|
||||
return ("Alias already exists (%s)." % address, 400)
|
||||
if existing_alias and update_if_exists == 'ignore':
|
||||
return ""
|
||||
|
||||
|
||||
cn="%s" % uuid.uuid4()
|
||||
dn="cn=%s,%s" % (cn, env.LDAP_ALIASES_BASE)
|
||||
if not description:
|
||||
@ -1202,12 +1199,12 @@ def add_mail_alias(address_utf8, description, forwards_to, permitted_senders, en
|
||||
description = "Catch-all for %s" % address
|
||||
else:
|
||||
description ="Mail alias %s" % address
|
||||
|
||||
|
||||
# when updating, ensure the description has a value because
|
||||
# the ldap schema does not allow an empty field
|
||||
else:
|
||||
description=" "
|
||||
|
||||
|
||||
attrs = {
|
||||
"mail": address if address == address_utf8.lower() else [ address, address_utf8 ],
|
||||
"description": description,
|
||||
@ -1222,7 +1219,7 @@ def add_mail_alias(address_utf8, description, forwards_to, permitted_senders, en
|
||||
attributes,
|
||||
[ 'mailGroup', 'namedProperties' ],
|
||||
attrs)
|
||||
|
||||
|
||||
if op == 'modify':
|
||||
return_status = "alias updated"
|
||||
else:
|
||||
@ -1231,9 +1228,9 @@ def add_mail_alias(address_utf8, description, forwards_to, permitted_senders, en
|
||||
|
||||
if verbose_result:
|
||||
return_status += ": " + address_utf8
|
||||
|
||||
|
||||
# add or modify permitted-senders group
|
||||
|
||||
|
||||
cn = '%s' % uuid.uuid4()
|
||||
dn = "cn=%s,%s" % (cn, env.LDAP_PERMITTED_SENDERS_BASE)
|
||||
attrs = {
|
||||
@ -1261,22 +1258,22 @@ def add_mail_alias(address_utf8, description, forwards_to, permitted_senders, en
|
||||
remove_mail_domain(env, domain_idna, validate=False)
|
||||
elif count_vfwd > 0:
|
||||
domain_added = add_mail_domain(env, domain_idna, validate=False)
|
||||
|
||||
|
||||
if domain_added:
|
||||
results = add_required_aliases(env, conn, domain_idna)
|
||||
for result in results:
|
||||
if isinstance(result, tuple):
|
||||
# error occurred
|
||||
return result
|
||||
elif result != '':
|
||||
elif result:
|
||||
return_status += "\n" + result
|
||||
|
||||
|
||||
if do_kick and domain_added:
|
||||
# Update things in case any new domains are added.
|
||||
return kick(env, return_status)
|
||||
else:
|
||||
return return_status
|
||||
|
||||
|
||||
|
||||
def remove_mail_alias(address_utf8, env, do_kick=True, auto=None, ignore_if_not_exists=False, verbose_result=False):
|
||||
# Remove an alias group and it's associated permitted senders
|
||||
@ -1292,7 +1289,7 @@ def remove_mail_alias(address_utf8, env, do_kick=True, auto=None, ignore_if_not_
|
||||
# http-status).
|
||||
#
|
||||
# If successful, the string "OK" is returned.
|
||||
|
||||
|
||||
# convert Unicode domain to IDNA
|
||||
address = sanitize_idn_email_address(address_utf8)
|
||||
|
||||
@ -1322,18 +1319,19 @@ def remove_mail_alias(address_utf8, env, do_kick=True, auto=None, ignore_if_not_
|
||||
if isinstance(result, tuple):
|
||||
# error occurred
|
||||
return result
|
||||
elif result != '':
|
||||
elif result:
|
||||
return_status += "\n" + result
|
||||
|
||||
if do_kick and domain_removed:
|
||||
# Update things in case any domains are removed.
|
||||
return kick(env, return_status)
|
||||
else:
|
||||
return return_status
|
||||
None
|
||||
|
||||
|
||||
def add_auto_aliases(aliases, env):
|
||||
conn, c = open_database(env, with_connection=True)
|
||||
c.execute("DELETE FROM auto_aliases");
|
||||
c.execute("DELETE FROM auto_aliases")
|
||||
for source, destination in aliases.items():
|
||||
c.execute("INSERT INTO auto_aliases (source, destination) VALUES (?, ?)", (source, destination))
|
||||
conn.commit()
|
||||
@ -1344,7 +1342,7 @@ def get_system_administrator(env):
|
||||
# def get_required_aliases(env):
|
||||
# # These are the aliases that must exist.
|
||||
# # Returns a set of email addresses.
|
||||
|
||||
|
||||
# aliases = set()
|
||||
|
||||
# # The system administrator alias is required.
|
||||
@ -1396,9 +1394,11 @@ def kick(env, mail_result=None):
|
||||
def validate_password(pw):
|
||||
# validate password
|
||||
if pw.strip() == "":
|
||||
raise ValueError("No password provided.")
|
||||
msg = "No password provided."
|
||||
raise ValueError(msg)
|
||||
if len(pw) < 8:
|
||||
raise ValueError("Passwords must be at least eight characters.")
|
||||
msg = "Passwords must be at least eight characters."
|
||||
raise ValueError(msg)
|
||||
|
||||
if __name__ == "__main__":
|
||||
import sys
|
||||
|
@ -22,8 +22,8 @@ def strip_order_prefix(rec, attributes):
|
||||
sorted in the record making the prefix superfluous.
|
||||
|
||||
For example, the function will change:
|
||||
totpSecret: {0}secret1
|
||||
totpSecret: {1}secret2
|
||||
totpSecret: {0}secret1
|
||||
totpSecret: {1}secret2
|
||||
to:
|
||||
totpSecret: secret1
|
||||
totpSecret: secret2
|
||||
@ -32,16 +32,16 @@ def strip_order_prefix(rec, attributes):
|
||||
'''
|
||||
for attr in attributes:
|
||||
# ignore attribute that doesn't exist
|
||||
if not attr in rec: continue
|
||||
if not attr in rec: continue
|
||||
# ..as well as None values and empty list
|
||||
if not rec[attr]: continue
|
||||
|
||||
|
||||
newvals = []
|
||||
for val in rec[attr]:
|
||||
i = val.find('}')
|
||||
newvals.append(val[i+1:])
|
||||
rec[attr] = newvals
|
||||
|
||||
|
||||
def get_mfa_user(email, env, conn=None):
|
||||
'''get the ldap record for the user along with all MFA-related
|
||||
attributes
|
||||
@ -49,7 +49,7 @@ def get_mfa_user(email, env, conn=None):
|
||||
'''
|
||||
user = find_mail_user(env, email, ['objectClass','totpSecret','totpMruToken','totpMruTokenTime','totpLabel'], conn)
|
||||
if not user:
|
||||
raise ValueError("User does not exist.")
|
||||
raise ValueError("User does not exist.")
|
||||
strip_order_prefix(user, ['totpSecret','totpMruToken','totpMruTokenTime','totpLabel'])
|
||||
return user
|
||||
|
||||
@ -104,7 +104,8 @@ multiple mfa schemes enabled of the same type.
|
||||
if type == "totp":
|
||||
mfa_totp.enable(user, secret, token, label, env)
|
||||
else:
|
||||
raise ValueError("Invalid MFA type.")
|
||||
msg = "Invalid MFA type."
|
||||
raise ValueError(msg)
|
||||
|
||||
def disable_mfa(email, mfa_id, env):
|
||||
'''disable a specific MFA scheme. `mfa_id` identifies the specific
|
||||
@ -121,7 +122,7 @@ def disable_mfa(email, mfa_id, env):
|
||||
return mfa_totp.disable(user, mfa_id, env)
|
||||
else:
|
||||
return False
|
||||
|
||||
|
||||
def validate_auth_mfa(email, request, env):
|
||||
# Validates that a login request satisfies any MFA modes
|
||||
# that have been enabled for the user's account. Returns
|
||||
|
@ -42,7 +42,7 @@ def time_ns():
|
||||
return time.time_ns()
|
||||
else:
|
||||
return int(time.time() * 1000000000)
|
||||
|
||||
|
||||
def get_state(user):
|
||||
state_list = []
|
||||
|
||||
@ -62,7 +62,8 @@ def enable(user, secret, token, label, env):
|
||||
# Sanity check with the provide current token.
|
||||
totp = pyotp.TOTP(secret)
|
||||
if not totp.verify(token, valid_window=1):
|
||||
raise ValueError("Invalid token.")
|
||||
msg = "Invalid token."
|
||||
raise ValueError(msg)
|
||||
|
||||
mods = {
|
||||
"totpSecret": user['totpSecret'].copy() + [secret],
|
||||
@ -72,7 +73,7 @@ def enable(user, secret, token, label, env):
|
||||
}
|
||||
if 'totpUser' not in user['objectClass']:
|
||||
mods['objectClass'] = user['objectClass'].copy() + ['totpUser']
|
||||
|
||||
|
||||
conn = open_database(env)
|
||||
conn.modify_record(user, mods)
|
||||
|
||||
@ -107,13 +108,13 @@ def disable(user, id, env):
|
||||
"totpSecret": None,
|
||||
"totpLabel": None
|
||||
}
|
||||
mods["objectClass"].remove("totpUser")
|
||||
mods["objectClass"].remove("totpUser")
|
||||
open_database(env).modify_record(user, mods)
|
||||
return True
|
||||
|
||||
else:
|
||||
# Disable totp at the index specified
|
||||
idx = index_from_id(user, id)
|
||||
idx = index_from_id(user, id)
|
||||
if idx<0 or idx>=len(user['totpSecret']):
|
||||
return False
|
||||
mods = {
|
||||
|
@ -13,7 +13,8 @@
|
||||
import os, os.path, re, shutil, subprocess, tempfile
|
||||
|
||||
from utils import shell, safe_domain_name, sort_domains
|
||||
import idna
|
||||
import functools
|
||||
import operator
|
||||
|
||||
# SELECTING SSL CERTIFICATES FOR USE IN WEB
|
||||
|
||||
@ -92,9 +93,8 @@ def get_ssl_certificates(env):
|
||||
for domain in cert_domains:
|
||||
# The primary hostname can only use a certificate mapped
|
||||
# to the system private key.
|
||||
if domain == env['PRIMARY_HOSTNAME']:
|
||||
if cert["private_key"]["filename"] != os.path.join(env['STORAGE_ROOT'], 'ssl', 'ssl_private_key.pem'):
|
||||
continue
|
||||
if domain == env['PRIMARY_HOSTNAME'] and cert["private_key"]["filename"] != os.path.join(env['STORAGE_ROOT'], 'ssl', 'ssl_private_key.pem'):
|
||||
continue
|
||||
|
||||
domains.setdefault(domain, []).append(cert)
|
||||
|
||||
@ -162,13 +162,12 @@ def get_domain_ssl_files(domain, ssl_certificates, env, allow_missing_cert=False
|
||||
"certificate_object": load_pem(load_cert_chain(ssl_certificate)[0]),
|
||||
}
|
||||
|
||||
if use_main_cert:
|
||||
if domain == env['PRIMARY_HOSTNAME']:
|
||||
# The primary domain must use the server certificate because
|
||||
# it is hard-coded in some service configuration files.
|
||||
return system_certificate
|
||||
if use_main_cert and domain == env['PRIMARY_HOSTNAME']:
|
||||
# The primary domain must use the server certificate because
|
||||
# it is hard-coded in some service configuration files.
|
||||
return system_certificate
|
||||
|
||||
wildcard_domain = re.sub("^[^\\.]+", "*", domain)
|
||||
wildcard_domain = re.sub(r"^[^\.]+", "*", domain)
|
||||
if domain in ssl_certificates:
|
||||
return ssl_certificates[domain]
|
||||
elif wildcard_domain in ssl_certificates:
|
||||
@ -224,7 +223,7 @@ def get_certificates_to_provision(env, limit_domains=None, show_valid_certs=True
|
||||
if not value: continue # IPv6 is not configured
|
||||
response = query_dns(domain, rtype)
|
||||
if response != normalize_ip(value):
|
||||
bad_dns.append("%s (%s)" % (response, rtype))
|
||||
bad_dns.append(f"{response} ({rtype})")
|
||||
|
||||
if bad_dns:
|
||||
domains_cant_provision[domain] = "The domain name does not resolve to this machine: " \
|
||||
@ -277,11 +276,11 @@ def provision_certificates(env, limit_domains):
|
||||
# primary domain listed in each certificate.
|
||||
from dns_update import get_dns_zones
|
||||
certs = { }
|
||||
for zone, zonefile in get_dns_zones(env):
|
||||
for zone, _zonefile in get_dns_zones(env):
|
||||
certs[zone] = [[]]
|
||||
for domain in sort_domains(domains, env):
|
||||
# Does the domain end with any domain we've seen so far.
|
||||
for parent in certs.keys():
|
||||
for parent in certs:
|
||||
if domain.endswith("." + parent):
|
||||
# Add this to the parent's list of domains.
|
||||
# Start a new group if the list already has
|
||||
@ -298,7 +297,7 @@ def provision_certificates(env, limit_domains):
|
||||
|
||||
# Flatten to a list of lists of domains (from a mapping). Remove empty
|
||||
# lists (zones with no domains that need certs).
|
||||
certs = sum(certs.values(), [])
|
||||
certs = functools.reduce(operator.iadd, certs.values(), [])
|
||||
certs = [_ for _ in certs if len(_) > 0]
|
||||
|
||||
# Prepare to provision.
|
||||
@ -426,7 +425,7 @@ def create_csr(domain, ssl_key, country_code, env):
|
||||
"openssl", "req", "-new",
|
||||
"-key", ssl_key,
|
||||
"-sha256",
|
||||
"-subj", "/C=%s/CN=%s" % (country_code, domain)])
|
||||
"-subj", f"/C={country_code}/CN={domain}"])
|
||||
|
||||
def install_cert(domain, ssl_cert, ssl_chain, env, raw=False):
|
||||
# Write the combined cert+chain to a temporary path and validate that it is OK.
|
||||
@ -462,8 +461,8 @@ def install_cert_copy_file(fn, env):
|
||||
from cryptography.hazmat.primitives import hashes
|
||||
from binascii import hexlify
|
||||
cert = load_pem(load_cert_chain(fn)[0])
|
||||
all_domains, cn = get_certificate_domains(cert)
|
||||
path = "%s-%s-%s.pem" % (
|
||||
_all_domains, cn = get_certificate_domains(cert)
|
||||
path = "{}-{}-{}.pem".format(
|
||||
safe_domain_name(cn), # common name, which should be filename safe because it is IDNA-encoded, but in case of a malformed cert make sure it's ok to use as a filename
|
||||
cert.not_valid_after.date().isoformat().replace("-", ""), # expiration date
|
||||
hexlify(cert.fingerprint(hashes.SHA256())).decode("ascii")[0:8], # fingerprint prefix
|
||||
@ -535,12 +534,12 @@ def check_certificate(domain, ssl_certificate, ssl_private_key, warn_if_expiring
|
||||
# First check that the domain name is one of the names allowed by
|
||||
# the certificate.
|
||||
if domain is not None:
|
||||
certificate_names, cert_primary_name = get_certificate_domains(cert)
|
||||
certificate_names, _cert_primary_name = get_certificate_domains(cert)
|
||||
|
||||
# Check that the domain appears among the acceptable names, or a wildcard
|
||||
# form of the domain name (which is a stricter check than the specs but
|
||||
# should work in normal cases).
|
||||
wildcard_domain = re.sub("^[^\\.]+", "*", domain)
|
||||
wildcard_domain = re.sub(r"^[^\.]+", "*", domain)
|
||||
if domain not in certificate_names and wildcard_domain not in certificate_names:
|
||||
return ("The certificate is for the wrong domain name. It is for %s."
|
||||
% ", ".join(sorted(certificate_names)), None)
|
||||
@ -551,7 +550,7 @@ def check_certificate(domain, ssl_certificate, ssl_private_key, warn_if_expiring
|
||||
with open(ssl_private_key, 'rb') as f:
|
||||
priv_key = load_pem(f.read())
|
||||
except ValueError as e:
|
||||
return ("The private key file %s is not a private key file: %s" % (ssl_private_key, str(e)), None)
|
||||
return (f"The private key file {ssl_private_key} is not a private key file: {e!s}", None)
|
||||
|
||||
if not isinstance(priv_key, RSAPrivateKey):
|
||||
return ("The private key file %s is not a private key file." % ssl_private_key, None)
|
||||
@ -582,7 +581,7 @@ def check_certificate(domain, ssl_certificate, ssl_private_key, warn_if_expiring
|
||||
import datetime
|
||||
now = datetime.datetime.utcnow()
|
||||
if not(cert.not_valid_before <= now <= cert.not_valid_after):
|
||||
return ("The certificate has expired or is not yet valid. It is valid from %s to %s." % (cert.not_valid_before, cert.not_valid_after), None)
|
||||
return (f"The certificate has expired or is not yet valid. It is valid from {cert.not_valid_before} to {cert.not_valid_after}.", None)
|
||||
|
||||
# Next validate that the certificate is valid. This checks whether the certificate
|
||||
# is self-signed, that the chain of trust makes sense, that it is signed by a CA
|
||||
@ -641,7 +640,8 @@ def load_cert_chain(pemfile):
|
||||
pem = f.read() + b"\n" # ensure trailing newline
|
||||
pemblocks = re.findall(re_pem, pem)
|
||||
if len(pemblocks) == 0:
|
||||
raise ValueError("File does not contain valid PEM data.")
|
||||
msg = "File does not contain valid PEM data."
|
||||
raise ValueError(msg)
|
||||
return pemblocks
|
||||
|
||||
def load_pem(pem):
|
||||
@ -652,9 +652,10 @@ def load_pem(pem):
|
||||
from cryptography.hazmat.backends import default_backend
|
||||
pem_type = re.match(b"-+BEGIN (.*?)-+[\r\n]", pem)
|
||||
if pem_type is None:
|
||||
raise ValueError("File is not a valid PEM-formatted file.")
|
||||
msg = "File is not a valid PEM-formatted file."
|
||||
raise ValueError(msg)
|
||||
pem_type = pem_type.group(1)
|
||||
if pem_type in (b"RSA PRIVATE KEY", b"PRIVATE KEY"):
|
||||
if pem_type in {b"RSA PRIVATE KEY", b"PRIVATE KEY"}:
|
||||
return serialization.load_pem_private_key(pem, password=None, backend=default_backend())
|
||||
if pem_type == b"CERTIFICATE":
|
||||
return load_pem_x509_certificate(pem, default_backend())
|
||||
|
@ -14,11 +14,10 @@
|
||||
# TLS certificates have been signed, etc., and if not tells the user
|
||||
# what to do next.
|
||||
|
||||
import sys, os, os.path, re, subprocess, datetime, multiprocessing.pool
|
||||
import sys, os, os.path, re, datetime, multiprocessing.pool
|
||||
import asyncio
|
||||
|
||||
import dns.reversename, dns.resolver
|
||||
import dateutil.parser, dateutil.tz
|
||||
import idna
|
||||
import psutil
|
||||
import postfix_mta_sts_resolver.resolver
|
||||
@ -101,7 +100,7 @@ def run_services_checks(env, output, pool):
|
||||
all_running = True
|
||||
fatal = False
|
||||
ret = pool.starmap(check_service, ((i, service, env) for i, service in enumerate(get_services())), chunksize=1)
|
||||
for i, running, fatal2, output2 in sorted(ret):
|
||||
for _i, running, fatal2, output2 in sorted(ret):
|
||||
if output2 is None: continue # skip check (e.g. no port was set, e.g. no sshd)
|
||||
all_running = all_running and running
|
||||
fatal = fatal or fatal2
|
||||
@ -137,7 +136,7 @@ def check_service(i, service, env):
|
||||
try:
|
||||
s.connect((ip, service["port"]))
|
||||
return True
|
||||
except OSError as e:
|
||||
except OSError:
|
||||
# timed out or some other odd error
|
||||
return False
|
||||
finally:
|
||||
@ -164,18 +163,17 @@ def check_service(i, service, env):
|
||||
output.print_error("%s is not running (port %d)." % (service['name'], service['port']))
|
||||
|
||||
# Why is nginx not running?
|
||||
if not running and service["port"] in (80, 443):
|
||||
if not running and service["port"] in {80, 443}:
|
||||
output.print_line(shell('check_output', ['nginx', '-t'], capture_stderr=True, trap=True)[1].strip())
|
||||
|
||||
# Service should be running locally.
|
||||
elif try_connect("127.0.0.1"):
|
||||
running = True
|
||||
else:
|
||||
# Service should be running locally.
|
||||
if try_connect("127.0.0.1"):
|
||||
running = True
|
||||
else:
|
||||
output.print_error("%s is not running (port %d)." % (service['name'], service['port']))
|
||||
output.print_error("%s is not running (port %d)." % (service['name'], service['port']))
|
||||
|
||||
# Flag if local DNS is not running.
|
||||
if not running and service["port"] == 53 and service["public"] == False:
|
||||
if not running and service["port"] == 53 and service["public"] is False:
|
||||
fatal = True
|
||||
|
||||
return (i, running, fatal, output)
|
||||
@ -207,7 +205,7 @@ def check_ufw(env, output):
|
||||
for service in get_services():
|
||||
if service["public"] and not is_port_allowed(ufw, service["port"]):
|
||||
not_allowed_ports += 1
|
||||
output.print_error("Port %s (%s) should be allowed in the firewall, please re-run the setup." % (service["port"], service["name"]))
|
||||
output.print_error("Port {} ({}) should be allowed in the firewall, please re-run the setup.".format(service["port"], service["name"]))
|
||||
|
||||
if not_allowed_ports == 0:
|
||||
output.print_ok("Firewall is active.")
|
||||
@ -225,10 +223,10 @@ def check_ssh_password(env, output):
|
||||
# the configuration file.
|
||||
if not os.path.exists("/etc/ssh/sshd_config"):
|
||||
return
|
||||
with open("/etc/ssh/sshd_config", "r") as f:
|
||||
with open("/etc/ssh/sshd_config", encoding="utf-8") as f:
|
||||
sshd = f.read()
|
||||
if re.search("\nPasswordAuthentication\s+yes", sshd) \
|
||||
or not re.search("\nPasswordAuthentication\s+no", sshd):
|
||||
if re.search("\nPasswordAuthentication\\s+yes", sshd) \
|
||||
or not re.search("\nPasswordAuthentication\\s+no", sshd):
|
||||
output.print_error("""The SSH server on this machine permits password-based login. A more secure
|
||||
way to log in is using a public key. Add your SSH public key to $HOME/.ssh/authorized_keys, check
|
||||
that you can log in without a password, set the option 'PasswordAuthentication no' in
|
||||
@ -249,7 +247,7 @@ def check_software_updates(env, output):
|
||||
else:
|
||||
output.print_error("There are %d software packages that can be updated." % len(pkgs))
|
||||
for p in pkgs:
|
||||
output.print_line("%s (%s)" % (p["package"], p["version"]))
|
||||
output.print_line("{} ({})".format(p["package"], p["version"]))
|
||||
|
||||
def check_system_aliases(env, output):
|
||||
# Check that the administrator alias exists since that's where all
|
||||
@ -281,8 +279,7 @@ def check_free_disk_space(rounded_values, env, output):
|
||||
except:
|
||||
backup_cache_count = 0
|
||||
if backup_cache_count > 1:
|
||||
output.print_warning("The backup cache directory {} has more than one backup target cache. Consider clearing this directory to save disk space."
|
||||
.format(backup_cache_path))
|
||||
output.print_warning(f"The backup cache directory {backup_cache_path} has more than one backup target cache. Consider clearing this directory to save disk space.")
|
||||
|
||||
def check_free_memory(rounded_values, env, output):
|
||||
# Check free memory.
|
||||
@ -308,7 +305,7 @@ def run_network_checks(env, output):
|
||||
# Stop if we cannot make an outbound connection on port 25. Many residential
|
||||
# networks block outbound port 25 to prevent their network from sending spam.
|
||||
# See if we can reach one of Google's MTAs with a 5-second timeout.
|
||||
code, ret = shell("check_call", ["/bin/nc", "-z", "-w5", "aspmx.l.google.com", "25"], trap=True)
|
||||
_code, ret = shell("check_call", ["/bin/nc", "-z", "-w5", "aspmx.l.google.com", "25"], trap=True)
|
||||
if ret == 0:
|
||||
output.print_ok("Outbound mail (SMTP port 25) is not blocked.")
|
||||
else:
|
||||
@ -321,18 +318,26 @@ def run_network_checks(env, output):
|
||||
# The user might have ended up on an IP address that was previously in use
|
||||
# by a spammer, or the user may be deploying on a residential network. We
|
||||
# will not be able to reliably send mail in these cases.
|
||||
|
||||
# See https://www.spamhaus.org/news/article/807/using-our-public-mirrors-check-your-return-codes-now. for
|
||||
# information on spamhaus return codes
|
||||
rev_ip4 = ".".join(reversed(env['PUBLIC_IP'].split('.')))
|
||||
zen = query_dns(rev_ip4+'.zen.spamhaus.org', 'A', nxdomain=None)
|
||||
if zen is None:
|
||||
output.print_ok("IP address is not blacklisted by zen.spamhaus.org.")
|
||||
elif zen == "[timeout]":
|
||||
output.print_warning("Connection to zen.spamhaus.org timed out. We could not determine whether your server's IP address is blacklisted. Please try again later.")
|
||||
output.print_warning("Connection to zen.spamhaus.org timed out. Could not determine whether this box's IP address is blacklisted. Please try again later.")
|
||||
elif zen == "[Not Set]":
|
||||
output.print_warning("Could not connect to zen.spamhaus.org. We could not determine whether your server's IP address is blacklisted. Please try again later.")
|
||||
output.print_warning("Could not connect to zen.spamhaus.org. Could not determine whether this box's IP address is blacklisted. Please try again later.")
|
||||
elif zen == "127.255.255.252":
|
||||
output.print_warning("Incorrect spamhaus query: %s. Could not determine whether this box's IP address is blacklisted." % (rev_ip4+'.zen.spamhaus.org'))
|
||||
elif zen == "127.255.255.254":
|
||||
output.print_warning("Mail-in-a-Box is configured to use a public DNS server. This is not supported by spamhaus. Could not determine whether this box's IP address is blacklisted.")
|
||||
elif zen == "127.255.255.255":
|
||||
output.print_warning("Too many queries have been performed on the spamhaus server. Could not determine whether this box's IP address is blacklisted.")
|
||||
else:
|
||||
output.print_error("""The IP address of this machine %s is listed in the Spamhaus Block List (code %s),
|
||||
which may prevent recipients from receiving your email. See http://www.spamhaus.org/query/ip/%s."""
|
||||
% (env['PUBLIC_IP'], zen, env['PUBLIC_IP']))
|
||||
output.print_error("""The IP address of this machine {} is listed in the Spamhaus Block List (code {}),
|
||||
which may prevent recipients from receiving your email. See http://www.spamhaus.org/query/ip/{}.""".format(env['PUBLIC_IP'], zen, env['PUBLIC_IP']))
|
||||
|
||||
def run_domain_checks(rounded_time, env, output, pool, domains_to_check=None):
|
||||
# Get the list of domains we handle mail for.
|
||||
@ -353,7 +358,7 @@ def run_domain_checks(rounded_time, env, output, pool, domains_to_check=None):
|
||||
domains_to_check = [
|
||||
d for d in domains_to_check
|
||||
if not (
|
||||
d.split(".", 1)[0] in ("www", "autoconfig", "autodiscover", "mta-sts")
|
||||
d.split(".", 1)[0] in {"www", "autoconfig", "autodiscover", "mta-sts"}
|
||||
and len(d.split(".", 1)) == 2
|
||||
and d.split(".", 1)[1] in domains_to_check
|
||||
)
|
||||
@ -435,10 +440,9 @@ def check_primary_hostname_dns(domain, env, output, dns_domains, dns_zonefiles):
|
||||
# If a DS record is set on the zone containing this domain, check DNSSEC now.
|
||||
has_dnssec = False
|
||||
for zone in dns_domains:
|
||||
if zone == domain or domain.endswith("." + zone):
|
||||
if query_dns(zone, "DS", nxdomain=None) is not None:
|
||||
has_dnssec = True
|
||||
check_dnssec(zone, env, output, dns_zonefiles, is_checking_primary=True)
|
||||
if (zone == domain or domain.endswith("." + zone)) and query_dns(zone, "DS", nxdomain=None) is not None:
|
||||
has_dnssec = True
|
||||
check_dnssec(zone, env, output, dns_zonefiles, is_checking_primary=True)
|
||||
|
||||
ip = query_dns(domain, "A")
|
||||
ns_ips = query_dns("ns1." + domain, "A") + '/' + query_dns("ns2." + domain, "A")
|
||||
@ -450,44 +454,41 @@ def check_primary_hostname_dns(domain, env, output, dns_domains, dns_zonefiles):
|
||||
# the nameserver, are reporting the right info --- but if the glue is incorrect this
|
||||
# will probably fail.
|
||||
if ns_ips == env['PUBLIC_IP'] + '/' + env['PUBLIC_IP']:
|
||||
output.print_ok("Nameserver glue records are correct at registrar. [ns1/ns2.%s ↦ %s]" % (env['PRIMARY_HOSTNAME'], env['PUBLIC_IP']))
|
||||
output.print_ok("Nameserver glue records are correct at registrar. [ns1/ns2.{} ↦ {}]".format(env['PRIMARY_HOSTNAME'], env['PUBLIC_IP']))
|
||||
|
||||
elif ip == env['PUBLIC_IP']:
|
||||
# The NS records are not what we expect, but the domain resolves correctly, so
|
||||
# the user may have set up external DNS. List this discrepancy as a warning.
|
||||
output.print_warning("""Nameserver glue records (ns1.%s and ns2.%s) should be configured at your domain name
|
||||
registrar as having the IP address of this box (%s). They currently report addresses of %s. If you have set up External DNS, this may be OK."""
|
||||
% (env['PRIMARY_HOSTNAME'], env['PRIMARY_HOSTNAME'], env['PUBLIC_IP'], ns_ips))
|
||||
output.print_warning("""Nameserver glue records (ns1.{} and ns2.{}) should be configured at your domain name
|
||||
registrar as having the IP address of this box ({}). They currently report addresses of {}. If you have set up External DNS, this may be OK.""".format(env['PRIMARY_HOSTNAME'], env['PRIMARY_HOSTNAME'], env['PUBLIC_IP'], ns_ips))
|
||||
|
||||
else:
|
||||
output.print_error("""Nameserver glue records are incorrect. The ns1.%s and ns2.%s nameservers must be configured at your domain name
|
||||
registrar as having the IP address %s. They currently report addresses of %s. It may take several hours for
|
||||
public DNS to update after a change."""
|
||||
% (env['PRIMARY_HOSTNAME'], env['PRIMARY_HOSTNAME'], env['PUBLIC_IP'], ns_ips))
|
||||
output.print_error("""Nameserver glue records are incorrect. The ns1.{} and ns2.{} nameservers must be configured at your domain name
|
||||
registrar as having the IP address {}. They currently report addresses of {}. It may take several hours for
|
||||
public DNS to update after a change.""".format(env['PRIMARY_HOSTNAME'], env['PRIMARY_HOSTNAME'], env['PUBLIC_IP'], ns_ips))
|
||||
|
||||
# Check that PRIMARY_HOSTNAME resolves to PUBLIC_IP[V6] in public DNS.
|
||||
ipv6 = query_dns(domain, "AAAA") if env.get("PUBLIC_IPV6") else None
|
||||
if ip == env['PUBLIC_IP'] and not (ipv6 and env['PUBLIC_IPV6'] and ipv6 != normalize_ip(env['PUBLIC_IPV6'])):
|
||||
output.print_ok("Domain resolves to box's IP address. [%s ↦ %s]" % (env['PRIMARY_HOSTNAME'], my_ips))
|
||||
output.print_ok("Domain resolves to box's IP address. [{} ↦ {}]".format(env['PRIMARY_HOSTNAME'], my_ips))
|
||||
else:
|
||||
output.print_error("""This domain must resolve to your box's IP address (%s) in public DNS but it currently resolves
|
||||
to %s. It may take several hours for public DNS to update after a change. This problem may result from other
|
||||
issues listed above."""
|
||||
% (my_ips, ip + ((" / " + ipv6) if ipv6 is not None else "")))
|
||||
output.print_error("""This domain must resolve to this box's IP address ({}) in public DNS but it currently resolves
|
||||
to {}. It may take several hours for public DNS to update after a change. This problem may result from other
|
||||
issues listed above.""".format(my_ips, ip + ((" / " + ipv6) if ipv6 is not None else "")))
|
||||
|
||||
|
||||
# Check reverse DNS matches the PRIMARY_HOSTNAME. Note that it might not be
|
||||
# a DNS zone if it is a subdomain of another domain we have a zone for.
|
||||
existing_rdns_v4 = query_dns(dns.reversename.from_address(env['PUBLIC_IP']), "PTR")
|
||||
existing_rdns_v6 = query_dns(dns.reversename.from_address(env['PUBLIC_IPV6']), "PTR") if env.get("PUBLIC_IPV6") else None
|
||||
if existing_rdns_v4 == domain and existing_rdns_v6 in (None, domain):
|
||||
output.print_ok("Reverse DNS is set correctly at ISP. [%s ↦ %s]" % (my_ips, env['PRIMARY_HOSTNAME']))
|
||||
if existing_rdns_v4 == domain and existing_rdns_v6 in {None, domain}:
|
||||
output.print_ok("Reverse DNS is set correctly at ISP. [{} ↦ {}]".format(my_ips, env['PRIMARY_HOSTNAME']))
|
||||
elif existing_rdns_v4 == existing_rdns_v6 or existing_rdns_v6 is None:
|
||||
output.print_error("""Your box's reverse DNS is currently %s, but it should be %s. Your ISP or cloud provider will have instructions
|
||||
on setting up reverse DNS for your box.""" % (existing_rdns_v4, domain) )
|
||||
output.print_error(f"""This box's reverse DNS is currently {existing_rdns_v4}, but it should be {domain}. Your ISP or cloud provider will have instructions
|
||||
on setting up reverse DNS for this box.""" )
|
||||
else:
|
||||
output.print_error("""Your box's reverse DNS is currently %s (IPv4) and %s (IPv6), but it should be %s. Your ISP or cloud provider will have instructions
|
||||
on setting up reverse DNS for your box.""" % (existing_rdns_v4, existing_rdns_v6, domain) )
|
||||
output.print_error(f"""This box's reverse DNS is currently {existing_rdns_v4} (IPv4) and {existing_rdns_v6} (IPv6), but it should be {domain}. Your ISP or cloud provider will have instructions
|
||||
on setting up reverse DNS for this box.""" )
|
||||
|
||||
# Check the TLSA record.
|
||||
tlsa_qname = "_25._tcp." + domain
|
||||
@ -501,9 +502,8 @@ def check_primary_hostname_dns(domain, env, output, dns_domains, dns_zonefiles):
|
||||
# since TLSA shouldn't be used without DNSSEC.
|
||||
output.print_warning("""The DANE TLSA record for incoming mail is not set. This is optional.""")
|
||||
else:
|
||||
output.print_error("""The DANE TLSA record for incoming mail (%s) is not correct. It is '%s' but it should be '%s'.
|
||||
It may take several hours for public DNS to update after a change."""
|
||||
% (tlsa_qname, tlsa25, tlsa25_expected))
|
||||
output.print_error(f"""The DANE TLSA record for incoming mail ({tlsa_qname}) is not correct. It is '{tlsa25}' but it should be '{tlsa25_expected}'.
|
||||
It may take several hours for public DNS to update after a change.""")
|
||||
|
||||
# Check that the hostmaster@ email address exists.
|
||||
check_alias_exists("Hostmaster contact address", "hostmaster@" + domain, env, output)
|
||||
@ -538,7 +538,7 @@ def check_dns_zone(domain, env, output, dns_zonefiles):
|
||||
secondary_ns = custom_secondary_ns or ["ns2." + env['PRIMARY_HOSTNAME']]
|
||||
|
||||
existing_ns = query_dns(domain, "NS")
|
||||
correct_ns = "; ".join(sorted(["ns1." + env['PRIMARY_HOSTNAME']] + secondary_ns))
|
||||
correct_ns = "; ".join(sorted(["ns1." + env["PRIMARY_HOSTNAME"], *secondary_ns]))
|
||||
ip = query_dns(domain, "A")
|
||||
|
||||
probably_external_dns = False
|
||||
@ -547,14 +547,12 @@ def check_dns_zone(domain, env, output, dns_zonefiles):
|
||||
output.print_ok("Nameservers are set correctly at registrar. [%s]" % correct_ns)
|
||||
elif ip == correct_ip:
|
||||
# The domain resolves correctly, so maybe the user is using External DNS.
|
||||
output.print_warning("""The nameservers set on this domain at your domain name registrar should be %s. They are currently %s.
|
||||
If you are using External DNS, this may be OK."""
|
||||
% (correct_ns, existing_ns) )
|
||||
output.print_warning(f"""The nameservers set on this domain at your domain name registrar should be {correct_ns}. They are currently {existing_ns}.
|
||||
If you are using External DNS, this may be OK.""" )
|
||||
probably_external_dns = True
|
||||
else:
|
||||
output.print_error("""The nameservers set on this domain are incorrect. They are currently %s. Use your domain name registrar's
|
||||
control panel to set the nameservers to %s."""
|
||||
% (existing_ns, correct_ns) )
|
||||
output.print_error(f"""The nameservers set on this domain are incorrect. They are currently {existing_ns}. Use your domain name registrar's
|
||||
control panel to set the nameservers to {correct_ns}.""" )
|
||||
|
||||
# Check that each custom secondary nameserver resolves the IP address.
|
||||
|
||||
@ -575,7 +573,7 @@ def check_dns_zone(domain, env, output, dns_zonefiles):
|
||||
elif ip is None:
|
||||
output.print_error("Secondary nameserver %s is not configured to resolve this domain." % ns)
|
||||
else:
|
||||
output.print_error("Secondary nameserver %s is not configured correctly. (It resolved this domain as %s. It should be %s.)" % (ns, ip, correct_ip))
|
||||
output.print_error(f"Secondary nameserver {ns} is not configured correctly. (It resolved this domain as {ip}. It should be {correct_ip}.)")
|
||||
|
||||
def check_dns_zone_suggestions(domain, env, output, dns_zonefiles, domains_with_a_records):
|
||||
# Warn if a custom DNS record is preventing this or the automatic www redirect from
|
||||
@ -604,7 +602,7 @@ def check_dnssec(domain, env, output, dns_zonefiles, is_checking_primary=False):
|
||||
expected_ds_records = { }
|
||||
ds_file = '/etc/nsd/zones/' + dns_zonefiles[domain] + '.ds'
|
||||
if not os.path.exists(ds_file): return # Domain is in our database but DNS has not yet been updated.
|
||||
with open(ds_file) as f:
|
||||
with open(ds_file, encoding="utf-8") as f:
|
||||
for rr_ds in f:
|
||||
rr_ds = rr_ds.rstrip()
|
||||
ds_keytag, ds_alg, ds_digalg, ds_digest = rr_ds.split("\t")[4].split(" ")
|
||||
@ -613,7 +611,7 @@ def check_dnssec(domain, env, output, dns_zonefiles, is_checking_primary=False):
|
||||
# record that we suggest using is for the KSK (and that's how the DS records were generated).
|
||||
# We'll also give the nice name for the key algorithm.
|
||||
dnssec_keys = load_env_vars_from_file(os.path.join(env['STORAGE_ROOT'], 'dns/dnssec/%s.conf' % alg_name_map[ds_alg]))
|
||||
with open(os.path.join(env['STORAGE_ROOT'], 'dns/dnssec/' + dnssec_keys['KSK'] + '.key'), 'r') as f:
|
||||
with open(os.path.join(env['STORAGE_ROOT'], 'dns/dnssec/' + dnssec_keys['KSK'] + '.key'), encoding="utf-8") as f:
|
||||
dnsssec_pubkey = f.read().split("\t")[3].split(" ")[3]
|
||||
|
||||
expected_ds_records[ (ds_keytag, ds_alg, ds_digalg, ds_digest) ] = {
|
||||
@ -646,10 +644,10 @@ def check_dnssec(domain, env, output, dns_zonefiles, is_checking_primary=False):
|
||||
#
|
||||
# But it may not be preferred. Only algorithm 13 is preferred. Warn if any of the
|
||||
# matched zones uses a different algorithm.
|
||||
if set(r[1] for r in matched_ds) == { '13' } and set(r[2] for r in matched_ds) <= { '2', '4' }: # all are alg 13 and digest type 2 or 4
|
||||
if {r[1] for r in matched_ds} == { '13' } and {r[2] for r in matched_ds} <= { '2', '4' }: # all are alg 13 and digest type 2 or 4
|
||||
output.print_ok("DNSSEC 'DS' record is set correctly at registrar.")
|
||||
return
|
||||
elif len([r for r in matched_ds if r[1] == '13' and r[2] in ( '2', '4' )]) > 0: # some but not all are alg 13
|
||||
elif len([r for r in matched_ds if r[1] == '13' and r[2] in { '2', '4' }]) > 0: # some but not all are alg 13
|
||||
output.print_ok("DNSSEC 'DS' record is set correctly at registrar. (Records using algorithm other than ECDSAP256SHA256 and digest types other than SHA-256/384 should be removed.)")
|
||||
return
|
||||
else: # no record uses alg 13
|
||||
@ -681,8 +679,8 @@ def check_dnssec(domain, env, output, dns_zonefiles, is_checking_primary=False):
|
||||
output.print_line("----------")
|
||||
output.print_line("Key Tag: " + ds_suggestion['keytag'])
|
||||
output.print_line("Key Flags: KSK / 257")
|
||||
output.print_line("Algorithm: %s / %s" % (ds_suggestion['alg'], ds_suggestion['alg_name']))
|
||||
output.print_line("Digest Type: %s / %s" % (ds_suggestion['digalg'], ds_suggestion['digalg_name']))
|
||||
output.print_line("Algorithm: {} / {}".format(ds_suggestion['alg'], ds_suggestion['alg_name']))
|
||||
output.print_line("Digest Type: {} / {}".format(ds_suggestion['digalg'], ds_suggestion['digalg_name']))
|
||||
output.print_line("Digest: " + ds_suggestion['digest'])
|
||||
output.print_line("Public Key: ")
|
||||
output.print_line(ds_suggestion['pubkey'], monospace=True)
|
||||
@ -693,7 +691,7 @@ def check_dnssec(domain, env, output, dns_zonefiles, is_checking_primary=False):
|
||||
output.print_line("")
|
||||
output.print_line("The DS record is currently set to:")
|
||||
for rr in sorted(ds):
|
||||
output.print_line("Key Tag: {0}, Algorithm: {1}, Digest Type: {2}, Digest: {3}".format(*rr))
|
||||
output.print_line("Key Tag: {}, Algorithm: {}, Digest Type: {}, Digest: {}".format(*rr))
|
||||
|
||||
def check_mail_domain(domain, env, output):
|
||||
# Check the MX record.
|
||||
@ -701,21 +699,19 @@ def check_mail_domain(domain, env, output):
|
||||
recommended_mx = "10 " + env['PRIMARY_HOSTNAME']
|
||||
mx = query_dns(domain, "MX", nxdomain=None)
|
||||
|
||||
if mx is None:
|
||||
mxhost = None
|
||||
elif mx == "[timeout]":
|
||||
if mx is None or mx == "[timeout]":
|
||||
mxhost = None
|
||||
else:
|
||||
# query_dns returns a semicolon-delimited list
|
||||
# of priority-host pairs.
|
||||
mxhost = mx.split('; ')[0].split(' ')[1]
|
||||
|
||||
if mxhost == None:
|
||||
if mxhost is None:
|
||||
# A missing MX record is okay on the primary hostname because
|
||||
# the primary hostname's A record (the MX fallback) is... itself,
|
||||
# which is what we want the MX to be.
|
||||
if domain == env['PRIMARY_HOSTNAME']:
|
||||
output.print_ok("Domain's email is directed to this domain. [%s has no MX record, which is ok]" % (domain,))
|
||||
output.print_ok(f"Domain's email is directed to this domain. [{domain} has no MX record, which is ok]")
|
||||
|
||||
# And a missing MX record is okay on other domains if the A record
|
||||
# matches the A record of the PRIMARY_HOSTNAME. Actually this will
|
||||
@ -723,17 +719,17 @@ def check_mail_domain(domain, env, output):
|
||||
else:
|
||||
domain_a = query_dns(domain, "A", nxdomain=None)
|
||||
primary_a = query_dns(env['PRIMARY_HOSTNAME'], "A", nxdomain=None)
|
||||
if domain_a != None and domain_a == primary_a:
|
||||
output.print_ok("Domain's email is directed to this domain. [%s has no MX record but its A record is OK]" % (domain,))
|
||||
if domain_a is not None and domain_a == primary_a:
|
||||
output.print_ok(f"Domain's email is directed to this domain. [{domain} has no MX record but its A record is OK]")
|
||||
else:
|
||||
output.print_error("""This domain's DNS MX record is not set. It should be '%s'. Mail will not
|
||||
output.print_error(f"""This domain's DNS MX record is not set. It should be '{recommended_mx}'. Mail will not
|
||||
be delivered to this box. It may take several hours for public DNS to update after a
|
||||
change. This problem may result from other issues listed here.""" % (recommended_mx,))
|
||||
change. This problem may result from other issues listed here.""")
|
||||
|
||||
elif mxhost == env['PRIMARY_HOSTNAME']:
|
||||
good_news = "Domain's email is directed to this domain. [%s ↦ %s]" % (domain, mx)
|
||||
good_news = f"Domain's email is directed to this domain. [{domain} ↦ {mx}]"
|
||||
if mx != recommended_mx:
|
||||
good_news += " This configuration is non-standard. The recommended configuration is '%s'." % (recommended_mx,)
|
||||
good_news += f" This configuration is non-standard. The recommended configuration is '{recommended_mx}'."
|
||||
output.print_ok(good_news)
|
||||
|
||||
# Check MTA-STS policy.
|
||||
@ -744,14 +740,14 @@ def check_mail_domain(domain, env, output):
|
||||
if policy[1].get("mx") == [env['PRIMARY_HOSTNAME']] and policy[1].get("mode") == "enforce": # policy[0] is the policyid
|
||||
output.print_ok("MTA-STS policy is present.")
|
||||
else:
|
||||
output.print_error("MTA-STS policy is present but has unexpected settings. [{}]".format(policy[1]))
|
||||
output.print_error(f"MTA-STS policy is present but has unexpected settings. [{policy[1]}]")
|
||||
else:
|
||||
output.print_error("MTA-STS policy is missing: {}".format(valid))
|
||||
output.print_error(f"MTA-STS policy is missing: {valid}")
|
||||
|
||||
else:
|
||||
output.print_error("""This domain's DNS MX record is incorrect. It is currently set to '%s' but should be '%s'. Mail will not
|
||||
output.print_error(f"""This domain's DNS MX record is incorrect. It is currently set to '{mx}' but should be '{recommended_mx}'. Mail will not
|
||||
be delivered to this box. It may take several hours for public DNS to update after a change. This problem may result from
|
||||
other issues listed here.""" % (mx, recommended_mx))
|
||||
other issues listed here.""")
|
||||
|
||||
# Check that the postmaster@ email address exists. Not required if the domain has a
|
||||
# catch-all address or domain alias.
|
||||
@ -761,17 +757,26 @@ def check_mail_domain(domain, env, output):
|
||||
# Stop if the domain is listed in the Spamhaus Domain Block List.
|
||||
# The user might have chosen a domain that was previously in use by a spammer
|
||||
# and will not be able to reliably send mail.
|
||||
|
||||
# See https://www.spamhaus.org/news/article/807/using-our-public-mirrors-check-your-return-codes-now. for
|
||||
# information on spamhaus return codes
|
||||
dbl = query_dns(domain+'.dbl.spamhaus.org', "A", nxdomain=None)
|
||||
if dbl is None:
|
||||
output.print_ok("Domain is not blacklisted by dbl.spamhaus.org.")
|
||||
elif dbl == "[timeout]":
|
||||
output.print_warning("Connection to dbl.spamhaus.org timed out. We could not determine whether the domain {} is blacklisted. Please try again later.".format(domain))
|
||||
output.print_warning(f"Connection to dbl.spamhaus.org timed out. Could not determine whether the domain {domain} is blacklisted. Please try again later.")
|
||||
elif dbl == "[Not Set]":
|
||||
output.print_warning("Could not connect to dbl.spamhaus.org. We could not determine whether the domain {} is blacklisted. Please try again later.".format(domain))
|
||||
output.print_warning(f"Could not connect to dbl.spamhaus.org. Could not determine whether the domain {domain} is blacklisted. Please try again later.")
|
||||
elif dbl == "127.255.255.252":
|
||||
output.print_warning("Incorrect spamhaus query: %s. Could not determine whether the domain %s is blacklisted." % (domain+'.dbl.spamhaus.org', domain))
|
||||
elif dbl == "127.255.255.254":
|
||||
output.print_warning("Mail-in-a-Box is configured to use a public DNS server. This is not supported by spamhaus. Could not determine whether the domain {} is blacklisted.".format(domain))
|
||||
elif dbl == "127.255.255.255":
|
||||
output.print_warning("Too many queries have been performed on the spamhaus server. Could not determine whether the domain {} is blacklisted.".format(domain))
|
||||
else:
|
||||
output.print_error("""This domain is listed in the Spamhaus Domain Block List (code %s),
|
||||
output.print_error(f"""This domain is listed in the Spamhaus Domain Block List (code {dbl}),
|
||||
which may prevent recipients from receiving your mail.
|
||||
See http://www.spamhaus.org/dbl/ and http://www.spamhaus.org/query/domain/%s.""" % (dbl, domain))
|
||||
See http://www.spamhaus.org/dbl/ and http://www.spamhaus.org/query/domain/{domain}.""")
|
||||
|
||||
def check_web_domain(domain, rounded_time, ssl_certificates, env, output):
|
||||
# See if the domain's A record resolves to our PUBLIC_IP. This is already checked
|
||||
@ -785,13 +790,13 @@ def check_web_domain(domain, rounded_time, ssl_certificates, env, output):
|
||||
if value == normalize_ip(expected):
|
||||
ok_values.append(value)
|
||||
else:
|
||||
output.print_error("""This domain should resolve to your box's IP address (%s %s) if you would like the box to serve
|
||||
webmail or a website on this domain. The domain currently resolves to %s in public DNS. It may take several hours for
|
||||
public DNS to update after a change. This problem may result from other issues listed here.""" % (rtype, expected, value))
|
||||
output.print_error(f"""This domain should resolve to this box's IP address ({rtype} {expected}) if you would like the box to serve
|
||||
webmail or a website on this domain. The domain currently resolves to {value} in public DNS. It may take several hours for
|
||||
public DNS to update after a change. This problem may result from other issues listed here.""")
|
||||
return
|
||||
|
||||
# If both A and AAAA are correct...
|
||||
output.print_ok("Domain resolves to this box's IP address. [%s ↦ %s]" % (domain, '; '.join(ok_values)))
|
||||
output.print_ok("Domain resolves to this box's IP address. [{} ↦ {}]".format(domain, '; '.join(ok_values)))
|
||||
|
||||
|
||||
# We need a TLS certificate for PRIMARY_HOSTNAME because that's where the
|
||||
@ -838,7 +843,7 @@ def query_dns(qname, rtype, nxdomain='[Not Set]', at=None, as_list=False):
|
||||
# be expressed in equivalent string forms. Canonicalize the form before
|
||||
# returning them. The caller should normalize any IP addresses the result
|
||||
# of this method is compared with.
|
||||
if rtype in ("A", "AAAA"):
|
||||
if rtype in {"A", "AAAA"}:
|
||||
response = [normalize_ip(str(r)) for r in response]
|
||||
|
||||
if as_list:
|
||||
@ -854,7 +859,7 @@ def check_ssl_cert(domain, rounded_time, ssl_certificates, env, output):
|
||||
# Check that TLS certificate is signed.
|
||||
|
||||
# Skip the check if the A record is not pointed here.
|
||||
if query_dns(domain, "A", None) not in (env['PUBLIC_IP'], None): return
|
||||
if query_dns(domain, "A", None) not in {env['PUBLIC_IP'], None}: return
|
||||
|
||||
# Where is the certificate file stored?
|
||||
tls_cert = get_domain_ssl_files(domain, ssl_certificates, env, allow_missing_cert=True)
|
||||
@ -928,18 +933,16 @@ def what_version_is_this(env):
|
||||
# Git may not be installed and Mail-in-a-Box may not have been cloned from github,
|
||||
# so this function may raise all sorts of exceptions.
|
||||
miab_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
|
||||
tag = shell("check_output", ["/usr/bin/git", "describe", "--always", "--abbrev=0"], env={"GIT_DIR": os.path.join(miab_dir, '.git')}).strip()
|
||||
return tag
|
||||
return shell("check_output", ["/usr/bin/git", "describe", "--always", "--abbrev=0"], env={"GIT_DIR": os.path.join(miab_dir, '.git')}).strip()
|
||||
|
||||
def get_latest_miab_version():
|
||||
# This pings https://mailinabox.email/setup.sh and extracts the tag named in
|
||||
# the script to determine the current product version.
|
||||
from urllib.request import urlopen, HTTPError, URLError
|
||||
from socket import timeout
|
||||
|
||||
try:
|
||||
return re.search(b'TAG=(.*)', urlopen("https://mailinabox.email/setup.sh?ping=1", timeout=5).read()).group(1).decode("utf8")
|
||||
except (HTTPError, URLError, timeout):
|
||||
except (TimeoutError, HTTPError, URLError):
|
||||
return None
|
||||
|
||||
def check_miab_version(env, output):
|
||||
@ -960,7 +963,7 @@ def check_miab_version(env, output):
|
||||
elif latest_ver is None:
|
||||
output.print_error("Latest Mail-in-a-Box LDAP version could not be determined. You are running version %s." % this_ver)
|
||||
else:
|
||||
output.print_error("A new version of Mail-in-a-Box LDAP is available. You are running version %s. The latest version is %s. For upgrade instructions, see https://mailinabox.email. "
|
||||
output.print_error(f"A new version of Mail-in-a-Box LDAP is available. You are running version {this_ver}. The latest version is {latest_ver}. For upgrade instructions, see https://mailinabox.email. "
|
||||
% (this_ver, latest_ver))
|
||||
|
||||
def run_and_output_changes(env, pool):
|
||||
@ -976,8 +979,11 @@ def run_and_output_changes(env, pool):
|
||||
# Load previously saved status checks.
|
||||
cache_fn = "/var/cache/mailinabox/status_checks.json"
|
||||
if os.path.exists(cache_fn):
|
||||
with open(cache_fn, 'r') as f:
|
||||
prev = json.load(f)
|
||||
with open(cache_fn, encoding="utf-8") as f:
|
||||
try:
|
||||
prev = json.load(f)
|
||||
except json.JSONDecodeError:
|
||||
prev = []
|
||||
|
||||
# execute hooks
|
||||
hook_data = {
|
||||
@ -1022,14 +1028,14 @@ def run_and_output_changes(env, pool):
|
||||
out.add_heading(category + " -- Previously:")
|
||||
elif op == "delete":
|
||||
out.add_heading(category + " -- Removed")
|
||||
if op in ("replace", "delete"):
|
||||
if op in {"replace", "delete"}:
|
||||
BufferedOutput(with_lines=prev_lines[i1:i2]).playback(out)
|
||||
|
||||
if op == "replace":
|
||||
out.add_heading(category + " -- Currently:")
|
||||
elif op == "insert":
|
||||
out.add_heading(category + " -- Added")
|
||||
if op in ("replace", "insert"):
|
||||
if op in {"replace", "insert"}:
|
||||
BufferedOutput(with_lines=cur_lines[j1:j2]).playback(out)
|
||||
|
||||
for category, prev_lines in prev_status.items():
|
||||
@ -1045,10 +1051,10 @@ def run_and_output_changes(env, pool):
|
||||
'output':out
|
||||
}
|
||||
hooks.exec_hooks('status_checks', hook_data)
|
||||
|
||||
|
||||
# Store the current status checks output for next time.
|
||||
os.makedirs(os.path.dirname(cache_fn), exist_ok=True)
|
||||
with open(cache_fn, "w") as f:
|
||||
with open(cache_fn, "w", encoding="utf-8") as f:
|
||||
json.dump(cur.buf, f, indent=True)
|
||||
|
||||
|
||||
@ -1086,8 +1092,8 @@ class FileOutput:
|
||||
|
||||
def print_block(self, message, first_line=" "):
|
||||
print(first_line, end='', file=self.buf)
|
||||
message = re.sub("\n\s*", " ", message)
|
||||
words = re.split("(\s+)", message)
|
||||
message = re.sub("\n\\s*", " ", message)
|
||||
words = re.split(r"(\s+)", message)
|
||||
linelen = 0
|
||||
for w in words:
|
||||
if self.width and (linelen + len(w) > self.width-1-len(first_line)):
|
||||
@ -1126,7 +1132,7 @@ class ConsoleOutput(FileOutput):
|
||||
class BufferedOutput:
|
||||
# Record all of the instance method calls so we can play them back later.
|
||||
def __init__(self, with_lines=None):
|
||||
self.buf = [] if not with_lines else with_lines
|
||||
self.buf = with_lines if with_lines else []
|
||||
def __getattr__(self, attr):
|
||||
if attr not in ("add_heading", "print_ok", "print_error", "print_warning", "print_info", "print_block", "print_line"):
|
||||
raise AttributeError
|
||||
|
@ -30,13 +30,13 @@ def load_environment():
|
||||
# It won't exist exist until migration 13 completes...
|
||||
if os.path.exists(os.path.join(env["STORAGE_ROOT"],"ldap/miab_ldap.conf")):
|
||||
load_env_vars_from_file(os.path.join(env["STORAGE_ROOT"],"ldap/miab_ldap.conf"), strip_quotes=True, merge_env=env)
|
||||
|
||||
|
||||
return env
|
||||
|
||||
def load_env_vars_from_file(fn, strip_quotes=False, merge_env=None):
|
||||
# Load settings from a KEY=VALUE file.
|
||||
env = Environment()
|
||||
with open(fn, 'r') as f:
|
||||
with open(fn, encoding="utf-8") as f:
|
||||
for line in f:
|
||||
env.setdefault(*line.strip().split("=", 1))
|
||||
if strip_quotes:
|
||||
@ -46,25 +46,25 @@ def load_env_vars_from_file(fn, strip_quotes=False, merge_env=None):
|
||||
return env
|
||||
|
||||
def save_environment(env):
|
||||
with open("/etc/mailinabox.conf", "w") as f:
|
||||
with open("/etc/mailinabox.conf", "w", encoding="utf-8") as f:
|
||||
for k, v in env.items():
|
||||
f.write("%s=%s\n" % (k, v))
|
||||
f.write(f"{k}={v}\n")
|
||||
|
||||
# THE SETTINGS FILE AT STORAGE_ROOT/settings.yaml.
|
||||
|
||||
def write_settings(config, env):
|
||||
import rtyaml
|
||||
fn = os.path.join(env['STORAGE_ROOT'], 'settings.yaml')
|
||||
with open(fn, "w") as f:
|
||||
with open(fn, "w", encoding="utf-8") as f:
|
||||
f.write(rtyaml.dump(config))
|
||||
|
||||
def load_settings(env):
|
||||
import rtyaml
|
||||
fn = os.path.join(env['STORAGE_ROOT'], 'settings.yaml')
|
||||
try:
|
||||
with open(fn, "r") as f:
|
||||
with open(fn, encoding="utf-8") as f:
|
||||
config = rtyaml.load(f)
|
||||
if not isinstance(config, dict): raise ValueError() # caught below
|
||||
if not isinstance(config, dict): raise ValueError # caught below
|
||||
return config
|
||||
except:
|
||||
return { }
|
||||
@ -85,7 +85,7 @@ def sort_domains(domain_names, env):
|
||||
# from shortest to longest since zones are always shorter than their
|
||||
# subdomains.
|
||||
zones = { }
|
||||
for domain in sorted(domain_names, key=lambda d : len(d)):
|
||||
for domain in sorted(domain_names, key=len):
|
||||
for z in zones.values():
|
||||
if domain.endswith("." + z):
|
||||
# We found a parent domain already in the list.
|
||||
@ -107,7 +107,7 @@ def sort_domains(domain_names, env):
|
||||
))
|
||||
|
||||
# Now sort the domain names that fall within each zone.
|
||||
domain_names = sorted(domain_names,
|
||||
return sorted(domain_names,
|
||||
key = lambda d : (
|
||||
# First by zone.
|
||||
zone_domains.index(zones[d]),
|
||||
@ -121,25 +121,26 @@ def sort_domains(domain_names, env):
|
||||
# Then in right-to-left lexicographic order of the .-separated parts of the name.
|
||||
list(reversed(d.split("."))),
|
||||
))
|
||||
|
||||
return domain_names
|
||||
|
||||
|
||||
def sort_email_addresses(email_addresses, env):
|
||||
email_addresses = set(email_addresses)
|
||||
domains = set(email.split("@", 1)[1] for email in email_addresses if "@" in email)
|
||||
domains = {email.split("@", 1)[1] for email in email_addresses if "@" in email}
|
||||
ret = []
|
||||
for domain in sort_domains(domains, env):
|
||||
domain_emails = set(email for email in email_addresses if email.endswith("@" + domain))
|
||||
domain_emails = {email for email in email_addresses if email.endswith("@" + domain)}
|
||||
ret.extend(sorted(domain_emails))
|
||||
email_addresses -= domain_emails
|
||||
ret.extend(sorted(email_addresses)) # whatever is left
|
||||
return ret
|
||||
|
||||
def shell(method, cmd_args, env={}, capture_stderr=False, return_bytes=False, trap=False, input=None):
|
||||
def shell(method, cmd_args, env=None, capture_stderr=False, return_bytes=False, trap=False, input=None):
|
||||
# A safe way to execute processes.
|
||||
# Some processes like apt-get require being given a sane PATH.
|
||||
import subprocess
|
||||
|
||||
if env is None:
|
||||
env = {}
|
||||
env.update({ "PATH": "/sbin:/bin:/usr/sbin:/usr/bin" })
|
||||
kwargs = {
|
||||
'env': env,
|
||||
@ -175,7 +176,7 @@ def du(path):
|
||||
# soft and hard links.
|
||||
total_size = 0
|
||||
seen = set()
|
||||
for dirpath, dirnames, filenames in os.walk(path):
|
||||
for dirpath, _dirnames, filenames in os.walk(path):
|
||||
for f in filenames:
|
||||
fp = os.path.join(dirpath, f)
|
||||
try:
|
||||
|
@ -34,16 +34,17 @@ def get_web_domains(env, include_www_redirects=True, include_auto=True, exclude_
|
||||
# Add 'www.' subdomains that we want to provide default redirects
|
||||
# to the main domain for. We'll add 'www.' to any DNS zones, i.e.
|
||||
# the topmost of each domain we serve.
|
||||
domains |= set('www.' + zone for zone, zonefile in get_dns_zones(env))
|
||||
domains |= {'www.' + zone for zone, zonefile in get_dns_zones(env)}
|
||||
|
||||
if 'mail' in categories and include_auto:
|
||||
# Add Autoconfiguration domains for domains that there are user accounts at:
|
||||
# 'autoconfig.' for Mozilla Thunderbird auto setup.
|
||||
# 'autodiscover.' for Activesync autodiscovery.
|
||||
domains |= set('autoconfig.' + maildomain for maildomain in get_mail_domains(env, users_only=True))
|
||||
domains |= set('autodiscover.' + maildomain for maildomain in get_mail_domains(env, users_only=True))
|
||||
# 'autodiscover.' for ActiveSync autodiscovery (Z-Push).
|
||||
domains |= {'autoconfig.' + maildomain for maildomain in get_mail_domains(env, users_only=True)}
|
||||
domains |= {'autodiscover.' + maildomain for maildomain in get_mail_domains(env, users_only=True)}
|
||||
|
||||
# 'mta-sts.' for MTA-STS support for all domains that have email addresses.
|
||||
domains |= set('mta-sts.' + maildomain for maildomain in get_mail_domains(env))
|
||||
domains |= {'mta-sts.' + maildomain for maildomain in get_mail_domains(env)}
|
||||
|
||||
if exclude_dns_elsewhere:
|
||||
# ...Unless the domain has an A/AAAA record that maps it to a different
|
||||
@ -56,15 +57,14 @@ def get_web_domains(env, include_www_redirects=True, include_auto=True, exclude_
|
||||
domains.add(env['PRIMARY_HOSTNAME'])
|
||||
|
||||
# Sort the list so the nginx conf gets written in a stable order.
|
||||
domains = sort_domains(domains, env)
|
||||
return sort_domains(domains, env)
|
||||
|
||||
return domains
|
||||
|
||||
def get_domains_with_a_records(env):
|
||||
domains = set()
|
||||
dns = get_custom_dns_config(env)
|
||||
for domain, rtype, value in dns:
|
||||
if rtype == "CNAME" or (rtype in ("A", "AAAA") and value not in ("local", env['PUBLIC_IP'])):
|
||||
if rtype == "CNAME" or (rtype in {"A", "AAAA"} and value not in {"local", env['PUBLIC_IP']}):
|
||||
domains.add(domain)
|
||||
return domains
|
||||
|
||||
@ -74,7 +74,7 @@ def get_web_domains_with_root_overrides(env):
|
||||
root_overrides = { }
|
||||
nginx_conf_custom_fn = os.path.join(env["STORAGE_ROOT"], "www/custom.yaml")
|
||||
if os.path.exists(nginx_conf_custom_fn):
|
||||
with open(nginx_conf_custom_fn, 'r') as f:
|
||||
with open(nginx_conf_custom_fn, encoding='utf-8') as f:
|
||||
custom_settings = rtyaml.load(f)
|
||||
for domain, settings in custom_settings.items():
|
||||
for type, value in [('redirect', settings.get('redirects', {}).get('/')),
|
||||
@ -89,7 +89,7 @@ def do_web_update(env):
|
||||
|
||||
# Helper for reading config files and templates
|
||||
def read_conf(conf_fn):
|
||||
with open(os.path.join(os.path.dirname(__file__), "../conf", conf_fn), "r") as f:
|
||||
with open(os.path.join(os.path.dirname(__file__), "../conf", conf_fn), encoding='utf-8') as f:
|
||||
return f.read()
|
||||
|
||||
# Build an nginx configuration file.
|
||||
@ -132,12 +132,12 @@ def do_web_update(env):
|
||||
# Did the file change? If not, don't bother writing & restarting nginx.
|
||||
nginx_conf_fn = "/etc/nginx/conf.d/local.conf"
|
||||
if os.path.exists(nginx_conf_fn):
|
||||
with open(nginx_conf_fn) as f:
|
||||
with open(nginx_conf_fn, encoding='utf-8') as f:
|
||||
if f.read() == nginx_conf:
|
||||
return ""
|
||||
|
||||
# Save the file.
|
||||
with open(nginx_conf_fn, "w") as f:
|
||||
with open(nginx_conf_fn, "w", encoding='utf-8') as f:
|
||||
f.write(nginx_conf)
|
||||
|
||||
# Kick nginx. Since this might be called from the web admin
|
||||
@ -169,13 +169,13 @@ def make_domain_config(domain, templates, ssl_certificates, env):
|
||||
with open(filepath, 'rb') as f:
|
||||
sha1.update(f.read())
|
||||
return sha1.hexdigest()
|
||||
nginx_conf_extra += "\t# ssl files sha1: %s / %s\n" % (hashfile(tls_cert["private-key"]), hashfile(tls_cert["certificate"]))
|
||||
nginx_conf_extra += "\t# ssl files sha1: {} / {}\n".format(hashfile(tls_cert["private-key"]), hashfile(tls_cert["certificate"]))
|
||||
|
||||
# Add in any user customizations in YAML format.
|
||||
hsts = "yes"
|
||||
nginx_conf_custom_fn = os.path.join(env["STORAGE_ROOT"], "www/custom.yaml")
|
||||
if os.path.exists(nginx_conf_custom_fn):
|
||||
with open(nginx_conf_custom_fn, 'r') as f:
|
||||
with open(nginx_conf_custom_fn, encoding='utf-8') as f:
|
||||
yaml = rtyaml.load(f)
|
||||
if domain in yaml:
|
||||
yaml = yaml[domain]
|
||||
@ -215,16 +215,16 @@ def make_domain_config(domain, templates, ssl_certificates, env):
|
||||
nginx_conf_extra += "\n\t\talias %s;" % alias
|
||||
nginx_conf_extra += "\n\t}\n"
|
||||
for path, url in yaml.get("redirects", {}).items():
|
||||
nginx_conf_extra += "\trewrite %s %s permanent;\n" % (path, url)
|
||||
nginx_conf_extra += f"\trewrite {path} {url} permanent;\n"
|
||||
|
||||
# override the HSTS directive type
|
||||
hsts = yaml.get("hsts", hsts)
|
||||
|
||||
# Add the HSTS header.
|
||||
if hsts == "yes":
|
||||
nginx_conf_extra += "\tadd_header Strict-Transport-Security \"max-age=15768000\" always;\n"
|
||||
nginx_conf_extra += '\tadd_header Strict-Transport-Security "max-age=15768000" always;\n'
|
||||
elif hsts == "preload":
|
||||
nginx_conf_extra += "\tadd_header Strict-Transport-Security \"max-age=15768000; includeSubDomains; preload\" always;\n"
|
||||
nginx_conf_extra += '\tadd_header Strict-Transport-Security "max-age=15768000; includeSubDomains; preload" always;\n'
|
||||
|
||||
# Add in any user customizations in the includes/ folder.
|
||||
nginx_conf_custom_include = os.path.join(env["STORAGE_ROOT"], "www", safe_domain_name(domain) + ".conf")
|
||||
@ -235,7 +235,7 @@ def make_domain_config(domain, templates, ssl_certificates, env):
|
||||
# Combine the pieces. Iteratively place each template into the "# ADDITIONAL DIRECTIVES HERE" placeholder
|
||||
# of the previous template.
|
||||
nginx_conf = "# ADDITIONAL DIRECTIVES HERE\n"
|
||||
for t in templates + [nginx_conf_extra]:
|
||||
for t in [*templates, nginx_conf_extra]:
|
||||
nginx_conf = re.sub("[ \t]*# ADDITIONAL DIRECTIVES HERE *\n", t, nginx_conf)
|
||||
|
||||
# Replace substitution strings in the template & return.
|
||||
@ -244,9 +244,8 @@ def make_domain_config(domain, templates, ssl_certificates, env):
|
||||
nginx_conf = nginx_conf.replace("$ROOT", root)
|
||||
nginx_conf = nginx_conf.replace("$SSL_KEY", tls_cert["private-key"])
|
||||
nginx_conf = nginx_conf.replace("$SSL_CERTIFICATE", tls_cert["certificate"])
|
||||
nginx_conf = nginx_conf.replace("$REDIRECT_DOMAIN", re.sub(r"^www\.", "", domain)) # for default www redirects to parent domain
|
||||
return nginx_conf.replace("$REDIRECT_DOMAIN", re.sub(r"^www\.", "", domain)) # for default www redirects to parent domain
|
||||
|
||||
return nginx_conf
|
||||
|
||||
def get_web_root(domain, env, test_exists=True):
|
||||
# Try STORAGE_ROOT/web/domain_name if it exists, but fall back to STORAGE_ROOT/web/default.
|
||||
@ -285,4 +284,3 @@ def get_web_domains_info(env):
|
||||
}
|
||||
for domain in get_web_domains(env)
|
||||
]
|
||||
|
||||
|
@ -8,9 +8,9 @@
|
||||
#####
|
||||
|
||||
from daemon import app
|
||||
import auth, utils
|
||||
import utils
|
||||
|
||||
app.logger.addHandler(utils.create_syslog_handler())
|
||||
|
||||
if __name__ == "__main__":
|
||||
app.run(port=10222)
|
||||
app.run(port=10222)
|
||||
|
@ -70,7 +70,7 @@ tools/editconf.py /etc/postfix/main.cf \
|
||||
myhostname=$PRIMARY_HOSTNAME\
|
||||
smtpd_banner="\$myhostname ESMTP Hi, I'm a Mail-in-a-Box (Ubuntu/Postfix; see https://mailinabox.email/)" \
|
||||
mydestination=localhost
|
||||
|
||||
|
||||
# Tweak some queue settings:
|
||||
# * Inform users when their e-mail delivery is delayed more than 3 hours (default is not to warn).
|
||||
# * Stop trying to send an undeliverable e-mail after 2 days (instead of 5), and for bounce messages just try for 1 day.
|
||||
@ -245,14 +245,15 @@ tools/editconf.py /etc/postfix/main.cf -e lmtp_destination_recipient_limit=
|
||||
# * `reject_unlisted_recipient`: Although Postfix will reject mail to unknown recipients, it's nicer to reject such mail ahead of greylisting rather than after.
|
||||
# * `check_policy_service`: Apply greylisting using postgrey.
|
||||
#
|
||||
# Note the spamhaus rbl return codes are taken into account as adviced here: https://docs.spamhaus.com/datasets/docs/source/40-real-world-usage/PublicMirrors/MTAs/020-Postfix.html
|
||||
# Notes: #NODOC
|
||||
# permit_dnswl_client can pass through mail from whitelisted IP addresses, which would be good to put before greylisting #NODOC
|
||||
# so these IPs get mail delivered quickly. But when an IP is not listed in the permit_dnswl_client list (i.e. it is not #NODOC
|
||||
# whitelisted) then postfix does a DEFER_IF_REJECT, which results in all "unknown user" sorts of messages turning into #NODOC
|
||||
# "450 4.7.1 Client host rejected: Service unavailable". This is a retry code, so the mail doesn't properly bounce. #NODOC
|
||||
tools/editconf.py /etc/postfix/main.cf \
|
||||
smtpd_sender_restrictions="reject_non_fqdn_sender,reject_unknown_sender_domain,reject_authenticated_sender_login_mismatch,reject_rhsbl_sender dbl.spamhaus.org" \
|
||||
smtpd_recipient_restrictions=permit_sasl_authenticated,permit_mynetworks,"reject_rbl_client zen.spamhaus.org",reject_unlisted_recipient,"check_policy_service unix:private/policy-spf","check_policy_service inet:127.0.0.1:10023"
|
||||
smtpd_sender_restrictions="reject_non_fqdn_sender,reject_unknown_sender_domain,reject_authenticated_sender_login_mismatch,reject_rhsbl_sender dbl.spamhaus.org=127.0.1.[2..99]" \
|
||||
smtpd_recipient_restrictions="permit_sasl_authenticated,permit_mynetworks,reject_rbl_client zen.spamhaus.org=127.0.0.[2..11],reject_unlisted_recipient,check_policy_service unix:private/policy-spf,check_policy_service inet:127.0.0.1:10023"
|
||||
|
||||
# Postfix connects to Postgrey on the 127.0.0.1 interface specifically. Ensure that
|
||||
# Postgrey listens on the same interface (and not IPv6, for instance).
|
||||
|
@ -75,18 +75,18 @@ rm -rf $assets_dir
|
||||
mkdir -p $assets_dir
|
||||
|
||||
# jQuery CDN URL
|
||||
jquery_version=2.1.4
|
||||
jquery_version=2.2.4
|
||||
jquery_url=https://code.jquery.com
|
||||
|
||||
# Get jQuery
|
||||
wget_verify $jquery_url/jquery-$jquery_version.min.js 43dc554608df885a59ddeece1598c6ace434d747 $assets_dir/jquery.min.js
|
||||
wget_verify $jquery_url/jquery-$jquery_version.min.js 69bb69e25ca7d5ef0935317584e6153f3fd9a88c $assets_dir/jquery.min.js
|
||||
|
||||
# Bootstrap CDN URL
|
||||
bootstrap_version=3.3.7
|
||||
bootstrap_version=3.4.1
|
||||
bootstrap_url=https://github.com/twbs/bootstrap/releases/download/v$bootstrap_version/bootstrap-$bootstrap_version-dist.zip
|
||||
|
||||
# Get Bootstrap
|
||||
wget_verify $bootstrap_url e6b1000b94e835ffd37f4c6dcbdad43f4b48a02a /tmp/bootstrap.zip
|
||||
wget_verify $bootstrap_url 0bb64c67c2552014d48ab4db81c2e8c01781f580 /tmp/bootstrap.zip
|
||||
unzip -q /tmp/bootstrap.zip -d $assets_dir
|
||||
mv $assets_dir/bootstrap-$bootstrap_version-dist $assets_dir/bootstrap
|
||||
rm -f /tmp/bootstrap.zip
|
||||
|
@ -19,6 +19,7 @@ import sys, os, os.path, glob, re, shutil
|
||||
|
||||
sys.path.insert(0, 'management')
|
||||
from utils import load_environment, load_env_vars_from_file, save_environment, shell
|
||||
import contextlib
|
||||
|
||||
def migration_1(env):
|
||||
# Re-arrange where we store SSL certificates. There was a typo also.
|
||||
@ -41,10 +42,8 @@ def migration_1(env):
|
||||
move_file(sslfn, domain_name, file_type)
|
||||
|
||||
# Move the old domains directory if it is now empty.
|
||||
try:
|
||||
with contextlib.suppress(Exception):
|
||||
os.rmdir(os.path.join( env["STORAGE_ROOT"], 'ssl/domains'))
|
||||
except:
|
||||
pass
|
||||
|
||||
def migration_2(env):
|
||||
# Delete the .dovecot_sieve script everywhere. This was formerly a copy of our spam -> Spam
|
||||
@ -178,7 +177,7 @@ def migration_12(env):
|
||||
dropcmd = "DROP TABLE %s" % table
|
||||
c.execute(dropcmd)
|
||||
except:
|
||||
print("Failed to drop table", table, e)
|
||||
print("Failed to drop table", table)
|
||||
# Save.
|
||||
conn.commit()
|
||||
conn.close()
|
||||
@ -215,7 +214,7 @@ def migration_miabldap_1(env):
|
||||
# maildrop: [email]
|
||||
# userPassword: [password]
|
||||
# mailaccess: [privilege] # multi-valued
|
||||
#
|
||||
#
|
||||
# aliases table:
|
||||
# for each row create an ldap entry of the form:
|
||||
# dn: cn=[uuid],ou=aliases,ou=Users,dc=mailinabox
|
||||
@ -229,15 +228,15 @@ def migration_miabldap_1(env):
|
||||
# objectClass: mailGroup
|
||||
# mail: [source]
|
||||
# member: [user-dn] # multi-valued
|
||||
|
||||
|
||||
print("Migrating users and aliases from sqlite to ldap")
|
||||
|
||||
|
||||
# Get the ldap server up and running
|
||||
shell("check_call", ["setup/ldap.sh", "-v"])
|
||||
|
||||
|
||||
import sqlite3, ldap3
|
||||
import migration_13 as m13
|
||||
|
||||
|
||||
# 2. get ldap site details (miab_ldap.conf was created by ldap.sh)
|
||||
ldapvars = load_env_vars_from_file(os.path.join(env["STORAGE_ROOT"], "ldap/miab_ldap.conf"), strip_quotes=True)
|
||||
ldap_base = ldapvars.LDAP_BASE
|
||||
@ -253,7 +252,7 @@ def migration_miabldap_1(env):
|
||||
conn = sqlite3.connect(os.path.join(env["STORAGE_ROOT"], "mail/users.sqlite"))
|
||||
ldap = ldap3.Connection('127.0.0.1', ldap_admin_dn, ldap_admin_pass, raise_exceptions=True)
|
||||
ldap.bind()
|
||||
|
||||
|
||||
# 4. perform the migration
|
||||
users=m13.create_users(env, conn, ldap, ldap_base, ldap_users_base, ldap_domains_base)
|
||||
aliases=m13.create_aliases(env, conn, ldap, ldap_aliases_base)
|
||||
@ -301,7 +300,7 @@ def migration_miabldap_2(env):
|
||||
"-LLL",
|
||||
"olcObjectClasses"
|
||||
])
|
||||
|
||||
|
||||
if "rfc822MailMember" in ret:
|
||||
def ldif_change_fn(ldif):
|
||||
return ldif.replace("rfc822MailMember: ", "mailMember: ")
|
||||
@ -327,10 +326,10 @@ def migration_miabldap_2(env):
|
||||
|
||||
print("Ensure all required aliases are created")
|
||||
m14.ensure_required_aliases(env, ldapvars, ldap)
|
||||
|
||||
|
||||
ldap.unbind()
|
||||
|
||||
|
||||
|
||||
def get_current_migration():
|
||||
ver = 0
|
||||
while True:
|
||||
@ -350,8 +349,8 @@ def run_migrations():
|
||||
migration_id_file = os.path.join(env['STORAGE_ROOT'], 'mailinabox.version')
|
||||
migration_id = None
|
||||
if os.path.exists(migration_id_file):
|
||||
with open(migration_id_file) as f:
|
||||
migration_id = f.read().strip();
|
||||
with open(migration_id_file, encoding='utf-8') as f:
|
||||
migration_id = f.read().strip()
|
||||
|
||||
if migration_id is None:
|
||||
# Load the legacy location of the migration ID. We'll drop support
|
||||
@ -360,7 +359,7 @@ def run_migrations():
|
||||
|
||||
if migration_id is None:
|
||||
print()
|
||||
print("%s file doesn't exists. Skipping migration..." % (migration_id_file,))
|
||||
print(f"{migration_id_file} file doesn't exists. Skipping migration...")
|
||||
return
|
||||
|
||||
ourver = int(migration_id)
|
||||
@ -391,7 +390,7 @@ def run_migrations():
|
||||
|
||||
# Write out our current version now. Do this sooner rather than later
|
||||
# in case of any problems.
|
||||
with open(migration_id_file, "w") as f:
|
||||
with open(migration_id_file, "w", encoding='utf-8') as f:
|
||||
f.write(str(ourver) + "\n")
|
||||
|
||||
# Delete the legacy location of this field.
|
||||
@ -422,7 +421,7 @@ def run_miabldap_migrations():
|
||||
print()
|
||||
print("%s file doesn't exists. Skipping migration..." % (migration_id_file,))
|
||||
return
|
||||
|
||||
|
||||
ourver = int(migration_id)
|
||||
|
||||
while True:
|
||||
@ -464,13 +463,12 @@ if __name__ == "__main__":
|
||||
elif sys.argv[-1] == "--migrate":
|
||||
# Perform migrations.
|
||||
env = load_environment()
|
||||
|
||||
|
||||
# if miab-ldap already installed, only run miab-ldap migrations
|
||||
if 'LDAP_USERS_BASE' in env:
|
||||
run_miabldap_migrations()
|
||||
|
||||
|
||||
# otherwise, run both
|
||||
else:
|
||||
run_migrations()
|
||||
run_miabldap_migrations()
|
||||
|
||||
|
@ -38,31 +38,48 @@ echo "Installing Nextcloud (contacts/calendar)..."
|
||||
# we automatically install intermediate versions as needed.
|
||||
# * The hash is the SHA1 hash of the ZIP package, which you can find by just running this script and
|
||||
# copying it from the error message when it doesn't match what is below.
|
||||
nextcloud_ver=25.0.7
|
||||
nextcloud_hash=a5a565c916355005c7b408dd41a1e53505e1a080
|
||||
nextcloud_ver=26.0.12
|
||||
nextcloud_hash=b55e9f51171c0a9b9ab3686cf5c8ad1a4292ca15
|
||||
|
||||
# Nextcloud apps
|
||||
# --------------
|
||||
# * Find the most recent tag that is compatible with the Nextcloud version above by
|
||||
# consulting the <dependencies>...<nextcloud> node at:
|
||||
# https://github.com/nextcloud-releases/contacts/blob/main/appinfo/info.xml
|
||||
# https://github.com/nextcloud-releases/calendar/blob/main/appinfo/info.xml
|
||||
# https://github.com/nextcloud/user_external/blob/master/appinfo/info.xml
|
||||
# * The hash is the SHA1 hash of the ZIP package, which you can find by just running this script and
|
||||
# copying it from the error message when it doesn't match what is below.
|
||||
contacts_ver=5.3.0
|
||||
contacts_hash=4b0a6666374e3b55cfd2ae9b72e1d458b87d4c8c
|
||||
# * Find the most recent tag that is compatible with the Nextcloud version above by:
|
||||
# https://github.com/nextcloud-releases/contacts/tags
|
||||
# https://github.com/nextcloud-releases/calendar/tags
|
||||
# https://github.com/nextcloud/user_external/tags
|
||||
#
|
||||
# * For these three packages, contact, calendar and user_external, the hash is the SHA1 hash of
|
||||
# the ZIP package, which you can find by just running this script and copying it from
|
||||
# the error message when it doesn't match what is below:
|
||||
|
||||
# Always ensure the versions are supported, see https://apps.nextcloud.com/apps/contacts
|
||||
contacts_ver=5.5.3
|
||||
contacts_hash=799550f38e46764d90fa32ca1a6535dccd8316e5
|
||||
|
||||
# Always ensure the versions are supported, see https://apps.nextcloud.com/apps/calendar
|
||||
calendar_ver=4.4.2
|
||||
calendar_hash=21a42e15806adc9b2618760ef94f1797ef399e2f
|
||||
calendar_ver=4.6.6
|
||||
calendar_hash=e34a71669a52d997e319d64a984dcd041389eb22
|
||||
|
||||
# And https://apps.nextcloud.com/apps/user_external
|
||||
# Always ensure the versions are supported, see https://apps.nextcloud.com/apps/user_external
|
||||
user_external_ver=3.2.0
|
||||
user_external_hash=a494073dcdecbbbc79a9c77f72524ac9994d2eec
|
||||
|
||||
# Clear prior packages and install dependencies from apt.
|
||||
# Developer advice (test plan)
|
||||
# ----------------------------
|
||||
# When upgrading above versions, how to test?
|
||||
#
|
||||
# 1. Enter your server instance (or on the Vagrant image)
|
||||
# 1. Git clone <your fork>
|
||||
# 2. Git checkout <your fork>
|
||||
# 3. Run `sudo ./setup/nextcloud.sh`
|
||||
# 4. Ensure the installation completes. If any hashes mismatch, correct them.
|
||||
# 5. Enter nextcloud web, run following tests:
|
||||
# 5.1 You still can create, edit and delete contacts
|
||||
# 5.2 You still can create, edit and delete calendar events
|
||||
# 5.3 You still can create, edit and delete users
|
||||
# 5.4 Go to Administration > Logs and ensure no new errors are shown
|
||||
|
||||
# Clear prior packages and install dependencies from apt.
|
||||
apt-get purge -qq -y owncloud* # we used to use the package manager
|
||||
|
||||
apt_install curl php${PHP_VER} php${PHP_VER}-fpm \
|
||||
@ -167,7 +184,7 @@ InstallNextcloud() {
|
||||
|
||||
# Current Nextcloud Version, #1623
|
||||
# Checking /usr/local/lib/owncloud/version.php shows version of the Nextcloud application, not the DB
|
||||
# $STORAGE_ROOT/owncloud is kept together even during a backup. It is better to rely on config.php than
|
||||
# $STORAGE_ROOT/owncloud is kept together even during a backup. It is better to rely on config.php than
|
||||
# version.php since the restore procedure can leave the system in a state where you have a newer Nextcloud
|
||||
# application version than the database.
|
||||
|
||||
@ -225,6 +242,11 @@ if [ ! -d /usr/local/lib/owncloud/ ] || [[ ! ${CURRENT_NEXTCLOUD_VER} =~ ^$nextc
|
||||
return 0
|
||||
fi
|
||||
|
||||
# Hint: whenever you bump, remember this:
|
||||
# - Run a server with the previous version
|
||||
# - On a new if-else block, copy the versions/hashes from the previous version
|
||||
# - Run sudo ./setup/start.sh on the new machine. Upon completion, test its basic functionalities.
|
||||
|
||||
if [[ ${CURRENT_NEXTCLOUD_VER} =~ ^20 ]]; then
|
||||
InstallNextcloud 21.0.7 f5c7079c5b56ce1e301c6a27c0d975d608bb01c9 4.0.7 45e7cf4bfe99cd8d03625cf9e5a1bb2e90549136 3.0.4 d0284b68135777ec9ca713c307216165b294d0fe
|
||||
CURRENT_NEXTCLOUD_VER="21.0.7"
|
||||
@ -241,6 +263,10 @@ if [ ! -d /usr/local/lib/owncloud/ ] || [[ ! ${CURRENT_NEXTCLOUD_VER} =~ ^$nextc
|
||||
InstallNextcloud 24.0.12 7aa5d61632c1ccf4ca3ff00fb6b295d318c05599 4.1.0 697f6b4a664e928d72414ea2731cb2c9d1dc3077 3.2.2 ce4030ab57f523f33d5396c6a81396d440756f5f 3.0.0 0df781b261f55bbde73d8c92da3f99397000972f
|
||||
CURRENT_NEXTCLOUD_VER="24.0.12"
|
||||
fi
|
||||
if [[ ${CURRENT_NEXTCLOUD_VER} =~ ^24 ]]; then
|
||||
InstallNextcloud 25.0.7 a5a565c916355005c7b408dd41a1e53505e1a080 5.3.0 4b0a6666374e3b55cfd2ae9b72e1d458b87d4c8c 4.4.2 21a42e15806adc9b2618760ef94f1797ef399e2f 3.2.0 a494073dcdecbbbc79a9c77f72524ac9994d2eec
|
||||
CURRENT_NEXTCLOUD_VER="25.0.7"
|
||||
fi
|
||||
fi
|
||||
|
||||
InstallNextcloud $nextcloud_ver $nextcloud_hash $contacts_ver $contacts_hash $calendar_ver $calendar_hash $user_external_ver $user_external_hash
|
||||
|
@ -1,3 +1,4 @@
|
||||
#!/bin/bash
|
||||
#####
|
||||
##### This file is part of Mail-in-a-Box-LDAP which is released under the
|
||||
##### terms of the GNU Affero General Public License as published by the
|
||||
@ -35,16 +36,16 @@ fi
|
||||
#
|
||||
# Skip the check if we appear to be running inside of Vagrant, because that's really just for testing.
|
||||
TOTAL_PHYSICAL_MEM=$(head -n 1 /proc/meminfo | awk '{print $2}')
|
||||
if [ $TOTAL_PHYSICAL_MEM -lt 490000 ]; then
|
||||
if [ "$TOTAL_PHYSICAL_MEM" -lt 490000 ]; then
|
||||
if [ ! -d /vagrant ]; then
|
||||
TOTAL_PHYSICAL_MEM=$(expr \( \( $TOTAL_PHYSICAL_MEM \* 1024 \) / 1000 \) / 1000)
|
||||
TOTAL_PHYSICAL_MEM=$(( TOTAL_PHYSICAL_MEM * 1024 / 1000 / 1000 ))
|
||||
echo "Your Mail-in-a-Box needs more memory (RAM) to function properly."
|
||||
echo "Please provision a machine with at least 512 MB, 1 GB recommended."
|
||||
echo "This machine has $TOTAL_PHYSICAL_MEM MB memory."
|
||||
exit
|
||||
fi
|
||||
fi
|
||||
if [ $TOTAL_PHYSICAL_MEM -lt 750000 ]; then
|
||||
if [ "$TOTAL_PHYSICAL_MEM" -lt 750000 ]; then
|
||||
echo "WARNING: Your Mail-in-a-Box has less than 768 MB of memory."
|
||||
echo " It might run unreliably when under heavy load."
|
||||
fi
|
||||
|
@ -49,8 +49,8 @@ apt_install php${PHP_VER}-ldap
|
||||
# https://github.com/mstilkerich/rcmcarddav/releases
|
||||
# The easiest way to get the package hashes is to run this script and get the hash from
|
||||
# the error message.
|
||||
VERSION=1.6.5
|
||||
HASH=326fcc206cddc00355e98d1e40fd0bcd9baec69f
|
||||
VERSION=1.6.6
|
||||
HASH=7705d2736890c49e7ae3ac75e3ae00ba56187056
|
||||
PERSISTENT_LOGIN_VERSION=version-5.3.0
|
||||
HTML5_NOTIFIER_VERSION=68d9ca194212e15b3c7225eb6085dbcf02fd13d7 # version 0.6.4+
|
||||
CARDDAV_VERSION=4.4.3
|
||||
@ -302,4 +302,3 @@ cat > /etc/cron.daily/mailinabox-roundcubemail << EOF
|
||||
cd $RCM_DIR && bin/cleandb.sh >/dev/null
|
||||
EOF
|
||||
chmod +x /etc/cron.daily/mailinabox-roundcubemail
|
||||
|
||||
|
@ -15,12 +15,12 @@
|
||||
# try to log in to.
|
||||
######################################################################
|
||||
|
||||
import sys, os, time, functools
|
||||
import sys, os, time
|
||||
|
||||
# parse command line
|
||||
|
||||
if len(sys.argv) != 4:
|
||||
print("Usage: tests/fail2ban.py \"ssh user@hostname\" hostname owncloud_user")
|
||||
print('Usage: tests/fail2ban.py "ssh user@hostname" hostname owncloud_user')
|
||||
sys.exit(1)
|
||||
|
||||
ssh_command, hostname, owncloud_user = sys.argv[1:4]
|
||||
@ -33,7 +33,6 @@ socket.setdefaulttimeout(10)
|
||||
class IsBlocked(Exception):
|
||||
"""Tests raise this exception when it appears that a fail2ban
|
||||
jail is in effect, i.e. on a connection refused error."""
|
||||
pass
|
||||
|
||||
def smtp_test():
|
||||
import smtplib
|
||||
@ -42,13 +41,14 @@ def smtp_test():
|
||||
server = smtplib.SMTP(hostname, 587)
|
||||
except ConnectionRefusedError:
|
||||
# looks like fail2ban worked
|
||||
raise IsBlocked()
|
||||
raise IsBlocked
|
||||
server.starttls()
|
||||
server.ehlo_or_helo_if_needed()
|
||||
|
||||
try:
|
||||
server.login("fakeuser", "fakepassword")
|
||||
raise Exception("authentication didn't fail")
|
||||
msg = "authentication didn't fail"
|
||||
raise Exception(msg)
|
||||
except smtplib.SMTPAuthenticationError:
|
||||
# athentication should fail
|
||||
pass
|
||||
@ -66,11 +66,12 @@ def imap_test():
|
||||
M = imaplib.IMAP4_SSL(hostname)
|
||||
except ConnectionRefusedError:
|
||||
# looks like fail2ban worked
|
||||
raise IsBlocked()
|
||||
raise IsBlocked
|
||||
|
||||
try:
|
||||
M.login("fakeuser", "fakepassword")
|
||||
raise Exception("authentication didn't fail")
|
||||
msg = "authentication didn't fail"
|
||||
raise Exception(msg)
|
||||
except imaplib.IMAP4.error:
|
||||
# authentication should fail
|
||||
pass
|
||||
@ -84,17 +85,18 @@ def pop_test():
|
||||
M = poplib.POP3_SSL(hostname)
|
||||
except ConnectionRefusedError:
|
||||
# looks like fail2ban worked
|
||||
raise IsBlocked()
|
||||
raise IsBlocked
|
||||
try:
|
||||
M.user('fakeuser')
|
||||
try:
|
||||
M.pass_('fakepassword')
|
||||
except poplib.error_proto as e:
|
||||
except poplib.error_proto:
|
||||
# Authentication should fail.
|
||||
M = None # don't .quit()
|
||||
return
|
||||
M.list()
|
||||
raise Exception("authentication didn't fail")
|
||||
msg = "authentication didn't fail"
|
||||
raise Exception(msg)
|
||||
finally:
|
||||
if M:
|
||||
M.quit()
|
||||
@ -108,11 +110,12 @@ def managesieve_test():
|
||||
M = imaplib.IMAP4(hostname, 4190)
|
||||
except ConnectionRefusedError:
|
||||
# looks like fail2ban worked
|
||||
raise IsBlocked()
|
||||
raise IsBlocked
|
||||
|
||||
try:
|
||||
M.login("fakeuser", "fakepassword")
|
||||
raise Exception("authentication didn't fail")
|
||||
msg = "authentication didn't fail"
|
||||
raise Exception(msg)
|
||||
except imaplib.IMAP4.error:
|
||||
# authentication should fail
|
||||
pass
|
||||
@ -138,17 +141,17 @@ def http_test(url, expected_status, postdata=None, qsargs=None, auth=None):
|
||||
headers={'User-Agent': 'Mail-in-a-Box fail2ban tester'},
|
||||
timeout=8,
|
||||
verify=False) # don't bother with HTTPS validation, it may not be configured yet
|
||||
except requests.exceptions.ConnectTimeout as e:
|
||||
raise IsBlocked()
|
||||
except requests.exceptions.ConnectTimeout:
|
||||
raise IsBlocked
|
||||
except requests.exceptions.ConnectionError as e:
|
||||
if "Connection refused" in str(e):
|
||||
raise IsBlocked()
|
||||
raise IsBlocked
|
||||
raise # some other unexpected condition
|
||||
|
||||
# return response status code
|
||||
if r.status_code != expected_status:
|
||||
r.raise_for_status() # anything but 200
|
||||
raise IOError("Got unexpected status code %s." % r.status_code)
|
||||
raise OSError("Got unexpected status code %s." % r.status_code)
|
||||
|
||||
# define how to run a test
|
||||
|
||||
@ -158,7 +161,7 @@ def restart_fail2ban_service(final=False):
|
||||
if not final:
|
||||
# Stop recidive jails during testing.
|
||||
command += " && sudo fail2ban-client stop recidive"
|
||||
os.system("%s \"%s\"" % (ssh_command, command))
|
||||
os.system(f'{ssh_command} "{command}"')
|
||||
|
||||
def testfunc_runner(i, testfunc, *args):
|
||||
print(i+1, end=" ", flush=True)
|
||||
@ -172,7 +175,6 @@ def run_test(testfunc, args, count, within_seconds, parallel):
|
||||
# run testfunc sequentially and still get to count requests within
|
||||
# the required time. So we split the requests across threads.
|
||||
|
||||
import requests.exceptions
|
||||
from multiprocessing import Pool
|
||||
|
||||
restart_fail2ban_service()
|
||||
@ -188,7 +190,7 @@ def run_test(testfunc, args, count, within_seconds, parallel):
|
||||
# Distribute the requests across the pool.
|
||||
asyncresults = []
|
||||
for i in range(count):
|
||||
ar = p.apply_async(testfunc_runner, [i, testfunc] + list(args))
|
||||
ar = p.apply_async(testfunc_runner, [i, testfunc, *list(args)])
|
||||
asyncresults.append(ar)
|
||||
|
||||
# Wait for all runs to finish.
|
||||
|
@ -16,7 +16,7 @@
|
||||
# where ipaddr is the IP address of your Mail-in-a-Box
|
||||
# and hostname is the domain name to check the DNS for.
|
||||
|
||||
import sys, re, difflib
|
||||
import sys, re
|
||||
import dns.reversename, dns.resolver
|
||||
|
||||
if len(sys.argv) < 3:
|
||||
@ -36,10 +36,10 @@ def test(server, description):
|
||||
("ns2." + primary_hostname, "A", ipaddr),
|
||||
("www." + hostname, "A", ipaddr),
|
||||
(hostname, "MX", "10 " + primary_hostname + "."),
|
||||
(hostname, "TXT", "\"v=spf1 mx -all\""),
|
||||
("mail._domainkey." + hostname, "TXT", "\"v=DKIM1; k=rsa; s=email; \" \"p=__KEY__\""),
|
||||
(hostname, "TXT", '"v=spf1 mx -all"'),
|
||||
("mail._domainkey." + hostname, "TXT", '"v=DKIM1; k=rsa; s=email; " "p=__KEY__"'),
|
||||
#("_adsp._domainkey." + hostname, "TXT", "\"dkim=all\""),
|
||||
("_dmarc." + hostname, "TXT", "\"v=DMARC1; p=quarantine;\""),
|
||||
("_dmarc." + hostname, "TXT", '"v=DMARC1; p=quarantine;"'),
|
||||
]
|
||||
return test2(tests, server, description)
|
||||
|
||||
@ -68,7 +68,7 @@ def test2(tests, server, description):
|
||||
response = ["[no value]"]
|
||||
response = ";".join(str(r) for r in response)
|
||||
response = re.sub(r"(\"p=).*(\")", r"\1__KEY__\2", response) # normalize DKIM key
|
||||
response = response.replace("\"\" ", "") # normalize TXT records (DNSSEC signing inserts empty text string components)
|
||||
response = response.replace('"" ', "") # normalize TXT records (DNSSEC signing inserts empty text string components)
|
||||
|
||||
# is it right?
|
||||
if response == expected_answer:
|
||||
@ -107,7 +107,7 @@ else:
|
||||
# And if that's OK, also check reverse DNS (the PTR record).
|
||||
if not test_ptr("8.8.8.8", "Google Public DNS (Reverse DNS)"):
|
||||
print ()
|
||||
print ("The reverse DNS for %s is not correct. Consult your ISP for how to set the reverse DNS (also called the PTR record) for %s to %s." % (hostname, hostname, ipaddr))
|
||||
print (f"The reverse DNS for {hostname} is not correct. Consult your ISP for how to set the reverse DNS (also called the PTR record) for {hostname} to {ipaddr}.")
|
||||
sys.exit(1)
|
||||
else:
|
||||
print ("And the reverse DNS for the domain is correct.")
|
||||
|
@ -83,7 +83,7 @@ while argi<len(sys.argv):
|
||||
argi+=2
|
||||
else:
|
||||
usage()
|
||||
|
||||
|
||||
if not smtpd:
|
||||
if len(sys.argv) - argi != 3: usage()
|
||||
host, login, pw = sys.argv[argi:argi+3]
|
||||
@ -222,13 +222,12 @@ if delete_msg:
|
||||
if not found:
|
||||
print("Test message not present in the inbox yet...")
|
||||
time.sleep(wait_cycle_sleep)
|
||||
|
||||
|
||||
M.close()
|
||||
M.logout()
|
||||
|
||||
|
||||
if not found:
|
||||
raise TimeoutError("Timeout waiting for message")
|
||||
|
||||
if send_msg and delete_msg:
|
||||
print("Test message sent & received successfully.")
|
||||
|
||||
|
@ -15,11 +15,11 @@ if len(sys.argv) < 3:
|
||||
sys.exit(1)
|
||||
|
||||
host, toaddr, fromaddr = sys.argv[1:4]
|
||||
msg = """From: %s
|
||||
To: %s
|
||||
msg = f"""From: {fromaddr}
|
||||
To: {toaddr}
|
||||
Subject: SMTP server test
|
||||
|
||||
This is a test message.""" % (fromaddr, toaddr)
|
||||
This is a test message."""
|
||||
|
||||
server = smtplib.SMTP(host, 25)
|
||||
server.set_debuglevel(1)
|
||||
|
12
tests/tls.py
12
tests/tls.py
@ -97,14 +97,14 @@ def sslyze(opts, port, ok_ciphers):
|
||||
|
||||
try:
|
||||
# Execute SSLyze.
|
||||
out = subprocess.check_output([SSLYZE] + common_opts + opts + [connection_string])
|
||||
out = subprocess.check_output([SSLYZE, *common_opts, *opts, connection_string])
|
||||
out = out.decode("utf8")
|
||||
|
||||
# Trim output to make better for storing in git.
|
||||
if "SCAN RESULTS FOR" not in out:
|
||||
# Failed. Just output the error.
|
||||
out = re.sub("[\w\W]*CHECKING HOST\(S\) AVAILABILITY\n\s*-+\n", "", out) # chop off header that shows the host we queried
|
||||
out = re.sub("[\w\W]*SCAN RESULTS FOR.*\n\s*-+\n", "", out) # chop off header that shows the host we queried
|
||||
out = re.sub("[\\w\\W]*CHECKING HOST\\(S\\) AVAILABILITY\n\\s*-+\n", "", out) # chop off header that shows the host we queried
|
||||
out = re.sub("[\\w\\W]*SCAN RESULTS FOR.*\n\\s*-+\n", "", out) # chop off header that shows the host we queried
|
||||
out = re.sub("SCAN COMPLETED IN .*", "", out)
|
||||
out = out.rstrip(" \n-") + "\n"
|
||||
|
||||
@ -114,8 +114,8 @@ def sslyze(opts, port, ok_ciphers):
|
||||
# Pull out the accepted ciphers list for each SSL/TLS protocol
|
||||
# version outputted.
|
||||
accepted_ciphers = set()
|
||||
for ciphers in re.findall(" Accepted:([\w\W]*?)\n *\n", out):
|
||||
accepted_ciphers |= set(re.findall("\n\s*(\S*)", ciphers))
|
||||
for ciphers in re.findall(" Accepted:([\\w\\W]*?)\n *\n", out):
|
||||
accepted_ciphers |= set(re.findall("\n\\s*(\\S*)", ciphers))
|
||||
|
||||
# Compare to what Mozilla recommends, for a given modernness-level.
|
||||
print(" Should Not Offer: " + (", ".join(sorted(accepted_ciphers-set(ok_ciphers))) or "(none -- good)"))
|
||||
@ -151,7 +151,7 @@ for cipher in csv.DictReader(io.StringIO(urllib.request.urlopen("https://raw.git
|
||||
client_compatibility = json.loads(urllib.request.urlopen("https://raw.githubusercontent.com/mail-in-a-box/user-agent-tls-capabilities/master/clients.json").read().decode("utf8"))
|
||||
cipher_clients = { }
|
||||
for client in client_compatibility:
|
||||
if len(set(client['protocols']) & set(["TLS 1.0", "TLS 1.1", "TLS 1.2"])) == 0: continue # does not support TLS
|
||||
if len(set(client['protocols']) & {"TLS 1.0", "TLS 1.1", "TLS 1.2"}) == 0: continue # does not support TLS
|
||||
for cipher in client['ciphers']:
|
||||
cipher_clients.setdefault(cipher_names.get(cipher), set()).add("/".join(x for x in [client['client']['name'], client['client']['version'], client['client']['platform']] if x))
|
||||
|
||||
|
19
tests/vagrant/Vagrantfile
vendored
19
tests/vagrant/Vagrantfile
vendored
@ -25,7 +25,7 @@ export FEATURE_MUNIN=false
|
||||
export EHDD_KEYFILE=$HOME/keyfile
|
||||
echo -n "boo" >$EHDD_KEYFILE
|
||||
tests/system-setup/remote-nextcloud-docker.sh || exit 1
|
||||
tests/runner.sh remote-nextcloud ehdd default || exit 2
|
||||
tests/runner.sh -no-smtp-remote remote-nextcloud ehdd default || exit 2
|
||||
SH
|
||||
end
|
||||
end
|
||||
@ -38,25 +38,26 @@ cd /mailinabox
|
||||
export PRIMARY_HOSTNAME=qa2.abc.com
|
||||
export FEATURE_MUNIN=false
|
||||
tests/system-setup/remote-nextcloud-docker.sh upgrade --populate=basic || exit 1
|
||||
tests/runner.sh remote-nextcloud upgrade-basic default || exit 2
|
||||
tests/runner.sh -no-smtp-remote remote-nextcloud upgrade-basic default || exit 2
|
||||
SH
|
||||
end
|
||||
|
||||
|
||||
# upgrade-from-upstream
|
||||
|
||||
|
||||
config.vm.define "upgrade-from-upstream" do |m1|
|
||||
m1.vm.provision :shell, :inline => <<-SH
|
||||
cd /mailinabox
|
||||
export PRIMARY_HOSTNAME=qa3.abc.com
|
||||
export UPSTREAM_TAG=main
|
||||
# TODO: change UPSTREAM_TAG to 'main' once upstream is installable
|
||||
export UPSTREAM_TAG=v67
|
||||
tests/system-setup/upgrade-from-upstream.sh --populate=basic --populate=totpuser || exit 1
|
||||
tests/runner.sh upgrade-basic upgrade-totpuser default || exit 2
|
||||
tests/runner.sh -no-smtp-remote upgrade-basic upgrade-totpuser default || exit 2
|
||||
SH
|
||||
end
|
||||
|
||||
# upgrade
|
||||
|
||||
|
||||
# this test is only needed when testing migrations from miabldap
|
||||
# to a newer miabldap with a migration step
|
||||
#
|
||||
@ -65,9 +66,11 @@ SH
|
||||
config.vm.define "upgrade" do |m1|
|
||||
m1.vm.provision :shell, :inline => <<-SH
|
||||
cd /mailinabox
|
||||
# TODO: remove DEB_PYTHON_INSTALL_LAYOUT once MIABLDAP_RELEASE_TAG >= v66 (see https://github.com/downtownallday/mailinabox-ldap/commit/371f5bc1b236de40a1ed5d9118140ee13fddf5dc)
|
||||
export DEB_PYTHON_INSTALL_LAYOUT='deb'
|
||||
export PRIMARY_HOSTNAME=upgrade.abc.com
|
||||
tests/system-setup/upgrade.sh --populate=basic --populate=totpuser || exit 1
|
||||
tests/runner.sh upgrade-basic upgrade-totpuser default || exit 2
|
||||
tests/runner.sh -no-smtp-remote upgrade-basic upgrade-totpuser default || exit 2
|
||||
SH
|
||||
end
|
||||
|
||||
@ -96,6 +99,6 @@ setup/start.sh
|
||||
SH
|
||||
end
|
||||
end
|
||||
|
||||
|
||||
|
||||
end
|
||||
|
@ -110,7 +110,7 @@ except:
|
||||
|
||||
found = set()
|
||||
buf = ""
|
||||
with open(filename, "r") as f:
|
||||
with open(filename, encoding="utf-8") as f:
|
||||
input_lines = list(f)
|
||||
cur_section = None
|
||||
|
||||
@ -119,7 +119,7 @@ while len(input_lines) > 0:
|
||||
|
||||
# If this configuration file uses folded lines, append any folded lines
|
||||
# into our input buffer.
|
||||
if folded_lines and line[0] not in (comment_char, " ", ""):
|
||||
if folded_lines and line[0] not in {comment_char, " ", ""}:
|
||||
while len(input_lines) > 0 and input_lines[0][0] in " \t":
|
||||
line += input_lines.pop(0)
|
||||
|
||||
@ -147,9 +147,9 @@ while len(input_lines) > 0:
|
||||
name, val = (settings[i].name, settings[i].val)
|
||||
flags = re.S | (re.I if case_insensitive_names else 0)
|
||||
m = re.match(
|
||||
"(\\s*)"
|
||||
+ "(" + re.escape(comment_char) + "\\s*)?"
|
||||
+ re.escape(name) + delimiter_re + "(.*?)\\s*$",
|
||||
r"(\s*)"
|
||||
+ "(" + re.escape(comment_char) + r"\s*)?"
|
||||
+ re.escape(name) + delimiter_re + r"(.*?)\s*$",
|
||||
line, flags)
|
||||
if not m: continue
|
||||
indent, is_comment, existing_val = m.groups()
|
||||
@ -206,7 +206,7 @@ if not ini_section or cur_section == ini_section.lower():
|
||||
|
||||
if not testing:
|
||||
# Write out the new file.
|
||||
with open(filename, "w") as f:
|
||||
with open(filename, "w", encoding="utf-8") as f:
|
||||
f.write(buf)
|
||||
else:
|
||||
# Just print the new file to stdout.
|
||||
|
@ -47,7 +47,7 @@ for date, ip in accesses:
|
||||
# Since logs are rotated, store the statistics permanently in a JSON file.
|
||||
# Load in the stats from an existing file.
|
||||
if os.path.exists(outfn):
|
||||
with open(outfn, "r") as f:
|
||||
with open(outfn, encoding="utf-8") as f:
|
||||
existing_data = json.load(f)
|
||||
for date, count in existing_data:
|
||||
if date not in by_date:
|
||||
@ -60,5 +60,5 @@ by_date = sorted(by_date.items())
|
||||
by_date.pop(-1)
|
||||
|
||||
# Write out.
|
||||
with open(outfn, "w") as f:
|
||||
with open(outfn, "w", encoding="utf-8") as f:
|
||||
json.dump(by_date, f, sort_keys=True, indent=True)
|
||||
|
@ -133,7 +133,7 @@ def generate_documentation():
|
||||
""")
|
||||
|
||||
parser = Source.parser()
|
||||
with open("setup/start.sh", "r") as start_file:
|
||||
with open("setup/start.sh", "r") as start_file:
|
||||
for line in start_file:
|
||||
try:
|
||||
fn = parser.parse_string(line).filename()
|
||||
|
Loading…
Reference in New Issue
Block a user