diff --git a/CHANGELOG.md b/CHANGELOG.md index f8c91d77..a09110cf 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,16 +6,71 @@ In Development Mail: +* Roundcube is updated to version 1.2.0. +* SSLv3 and RC4 are now no longer supported in incoming and outgoing mail (SMTP port 25). + +v0.18c (June 2, 2016) +--------------------- + +* Domain aliases (and misconfigured aliases/catch-alls with non-existent local targets) would accept mail and deliver it to new mailbox folders on disk even if the target address didn't correspond with an existing mail user, instead of rejecting the mail. This issue was introduced in v0.18. +* The Munin Monitoring link in the control panel now opens a new window. +* Added an undocumented before-backup script. + +v0.18b (May 16, 2016) +--------------------- + +* Fixed a Roundcube user accounts issue introduced in v0.18. + +v0.18 (May 15, 2016) +-------------------- + +ownCloud: + +* Updated to ownCloud to 8.2.3 + +Mail: + +* Roundcube is updated to version 1.1.5 and the Roundcube login screen now says "[hostname] Webmail" instead of "Mail-in-a-Box/Roundcube webmail". * Fixed a long-standing issue with training the spam filter not working (because of a file permissions issue). Control panel: * Munin system monitoring graphs are now zoomable. -* When a reboot is required (due to Ubuntu security updates automatically installed), a Reboot Box button now appears. +* When a reboot is required (due to Ubuntu security updates automatically installed), a Reboot Box button now appears on the System Status Checks page of the control panel. +* It is now possible to add SRV and secondary MX records in the Custom DNS page. +* Other minor fixes. + +System: + +* The fail2ban recidive jail, which blocks long-duration brute force attacks, now no longer sends the administrator emails (which were not helpful). Setup: +* The system hostname is now set during setup. * A swap file is now created if system memory is less than 2GB, 5GB of free disk space is available, and if no swap file yet exists. +* We now install Roundcube from the official GitHub repository instead of our own mirror, which we had previously created to solve problems with SourceForge. +* DKIM was incorrectly set up on machines where "localhost" was defined as something other than "127.0.0.1". + +v0.17c (April 1, 2016) +---------------------- + +This update addresses some minor security concerns and some installation issues. + +ownCoud: + +* Block web access to the configuration parameters (config.php). There is no immediate impact (see [#776](https://github.com/mail-in-a-box/mailinabox/pull/776)), although advanced users may want to take note. + +Mail: + +* Roundcube html5_notifier plugin updated from version 0.6 to 0.6.2 to fix Roundcube getting stuck for some people. + +Control panel: + +* Prevent click-jacking of the management interface by adding HTTP headers. +* Failed login no longer reveals whether an account exists on the system. + +Setup: + * Setup dialogs did not appear correctly when connecting to SSH using Putty on Windows. * We now install Roundcube from our own mirror because Sourceforge's downloads experience frequent intermittant unavailability. diff --git a/README.md b/README.md index d8329054..ad912a28 100644 --- a/README.md +++ b/README.md @@ -59,7 +59,7 @@ by me: $ curl -s https://keybase.io/joshdata/key.asc | gpg --import gpg: key C10BDD81: public key "Joshua Tauberer " imported - $ git verify-tag v0.17b + $ git verify-tag v0.18c gpg: Signature made ..... using RSA key ID C10BDD81 gpg: Good signature from "Joshua Tauberer " gpg: WARNING: This key is not certified with a trusted signature! @@ -72,7 +72,7 @@ and on my [personal homepage](https://razor.occams.info/). (Of course, if this r Checkout the tag corresponding to the most recent release: - $ git checkout v0.17b + $ git checkout v0.18c Begin the installation. diff --git a/conf/fail2ban/jail.local b/conf/fail2ban/jail.local index b9340e52..dc338803 100644 --- a/conf/fail2ban/jail.local +++ b/conf/fail2ban/jail.local @@ -23,7 +23,19 @@ enabled = true filter = dovecotimap findtime = 30 maxretry = 20 +logpath = /var/log/mail.log [recidive] enabled = true maxretry = 10 +action = iptables-allports[name=recidive] +# In the recidive section of jail.conf the action contains: +# +# action = iptables-allports[name=recidive] +# sendmail-whois-lines[name=recidive, logpath=/var/log/fail2ban.log] +# +# The last line on the action will sent an email to the configured address. This mail will +# notify the administrator that someone has been repeatedly triggering one of the other jails. +# By default we don't configure this address and no action is required from the admin anyway. +# So the notification is ommited. This will prevent message appearing in the mail.log that mail +# can't be delivered to fail2ban@$HOSTNAME. diff --git a/conf/nginx-primaryonly.conf b/conf/nginx-primaryonly.conf index 8fd546af..55c80eba 100644 --- a/conf/nginx-primaryonly.conf +++ b/conf/nginx-primaryonly.conf @@ -18,8 +18,11 @@ rewrite ^(/cloud/core/doc/[^\/]+/)$ $1/index.html; location /cloud/ { alias /usr/local/lib/owncloud/; - location ~ ^/(data|config|\.ht|db_structure\.xml|README) { - deny all; + location ~ ^/cloud/(build|tests|config|lib|3rdparty|templates|data|README)/ { + deny all; + } + location ~ ^/cloud/(?:\.|autotest|occ|issue|indie|db_|console) { + deny all; } } location ~ ^(/cloud)((?:/ocs)?/[^/]+\.php)(/.*)?$ { diff --git a/conf/zpush/backend_caldav.php b/conf/zpush/backend_caldav.php index 7bddded9..b10ebc3e 100644 --- a/conf/zpush/backend_caldav.php +++ b/conf/zpush/backend_caldav.php @@ -6,7 +6,7 @@ ************************************************/ define('CALDAV_PROTOCOL', 'https'); -define('CALDAV_SERVER', 'localhost'); +define('CALDAV_SERVER', '127.0.0.1'); define('CALDAV_PORT', '443'); define('CALDAV_PATH', '/caldav/calendars/%u/'); define('CALDAV_PERSONAL', 'PRINCIPAL'); diff --git a/conf/zpush/backend_carddav.php b/conf/zpush/backend_carddav.php index edf32901..4b166ad5 100644 --- a/conf/zpush/backend_carddav.php +++ b/conf/zpush/backend_carddav.php @@ -7,7 +7,7 @@ define('CARDDAV_PROTOCOL', 'https'); /* http or https */ -define('CARDDAV_SERVER', 'localhost'); +define('CARDDAV_SERVER', '127.0.0.1'); define('CARDDAV_PORT', '443'); define('CARDDAV_PATH', '/carddav/addressbooks/%u/'); define('CARDDAV_DEFAULT_PATH', '/carddav/addressbooks/%u/contacts/'); /* subdirectory of the main path */ diff --git a/conf/zpush/backend_imap.php b/conf/zpush/backend_imap.php index 3f69f53e..84dc7358 100644 --- a/conf/zpush/backend_imap.php +++ b/conf/zpush/backend_imap.php @@ -5,7 +5,7 @@ * Descr : IMAP backend configuration file ************************************************/ -define('IMAP_SERVER', 'localhost'); +define('IMAP_SERVER', '127.0.0.1'); define('IMAP_PORT', 993); define('IMAP_OPTIONS', '/ssl/norsh/novalidate-cert'); define('IMAP_DEFAULTFROM', ''); @@ -44,7 +44,7 @@ define('IMAP_FROM_LDAP_FROM', '#givenname #sn <#mail>'); define('IMAP_SMTP_METHOD', 'sendmail'); global $imap_smtp_params; -$imap_smtp_params = array('host' => 'ssl://localhost', 'port' => 587, 'auth' => true, 'username' => 'imap_username', 'password' => 'imap_password'); +$imap_smtp_params = array('host' => 'ssl://127.0.0.1', 'port' => 587, 'auth' => true, 'username' => 'imap_username', 'password' => 'imap_password'); define('MAIL_MIMEPART_CRLF', "\r\n"); diff --git a/management/backup.py b/management/backup.py index 2bb499ef..be87a429 100755 --- a/management/backup.py +++ b/management/backup.py @@ -2,9 +2,10 @@ # This script performs a backup of all user data: # 1) System services are stopped. -# 2) An incremental encrypted backup is made using duplicity. -# 3) The stopped services are restarted. -# 4) STORAGE_ROOT/backup/after-backup is executd if it exists. +# 2) STORAGE_ROOT/backup/before-backup is executed if it exists. +# 3) An incremental encrypted backup is made using duplicity. +# 4) The stopped services are restarted. +# 5) STORAGE_ROOT/backup/after-backup is executed if it exists. import os, os.path, shutil, glob, re, datetime, sys import dateutil.parser, dateutil.relativedelta, dateutil.tz @@ -258,6 +259,15 @@ def perform_backup(full_backup): service_command("postfix", "stop", quit=True) service_command("dovecot", "stop", quit=True) + # Execute a pre-backup script that copies files outside the homedir. + # Run as the STORAGE_USER user, not as root. Pass our settings in + # environment variables so the script has access to STORAGE_ROOT. + pre_script = os.path.join(backup_root, 'before-backup') + if os.path.exists(pre_script): + shell('check_call', + ['su', env['STORAGE_USER'], '-c', pre_script, config["target"]], + env=env) + # Run a backup of STORAGE_ROOT (but excluding the backups themselves!). # --allow-source-mismatch is needed in case the box's hostname is changed # after the first backup. See #396. diff --git a/management/daemon.py b/management/daemon.py index 690f8b0f..5400925f 100755 --- a/management/daemon.py +++ b/management/daemon.py @@ -43,7 +43,7 @@ def authorized_personnel_only(viewfunc): except ValueError as e: # Authentication failed. privs = [] - error = str(e) + error = "Incorrect username or password" # Authorized to access an API view? if "admin" in privs: @@ -119,7 +119,7 @@ def me(): except ValueError as e: return json_response({ "status": "invalid", - "reason": str(e), + "reason": "Incorrect username or password", }) resp = { diff --git a/management/email_administrator.py b/management/email_administrator.py index 84d27460..b16fda1d 100755 --- a/management/email_administrator.py +++ b/management/email_administrator.py @@ -33,7 +33,7 @@ msg['Subject'] = "[%s] %s" % (env['PRIMARY_HOSTNAME'], subject) msg.set_payload(content, "UTF-8") # send -smtpclient = smtplib.SMTP('localhost', 25) +smtpclient = smtplib.SMTP('127.0.0.1', 25) smtpclient.ehlo() smtpclient.sendmail( admin_addr, # MAIL FROM diff --git a/management/mail_log.py b/management/mail_log.py index 22fb87af..0a9d5099 100755 --- a/management/mail_log.py +++ b/management/mail_log.py @@ -1,136 +1,211 @@ #!/usr/bin/python3 +import os.path +import re from collections import defaultdict -import re, os.path + import dateutil.parser import mailconfig import utils + def scan_mail_log(logger, env): - collector = { - "other-services": set(), - "imap-logins": { }, - "postgrey": { }, - "rejected-mail": { }, - "activity-by-hour": { "imap-logins": defaultdict(int), "smtp-sends": defaultdict(int) }, - } + """ Scan the system's mail log files and collect interesting data - collector["real_mail_addresses"] = set(mailconfig.get_mail_users(env)) | set(alias[0] for alias in mailconfig.get_mail_aliases(env)) + This function scans the 2 most recent mail log files in /var/log/. - for fn in ('/var/log/mail.log.1', '/var/log/mail.log'): - if not os.path.exists(fn): continue - with open(fn, 'rb') as log: - for line in log: - line = line.decode("utf8", errors='replace') - scan_mail_log_line(line.strip(), collector) + Args: + logger (ConsoleOutput): Object used for writing messages to the console + env (dict): Dictionary containing MiaB settings + """ - if collector["imap-logins"]: - logger.add_heading("Recent IMAP Logins") - logger.print_block("The most recent login from each remote IP adddress is show.") - for k in utils.sort_email_addresses(collector["imap-logins"], env): - for ip, date in sorted(collector["imap-logins"][k].items(), key = lambda kv : kv[1]): - logger.print_line(k + "\t" + str(date) + "\t" + ip) + collector = { + "other-services": set(), + "imap-logins": {}, + "pop3-logins": {}, + "postgrey": {}, + "rejected-mail": {}, + "activity-by-hour": { + "imap-logins": defaultdict(int), + "pop3-logins": defaultdict(int), + "smtp-sends": defaultdict(int), + "smtp-receives": defaultdict(int), + }, + "real_mail_addresses": ( + set(mailconfig.get_mail_users(env)) | set(alias[0] for alias in mailconfig.get_mail_aliases(env)) + ) + } - if collector["postgrey"]: - logger.add_heading("Greylisted Mail") - logger.print_block("The following mail was greylisted, meaning the emails were temporarily rejected. Legitimate senders will try again within ten minutes.") - logger.print_line("recipient" + "\t" + "received" + "\t" + "sender" + "\t" + "delivered") - for recipient in utils.sort_email_addresses(collector["postgrey"], env): - for (client_address, sender), (first_date, delivered_date) in sorted(collector["postgrey"][recipient].items(), key = lambda kv : kv[1][0]): - logger.print_line(recipient + "\t" + str(first_date) + "\t" + sender + "\t" + (("delivered " + str(delivered_date)) if delivered_date else "no retry yet")) + for fn in ('/var/log/mail.log.1', '/var/log/mail.log'): + if not os.path.exists(fn): + continue + with open(fn, 'rb') as log: + for line in log: + line = line.decode("utf8", errors='replace') + scan_mail_log_line(line.strip(), collector) - if collector["rejected-mail"]: - logger.add_heading("Rejected Mail") - logger.print_block("The following incoming mail was rejected.") - for k in utils.sort_email_addresses(collector["rejected-mail"], env): - for date, sender, message in collector["rejected-mail"][k]: - logger.print_line(k + "\t" + str(date) + "\t" + sender + "\t" + message) + if collector["imap-logins"]: + logger.add_heading("Recent IMAP Logins") + logger.print_block("The most recent login from each remote IP adddress is shown.") + for k in utils.sort_email_addresses(collector["imap-logins"], env): + for ip, date in sorted(collector["imap-logins"][k].items(), key=lambda kv: kv[1]): + logger.print_line(k + "\t" + str(date) + "\t" + ip) - logger.add_heading("Activity by Hour") - for h in range(24): - logger.print_line("%d\t%d\t%d" % (h, collector["activity-by-hour"]["imap-logins"][h], collector["activity-by-hour"]["smtp-sends"][h] )) + if collector["pop3-logins"]: + logger.add_heading("Recent POP3 Logins") + logger.print_block("The most recent login from each remote IP adddress is shown.") + for k in utils.sort_email_addresses(collector["pop3-logins"], env): + for ip, date in sorted(collector["pop3-logins"][k].items(), key=lambda kv: kv[1]): + logger.print_line(k + "\t" + str(date) + "\t" + ip) + + if collector["postgrey"]: + logger.add_heading("Greylisted Mail") + logger.print_block("The following mail was greylisted, meaning the emails were temporarily rejected. " + "Legitimate senders will try again within ten minutes.") + logger.print_line("recipient" + "\t" + "received" + 3 * "\t" + "sender" + 6 * "\t" + "delivered") + for recipient in utils.sort_email_addresses(collector["postgrey"], env): + sorted_recipients = sorted(collector["postgrey"][recipient].items(), key=lambda kv: kv[1][0]) + for (client_address, sender), (first_date, delivered_date) in sorted_recipients: + logger.print_line( + recipient + "\t" + str(first_date) + "\t" + sender + "\t" + + (("delivered " + str(delivered_date)) if delivered_date else "no retry yet") + ) + + if collector["rejected-mail"]: + logger.add_heading("Rejected Mail") + logger.print_block("The following incoming mail was rejected.") + for k in utils.sort_email_addresses(collector["rejected-mail"], env): + for date, sender, message in collector["rejected-mail"][k]: + logger.print_line(k + "\t" + str(date) + "\t" + sender + "\t" + message) + + logger.add_heading("Activity by Hour") + logger.print_block("Dovecot logins and Postfix mail traffic per hour.") + logger.print_block("Hour\tIMAP\tPOP3\tSent\tReceived") + for h in range(24): + logger.print_line( + "%d\t%d\t\t%d\t\t%d\t\t%d" % ( + h, + collector["activity-by-hour"]["imap-logins"][h], + collector["activity-by-hour"]["pop3-logins"][h], + collector["activity-by-hour"]["smtp-sends"][h], + collector["activity-by-hour"]["smtp-receives"][h], + ) + ) + + if len(collector["other-services"]) > 0: + logger.add_heading("Other") + logger.print_block("Unrecognized services in the log: " + ", ".join(collector["other-services"])) - if len(collector["other-services"]) > 0: - logger.add_heading("Other") - logger.print_block("Unrecognized services in the log: " + ", ".join(collector["other-services"])) def scan_mail_log_line(line, collector): - m = re.match(r"(\S+ \d+ \d+:\d+:\d+) (\S+) (\S+?)(\[\d+\])?: (.*)", line) - if not m: return + """ Scan a log line and extract interesting data """ - date, system, service, pid, log = m.groups() - date = dateutil.parser.parse(date) - - if service == "dovecot": - scan_dovecot_line(date, log, collector) + m = re.match(r"(\S+ \d+ \d+:\d+:\d+) (\S+) (\S+?)(\[\d+\])?: (.*)", line) - elif service == "postgrey": - scan_postgrey_line(date, log, collector) + if not m: + return - elif service == "postfix/smtpd": - scan_postfix_smtpd_line(date, log, collector) + date, system, service, pid, log = m.groups() + date = dateutil.parser.parse(date) - elif service == "postfix/submission/smtpd": - scan_postfix_submission_line(date, log, collector) + if service == "dovecot": + scan_dovecot_line(date, log, collector) + elif service == "postgrey": + scan_postgrey_line(date, log, collector) + elif service == "postfix/smtpd": + scan_postfix_smtpd_line(date, log, collector) + elif service == "postfix/cleanup": + scan_postfix_cleanup_line(date, log, collector) + elif service == "postfix/submission/smtpd": + scan_postfix_submission_line(date, log, collector) + elif service in ("postfix/qmgr", "postfix/pickup", "postfix/cleanup", "postfix/scache", "spampd", "postfix/anvil", + "postfix/master", "opendkim", "postfix/lmtp", "postfix/tlsmgr"): + # nothing to look at + pass + else: + collector["other-services"].add(service) - elif service in ("postfix/qmgr", "postfix/pickup", "postfix/cleanup", - "postfix/scache", "spampd", "postfix/anvil", "postfix/master", - "opendkim", "postfix/lmtp", "postfix/tlsmgr"): - # nothing to look at - pass - else: - collector["other-services"].add(service) +def scan_dovecot_line(date, line, collector): + """ Scan a dovecot log line and extract interesting data """ + + m = re.match("(imap|pop3)-login: Login: user=<(.*?)>, method=PLAIN, rip=(.*?),", line) + + if m: + prot, login, ip = m.group(1), m.group(2), m.group(3) + logins_key = "%s-logins" % prot + if ip != "127.0.0.1": # local login from webmail/zpush + collector[logins_key].setdefault(login, {})[ip] = date + collector["activity-by-hour"][logins_key][date.hour] += 1 -def scan_dovecot_line(date, log, collector): - m = re.match("imap-login: Login: user=<(.*?)>, method=PLAIN, rip=(.*?),", log) - if m: - login, ip = m.group(1), m.group(2) - if ip != "127.0.0.1": # local login from webmail/zpush - collector["imap-logins"].setdefault(login, {})[ip] = date - collector["activity-by-hour"]["imap-logins"][date.hour] += 1 def scan_postgrey_line(date, log, collector): - m = re.match("action=(greylist|pass), reason=(.*?), (?:delay=\d+, )?client_name=(.*), client_address=(.*), sender=(.*), recipient=(.*)", log) - if m: - action, reason, client_name, client_address, sender, recipient = m.groups() - key = (client_address, sender) - if action == "greylist" and reason == "new": - collector["postgrey"].setdefault(recipient, {})[key] = (date, None) - elif action == "pass" and reason == "triplet found" and key in collector["postgrey"].get(recipient, {}): - collector["postgrey"][recipient][key] = (collector["postgrey"][recipient][key][0], date) + """ Scan a postgrey log line and extract interesting data """ + + m = re.match("action=(greylist|pass), reason=(.*?), (?:delay=\d+, )?client_name=(.*), client_address=(.*), " + "sender=(.*), recipient=(.*)", + log) + + if m: + action, reason, client_name, client_address, sender, recipient = m.groups() + key = (client_address, sender) + if action == "greylist" and reason == "new": + collector["postgrey"].setdefault(recipient, {})[key] = (date, None) + elif action == "pass" and reason == "triplet found" and key in collector["postgrey"].get(recipient, {}): + collector["postgrey"][recipient][key] = (collector["postgrey"][recipient][key][0], date) + def scan_postfix_smtpd_line(date, log, collector): - m = re.match("NOQUEUE: reject: RCPT from .*?: (.*?); from=<(.*?)> to=<(.*?)>", log) - if m: - message, sender, recipient = m.groups() - if recipient in collector["real_mail_addresses"]: - # only log mail to real recipients + """ Scan a postfix smtpd log line and extract interesting data """ - # skip this, is reported in the greylisting report - if "Recipient address rejected: Greylisted" in message: - return + # Check if the incomming mail was rejected - # simplify this one - m = re.search(r"Client host \[(.*?)\] blocked using zen.spamhaus.org; (.*)", message) - if m: - message = "ip blocked: " + m.group(2) + m = re.match("NOQUEUE: reject: RCPT from .*?: (.*?); from=<(.*?)> to=<(.*?)>", log) - # simplify this one too - m = re.search(r"Sender address \[.*@(.*)\] blocked using dbl.spamhaus.org; (.*)", message) - if m: - message = "domain blocked: " + m.group(2) + if m: + message, sender, recipient = m.groups() + if recipient in collector["real_mail_addresses"]: + # only log mail to real recipients - collector["rejected-mail"].setdefault(recipient, []).append( (date, sender, message) ) + # skip this, if reported in the greylisting report + if "Recipient address rejected: Greylisted" in message: + return + + # simplify this one + m = re.search(r"Client host \[(.*?)\] blocked using zen.spamhaus.org; (.*)", message) + if m: + message = "ip blocked: " + m.group(2) + + # simplify this one too + m = re.search(r"Sender address \[.*@(.*)\] blocked using dbl.spamhaus.org; (.*)", message) + if m: + message = "domain blocked: " + m.group(2) + + collector["rejected-mail"].setdefault(recipient, []).append((date, sender, message)) + + +def scan_postfix_cleanup_line(date, _, collector): + """ Scan a postfix cleanup log line and extract interesting data + + It is assumed that every log of postfix/cleanup indicates an email that was successfulfy received by Postfix. + + """ + + collector["activity-by-hour"]["smtp-receives"][date.hour] += 1 def scan_postfix_submission_line(date, log, collector): - m = re.match("([A-Z0-9]+): client=(\S+), sasl_method=PLAIN, sasl_username=(\S+)", log) - if m: - procid, client, user = m.groups() - collector["activity-by-hour"]["smtp-sends"][date.hour] += 1 + """ Scan a postfix submission log line and extract interesting data """ + + m = re.match("([A-Z0-9]+): client=(\S+), sasl_method=PLAIN, sasl_username=(\S+)", log) + + if m: + # procid, client, user = m.groups() + collector["activity-by-hour"]["smtp-sends"][date.hour] += 1 + if __name__ == "__main__": - from status_checks import ConsoleOutput - env = utils.load_environment() - scan_mail_log(ConsoleOutput(), env) + from status_checks import ConsoleOutput + + env_vars = utils.load_environment() + scan_mail_log(ConsoleOutput(), env_vars) diff --git a/management/templates/custom-dns.html b/management/templates/custom-dns.html index f1244810..bd5643c3 100644 --- a/management/templates/custom-dns.html +++ b/management/templates/custom-dns.html @@ -36,6 +36,7 @@ + diff --git a/management/templates/index.html b/management/templates/index.html index ef0821fd..09684774 100644 --- a/management/templates/index.html +++ b/management/templates/index.html @@ -9,7 +9,7 @@ - + - + @@ -93,7 +93,7 @@
  • Advanced Pages
  • Custom DNS
  • External DNS
    • -
  • Munin Monitoring
    • +
  • Munin Monitoring
  • @@ -192,7 +192,7 @@ - +