Merge master into fail2ban

CHANGELOG.md

@@ -1,6 +1,24 @@
 CHANGELOG
 =========
 
+In Development
+--------------
+
+Mail:
+
+* Fixed a long-standing issue with training the spam filter not working (because of a file permissions issue).
+
+Control panel:
+
+* Munin system monitoring graphs are now zoomable.
+* When a reboot is required (due to Ubuntu security updates automatically installed), a Reboot Box button now appears.
+
+Setup:
+
+* A swap file is now created if system memory is less than 2GB, 5GB of free disk space is available, and if no swap file yet exists.
+* Setup dialogs did not appear correctly when connecting to SSH using Putty on Windows.
+* We now install Roundcube from our own mirror because Sourceforge's downloads experience frequent intermittant unavailability.
+
 v0.17b (March 1, 2016)
 ----------------------
 

conf/nginx-primaryonly.conf

@@ -6,6 +6,9 @@
 	location /admin/ {
 		proxy_pass http://127.0.0.1:10222/;
 		proxy_set_header X-Forwarded-For $remote_addr;
+		add_header X-Frame-Options "DENY";
+		add_header X-Content-Type-Options nosniff;
+		add_header Content-Security-Policy "frame-ancestors 'none';";
 	}
 
 	# ownCloud configuration.

management/daemon.py

@@ -1,10 +1,11 @@
 #!/usr/bin/python3
 
 import os, os.path, re, json, time
+import subprocess
 
 from functools import wraps
 
-from flask import Flask, request, render_template, abort, Response, send_from_directory
+from flask import Flask, request, render_template, abort, Response, send_from_directory, make_response
 
 import auth, utils, multiprocessing.pool
 from mailconfig import get_mail_users, get_mail_users_ex, get_admins, add_mail_user, set_mail_password, remove_mail_user
@@ -459,6 +460,27 @@ def do_updates():
 		"DEBIAN_FRONTEND": "noninteractive"
 	})
 
+
+@app.route('/system/reboot', methods=["GET"])
+@authorized_personnel_only
+def needs_reboot():
+	from status_checks import is_reboot_needed_due_to_package_installation
+	if is_reboot_needed_due_to_package_installation():
+		return json_response(True)
+	else:
+		return json_response(False)
+
+@app.route('/system/reboot', methods=["POST"])
+@authorized_personnel_only
+def do_reboot():
+	# To keep the attack surface low, we don't allow a remote reboot if one isn't necessary.
+	from status_checks import is_reboot_needed_due_to_package_installation
+	if is_reboot_needed_due_to_package_installation():
+		return utils.shell("check_output", ["/sbin/shutdown", "-r", "now"], capture_stderr=True)
+	else:
+		return "No reboot is required, so it is not allowed."
+
+
 @app.route('/system/backup/status')
 @authorized_personnel_only
 def backup_status():
@@ -510,11 +532,69 @@ def munin(filename=""):
 	if filename == "": filename = "index.html"
 	return send_from_directory("/var/cache/munin/www", filename)
 
+@app.route('/munin/cgi-graph/<path:filename>')
+@authorized_personnel_only
+def munin_cgi(filename):
+	""" Relay munin cgi dynazoom requests
+	/usr/lib/munin/cgi/munin-cgi-graph is a perl cgi script in the munin package
+	that is responsible for generating binary png images _and_ associated HTTP
+	headers based on parameters in the requesting URL. All output is written
+	to stdout which munin_cgi splits into response headers and binary response
+	data.
+	munin-cgi-graph reads environment variables as well as passed input to determine
+	what it should do. It expects a path to be in the env-var PATH_INFO, and a
+	querystring to be in the env-var QUERY_STRING as well as passed as input to the
+	command.
+	munin-cgi-graph has several failure modes. Some write HTTP Status headers and
+	others return nonzero exit codes.
+	Situating munin_cgi between the user-agent and munin-cgi-graph enables keeping
+	the cgi script behind mailinabox's auth mechanisms and avoids additional
+	support infrastructure like spawn-fcgi.
+	"""
+
+	COMMAND = 'su - munin --preserve-environment --shell=/bin/bash -c /usr/lib/munin/cgi/munin-cgi-graph "%s"'
+	# su changes user, we use the munin user here
+	# --preserve-environment retains the environment, which is where Popen's `env` data is
+	# --shell=/bin/bash ensures the shell used is bash
+	# -c "/usr/lib/munin/cgi/munin-cgi-graph" passes the command to run as munin
+	# "%s" is a placeholder for where the request's querystring will be added
+
+	if filename == "":
+		return ("a path must be specified", 404)
+
+	query_str = request.query_string.decode("utf-8", 'ignore')
+
+	env = {'PATH_INFO': '/%s/' % filename, 'QUERY_STRING': query_str}
+	cmd = COMMAND % query_str
+	code, binout = utils.shell('check_output',
+							   cmd.split(' ', 5),
+							   # Using a maxsplit of 5 keeps the last 2 arguments together
+							   input=query_str.encode('UTF-8'),
+							   env=env,
+							   return_bytes=True,
+							   trap=True)
+
+	if code != 0:
+		# nonzero returncode indicates error
+		app.logger.error("munin_cgi: munin-cgi-graph returned nonzero exit code, %s", process.returncode)
+		return ("error processing graph image", 500)
+
+	# /usr/lib/munin/cgi/munin-cgi-graph returns both headers and binary png when successful.
+	# A double-Windows-style-newline always indicates the end of HTTP headers.
+	headers, image_bytes = binout.split(b'\r\n\r\n', 1)
+	response = make_response(image_bytes)
+	for line in headers.splitlines():
+		name, value = line.decode("utf8").split(':', 1)
+		response.headers[name] = value
+	if 'Status' in response.headers and '404' in response.headers['Status']:
+		app.logger.warning("munin_cgi: munin-cgi-graph returned 404 status code. PATH_INFO=%s", env['PATH_INFO'])
+	return response
+
 def log_failed_login(request):
-	# We need to figure out the ip to list in the message, all our calls are routed 
+	# We need to figure out the ip to list in the message, all our calls are routed
-	# through nginx who will put the original ip in X-Forwarded-For. 
+	# through nginx who will put the original ip in X-Forwarded-For.
 	# During setup we call the management interface directly to determine the user
-	# status. So we can't always use X-Forwarded-For because during setup that header 
+	# status. So we can't always use X-Forwarded-For because during setup that header
 	# will not be present.
 	if request.headers.getlist("X-Forwarded-For"):
 		ip = request.headers.getlist("X-Forwarded-For")[0]
@@ -525,6 +605,7 @@ def log_failed_login(request):
 	# message.
 	app.logger.warning( "MIAB: Failed login attempt from ip %s - timestamp %s" % (ip, time.time()))
 
+
 # APP
 
 if __name__ == '__main__':

management/dns_update.py

@@ -175,9 +175,6 @@ def build_zone(domain, all_domains, additional_records, www_redirect_domains, en
 		for value in build_sshfp_records():
 			records.append((None, "SSHFP", value, "Optional. Provides an out-of-band method for verifying an SSH key before connecting. Use 'VerifyHostKeyDNS yes' (or 'VerifyHostKeyDNS ask') when connecting with ssh."))
 
-	# The MX record says where email for the domain should be delivered: Here!
-	records.append((None,  "MX",  "10 %s." % env["PRIMARY_HOSTNAME"], "Required. Specifies the hostname (and priority) of the machine that handles @%s mail." % domain))
-
 	# Add DNS records for any subdomains of this domain. We should not have a zone for
 	# both a domain and one of its subdomains.
 	subdomains = [d for d in all_domains if d.endswith("." + domain)]
@@ -244,6 +241,10 @@ def build_zone(domain, all_domains, additional_records, www_redirect_domains, en
 	# Don't pin the list of records that has_rec checks against anymore.
 	has_rec_base = records
 
+	# The MX record says where email for the domain should be delivered: Here!
+	if not has_rec(None, "MX", prefix="10 "):
+		records.append((None,  "MX",  "10 %s." % env["PRIMARY_HOSTNAME"], "Required. Specifies the hostname (and priority) of the machine that handles @%s mail." % domain))
+
 	# SPF record: Permit the box ('mx', see above) to send mail on behalf of
 	# the domain, and no one else.
 	# Skip if the user has set a custom SPF record.

management/status_checks.py

@@ -185,10 +185,13 @@ def check_ssh_password(env, output):
 	else:
 		output.print_ok("SSH disallows password-based login.")
 
+def is_reboot_needed_due_to_package_installation():
+	return os.path.exists("/var/run/reboot-required")
+
 def check_software_updates(env, output):
 	# Check for any software package updates.
 	pkgs = list_apt_updates(apt_update=False)
-	if os.path.exists("/var/run/reboot-required"):
+	if is_reboot_needed_due_to_package_installation():
 		output.print_error("System updates have been installed and a reboot of the machine is required.")
 	elif len(pkgs) == 0:
 		output.print_ok("System software is up to date.")

management/templates/system-status.html

@@ -34,19 +34,23 @@
   font-family: monospace;
   white-space: pre-wrap;
 }
-
-#system-privacy-setting {
-  float: right;
-  max-width: 20em;
-  margin-bottom: 1em;
-}
 </style>
 
+<div class="row">
+	<div class="col-md-push-9 col-md-3">
+
+<div id="system-reboot-required" style="display: none; margin-bottom: 1em;">
+	<button type="button" class="btn btn-danger" onclick="confirm_reboot(); return false;">Reboot Box</button>
+	<div>No reboot is necessary.</div>
+</div>
+
 <div id="system-privacy-setting" style="display: none">
 	<div><a onclick="return enable_privacy(!current_privacy_setting)" href="#"><span>Enable/Disable</span> New-Version Check</a></div>
 	<p style="line-height: 125%"><small>(When enabled, status checks phone-home to check for a new release of Mail-in-a-Box.)</small></p>
 </div>
 
+	</div> <!-- /col -->
+	<div class="col-md-pull-3 col-md-8">
 
 <table id="system-checks" class="table" style="max-width: 60em">
   <thead>
@@ -55,6 +59,9 @@
   </tbody>
 </table>
 
+	</div> <!-- /col -->
+</div> <!-- /row -->
+
 <script>
 function show_system_status() {
   $('#system-checks tbody').html("<tr><td colspan='2' class='text-muted'>Loading...</td></tr>")
@@ -70,6 +77,16 @@ function show_system_status() {
       $('#system-privacy-setting p').toggle(r);
     });
 
+  api(
+    "/system/reboot",
+    "GET",
+    { },
+    function(r) {
+      $('#system-reboot-required').show(); // show when r becomes available
+      $('#system-reboot-required').find('button').toggle(r);
+      $('#system-reboot-required').find('div').toggle(!r);
+    });
+
   api(
     "/system/status",
     "POST",
@@ -122,4 +139,22 @@ function enable_privacy(status) {
     });
   return false; // disable link
 }
+
+function confirm_reboot() {
+  show_modal_confirm(
+    "Reboot",
+    $("<p>This will reboot your Mail-in-a-Box <code>{{hostname}}</code>.</p> <p>Until the machine is fully restarted, your users will not be able to send and receive email, and you will not be able to connect to this control panel or with SSH. The reboot cannot be cancelled.</p>"),
+    "Reboot Now",
+    function() {
+      api(
+        "/system/reboot",
+        "POST",
+        { },
+        function(r) {
+          var msg = "<p>Please reload this page after a minute or so.</p>";
+          if (r) msg = "<p>The reboot command said:</p> <pre>" + $("<pre/>").text(r).html() + "</pre>"; // successful reboots don't produce any output; the output must be HTML-escaped
+          show_modal_error("Reboot", msg);
+        });
+    });
+}
 </script>

setup/munin.sh

@@ -7,7 +7,8 @@ source /etc/mailinabox.conf # load global vars
 
 # install Munin
 echo "Installing Munin (system monitoring)..."
-apt_install munin munin-node
+apt_install munin munin-node libcgi-fast-perl
+# libcgi-fast-perl is needed by /usr/lib/munin/cgi/munin-cgi-graph
 
 # edit config
 cat > /etc/munin/munin.conf <<EOF;
@@ -19,6 +20,9 @@ tmpldir /etc/munin/templates
 
 includedir /etc/munin/munin-conf.d
 
+# path dynazoom uses for requests
+cgiurl_graph /admin/munin/cgi-graph
+
 # a simple host tree
 [$PRIMARY_HOSTNAME]
 address 127.0.0.1
@@ -29,6 +33,10 @@ contact.admin.command mail -s "Munin notification ${var:host}" administrator@$PR
 contact.admin.always_send warning critical
 EOF
 
+# The Debian installer touches these files and chowns them to www-data:adm for use with spawn-fcgi
+chown munin. /var/log/munin/munin-cgi-html.log
+chown munin. /var/log/munin/munin-cgi-graph.log
+
 # ensure munin-node knows the name of this machine
 tools/editconf.py /etc/munin/munin-node.conf -s \
 	host_name=$PRIMARY_HOSTNAME

setup/preflight.sh

@@ -46,3 +46,17 @@ if [ -e ~/.wgetrc ]; then
 	echo "Mail-in-a-Box expects no overrides to wget defaults, ~/.wgetrc exists"
 	exit
 fi
+
+# Check that we are running on x86_64, any other architecture is unsupported and
+# will fail later in the setup when we try to install the custom build lucene packages.
+#
+# Set ARM=1 to ignore this check if you have built the packages yourself. If you do this
+# you are on your own!
+ARCHITECTURE=$(uname -m)
+if [ "$ARCHITECTURE" != "x86_64" ]; then
+if [ -z "$ARM" ]; then
+	echo "Mail-in-a-Box only supports x86_64 and will not work on any other architecture, like ARM."
+	echo "Your architecture is $ARCHITECTURE"
+	exit
+fi
+fi

setup/spamassassin.sh

@@ -78,9 +78,13 @@ tools/editconf.py /etc/spamassassin/local.cf -s \
 # * Writable by the debian-spamd user, which runs /etc/cron.daily/spamassassin.
 #
 # We'll have these files owned by spampd and grant access to the other two processes.
+#
+# Spamassassin will change the access rights back to the defaults, so we must also configure
+# the filemode in the config file.
 
 tools/editconf.py /etc/spamassassin/local.cf -s \
-	bayes_path=$STORAGE_ROOT/mail/spamassassin/bayes
+	bayes_path=$STORAGE_ROOT/mail/spamassassin/bayes \
+	bayes_file_mode=0660
 
 mkdir -p $STORAGE_ROOT/mail/spamassassin
 chown -R spampd:spampd $STORAGE_ROOT/mail/spamassassin

setup/start.sh

@@ -24,6 +24,9 @@ export LC_ALL=en_US.UTF-8
 export LANG=en_US.UTF-8
 export LC_TYPE=en_US.UTF-8
 
+# Fix so line drawing characters are shown correctly in Putty on Windows. See #744.
+export NCURSES_NO_UTF8_ACS=1
+
 # Recall the last settings used if we're running this a second time.
 if [ -f /etc/mailinabox.conf ]; then
 	# Run any system migrations before proceeding. Since this is a second run,

setup/system.sh

@@ -4,6 +4,60 @@ source setup/functions.sh # load our functions
 # Basic System Configuration
 # -------------------------
 
+# ### Add swap space to the system
+
+# If the physical memory of the system is below 2GB it is wise to create a
+# swap file. This will make the system more resiliant to memory spikes and
+# prevent for instance spam filtering from crashing
+
+# We will create a 1G file, this should be a good balance between disk usage
+# and buffers for the system. We will only allocate this file if there is more
+# than 5GB of disk space available
+
+# The following checks are performed:
+# - Check if swap is currently mountend by looking at /proc/swaps
+# - Check if the user intents to activate swap on next boot by checking fstab entries.
+# - Check if a swapfile already exists
+# - Check if the root file system is not btrfs, might be an incompatible version with
+#   swapfiles. User should hanle it them selves.
+# - Check the memory requirements
+# - Check available diskspace
+
+# See https://www.digitalocean.com/community/tutorials/how-to-add-swap-on-ubuntu-14-04
+# for reference
+
+SWAP_MOUNTED=$(cat /proc/swaps | tail -n+2)
+SWAP_IN_FSTAB=$(grep "swap" /etc/fstab)
+ROOT_IS_BTRFS=$(grep "\/ .*btrfs" /proc/mounts)
+TOTAL_PHYSICAL_MEM=$(head -n 1 /proc/meminfo | awk '{print $2}')
+AVAILABLE_DISK_SPACE=$(df / --output=avail | tail -n 1)
+if
+	[ -z "$SWAP_MOUNTED" ] &&
+	[ -z "$SWAP_IN_FSTAB" ] &&
+	[ ! -e /swapfile ] &&
+	[ -z "$ROOT_IS_BTRFS" ] &&
+	[ $TOTAL_PHYSICAL_MEM -lt 1900000 ] &&
+	[ $AVAILABLE_DISK_SPACE -gt 5242880 ]
+then
+	echo "Adding a swap file to the system..."
+
+	# Allocate and activate the swap file. Allocate in 1KB chuncks
+	# doing it in one go, could fail on low memory systems
+	dd if=/dev/zero of=/swapfile bs=1024 count=$[1024*1024] status=none
+	if [ -e /swapfile ]; then
+		chmod 600 /swapfile
+		hide_output mkswap /swapfile
+		swapon /swapfile
+	fi
+
+	# Check if swap is mounted then activate on boot
+	if swapon -s | grep -q "\/swapfile"; then
+		echo "/swapfile   none    swap    sw    0   0" >> /etc/fstab
+	else
+		echo "ERROR: Swap allocation failed"
+	fi
+fi
+
 # ### Add Mail-in-a-Box's PPA.
 
 # We've built several .deb packages on our own that we want to include.

setup/webmail.sh

@@ -51,7 +51,7 @@ fi
 if [ $needs_update == 1 ]; then
 	# install roundcube
 	wget_verify \
-		https://downloads.sourceforge.net/project/roundcubemail/roundcubemail/$VERSION/roundcubemail-$VERSION.tar.gz \
+		https://s3.amazonaws.com/joshdata/mail-in-a-box/public/roundcubemail-$VERSION.tar.gz \
 		$HASH \
 		/tmp/roundcube.tgz
 	tar -C /usr/local/lib --no-same-owner -zxf /tmp/roundcube.tgz