From 6d6f3ea3919d81abfd1a6eddd8787c98b761e1bb Mon Sep 17 00:00:00 2001 From: mike Date: Wed, 13 Jan 2016 22:20:33 -0500 Subject: [PATCH 001/165] Added ability to use munin's dynazoom --- management/daemon.py | 79 ++++++++++++++++++++++++++++++++++++++++++-- setup/munin.sh | 3 ++ 2 files changed, 80 insertions(+), 2 deletions(-) diff --git a/management/daemon.py b/management/daemon.py index 1099a596..bc19e2ec 100755 --- a/management/daemon.py +++ b/management/daemon.py @@ -1,10 +1,10 @@ #!/usr/bin/python3 import os, os.path, re, json - +import subprocess from functools import wraps -from flask import Flask, request, render_template, abort, Response, send_from_directory +from flask import Flask, request, render_template, abort, Response, send_from_directory, make_response import auth, utils from mailconfig import get_mail_users, get_mail_users_ex, get_admins, add_mail_user, set_mail_password, remove_mail_user @@ -507,6 +507,81 @@ def munin(filename=""): if filename == "": filename = "index.html" return send_from_directory("/var/cache/munin/www", filename) +# MUNIN CGI-GRAPH + +@app.route('/munin/cgi-graph/') +@app.route('/munin/cgi-graph/') +@authorized_personnel_only +def munin_cgi(filename=""): + """ Relay munin cgi dynazoom requests + + /usr/lib/munin/cgi/munin-cgi-graph is a perl cgi script in the munin package + that is responsible for generating binary png images _and_ associated HTTP + headers based on parameters in the requesting URL. All output is written + to stdout which munin_cgi splits into response headers and binary response + data. + + munin-cgi-graph reads environment variables as well as passed input to determin + what it should do. It expects a path to be in the env-var PATH_INFO, and a + querystring to be in the env-var QUERY_STRING as well as passed as input to the + command. + + munin-cgi-graph has several failure modes. Some write HTTP 404 Status headers + and others return nonzero exit codes. munin_cgi has some basic handling, and + logs errors to app.logger. + + = Reasoning = + Situating munin_cgi between the user-agent and munin-cgi-graph enables keeping + the cgi script behind mailinabox's auth mechanisms and avoids additional + support infrastructure like spawn-fcgi. + + = Configuration = + A single configuration change is all that is required to enable the + functionality of munin_cgi. In the munin.conf file (/etc/munin/munin.conf) add + the following line above your server listings: + `cgiurl_graph /admin/munin/cgi-graph` + This will tell munin to override the default path for dynazoom requests. + """ + + COMMAND = 'su - munin --preserve-environment --shell=/bin/bash -c "/usr/lib/munin/cgi/munin-cgi-graph \'%s\'"' + # su changes user, we use the munin user here + # --preserve-environment retains the environment, which is where Popen's `env` data is + # --shell=/bin/bash ensures the shell used is bash + # -c "/usr/lib/munin/cgi/munin-cgi-graph" passes the command to run as munin + # \'%s\' is a placeholder for where the request's querystring will be added + + if filename == "": + return ("a path must be specified", 404) + + query_str = request.query_string.decode("utf-8", 'ignore') + + env = {'PATH_INFO': '/%s/' % filename, 'QUERY_STRING': query_str} + cmd = COMMAND % (query_str if not query_str.startswith('&') else query_str[1:]) + process = subprocess.Popen(cmd, env=env, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, close_fds=True) + stdout, stderr = process.communicate() + + if process.returncode != 0: + # nonzero returncode indicates error + app.logger.error("munin_cgi: munin-cgi-graph returned nonzero exit code, %s", process.returncode) + return ("error processing graph image", 500) + + # /usr/lib/munin/cgi/munin-cgi-graph returns both headers and binary png when successful. + # Per http://www.libpng.org/pub/png/spec/1.2/PNG-Structure.html PNG files always start + # with the same 8 bytes (137 80 78 71 13 10 26 10) or b'\x89PNG\r\n\x1a\n' So we split + # the output of munin-cgi-graph where the PNG begins + bin_start = stdout.find(b'\x89PNG\r\n\x1a\n') + str_headers = stdout[:bin_start].decode("utf-8") + # decode the byte str containing response headers + bin_image = stdout[bin_start:] + response = make_response(bin_image) + for line in str_headers.splitlines(): + if line: + name, value = line.split(':',1) + response.headers[name] = value + if 'Status' in response.headers and '404' in response.headers['Status']: + app.logger.warning("munin_cgi: munin-cgi-graph returned 404 status code. PATH_INFO=%s", env['PATH_INFO']) + return response + # APP if __name__ == '__main__': diff --git a/setup/munin.sh b/setup/munin.sh index 0cee9bad..0f2dec04 100755 --- a/setup/munin.sh +++ b/setup/munin.sh @@ -19,6 +19,9 @@ tmpldir /etc/munin/templates includedir /etc/munin/munin-conf.d +# path dynazoom uses for requests +cgiurl_graph /admin/munin/cgi-graph + # a simple host tree [$PRIMARY_HOSTNAME] address 127.0.0.1 From 8932aaf4ef3d24b7009419c89cbf7919627680ee Mon Sep 17 00:00:00 2001 From: mike Date: Wed, 13 Jan 2016 23:55:45 -0500 Subject: [PATCH 002/165] needed libcgi-fast-perl and chown log files --- management/daemon.py | 19 +++---------------- setup/munin.sh | 7 ++++++- 2 files changed, 9 insertions(+), 17 deletions(-) diff --git a/management/daemon.py b/management/daemon.py index bc19e2ec..bd917692 100755 --- a/management/daemon.py +++ b/management/daemon.py @@ -514,33 +514,20 @@ def munin(filename=""): @authorized_personnel_only def munin_cgi(filename=""): """ Relay munin cgi dynazoom requests - /usr/lib/munin/cgi/munin-cgi-graph is a perl cgi script in the munin package that is responsible for generating binary png images _and_ associated HTTP headers based on parameters in the requesting URL. All output is written to stdout which munin_cgi splits into response headers and binary response data. - - munin-cgi-graph reads environment variables as well as passed input to determin + munin-cgi-graph reads environment variables as well as passed input to determine what it should do. It expects a path to be in the env-var PATH_INFO, and a querystring to be in the env-var QUERY_STRING as well as passed as input to the command. - - munin-cgi-graph has several failure modes. Some write HTTP 404 Status headers - and others return nonzero exit codes. munin_cgi has some basic handling, and - logs errors to app.logger. - - = Reasoning = + munin-cgi-graph has several failure modes. Some write HTTP Status headers and + others return nonzero exit codes. Situating munin_cgi between the user-agent and munin-cgi-graph enables keeping the cgi script behind mailinabox's auth mechanisms and avoids additional support infrastructure like spawn-fcgi. - - = Configuration = - A single configuration change is all that is required to enable the - functionality of munin_cgi. In the munin.conf file (/etc/munin/munin.conf) add - the following line above your server listings: - `cgiurl_graph /admin/munin/cgi-graph` - This will tell munin to override the default path for dynazoom requests. """ COMMAND = 'su - munin --preserve-environment --shell=/bin/bash -c "/usr/lib/munin/cgi/munin-cgi-graph \'%s\'"' diff --git a/setup/munin.sh b/setup/munin.sh index 0f2dec04..b1c5c8ba 100755 --- a/setup/munin.sh +++ b/setup/munin.sh @@ -7,7 +7,8 @@ source /etc/mailinabox.conf # load global vars # install Munin echo "Installing Munin (system monitoring)..." -apt_install munin munin-node +apt_install munin munin-node libcgi-fast-perl +# libcgi-fast-perl is needed by /usr/lib/munin/cgi/munin-cgi-graph # edit config cat > /etc/munin/munin.conf < Date: Thu, 14 Jan 2016 10:24:04 -0500 Subject: [PATCH 003/165] Use utils.shell instead of subprocess.Popen --- management/daemon.py | 25 ++++++++++++++++--------- 1 file changed, 16 insertions(+), 9 deletions(-) diff --git a/management/daemon.py b/management/daemon.py index bd917692..fd2d6b4d 100755 --- a/management/daemon.py +++ b/management/daemon.py @@ -530,24 +530,31 @@ def munin_cgi(filename=""): support infrastructure like spawn-fcgi. """ - COMMAND = 'su - munin --preserve-environment --shell=/bin/bash -c "/usr/lib/munin/cgi/munin-cgi-graph \'%s\'"' + COMMAND = 'su - munin --preserve-environment --shell=/bin/bash -c /usr/lib/munin/cgi/munin-cgi-graph "%s"' # su changes user, we use the munin user here # --preserve-environment retains the environment, which is where Popen's `env` data is # --shell=/bin/bash ensures the shell used is bash # -c "/usr/lib/munin/cgi/munin-cgi-graph" passes the command to run as munin - # \'%s\' is a placeholder for where the request's querystring will be added + # "%s" is a placeholder for where the request's querystring will be added if filename == "": return ("a path must be specified", 404) query_str = request.query_string.decode("utf-8", 'ignore') + query_str = query_str[1:] if query_str.startswith('&') else query_str[1:] + # I don't know if this is strictly necessary env = {'PATH_INFO': '/%s/' % filename, 'QUERY_STRING': query_str} - cmd = COMMAND % (query_str if not query_str.startswith('&') else query_str[1:]) - process = subprocess.Popen(cmd, env=env, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, close_fds=True) - stdout, stderr = process.communicate() + cmd = COMMAND % query_str + code, binout = utils.shell('check_output', + cmd.split(' ', 5), + # Using a maxsplit of 5 keeps the last 2 arguments together + input=query_str.encode('UTF-8'), + env=env, + return_bytes=True, + trap=True) - if process.returncode != 0: + if code != 0: # nonzero returncode indicates error app.logger.error("munin_cgi: munin-cgi-graph returned nonzero exit code, %s", process.returncode) return ("error processing graph image", 500) @@ -556,10 +563,10 @@ def munin_cgi(filename=""): # Per http://www.libpng.org/pub/png/spec/1.2/PNG-Structure.html PNG files always start # with the same 8 bytes (137 80 78 71 13 10 26 10) or b'\x89PNG\r\n\x1a\n' So we split # the output of munin-cgi-graph where the PNG begins - bin_start = stdout.find(b'\x89PNG\r\n\x1a\n') - str_headers = stdout[:bin_start].decode("utf-8") + bin_start = binout.find(b'\x89PNG\r\n\x1a\n') + str_headers = binout[:bin_start].decode("utf-8") # decode the byte str containing response headers - bin_image = stdout[bin_start:] + bin_image = binout[bin_start:] response = make_response(bin_image) for line in str_headers.splitlines(): if line: From 8ea2f5a766223fc2b12c7dfc394f578b3d88253e Mon Sep 17 00:00:00 2001 From: Michael Kroes Date: Thu, 25 Feb 2016 21:56:27 +0100 Subject: [PATCH 004/165] Allow a server to be rebooted when a reboot is required --- management/daemon.py | 18 ++++++++++++- management/templates/system-status.html | 34 +++++++++++++++++++++++-- 2 files changed, 49 insertions(+), 3 deletions(-) diff --git a/management/daemon.py b/management/daemon.py index 6de0a59a..823bce5e 100755 --- a/management/daemon.py +++ b/management/daemon.py @@ -391,7 +391,6 @@ def ssl_provision_certs(): def web_get_domains(): from web_update import get_web_domains_info return json_response(get_web_domains_info(env)) - @app.route('/web/update', methods=['POST']) @authorized_personnel_only def web_update(): @@ -456,6 +455,23 @@ def do_updates(): "DEBIAN_FRONTEND": "noninteractive" }) +@app.route('/system/reboot', methods=["GET"]) +@authorized_personnel_only +def needs_reboot(): + if os.path.isfile("/var/run/reboot-required"): + return json_response(True) + else: + return json_response(False) + +@app.route('/system/reboot', methods=["POST"]) +@authorized_personnel_only +def do_reboot(): + if os.path.isfile("/var/run/reboot-required"): + return utils.shell("check_output", ["/sbin/shutdown", "-r", "now"], capture_stderr=True) + else: + return "No reboot is required" + + @app.route('/system/backup/status') @authorized_personnel_only def backup_status(): diff --git a/management/templates/system-status.html b/management/templates/system-status.html index 9fe9d08e..efc8ad90 100644 --- a/management/templates/system-status.html +++ b/management/templates/system-status.html @@ -1,5 +1,9 @@

System Status Checks

+ + +
+
+ + + +
+
+ @@ -61,6 +59,9 @@
+
+
+ - + - +