From 6d6f3ea3919d81abfd1a6eddd8787c98b761e1bb Mon Sep 17 00:00:00 2001 From: mike Date: Wed, 13 Jan 2016 22:20:33 -0500 Subject: [PATCH 1/4] Added ability to use munin's dynazoom --- management/daemon.py | 79 ++++++++++++++++++++++++++++++++++++++++++-- setup/munin.sh | 3 ++ 2 files changed, 80 insertions(+), 2 deletions(-) diff --git a/management/daemon.py b/management/daemon.py index 1099a59..bc19e2e 100755 --- a/management/daemon.py +++ b/management/daemon.py @@ -1,10 +1,10 @@ #!/usr/bin/python3 import os, os.path, re, json - +import subprocess from functools import wraps -from flask import Flask, request, render_template, abort, Response, send_from_directory +from flask import Flask, request, render_template, abort, Response, send_from_directory, make_response import auth, utils from mailconfig import get_mail_users, get_mail_users_ex, get_admins, add_mail_user, set_mail_password, remove_mail_user @@ -507,6 +507,81 @@ def munin(filename=""): if filename == "": filename = "index.html" return send_from_directory("/var/cache/munin/www", filename) +# MUNIN CGI-GRAPH + +@app.route('/munin/cgi-graph/') +@app.route('/munin/cgi-graph/') +@authorized_personnel_only +def munin_cgi(filename=""): + """ Relay munin cgi dynazoom requests + + /usr/lib/munin/cgi/munin-cgi-graph is a perl cgi script in the munin package + that is responsible for generating binary png images _and_ associated HTTP + headers based on parameters in the requesting URL. All output is written + to stdout which munin_cgi splits into response headers and binary response + data. + + munin-cgi-graph reads environment variables as well as passed input to determin + what it should do. It expects a path to be in the env-var PATH_INFO, and a + querystring to be in the env-var QUERY_STRING as well as passed as input to the + command. + + munin-cgi-graph has several failure modes. Some write HTTP 404 Status headers + and others return nonzero exit codes. munin_cgi has some basic handling, and + logs errors to app.logger. + + = Reasoning = + Situating munin_cgi between the user-agent and munin-cgi-graph enables keeping + the cgi script behind mailinabox's auth mechanisms and avoids additional + support infrastructure like spawn-fcgi. + + = Configuration = + A single configuration change is all that is required to enable the + functionality of munin_cgi. In the munin.conf file (/etc/munin/munin.conf) add + the following line above your server listings: + `cgiurl_graph /admin/munin/cgi-graph` + This will tell munin to override the default path for dynazoom requests. + """ + + COMMAND = 'su - munin --preserve-environment --shell=/bin/bash -c "/usr/lib/munin/cgi/munin-cgi-graph \'%s\'"' + # su changes user, we use the munin user here + # --preserve-environment retains the environment, which is where Popen's `env` data is + # --shell=/bin/bash ensures the shell used is bash + # -c "/usr/lib/munin/cgi/munin-cgi-graph" passes the command to run as munin + # \'%s\' is a placeholder for where the request's querystring will be added + + if filename == "": + return ("a path must be specified", 404) + + query_str = request.query_string.decode("utf-8", 'ignore') + + env = {'PATH_INFO': '/%s/' % filename, 'QUERY_STRING': query_str} + cmd = COMMAND % (query_str if not query_str.startswith('&') else query_str[1:]) + process = subprocess.Popen(cmd, env=env, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, close_fds=True) + stdout, stderr = process.communicate() + + if process.returncode != 0: + # nonzero returncode indicates error + app.logger.error("munin_cgi: munin-cgi-graph returned nonzero exit code, %s", process.returncode) + return ("error processing graph image", 500) + + # /usr/lib/munin/cgi/munin-cgi-graph returns both headers and binary png when successful. + # Per http://www.libpng.org/pub/png/spec/1.2/PNG-Structure.html PNG files always start + # with the same 8 bytes (137 80 78 71 13 10 26 10) or b'\x89PNG\r\n\x1a\n' So we split + # the output of munin-cgi-graph where the PNG begins + bin_start = stdout.find(b'\x89PNG\r\n\x1a\n') + str_headers = stdout[:bin_start].decode("utf-8") + # decode the byte str containing response headers + bin_image = stdout[bin_start:] + response = make_response(bin_image) + for line in str_headers.splitlines(): + if line: + name, value = line.split(':',1) + response.headers[name] = value + if 'Status' in response.headers and '404' in response.headers['Status']: + app.logger.warning("munin_cgi: munin-cgi-graph returned 404 status code. PATH_INFO=%s", env['PATH_INFO']) + return response + # APP if __name__ == '__main__': diff --git a/setup/munin.sh b/setup/munin.sh index 0cee9ba..0f2dec0 100755 --- a/setup/munin.sh +++ b/setup/munin.sh @@ -19,6 +19,9 @@ tmpldir /etc/munin/templates includedir /etc/munin/munin-conf.d +# path dynazoom uses for requests +cgiurl_graph /admin/munin/cgi-graph + # a simple host tree [$PRIMARY_HOSTNAME] address 127.0.0.1 From 8932aaf4ef3d24b7009419c89cbf7919627680ee Mon Sep 17 00:00:00 2001 From: mike Date: Wed, 13 Jan 2016 23:55:45 -0500 Subject: [PATCH 2/4] needed libcgi-fast-perl and chown log files --- management/daemon.py | 19 +++---------------- setup/munin.sh | 7 ++++++- 2 files changed, 9 insertions(+), 17 deletions(-) diff --git a/management/daemon.py b/management/daemon.py index bc19e2e..bd91769 100755 --- a/management/daemon.py +++ b/management/daemon.py @@ -514,33 +514,20 @@ def munin(filename=""): @authorized_personnel_only def munin_cgi(filename=""): """ Relay munin cgi dynazoom requests - /usr/lib/munin/cgi/munin-cgi-graph is a perl cgi script in the munin package that is responsible for generating binary png images _and_ associated HTTP headers based on parameters in the requesting URL. All output is written to stdout which munin_cgi splits into response headers and binary response data. - - munin-cgi-graph reads environment variables as well as passed input to determin + munin-cgi-graph reads environment variables as well as passed input to determine what it should do. It expects a path to be in the env-var PATH_INFO, and a querystring to be in the env-var QUERY_STRING as well as passed as input to the command. - - munin-cgi-graph has several failure modes. Some write HTTP 404 Status headers - and others return nonzero exit codes. munin_cgi has some basic handling, and - logs errors to app.logger. - - = Reasoning = + munin-cgi-graph has several failure modes. Some write HTTP Status headers and + others return nonzero exit codes. Situating munin_cgi between the user-agent and munin-cgi-graph enables keeping the cgi script behind mailinabox's auth mechanisms and avoids additional support infrastructure like spawn-fcgi. - - = Configuration = - A single configuration change is all that is required to enable the - functionality of munin_cgi. In the munin.conf file (/etc/munin/munin.conf) add - the following line above your server listings: - `cgiurl_graph /admin/munin/cgi-graph` - This will tell munin to override the default path for dynazoom requests. """ COMMAND = 'su - munin --preserve-environment --shell=/bin/bash -c "/usr/lib/munin/cgi/munin-cgi-graph \'%s\'"' diff --git a/setup/munin.sh b/setup/munin.sh index 0f2dec0..b1c5c8b 100755 --- a/setup/munin.sh +++ b/setup/munin.sh @@ -7,7 +7,8 @@ source /etc/mailinabox.conf # load global vars # install Munin echo "Installing Munin (system monitoring)..." -apt_install munin munin-node +apt_install munin munin-node libcgi-fast-perl +# libcgi-fast-perl is needed by /usr/lib/munin/cgi/munin-cgi-graph # edit config cat > /etc/munin/munin.conf < Date: Thu, 14 Jan 2016 10:24:04 -0500 Subject: [PATCH 3/4] Use utils.shell instead of subprocess.Popen --- management/daemon.py | 25 ++++++++++++++++--------- 1 file changed, 16 insertions(+), 9 deletions(-) diff --git a/management/daemon.py b/management/daemon.py index bd91769..fd2d6b4 100755 --- a/management/daemon.py +++ b/management/daemon.py @@ -530,24 +530,31 @@ def munin_cgi(filename=""): support infrastructure like spawn-fcgi. """ - COMMAND = 'su - munin --preserve-environment --shell=/bin/bash -c "/usr/lib/munin/cgi/munin-cgi-graph \'%s\'"' + COMMAND = 'su - munin --preserve-environment --shell=/bin/bash -c /usr/lib/munin/cgi/munin-cgi-graph "%s"' # su changes user, we use the munin user here # --preserve-environment retains the environment, which is where Popen's `env` data is # --shell=/bin/bash ensures the shell used is bash # -c "/usr/lib/munin/cgi/munin-cgi-graph" passes the command to run as munin - # \'%s\' is a placeholder for where the request's querystring will be added + # "%s" is a placeholder for where the request's querystring will be added if filename == "": return ("a path must be specified", 404) query_str = request.query_string.decode("utf-8", 'ignore') + query_str = query_str[1:] if query_str.startswith('&') else query_str[1:] + # I don't know if this is strictly necessary env = {'PATH_INFO': '/%s/' % filename, 'QUERY_STRING': query_str} - cmd = COMMAND % (query_str if not query_str.startswith('&') else query_str[1:]) - process = subprocess.Popen(cmd, env=env, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, close_fds=True) - stdout, stderr = process.communicate() + cmd = COMMAND % query_str + code, binout = utils.shell('check_output', + cmd.split(' ', 5), + # Using a maxsplit of 5 keeps the last 2 arguments together + input=query_str.encode('UTF-8'), + env=env, + return_bytes=True, + trap=True) - if process.returncode != 0: + if code != 0: # nonzero returncode indicates error app.logger.error("munin_cgi: munin-cgi-graph returned nonzero exit code, %s", process.returncode) return ("error processing graph image", 500) @@ -556,10 +563,10 @@ def munin_cgi(filename=""): # Per http://www.libpng.org/pub/png/spec/1.2/PNG-Structure.html PNG files always start # with the same 8 bytes (137 80 78 71 13 10 26 10) or b'\x89PNG\r\n\x1a\n' So we split # the output of munin-cgi-graph where the PNG begins - bin_start = stdout.find(b'\x89PNG\r\n\x1a\n') - str_headers = stdout[:bin_start].decode("utf-8") + bin_start = binout.find(b'\x89PNG\r\n\x1a\n') + str_headers = binout[:bin_start].decode("utf-8") # decode the byte str containing response headers - bin_image = stdout[bin_start:] + bin_image = binout[bin_start:] response = make_response(bin_image) for line in str_headers.splitlines(): if line: From bd86d44c8b5949a2a3817e2a657933cf21206a10 Mon Sep 17 00:00:00 2001 From: Joshua Tauberer Date: Wed, 23 Mar 2016 16:07:34 -0400 Subject: [PATCH 4/4] simplify the munin_cgi wrapper / add changelog entry --- CHANGELOG.md | 2 ++ management/daemon.py | 25 +++++++------------------ 2 files changed, 9 insertions(+), 18 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ff42710..36234c2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,8 @@ CHANGELOG ========= +* Munin system monitoring graphs are now zoomable. + In Development -------------- diff --git a/management/daemon.py b/management/daemon.py index fd2d6b4..0d54d32 100755 --- a/management/daemon.py +++ b/management/daemon.py @@ -507,12 +507,9 @@ def munin(filename=""): if filename == "": filename = "index.html" return send_from_directory("/var/cache/munin/www", filename) -# MUNIN CGI-GRAPH - -@app.route('/munin/cgi-graph/') @app.route('/munin/cgi-graph/') @authorized_personnel_only -def munin_cgi(filename=""): +def munin_cgi(filename): """ Relay munin cgi dynazoom requests /usr/lib/munin/cgi/munin-cgi-graph is a perl cgi script in the munin package that is responsible for generating binary png images _and_ associated HTTP @@ -541,8 +538,6 @@ def munin_cgi(filename=""): return ("a path must be specified", 404) query_str = request.query_string.decode("utf-8", 'ignore') - query_str = query_str[1:] if query_str.startswith('&') else query_str[1:] - # I don't know if this is strictly necessary env = {'PATH_INFO': '/%s/' % filename, 'QUERY_STRING': query_str} cmd = COMMAND % query_str @@ -560,18 +555,12 @@ def munin_cgi(filename=""): return ("error processing graph image", 500) # /usr/lib/munin/cgi/munin-cgi-graph returns both headers and binary png when successful. - # Per http://www.libpng.org/pub/png/spec/1.2/PNG-Structure.html PNG files always start - # with the same 8 bytes (137 80 78 71 13 10 26 10) or b'\x89PNG\r\n\x1a\n' So we split - # the output of munin-cgi-graph where the PNG begins - bin_start = binout.find(b'\x89PNG\r\n\x1a\n') - str_headers = binout[:bin_start].decode("utf-8") - # decode the byte str containing response headers - bin_image = binout[bin_start:] - response = make_response(bin_image) - for line in str_headers.splitlines(): - if line: - name, value = line.split(':',1) - response.headers[name] = value + # A double-Windows-style-newline always indicates the end of HTTP headers. + headers, image_bytes = binout.split(b'\r\n\r\n', 1) + response = make_response(image_bytes) + for line in headers.splitlines(): + name, value = line.decode("utf8").split(':', 1) + response.headers[name] = value if 'Status' in response.headers and '404' in response.headers['Status']: app.logger.warning("munin_cgi: munin-cgi-graph returned 404 status code. PATH_INFO=%s", env['PATH_INFO']) return response