From c1768632e127ab0fc2277b66c96cb9e0294cfb22 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fr=C3=A9d=C3=A9ric=20P=C3=A9ters?= Date: Sun, 29 Oct 2017 17:31:03 +0100 Subject: [PATCH] general: add support for prometheus stats (#19766) --- debian/control | 3 +- debian/debian_config_common.py | 3 + hobo/middleware/__init__.py | 1 + hobo/middleware/stats.py | 106 +++++++++++++++++++++++++++++++++ requirements.txt | 1 + setup.py | 1 + 6 files changed, 114 insertions(+), 1 deletion(-) create mode 100644 hobo/middleware/stats.py diff --git a/debian/control b/debian/control index 67cb71e..6569d6d 100644 --- a/debian/control +++ b/debian/control @@ -14,7 +14,8 @@ Depends: ${misc:Depends}, python-requests, python-raven, python-graypy, - python-apt + python-apt, + python-prometheus-client Recommends: python-django (>= 1.8), python-gadjo, python-django-mellon (>= 1.2.22.26), diff --git a/debian/debian_config_common.py b/debian/debian_config_common.py index 212e06a..e4d2a22 100644 --- a/debian/debian_config_common.py +++ b/debian/debian_config_common.py @@ -304,6 +304,9 @@ MIDDLEWARE_CLASSES = ( 'hobo.middleware.xforwardedfor.XForwardedForMiddleware', ) + MIDDLEWARE_CLASSES +MIDDLEWARE_CLASSES = MIDDLEWARE_CLASSES + ( + 'hobo.middleware.PrometheusStatsMiddleware',) + HOBO_MANAGER_HOMEPAGE_URL_VAR = 'portal_agent_url' HOBO_MANAGER_HOMEPAGE_TITLE_VAR = 'portal_agent_title' diff --git a/hobo/middleware/__init__.py b/hobo/middleware/__init__.py index 7352899..5e4b3a2 100644 --- a/hobo/middleware/__init__.py +++ b/hobo/middleware/__init__.py @@ -1,2 +1,3 @@ from .version import VersionMiddleware from .cors import CORSMiddleware +from .stats import PrometheusStatsMiddleware diff --git a/hobo/middleware/stats.py b/hobo/middleware/stats.py new file mode 100644 index 0000000..08db097 --- /dev/null +++ b/hobo/middleware/stats.py @@ -0,0 +1,106 @@ +# hobo - portal to configure and deploy applications +# Copyright (C) 2017 Entr'ouvert +# +# This program is free software: you can redistribute it and/or modify it +# under the terms of the GNU Affero General Public License as published +# by the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +import inspect +import time + +import django +from django.db import connection +from django.http import HttpResponse + +import prometheus_client + +if django.VERSION >= (1, 10, 0): + from django.utils.deprecation import MiddlewareMixin +else: + MiddlewareMixin = object + + +requests_total_by_host_view_status_method = prometheus_client.Counter( + 'django_http_requests_total_by_host_view_status_method', + 'Count of requests by host, view, status, method.', + ['host', 'view', 'status', 'method']) + +requests_bytes_by_host_view_status_method = prometheus_client.Summary( + 'django_http_requests_bytes_by_host_view_status_method', + 'Bytes of requests by host, view, status, method.', + ['host', 'view', 'status', 'method']) + +requests_times_by_host_view_status_method = prometheus_client.Summary( + 'django_http_requests_times_by_host_view_status_method', + 'Duration of requests by host, view, status, method.', + ['host', 'view', 'status', 'method']) + +requests_queries_count_by_host_view_status_method = prometheus_client.Summary( + 'django_http_requests_queries_count_by_host_view_status_method', + 'Query count by host, view, status, method.', + ['host', 'view', 'status', 'method']) + +requests_queries_time_by_host_view_status_method = prometheus_client.Summary( + 'django_http_requests_queries_time_by_host_view_status_method', + 'Query time by host, view, status, method.', + ['host', 'view', 'status', 'method']) + + + +class PrometheusStatsMiddleware(MiddlewareMixin): + def process_request(self, request): + if request.method == 'GET' and request.path == '/__metrics__/': + return HttpResponse(prometheus_client.generate_latest(), + content_type=prometheus_client.CONTENT_TYPE_LATEST) + + connection.force_debug_cursor = True # to count queries + request._stats_t0 = time.time() + return None + + def process_view(self, request, view_func, view_args, view_kwargs): + view = view_func + if not inspect.isfunction(view_func): + view = view.__class__ + try: + request._view_name = '%s.%s' % (view.__module__, view.__name__) + except AttributeError: + pass + + def process_response(self, request, response): + if not hasattr(request, '_stats_t0'): + return response + http_method = request.method + host_name = request.get_host() + total_time = time.time() - request._stats_t0 + + status_code = response.status_code + view_name = getattr(request, '_view_name', '') + requests_total_by_host_view_status_method.labels( + host_name, view_name, status_code, http_method).inc() + + if hasattr(response, 'content'): + response_size = len(response.content) + requests_bytes_by_host_view_status_method.labels( + host_name, view_name, status_code, http_method).observe(response_size) + + requests_times_by_host_view_status_method.labels( + host_name, view_name, status_code, http_method).observe(total_time) + + if connection.queries_logged: + sql_queries_count = len(connection.queries) + sql_queries_time = sum([float(x['time']) for x in connection.queries]) + requests_queries_count_by_host_view_status_method.labels( + host_name, view_name, status_code, http_method).observe(sql_queries_count) + requests_queries_time_by_host_view_status_method.labels( + host_name, view_name, status_code, http_method).observe(sql_queries_time) + + return response diff --git a/requirements.txt b/requirements.txt index dc0791a..aa381f3 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,3 +2,4 @@ django>=1.8,<1.9 -e git+http://repos.entrouvert.org/gadjo.git/#egg=gadjo celery<4 django-mellon +prometheus_client diff --git a/setup.py b/setup.py index 82d8764..490da54 100644 --- a/setup.py +++ b/setup.py @@ -100,6 +100,7 @@ setup( 'celery<4', 'django-mellon', 'django-tenant-schemas', + 'prometheus_client', ], zip_safe=False, cmdclass={