diff --git a/LICENSE.rst b/LICENSE.rst new file mode 100644 index 0000000..d3564e9 --- /dev/null +++ b/LICENSE.rst @@ -0,0 +1,3 @@ +BSD and MPL + +Todo: portions of this is from commonware diff --git a/README.rst b/README.rst new file mode 100644 index 0000000..9033506 --- /dev/null +++ b/README.rst @@ -0,0 +1,50 @@ +Todo: write this + + + +A tool that mashes up django-debug-toolbar, graphite, statsd and pystatsd. + +Before you can think about getting this to work you'll need: + +- A graphite server running and processing the data from statsd + +- Some Django middleware or code that sends the data to statsd + +Pystatsd: https://github.com/andymckay/pystatsd + +Graphite: http://graphite.wikidot.com/installation + +Django debug toolbar: https://github.com/django-debug-toolbar/django-debug-toolbar + +An example Django app that logs to statsd on each request can be found in +nuggets: https://github.com/mozilla/nuggets + +It works by adding the following to your middleware:: + + MIDDLEWARE_CLASSES = ( + 'commonware.response.middleware.GraphiteRequestTimingMiddleware', + 'commonware.response.middleware.GraphiteMiddleware', + ) + +If you've got that setup, to your settings, add the following:: + + DEBUG_TOOLBAR_PANELS = ( + ... + 'toolbar_statsd.panel.StatsdPanel' + ) + + STATSD_CLIENT = 'toolbar_statsd.panel' + + TOOLBAR_STATSD = { + 'graphite': 'http://your.graphite.server', + 'roots': ['root.key.for.dev', 'root.key.for.stage'] + } + + INSTALLED_APPS = ( + ... + 'toolbar_statsd' + ) + +Notes: django-debug-toolbar middleware must come *after* graphite middleware. + +See: example.png for an example of the fun that can be had. diff --git a/django_statsd/__init__.py b/django_statsd/__init__.py new file mode 100644 index 0000000..2e1dc6a --- /dev/null +++ b/django_statsd/__init__.py @@ -0,0 +1 @@ +from statsd import statsd, get_client diff --git a/README b/django_statsd/clients/__init__.py similarity index 100% rename from README rename to django_statsd/clients/__init__.py diff --git a/django_statsd/clients/null.py b/django_statsd/clients/null.py new file mode 100644 index 0000000..e59b54f --- /dev/null +++ b/django_statsd/clients/null.py @@ -0,0 +1,8 @@ +from statsd.client import StatsClient + + +class StatsClient(StatsClient): + """A null client that does nothing.""" + + def _send(self, stat, value, rate): + pass diff --git a/django_statsd/clients/toolbar.py b/django_statsd/clients/toolbar.py new file mode 100644 index 0000000..5abf4e0 --- /dev/null +++ b/django_statsd/clients/toolbar.py @@ -0,0 +1,18 @@ +from statsd.client import StatsClient +from django.contrib.humanize.templatetags.humanize import intcomma + + +class StatsClient(StatsClient): + """A client that pushes things into a local cache.""" + + def __init__(self, *args, **kw): + super(StatsClient, self).__init__(*args, **kw) + self.reset() + + def reset(self): + self.cache = [] + + def _send(self, stat, value, rate): + num, scale = value.split('|') + value = '%s%s' % (intcomma(int(num)), scale) + self.cache.append([stat, value, rate]) diff --git a/django_statsd/middleware.py b/django_statsd/middleware.py new file mode 100644 index 0000000..6b01bcb --- /dev/null +++ b/django_statsd/middleware.py @@ -0,0 +1,47 @@ +from django_statsd import statsd +import inspect +import time + +class GraphiteMiddleware(object): + + def process_response(self, request, response): + statsd.incr('response.%s' % response.status_code) + if hasattr(request, 'user') and request.user.is_authenticated(): + statsd.incr('response.auth.%s' % response.status_code) + return response + + def process_exception(self, request, exception): + statsd.incr('response.500') + if hasattr(request, 'user') and request.user.is_authenticated(): + statsd.incr('response.auth.500') + + +class GraphiteRequestTimingMiddleware(object): + """statsd's timing data per view.""" + + def process_view(self, request, view_func, view_args, view_kwargs): + view = view_func + if not inspect.isfunction(view_func): + view = view.__class__ + try: + request._view_module = view.__module__ + request._view_name = view.__name__ + request._start_time = time.time() + except AttributeError: + pass + + def process_response(self, request, response): + self._record_time(request) + return response + + def process_exception(self, request, exception): + self._record_time(request) + + def _record_time(self, request): + if hasattr(request, '_start_time'): + ms = int((time.time() - request._start_time) * 1000) + data = dict(module=request._view_module, name=request._view_name, + method=request.method) + statsd.timing('view.{module}.{name}.{method}'.format(**data), ms) + statsd.timing('view.{module}.{method}'.format(**data), ms) + statsd.timing('view.{method}'.format(**data), ms) diff --git a/django_statsd/panel.py b/django_statsd/panel.py new file mode 100644 index 0000000..a2220c2 --- /dev/null +++ b/django_statsd/panel.py @@ -0,0 +1,38 @@ +from django.conf import settings +from django.template.loader import render_to_string +from django.utils.translation import ugettext_lazy as _, ungettext + +from debug_toolbar.panels import DebugPanel +from django_statsd.statsd import statsd + + +class StatsdPanel(DebugPanel): + + name = 'Statsd' + has_content = True + + def __init__(self, *args, **kw): + super(StatsdPanel, self).__init__(*args, **kw) + self.statsd = statsd + self.statsd.reset() + + def nav_title(self): + return _('Statsd') + + def nav_subtitle(self): + length = len(self.statsd.cache) + return ungettext('%s record', '%s records', length) % length + + def title(self): + return _('Statsd') + + def url(self): + return '' + + def content(self): + context = self.context.copy() + context.update(settings.TOOLBAR_STATSD) + context['statsd'] = self.statsd.cache + return render_to_string('toolbar_statsd/statsd.html', context) + + diff --git a/django_statsd/statsd.py b/django_statsd/statsd.py new file mode 100644 index 0000000..ac607f7 --- /dev/null +++ b/django_statsd/statsd.py @@ -0,0 +1,14 @@ +from django.utils.importlib import import_module +from django.conf import settings + +_statsd = None + +def get_client(): + client = getattr(settings, 'STATSD_CLIENT', 'statsd.client') + host = getattr(settings, 'STATSD_HOST', 'localhost') + port = getattr(settings, 'STATSD_PORT', 8125) + prefix = getattr(settings, 'STATSD_PREFIX', None) + return import_module(client).StatsClient(host, port, prefix) + +statsd = get_client() + diff --git a/django_statsd/templates/toolbar_statsd/statsd.html b/django_statsd/templates/toolbar_statsd/statsd.html new file mode 100644 index 0000000..e0bf9c2 --- /dev/null +++ b/django_statsd/templates/toolbar_statsd/statsd.html @@ -0,0 +1,45 @@ + + + + + + + + + + {% for record in statsd %} + + + + + + {% endfor %} + +
StatValueCount
{{ record.0 }}{{ record.1 }}{{ record.2 }}
+ +
+ + + diff --git a/django_statsd/tests.py b/django_statsd/tests.py new file mode 100644 index 0000000..1ef605c --- /dev/null +++ b/django_statsd/tests.py @@ -0,0 +1,104 @@ +from django.conf import settings +from django.http import HttpResponse +from django.test.client import RequestFactory + +import mock +from nose.tools import eq_ + +from django_statsd import statsd, get_client, middleware + +import unittest + + +@mock.patch.object(middleware.statsd, 'incr') +class TestIncr(unittest.TestCase): + + def setUp(self): + self.req = RequestFactory().get('/') + self.res = HttpResponse() + + def test_graphite_response(self, incr): + gmw = middleware.GraphiteMiddleware() + gmw.process_response(self.req, self.res) + assert incr.called + + def test_graphite_response_authenticated(self, incr): + self.req.user = mock.Mock() + self.req.user.is_authenticated.return_value = True + gmw = middleware.GraphiteMiddleware() + gmw.process_response(self.req, self.res) + eq_(incr.call_count, 2) + + def test_graphite_exception(self, incr): + gmw = middleware.GraphiteMiddleware() + gmw.process_exception(self.req, None) + assert incr.called + + def test_graphite_exception_authenticated(self, incr): + self.req.user = mock.Mock() + self.req.user.is_authenticated.return_value = True + gmw = middleware.GraphiteMiddleware() + gmw.process_exception(self.req, None) + eq_(incr.call_count, 2) + + +@mock.patch.object(middleware.statsd, 'timing') +class TestTiming(unittest.TestCase): + + def setUp(self): + self.req = RequestFactory().get('/') + self.res = HttpResponse() + + def test_request_timing(self, timing): + func = lambda x: x + gmw = middleware.GraphiteRequestTimingMiddleware() + gmw.process_view(self.req, func, tuple(), dict()) + gmw.process_response(self.req, self.res) + eq_(timing.call_count, 3) + names = ['view.%s.%s.GET' % (func.__module__, func.__name__), + 'view.%s.GET' % func.__module__, + 'view.GET'] + for expected, (args, kwargs) in zip(names, timing.call_args_list): + eq_(expected, args[0]) + + def test_request_timing_exception(self, timing): + func = lambda x: x + gmw = middleware.GraphiteRequestTimingMiddleware() + gmw.process_view(self.req, func, tuple(), dict()) + gmw.process_exception(self.req, self.res) + eq_(timing.call_count, 3) + names = ['view.%s.%s.GET' % (func.__module__, func.__name__), + 'view.%s.GET' % func.__module__, + 'view.GET'] + for expected, (args, kwargs) in zip(names, timing.call_args_list): + eq_(expected, args[0]) + + +class TestClient(unittest.TestCase): + + @mock.patch_object(settings, 'STATSD_CLIENT', 'statsd.client') + def test_normal(self): + eq_(get_client().__module__, 'statsd.client') + + @mock.patch_object(settings, 'STATSD_CLIENT', + 'django_statsd.clients.null') + def test_null(self): + eq_(get_client().__module__, 'django_statsd.clients.null') + + @mock.patch_object(settings, 'STATSD_CLIENT', + 'django_statsd.clients.toolbar') + def test_toolbar(self): + eq_(get_client().__module__, 'django_statsd.clients.toolbar') + + @mock.patch_object(settings, 'STATSD_CLIENT', + 'django_statsd.clients.toolbar') + def test_toolbar_send(self): + client = get_client() + eq_(client.cache, []) + client.incr('testing') + eq_(client.cache, [['testing', u'1c', 1]]) + + + + + diff --git a/example.png b/example.png new file mode 100644 index 0000000..a125394 Binary files /dev/null and b/example.png differ diff --git a/setup.py b/setup.py new file mode 100644 index 0000000..4b0d907 --- /dev/null +++ b/setup.py @@ -0,0 +1,24 @@ +import os +from setuptools import setup, find_packages + +def read(fname): + return open(os.path.join(os.path.dirname(__file__), fname)).read() + +setup( + name='django-statsd', + version='0.1', + description='', + long_description=read('README.rst'), + author='Andy McKay', + author_email='andym@mozilla.com', + license='BSD', + packages=['django_statsd'], + url='', + package_data = {'django_statsd': ['templates/django_statsd/*.html']}, + classifiers=[ + 'Intended Audience :: Developers', + 'Natural Language :: English', + 'Operating System :: OS Independent', + 'Framework :: Django' + ], + )