clients, middleware and tests

LICENSE.rst

@@ -0,0 +1,3 @@
+BSD and MPL
+
+Todo: portions of this is from commonware

README.rst

@@ -0,0 +1,50 @@
+Todo: write this
+
+
+
+A tool that mashes up django-debug-toolbar, graphite, statsd and pystatsd.
+
+Before you can think about getting this to work you'll need:
+
+- A graphite server running and processing the data from statsd
+
+- Some Django middleware or code that sends the data to statsd
+
+Pystatsd: https://github.com/andymckay/pystatsd
+
+Graphite: http://graphite.wikidot.com/installation
+
+Django debug toolbar: https://github.com/django-debug-toolbar/django-debug-toolbar
+
+An example Django app that logs to statsd on each request can be found in
+nuggets: https://github.com/mozilla/nuggets
+
+It works by adding the following to your middleware::
+
+    MIDDLEWARE_CLASSES = (
+        'commonware.response.middleware.GraphiteRequestTimingMiddleware',
+        'commonware.response.middleware.GraphiteMiddleware',
+    )
+
+If you've got that setup, to your settings, add the following::
+
+    DEBUG_TOOLBAR_PANELS = (
+        ...
+        'toolbar_statsd.panel.StatsdPanel'
+    )
+
+    STATSD_CLIENT = 'toolbar_statsd.panel'
+
+    TOOLBAR_STATSD = {
+        'graphite': 'http://your.graphite.server',
+        'roots': ['root.key.for.dev', 'root.key.for.stage']
+    }
+
+    INSTALLED_APPS = (
+        ...
+        'toolbar_statsd'
+    )
+
+Notes: django-debug-toolbar middleware must come *after* graphite middleware.
+
+See: example.png for an example of the fun that can be had.

django_statsd/__init__.py

@@ -0,0 +1 @@
+from statsd import statsd, get_client

django_statsd/clients/null.py

@@ -0,0 +1,8 @@
+from statsd.client import StatsClient
+
+
+class StatsClient(StatsClient):
+    """A null client that does nothing."""
+
+    def _send(self, stat, value, rate):
+        pass

django_statsd/clients/toolbar.py

@@ -0,0 +1,18 @@
+from statsd.client import StatsClient
+from django.contrib.humanize.templatetags.humanize import intcomma
+
+
+class StatsClient(StatsClient):
+    """A client that pushes things into a local cache."""
+
+    def __init__(self, *args, **kw):
+        super(StatsClient, self).__init__(*args, **kw)
+        self.reset()
+
+    def reset(self):
+        self.cache = []
+
+    def _send(self, stat, value, rate):
+        num, scale = value.split('|')
+        value = '%s%s' % (intcomma(int(num)), scale)
+        self.cache.append([stat, value, rate])

django_statsd/middleware.py

@@ -0,0 +1,47 @@
+from django_statsd import statsd
+import inspect
+import time
+
+class GraphiteMiddleware(object):
+
+    def process_response(self, request, response):
+        statsd.incr('response.%s' % response.status_code)
+        if hasattr(request, 'user') and request.user.is_authenticated():
+            statsd.incr('response.auth.%s' % response.status_code)
+        return response
+
+    def process_exception(self, request, exception):
+        statsd.incr('response.500')
+        if hasattr(request, 'user') and request.user.is_authenticated():
+            statsd.incr('response.auth.500')
+
+
+class GraphiteRequestTimingMiddleware(object):
+    """statsd's timing data per view."""
+
+    def process_view(self, request, view_func, view_args, view_kwargs):
+        view = view_func
+        if not inspect.isfunction(view_func):
+            view = view.__class__
+        try:
+            request._view_module = view.__module__
+            request._view_name = view.__name__
+            request._start_time = time.time()
+        except AttributeError:
+            pass
+
+    def process_response(self, request, response):
+        self._record_time(request)
+        return response
+
+    def process_exception(self, request, exception):
+        self._record_time(request)
+
+    def _record_time(self, request):
+        if hasattr(request, '_start_time'):
+            ms = int((time.time() - request._start_time) * 1000)
+            data = dict(module=request._view_module, name=request._view_name,
+                        method=request.method)
+            statsd.timing('view.{module}.{name}.{method}'.format(**data), ms)
+            statsd.timing('view.{module}.{method}'.format(**data), ms)
+            statsd.timing('view.{method}'.format(**data), ms)

django_statsd/panel.py

@@ -0,0 +1,38 @@
+from django.conf import settings
+from django.template.loader import render_to_string
+from django.utils.translation import ugettext_lazy as _, ungettext
+
+from debug_toolbar.panels import DebugPanel
+from django_statsd.statsd import statsd
+
+
+class StatsdPanel(DebugPanel):
+
+    name = 'Statsd'
+    has_content = True
+
+    def __init__(self, *args, **kw):
+        super(StatsdPanel, self).__init__(*args, **kw)
+        self.statsd = statsd
+        self.statsd.reset()
+
+    def nav_title(self):
+        return _('Statsd')
+
+    def nav_subtitle(self):
+        length = len(self.statsd.cache)
+        return ungettext('%s record', '%s records', length) % length
+
+    def title(self):
+        return _('Statsd')
+
+    def url(self):
+        return ''
+
+    def content(self):
+        context = self.context.copy()
+        context.update(settings.TOOLBAR_STATSD)
+        context['statsd'] = self.statsd.cache
+        return render_to_string('toolbar_statsd/statsd.html', context)
+
+

django_statsd/statsd.py

@@ -0,0 +1,14 @@
+from django.utils.importlib import import_module
+from django.conf import settings
+
+_statsd = None
+
+def get_client():
+    client = getattr(settings, 'STATSD_CLIENT', 'statsd.client')
+    host = getattr(settings, 'STATSD_HOST', 'localhost')
+    port = getattr(settings, 'STATSD_PORT', 8125)
+    prefix = getattr(settings, 'STATSD_PREFIX', None)
+    return import_module(client).StatsClient(host, port, prefix)
+
+statsd = get_client()
+

django_statsd/templates/toolbar_statsd/statsd.html

@@ -0,0 +1,45 @@
+<table id="statsd"
+  data-graphite="{{ graphite }}"
+  data-roots="{% for root in roots %}{{ root }}{% if not forloop.last %}|{% endif %}{% endfor %}">
+  <thead>
+    <tr>
+      <th>Stat</th>
+      <th>Value</th>
+      <th>Count</th>
+    </tr>
+  </thead>
+  <tbody>
+    {% for record in statsd %}
+    <tr>
+      <td><a href="#" class="statsd" data-key="{{ record.0 }}">{{ record.0 }}</a></td>
+      <td>{{ record.1 }}</td>
+      <td>{{ record.2 }}</td>
+    </tr>
+    {% endfor %}
+  </tbody>
+</table>
+
+<div id="graphs" img="graphite?width=586&amp;height=308&amp;target=root.key.lower&amp;target=root.key.mean&amp;target=root.key.upper_90&amp;target=scale(root.key.count,0.1)&amp;from=-24hours&amp;title=24 hours" />
+
+<script type="text/javascript">
+  // TODO: inlining is bad, this should be external.
+
+$(document).ready(function() {
+    var graphite = $('#statsd').attr('data-graphite'),
+        roots = $('#statsd').attr('data-roots').split('|'),
+        target = $('#graphs'),
+        img = target.attr('img');
+
+    $('a.statsd').click(function() {
+        var that = $(this);
+        target.html('');
+        $.each(roots, function(root) {
+            var custom = img.replace('graphite', graphite, 'g')
+                            .replace('root', roots[root], 'g')
+                            .replace('key', that.attr('data-key'), 'g');
+            target.append('<p><b>' + roots[root] + '.' + that.attr('data-key') + '</b></p><img src="' + custom + '">');
+        })
+    });
+})
+</script>
+

django_statsd/tests.py

@@ -0,0 +1,104 @@
+from django.conf import settings
+from django.http import HttpResponse
+from django.test.client import RequestFactory
+
+import mock
+from nose.tools import eq_
+
+from django_statsd import statsd, get_client, middleware
+
+import unittest
+
+
+@mock.patch.object(middleware.statsd, 'incr')
+class TestIncr(unittest.TestCase):
+
+    def setUp(self):
+        self.req = RequestFactory().get('/')
+        self.res = HttpResponse()
+
+    def test_graphite_response(self, incr):
+        gmw = middleware.GraphiteMiddleware()
+        gmw.process_response(self.req, self.res)
+        assert incr.called
+
+    def test_graphite_response_authenticated(self, incr):
+        self.req.user = mock.Mock()
+        self.req.user.is_authenticated.return_value = True
+        gmw = middleware.GraphiteMiddleware()
+        gmw.process_response(self.req, self.res)
+        eq_(incr.call_count, 2)
+
+    def test_graphite_exception(self, incr):
+        gmw = middleware.GraphiteMiddleware()
+        gmw.process_exception(self.req, None)
+        assert incr.called
+
+    def test_graphite_exception_authenticated(self, incr):
+        self.req.user = mock.Mock()
+        self.req.user.is_authenticated.return_value = True
+        gmw = middleware.GraphiteMiddleware()
+        gmw.process_exception(self.req, None)
+        eq_(incr.call_count, 2)
+
+
+@mock.patch.object(middleware.statsd, 'timing')
+class TestTiming(unittest.TestCase):
+
+    def setUp(self):
+        self.req = RequestFactory().get('/')
+        self.res = HttpResponse()
+
+    def test_request_timing(self, timing):
+        func = lambda x: x
+        gmw = middleware.GraphiteRequestTimingMiddleware()
+        gmw.process_view(self.req, func, tuple(), dict())
+        gmw.process_response(self.req, self.res)
+        eq_(timing.call_count, 3)
+        names = ['view.%s.%s.GET' % (func.__module__, func.__name__),
+                 'view.%s.GET' % func.__module__,
+                 'view.GET']
+        for expected, (args, kwargs) in zip(names, timing.call_args_list):
+            eq_(expected, args[0])
+
+    def test_request_timing_exception(self, timing):
+        func = lambda x: x
+        gmw = middleware.GraphiteRequestTimingMiddleware()
+        gmw.process_view(self.req, func, tuple(), dict())
+        gmw.process_exception(self.req, self.res)
+        eq_(timing.call_count, 3)
+        names = ['view.%s.%s.GET' % (func.__module__, func.__name__),
+                 'view.%s.GET' % func.__module__,
+                 'view.GET']
+        for expected, (args, kwargs) in zip(names, timing.call_args_list):
+            eq_(expected, args[0])
+
+
+class TestClient(unittest.TestCase):
+
+    @mock.patch_object(settings, 'STATSD_CLIENT', 'statsd.client')
+    def test_normal(self):
+        eq_(get_client().__module__, 'statsd.client')
+
+    @mock.patch_object(settings, 'STATSD_CLIENT',
+                       'django_statsd.clients.null')
+    def test_null(self):
+        eq_(get_client().__module__, 'django_statsd.clients.null')
+
+    @mock.patch_object(settings, 'STATSD_CLIENT',
+                       'django_statsd.clients.toolbar')
+    def test_toolbar(self):
+        eq_(get_client().__module__, 'django_statsd.clients.toolbar')
+
+    @mock.patch_object(settings, 'STATSD_CLIENT',
+                       'django_statsd.clients.toolbar')
+    def test_toolbar_send(self):
+        client = get_client()
+        eq_(client.cache, [])
+        client.incr('testing')
+        eq_(client.cache, [['testing', u'1c', 1]])
+
+
+
+
+

setup.py

@@ -0,0 +1,24 @@
+import os
+from setuptools import setup, find_packages
+
+def read(fname):
+    return open(os.path.join(os.path.dirname(__file__), fname)).read()
+
+setup(
+    name='django-statsd',
+    version='0.1',
+    description='',
+    long_description=read('README.rst'),
+    author='Andy McKay',
+    author_email='andym@mozilla.com',
+    license='BSD',
+    packages=['django_statsd'],
+    url='',
+    package_data = {'django_statsd': ['templates/django_statsd/*.html']},
+    classifiers=[
+        'Intended Audience :: Developers',
+        'Natural Language :: English',
+        'Operating System :: OS Independent',
+        'Framework :: Django'
+        ],
+    )