fix up the db and cache patches and do summations in the template

README.rst

@@ -1,52 +1,77 @@
-Credits:
-- jbalogh and jsocol for statsd and commonware.
-- robhudson for django-debug-toolbar
+Integration between statsd and django. It allows you to use different clients,
+sends timings as middleware and integrates with django debug toolbar.
 
-Settings:
+Requirement, pystatsd: https://github.com/jsocol/pystatsd
 
-Adding in the middleware::
+First off, pick your client, one of:
+
+- django_statsd.clients.null  (does nothing, good for development)
+- django_statsd.clients.toolbar  (use for the toolbar)
+- django_statsd.clients.statsd  (use for production, just passes through to pystatsd)
+
+Usage
+-----
+
+To send timings from your code, use just like statsd, but change your imports
+to read::
+
+        from django_statsd.clients import statsd
+
+For example::
+
+        from django_statsd.clients import statsd
+        statsd.incr('response.200')
+
+To send timings or counts with every request, add in some middleware::
 
         MIDDLEWARE_CLASSES = (
-                'debug_toolbar.middleware.DebugToolbarMiddleware',
                 'django_statsd.middleware.GraphiteRequestTimingMiddleware',
                 'django_statsd.middleware.GraphiteMiddleware',
                 ) + MIDDLEWARE_CLASSES
 
-
-Add in the panel, you must remove the SQL panel (hopefully this will get
-fixed)::
-
-        DEBUG_TOOLBAR_PANELS = (
-        ...
-        #    'debug_toolbar.panels.sql.SQLDebugPanel',
-             'django_statsd.panel.StatsdPanel',
-        )
-
-Pick your client, one of:
-
-- django_statsd.clients.null  (does nothing)
-- django_statsd.clients.toolbar  (use for the toolbar)
-- django_statsd.clients.statsd  (use for production)
-
-For the toolbar::
-
-        STATSD_CLIENT = 'django_statsd.clients.toolbar'
-
-Pick your patches::
+To get timings for your database or your cache, put in some monkeypatches::
 
         STATSD_PATCHES = [
                 'django_statsd.patches.db',
                 'django_statsd.patches.cache',
         ]
 
+Toolbar integration
+-------------------
+
+This will show you the statsd timings in the toolbar::
+
+        MIDDLEWARE_CLASSES = (
+                'debug_toolbar.middleware.DebugToolbarMiddleware',
+                ) + MIDDLEWARE_CLASSES
+
+Note: this must go before the GraphiteMiddleware.
+
+Add in the panel::
+
+        DEBUG_TOOLBAR_PANELS = (
+             ...
+             'django_statsd.panel.StatsdPanel',
+        )
+
+Set the client::
+
+        STATSD_CLIENT = 'django_statsd.clients.toolbar'
+
+Finally, to show data from a graphite server, if you have one, link it up::
+
 Configure where the toolbar shows graphs::
 
         TOOLBAR_STATSD = {
                 'graphite': 'https://graphite-phx.mozilla.org/render/',
                 'roots': {
-                'timers': ['stats.timers.addons-dev', 'stats.timers.addons'],
-                 'counts': ['stats.addons-dev', 'stats.addons']
+                        'timers': ['stats.timers.addons-dev', 'stats.timers.addons'],
+                        'counts': ['stats.addons-dev', 'stats.addons']
                 }
         }
 
-Phew.
+Credits:
+
+- jbalogh and jsocol for statsd and commonware, which I just ripped out
+  and put in here.
+- robhudson for django-debug-toolbar

django_statsd/__init__.py

@@ -1,3 +1,2 @@
-from statsd import statsd, get_client
 from django_statsd import patches
-
+from django_statsd import clients

django_statsd/clients/__init__.py

@@ -0,0 +1,18 @@
+from django.utils.importlib import import_module
+from django.conf import settings
+
+_statsd = None
+
+
+def get_client():
+    print 'getting client'
+    client = getattr(settings, 'STATSD_CLIENT', 'statsd.client')
+    host = getattr(settings, 'STATSD_HOST', 'localhost')
+    port = getattr(settings, 'STATSD_PORT', 8125)
+    prefix = getattr(settings, 'STATSD_PREFIX', None)
+    return import_module(client).StatsClient(host, port, prefix)
+
+if not _statsd:
+    _statsd = get_client()
+
+statsd = _statsd

django_statsd/clients/normal.py

@@ -0,0 +1 @@
+from statsd.client import StatsClient

django_statsd/clients/toolbar.py

@@ -9,10 +9,25 @@ class StatsClient(StatsClient):
         self.reset()
 
     def reset(self):
-        self.cache = []
+        self.cache = {}
+
+    def timing(self, stat, delta, rate=1):
+        """Send new timing information. `delta` is in milliseconds."""
+        stat = '%s|timing' % stat
+        self.cache.setdefault(stat, [])
+        self.cache[stat].append([delta, rate])
+
+    def incr(self, stat, count=1, rate=1):
+        """Increment a stat by `count`."""
+        stat = '%s|count' % stat
+        self.cache.setdefault(stat, [])
+        self.cache[stat].append([count, rate])
+
+    def decr(self, stat, count=1, rate=1):
+        """Decrement a stat by `count`."""
+        stat = '%s|count' % stat
+        self.cache.setdefault(stat, [])
+        self.cache[stat].append([-count, rate])
 
     def _send(self, stat, value, rate):
-        type_ = 'timer'
-        if value.split('|')[1] == 'c':
-            type_ = 'count'
-        self.cache.append([stat, value, rate, type_])
+        pass

django_statsd/middleware.py

@@ -1,4 +1,4 @@
-from django_statsd import statsd
+from django_statsd.clients import statsd
 import inspect
 import time
 

django_statsd/panel.py

@@ -3,9 +3,23 @@ from django.template.loader import render_to_string
 from django.utils.translation import ugettext_lazy as _, ungettext
 
 from debug_toolbar.panels import DebugPanel
-from django_statsd.statsd import statsd
+from django_statsd.clients import statsd
 
 
+def munge(stats):
+    # Munge the stats back into something easy for a template.
+    results = []
+    for stat in sorted(stats.keys()):
+        values = stats[stat]
+        name, type_ = stat.split('|')
+        total = sum([x * y for x, y in values])
+        data = {'name': name, 'type': type_,
+                'count': len(values),
+                'total': total,
+                'values': values}
+        results.append(data)
+    return results
+
 class StatsdPanel(DebugPanel):
 
     name = 'Statsd'
@@ -40,7 +54,7 @@ class StatsdPanel(DebugPanel):
             for key in ['timers', 'counts']:
                 context[key] = config['roots'][key]
         context['graphite'] = config.get('graphite')
-        context['statsd'] = self.statsd.cache
+        context['statsd'] = munge(self.statsd.cache)
         return render_to_string('toolbar_statsd/statsd.html', context)
 
 

django_statsd/patches/__init__.py

@@ -4,6 +4,5 @@ from django.utils.importlib import import_module
 
 patches = settings.STATSD_PATCHES
 for patch in patches:
-    print '*', patch
     import_module(patch).patch()
 

django_statsd/patches/cache.py

@@ -1,35 +1,22 @@
 from django.core import cache
 from django.core.cache.backends.base import BaseCache
 
-from django_statsd import statsd
+from django_statsd.patches.utils import wrap
+
+
+def key(cache, attr):
+    return 'cache.%s.%s' % (cache.__module__.split('.')[-1], attr)
+
 
 class StatsdTracker(BaseCache):
 
     def __init__(self, cache):
         self.cache = cache
 
-    def _key(self):
-        return 'cache'
-
-    def add(self, *args, **kw):
-        with statsd.timer('%s.add' % self._key()):
-            return self.cache.add(*args, **kw)
-
-    def get(self, *args, **kw):
-        with statsd.timer('%s.get' % self._key()):
-            return self.cache.get(*args, **kw)
-
-    def set(self, *args, **kw):
-        with statsd.timer('%s.set' % self._key()):
-            return self.cache.set(*args, **kw)
-
-    def delete(self, *args, **kw):
-        with statsd.timer('%s.delete' % self._key()):
-            return super(StatsdTracker, self).delete(*args, **kw)
-
-    def get_many(self, *args, **kw):
-        with statsd.timer('%s.get_many' % self._key()):
-            return super(StatsdTracker, self).get_many(*args, **kw)
+    def __getattribute__(self, attr):
+        if attr == 'cache':
+            return BaseCache.__getattribute__(self, attr)
+        return wrap(getattr(self.cache, attr), key(self.cache, attr))
 
 
 def patch():

django_statsd/patches/db.py

@@ -1,20 +1,61 @@
 from django.db.backends import util
-from django_statsd import statsd
 
-class StatsdTracker(util.CursorDebugWrapper):
+from debug_toolbar.toolbar import loader
+from django_statsd.patches.utils import wrap
 
-    def _key(self):
-        return '%s.%s' % (self.db.client.executable_name, self.db.alias)
 
-    def execute(self, *args, **kw):
-        with statsd.timer('%s.execute' % self._key()):
-            return super(StatsdTracker, self).execute(*args, **kw)
+def key(db, attr):
+    return 'db.%s.%s.%s' % (db.client.executable_name, db.alias, attr)
+
+
+def __getattr__(self, attr):
+    """
+    The CursorWrapper is a pretty small wrapper around the cursor.
+    If you are NOT in debug mode, this is the wrapper that's used.
+    Sadly if it's in debug mode, we get a different wrapper.
+    """
+    if self.db.is_managed():
+        self.db.set_dirty()
+    if attr in self.__dict__:
+        return self.__dict__[attr]
+    else:
+        if attr in ['execute', 'executemany']:
+            return wrap(getattr(self.cursor, attr), key(self.db, attr))
+        return getattr(self.cursor, attr)
+
+
+def wrap_class(base):
+    class Wrapper(base):
+        def execute(self, *args, **kw):
+            return wrap(super(Wrapper, self).execute,
+                        key(self.db, 'execute'))(*args, **kw)
+
+        def executemany(self, *args, **kw):
+            return wrap(super(Wrapper, self).executemany,
+                        key(self.db, 'executemany'))(*args, **kw)
+
+    return Wrapper
+
+# This is where the fun begins. django-debug-toolbar monkey patches
+# util.CursorDebugWrapper so you can't monkey patch that if you want to
+# see the SQL queries that it spits out.
+old = loader.DebugToolbar.load_panels
+
+# Even better, it only does this in the middleware, which is after the patch
+# so we monkey patch the loading of the patches. So once the toolbar has done
+# it's stuff we write out patch on top of it.
+def callback(self, *args, **kw):
+    res = old(self, *args, **kw)
+    from debug_toolbar.panels import sql
+    # This will rewrap so this will work when DEBUG = True and the SQL Panel
+    # is used in the toolbar.
+    util.CursorDebugWrapper = wrap_class(sql.DatabaseStatTracker)
+    return res
 
-    def executemany(self, *args, **kw):
-        with statsd.timer('%s.executemany' % self._key()):
-            return super(StatsdTracker, self).executemany(*args, **kw)
 
-# TODO: make this work when DEBUG = False
-# TODO: play nicer with debug toolbar
 def patch():
-    util.CursorDebugWrapper = StatsdTracker
+    loader.DebugToolbar.load_panels = callback
+    # So that it will work when DEBUG = True.
+    util.CursorDebugWrapper = wrap_class(util.CursorDebugWrapper)
+    # So that it will work when DEBUG = False.
+    util.CursorWrapper.__getattr__ = __getattr__

django_statsd/patches/utils.py

@@ -0,0 +1,12 @@
+from django_statsd.clients import statsd
+from functools import partial
+
+
+def wrapped(method, key, *args, **kw):
+    with statsd.timer(key):
+        return method(*args, **kw)
+
+
+def wrap(method, key, *args, **kw):
+    return partial(wrapped, method, key, *args, **kw)
+

django_statsd/statsd.py

@@ -1,18 +0,0 @@
-from django.utils.importlib import import_module
-from django.conf import settings
-
-_statsd = None
-
-
-def get_client():
-    client = getattr(settings, 'STATSD_CLIENT', 'statsd.client')
-    host = getattr(settings, 'STATSD_HOST', 'localhost')
-    port = getattr(settings, 'STATSD_PORT', 8125)
-    prefix = getattr(settings, 'STATSD_PREFIX', None)
-    return import_module(client).StatsClient(host, port, prefix)
-
-if not _statsd:
-    _statsd = get_client()
-
-statsd = _statsd
-

django_statsd/templates/toolbar_statsd/statsd.html

@@ -5,16 +5,26 @@
   <thead>
     <tr>
       <th>Stat</th>
-      <th>Value</th>
+      <th>Total</th>
       <th>Count</th>
+      <th>Values</th>
     </tr>
   </thead>
   <tbody>
     {% for record in statsd %}
     <tr>
-      <td><a href="#" class="statsd" data-key="{{ record.0 }}" data-type="{{ record.3 }}">{{ record.0 }}</a></td>
-      <td>{{ record.1 }}</td>
-      <td>{{ record.2 }}</td>
+      <td><a href="#" class="statsd" data-key="{{ record.name }}" data-type="{{ record.type }}">{{ record.name }}</a></td>
+      <td>{{ record.total }}{% if record.type == 'timing' %}ms{% endif %}</td>
+      <td>{{ record.count }}</td>
+      <td>
+        {% if record.count > 1 %}
+          <span class="values" data-count="{{ record.count }}">
+            {% for value in record.values %}
+              {{ value.0 }}{% if record.type == 'timing' %}ms{% endif %}|{{ value.1 }}{% if not forloop.last %},{% endif %}
+            {% endfor %}
+          </span>
+        {% endif %}
+      </td>
     </tr>
     {% endfor %}
   </tbody>