clients, middleware and tests
This commit is contained in:
parent
5b786e9c4d
commit
c4638d3045
|
@ -0,0 +1,3 @@
|
||||||
|
BSD and MPL
|
||||||
|
|
||||||
|
Todo: portions of this is from commonware
|
|
@ -0,0 +1,50 @@
|
||||||
|
Todo: write this
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
A tool that mashes up django-debug-toolbar, graphite, statsd and pystatsd.
|
||||||
|
|
||||||
|
Before you can think about getting this to work you'll need:
|
||||||
|
|
||||||
|
- A graphite server running and processing the data from statsd
|
||||||
|
|
||||||
|
- Some Django middleware or code that sends the data to statsd
|
||||||
|
|
||||||
|
Pystatsd: https://github.com/andymckay/pystatsd
|
||||||
|
|
||||||
|
Graphite: http://graphite.wikidot.com/installation
|
||||||
|
|
||||||
|
Django debug toolbar: https://github.com/django-debug-toolbar/django-debug-toolbar
|
||||||
|
|
||||||
|
An example Django app that logs to statsd on each request can be found in
|
||||||
|
nuggets: https://github.com/mozilla/nuggets
|
||||||
|
|
||||||
|
It works by adding the following to your middleware::
|
||||||
|
|
||||||
|
MIDDLEWARE_CLASSES = (
|
||||||
|
'commonware.response.middleware.GraphiteRequestTimingMiddleware',
|
||||||
|
'commonware.response.middleware.GraphiteMiddleware',
|
||||||
|
)
|
||||||
|
|
||||||
|
If you've got that setup, to your settings, add the following::
|
||||||
|
|
||||||
|
DEBUG_TOOLBAR_PANELS = (
|
||||||
|
...
|
||||||
|
'toolbar_statsd.panel.StatsdPanel'
|
||||||
|
)
|
||||||
|
|
||||||
|
STATSD_CLIENT = 'toolbar_statsd.panel'
|
||||||
|
|
||||||
|
TOOLBAR_STATSD = {
|
||||||
|
'graphite': 'http://your.graphite.server',
|
||||||
|
'roots': ['root.key.for.dev', 'root.key.for.stage']
|
||||||
|
}
|
||||||
|
|
||||||
|
INSTALLED_APPS = (
|
||||||
|
...
|
||||||
|
'toolbar_statsd'
|
||||||
|
)
|
||||||
|
|
||||||
|
Notes: django-debug-toolbar middleware must come *after* graphite middleware.
|
||||||
|
|
||||||
|
See: example.png for an example of the fun that can be had.
|
|
@ -0,0 +1 @@
|
||||||
|
from statsd import statsd, get_client
|
|
@ -0,0 +1,8 @@
|
||||||
|
from statsd.client import StatsClient
|
||||||
|
|
||||||
|
|
||||||
|
class StatsClient(StatsClient):
|
||||||
|
"""A null client that does nothing."""
|
||||||
|
|
||||||
|
def _send(self, stat, value, rate):
|
||||||
|
pass
|
|
@ -0,0 +1,18 @@
|
||||||
|
from statsd.client import StatsClient
|
||||||
|
from django.contrib.humanize.templatetags.humanize import intcomma
|
||||||
|
|
||||||
|
|
||||||
|
class StatsClient(StatsClient):
|
||||||
|
"""A client that pushes things into a local cache."""
|
||||||
|
|
||||||
|
def __init__(self, *args, **kw):
|
||||||
|
super(StatsClient, self).__init__(*args, **kw)
|
||||||
|
self.reset()
|
||||||
|
|
||||||
|
def reset(self):
|
||||||
|
self.cache = []
|
||||||
|
|
||||||
|
def _send(self, stat, value, rate):
|
||||||
|
num, scale = value.split('|')
|
||||||
|
value = '%s%s' % (intcomma(int(num)), scale)
|
||||||
|
self.cache.append([stat, value, rate])
|
|
@ -0,0 +1,47 @@
|
||||||
|
from django_statsd import statsd
|
||||||
|
import inspect
|
||||||
|
import time
|
||||||
|
|
||||||
|
class GraphiteMiddleware(object):
|
||||||
|
|
||||||
|
def process_response(self, request, response):
|
||||||
|
statsd.incr('response.%s' % response.status_code)
|
||||||
|
if hasattr(request, 'user') and request.user.is_authenticated():
|
||||||
|
statsd.incr('response.auth.%s' % response.status_code)
|
||||||
|
return response
|
||||||
|
|
||||||
|
def process_exception(self, request, exception):
|
||||||
|
statsd.incr('response.500')
|
||||||
|
if hasattr(request, 'user') and request.user.is_authenticated():
|
||||||
|
statsd.incr('response.auth.500')
|
||||||
|
|
||||||
|
|
||||||
|
class GraphiteRequestTimingMiddleware(object):
|
||||||
|
"""statsd's timing data per view."""
|
||||||
|
|
||||||
|
def process_view(self, request, view_func, view_args, view_kwargs):
|
||||||
|
view = view_func
|
||||||
|
if not inspect.isfunction(view_func):
|
||||||
|
view = view.__class__
|
||||||
|
try:
|
||||||
|
request._view_module = view.__module__
|
||||||
|
request._view_name = view.__name__
|
||||||
|
request._start_time = time.time()
|
||||||
|
except AttributeError:
|
||||||
|
pass
|
||||||
|
|
||||||
|
def process_response(self, request, response):
|
||||||
|
self._record_time(request)
|
||||||
|
return response
|
||||||
|
|
||||||
|
def process_exception(self, request, exception):
|
||||||
|
self._record_time(request)
|
||||||
|
|
||||||
|
def _record_time(self, request):
|
||||||
|
if hasattr(request, '_start_time'):
|
||||||
|
ms = int((time.time() - request._start_time) * 1000)
|
||||||
|
data = dict(module=request._view_module, name=request._view_name,
|
||||||
|
method=request.method)
|
||||||
|
statsd.timing('view.{module}.{name}.{method}'.format(**data), ms)
|
||||||
|
statsd.timing('view.{module}.{method}'.format(**data), ms)
|
||||||
|
statsd.timing('view.{method}'.format(**data), ms)
|
|
@ -0,0 +1,38 @@
|
||||||
|
from django.conf import settings
|
||||||
|
from django.template.loader import render_to_string
|
||||||
|
from django.utils.translation import ugettext_lazy as _, ungettext
|
||||||
|
|
||||||
|
from debug_toolbar.panels import DebugPanel
|
||||||
|
from django_statsd.statsd import statsd
|
||||||
|
|
||||||
|
|
||||||
|
class StatsdPanel(DebugPanel):
|
||||||
|
|
||||||
|
name = 'Statsd'
|
||||||
|
has_content = True
|
||||||
|
|
||||||
|
def __init__(self, *args, **kw):
|
||||||
|
super(StatsdPanel, self).__init__(*args, **kw)
|
||||||
|
self.statsd = statsd
|
||||||
|
self.statsd.reset()
|
||||||
|
|
||||||
|
def nav_title(self):
|
||||||
|
return _('Statsd')
|
||||||
|
|
||||||
|
def nav_subtitle(self):
|
||||||
|
length = len(self.statsd.cache)
|
||||||
|
return ungettext('%s record', '%s records', length) % length
|
||||||
|
|
||||||
|
def title(self):
|
||||||
|
return _('Statsd')
|
||||||
|
|
||||||
|
def url(self):
|
||||||
|
return ''
|
||||||
|
|
||||||
|
def content(self):
|
||||||
|
context = self.context.copy()
|
||||||
|
context.update(settings.TOOLBAR_STATSD)
|
||||||
|
context['statsd'] = self.statsd.cache
|
||||||
|
return render_to_string('toolbar_statsd/statsd.html', context)
|
||||||
|
|
||||||
|
|
|
@ -0,0 +1,14 @@
|
||||||
|
from django.utils.importlib import import_module
|
||||||
|
from django.conf import settings
|
||||||
|
|
||||||
|
_statsd = None
|
||||||
|
|
||||||
|
def get_client():
|
||||||
|
client = getattr(settings, 'STATSD_CLIENT', 'statsd.client')
|
||||||
|
host = getattr(settings, 'STATSD_HOST', 'localhost')
|
||||||
|
port = getattr(settings, 'STATSD_PORT', 8125)
|
||||||
|
prefix = getattr(settings, 'STATSD_PREFIX', None)
|
||||||
|
return import_module(client).StatsClient(host, port, prefix)
|
||||||
|
|
||||||
|
statsd = get_client()
|
||||||
|
|
|
@ -0,0 +1,45 @@
|
||||||
|
<table id="statsd"
|
||||||
|
data-graphite="{{ graphite }}"
|
||||||
|
data-roots="{% for root in roots %}{{ root }}{% if not forloop.last %}|{% endif %}{% endfor %}">
|
||||||
|
<thead>
|
||||||
|
<tr>
|
||||||
|
<th>Stat</th>
|
||||||
|
<th>Value</th>
|
||||||
|
<th>Count</th>
|
||||||
|
</tr>
|
||||||
|
</thead>
|
||||||
|
<tbody>
|
||||||
|
{% for record in statsd %}
|
||||||
|
<tr>
|
||||||
|
<td><a href="#" class="statsd" data-key="{{ record.0 }}">{{ record.0 }}</a></td>
|
||||||
|
<td>{{ record.1 }}</td>
|
||||||
|
<td>{{ record.2 }}</td>
|
||||||
|
</tr>
|
||||||
|
{% endfor %}
|
||||||
|
</tbody>
|
||||||
|
</table>
|
||||||
|
|
||||||
|
<div id="graphs" img="graphite?width=586&height=308&target=root.key.lower&target=root.key.mean&target=root.key.upper_90&target=scale(root.key.count,0.1)&from=-24hours&title=24 hours" />
|
||||||
|
|
||||||
|
<script type="text/javascript">
|
||||||
|
// TODO: inlining is bad, this should be external.
|
||||||
|
|
||||||
|
$(document).ready(function() {
|
||||||
|
var graphite = $('#statsd').attr('data-graphite'),
|
||||||
|
roots = $('#statsd').attr('data-roots').split('|'),
|
||||||
|
target = $('#graphs'),
|
||||||
|
img = target.attr('img');
|
||||||
|
|
||||||
|
$('a.statsd').click(function() {
|
||||||
|
var that = $(this);
|
||||||
|
target.html('');
|
||||||
|
$.each(roots, function(root) {
|
||||||
|
var custom = img.replace('graphite', graphite, 'g')
|
||||||
|
.replace('root', roots[root], 'g')
|
||||||
|
.replace('key', that.attr('data-key'), 'g');
|
||||||
|
target.append('<p><b>' + roots[root] + '.' + that.attr('data-key') + '</b></p><img src="' + custom + '">');
|
||||||
|
})
|
||||||
|
});
|
||||||
|
})
|
||||||
|
</script>
|
||||||
|
|
|
@ -0,0 +1,104 @@
|
||||||
|
from django.conf import settings
|
||||||
|
from django.http import HttpResponse
|
||||||
|
from django.test.client import RequestFactory
|
||||||
|
|
||||||
|
import mock
|
||||||
|
from nose.tools import eq_
|
||||||
|
|
||||||
|
from django_statsd import statsd, get_client, middleware
|
||||||
|
|
||||||
|
import unittest
|
||||||
|
|
||||||
|
|
||||||
|
@mock.patch.object(middleware.statsd, 'incr')
|
||||||
|
class TestIncr(unittest.TestCase):
|
||||||
|
|
||||||
|
def setUp(self):
|
||||||
|
self.req = RequestFactory().get('/')
|
||||||
|
self.res = HttpResponse()
|
||||||
|
|
||||||
|
def test_graphite_response(self, incr):
|
||||||
|
gmw = middleware.GraphiteMiddleware()
|
||||||
|
gmw.process_response(self.req, self.res)
|
||||||
|
assert incr.called
|
||||||
|
|
||||||
|
def test_graphite_response_authenticated(self, incr):
|
||||||
|
self.req.user = mock.Mock()
|
||||||
|
self.req.user.is_authenticated.return_value = True
|
||||||
|
gmw = middleware.GraphiteMiddleware()
|
||||||
|
gmw.process_response(self.req, self.res)
|
||||||
|
eq_(incr.call_count, 2)
|
||||||
|
|
||||||
|
def test_graphite_exception(self, incr):
|
||||||
|
gmw = middleware.GraphiteMiddleware()
|
||||||
|
gmw.process_exception(self.req, None)
|
||||||
|
assert incr.called
|
||||||
|
|
||||||
|
def test_graphite_exception_authenticated(self, incr):
|
||||||
|
self.req.user = mock.Mock()
|
||||||
|
self.req.user.is_authenticated.return_value = True
|
||||||
|
gmw = middleware.GraphiteMiddleware()
|
||||||
|
gmw.process_exception(self.req, None)
|
||||||
|
eq_(incr.call_count, 2)
|
||||||
|
|
||||||
|
|
||||||
|
@mock.patch.object(middleware.statsd, 'timing')
|
||||||
|
class TestTiming(unittest.TestCase):
|
||||||
|
|
||||||
|
def setUp(self):
|
||||||
|
self.req = RequestFactory().get('/')
|
||||||
|
self.res = HttpResponse()
|
||||||
|
|
||||||
|
def test_request_timing(self, timing):
|
||||||
|
func = lambda x: x
|
||||||
|
gmw = middleware.GraphiteRequestTimingMiddleware()
|
||||||
|
gmw.process_view(self.req, func, tuple(), dict())
|
||||||
|
gmw.process_response(self.req, self.res)
|
||||||
|
eq_(timing.call_count, 3)
|
||||||
|
names = ['view.%s.%s.GET' % (func.__module__, func.__name__),
|
||||||
|
'view.%s.GET' % func.__module__,
|
||||||
|
'view.GET']
|
||||||
|
for expected, (args, kwargs) in zip(names, timing.call_args_list):
|
||||||
|
eq_(expected, args[0])
|
||||||
|
|
||||||
|
def test_request_timing_exception(self, timing):
|
||||||
|
func = lambda x: x
|
||||||
|
gmw = middleware.GraphiteRequestTimingMiddleware()
|
||||||
|
gmw.process_view(self.req, func, tuple(), dict())
|
||||||
|
gmw.process_exception(self.req, self.res)
|
||||||
|
eq_(timing.call_count, 3)
|
||||||
|
names = ['view.%s.%s.GET' % (func.__module__, func.__name__),
|
||||||
|
'view.%s.GET' % func.__module__,
|
||||||
|
'view.GET']
|
||||||
|
for expected, (args, kwargs) in zip(names, timing.call_args_list):
|
||||||
|
eq_(expected, args[0])
|
||||||
|
|
||||||
|
|
||||||
|
class TestClient(unittest.TestCase):
|
||||||
|
|
||||||
|
@mock.patch_object(settings, 'STATSD_CLIENT', 'statsd.client')
|
||||||
|
def test_normal(self):
|
||||||
|
eq_(get_client().__module__, 'statsd.client')
|
||||||
|
|
||||||
|
@mock.patch_object(settings, 'STATSD_CLIENT',
|
||||||
|
'django_statsd.clients.null')
|
||||||
|
def test_null(self):
|
||||||
|
eq_(get_client().__module__, 'django_statsd.clients.null')
|
||||||
|
|
||||||
|
@mock.patch_object(settings, 'STATSD_CLIENT',
|
||||||
|
'django_statsd.clients.toolbar')
|
||||||
|
def test_toolbar(self):
|
||||||
|
eq_(get_client().__module__, 'django_statsd.clients.toolbar')
|
||||||
|
|
||||||
|
@mock.patch_object(settings, 'STATSD_CLIENT',
|
||||||
|
'django_statsd.clients.toolbar')
|
||||||
|
def test_toolbar_send(self):
|
||||||
|
client = get_client()
|
||||||
|
eq_(client.cache, [])
|
||||||
|
client.incr('testing')
|
||||||
|
eq_(client.cache, [['testing', u'1c', 1]])
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
Binary file not shown.
After Width: | Height: | Size: 148 KiB |
|
@ -0,0 +1,24 @@
|
||||||
|
import os
|
||||||
|
from setuptools import setup, find_packages
|
||||||
|
|
||||||
|
def read(fname):
|
||||||
|
return open(os.path.join(os.path.dirname(__file__), fname)).read()
|
||||||
|
|
||||||
|
setup(
|
||||||
|
name='django-statsd',
|
||||||
|
version='0.1',
|
||||||
|
description='',
|
||||||
|
long_description=read('README.rst'),
|
||||||
|
author='Andy McKay',
|
||||||
|
author_email='andym@mozilla.com',
|
||||||
|
license='BSD',
|
||||||
|
packages=['django_statsd'],
|
||||||
|
url='',
|
||||||
|
package_data = {'django_statsd': ['templates/django_statsd/*.html']},
|
||||||
|
classifiers=[
|
||||||
|
'Intended Audience :: Developers',
|
||||||
|
'Natural Language :: English',
|
||||||
|
'Operating System :: OS Independent',
|
||||||
|
'Framework :: Django'
|
||||||
|
],
|
||||||
|
)
|
Loading…
Reference in New Issue