Release 0.3.

* Added initial unit tests for service, client.
 * Created stateful timer and counter client classes to simplify usage.
 * Finished key sanitization.
 * Bumped to alpha status.
 * Removed 'test.py', updated README with usage examples.
This commit is contained in:
Patrick Hensley 2011-06-29 10:28:44 -04:00
parent dcc4f2413d
commit a0e07e48cd
9 changed files with 313 additions and 35 deletions

View File

@ -1,15 +1,48 @@
gstatsd - A statsd service implementation in Python + gevent.
License: Apache 2.0
If you are unfamiliar with statsd, you can read [why statsd exists][etsy post],
or look at the [NodeJS statsd implementation][etsy repo].
Usage:
------
License: [Apache 2.0][license]
Requirements
------------
* [Python][python] - I'm testing on 2.6/2.7 at the moment.
* [gevent][gevent] - A libevent wrapper.
* [distribute][distribute] - (or setuptools) for builds.
Using gstatsd
-------------
Show gstatsd help:
% gstatsd -h
Options:
Usage: gstatsd [options]
A statsd service in Python + gevent.
Options:
--version show program's version number and exit
-h, --help show this help message and exit
-b BIND_ADDR, --bind=BIND_ADDR
bind [host]:port (host defaults to '')
-d DEST_ADDR, --dest=DEST_ADDR
receiver [backend:]host:port (backend defaults to
'graphite')
-v increase verbosity (currently used for debugging)
-f INTERVAL, --flush=INTERVAL
flush interval, in seconds (default 10)
-p PERCENT, --percent=PERCENT
percent threshold (default 90)
-l, --list list supported backends
-D, --daemonize daemonize the service
Start gstatsd and send stats to port 9100 every 5 seconds:
% gstatsd -d :9100 -f 5
@ -18,3 +51,48 @@ Bind listener to host 'hostname' port 8126:
% gstatsd -b hostname:8126 -d :9100 -f 5
To send the stats to multiple graphite servers, specify multiple destinations:
% gstatsd -b :8125 -d stats1:9100 stats2:9100
Using the client
----------------
The code example below demonstrates using the low-level client interface:
from gstatsd import client
# location of the statsd server
hostport = ('', 8125)
raw = client.StatsClient(hostport)
# add 1 to the 'foo' bucket
raw.increment('foo')
# timer 'bar' took 25ms to complete
raw.timer('bar', 25)
You may prefer to use the stateful client:
# wraps the raw client
cli = client.Stats(raw)
timer = cli.get_timer('foo')
timer.start()
... do some work ..
# when .stop() is called, the stat is sent to the server
timer.stop()
[python]: http://www.python.org/
[gevent]: http://www.gevent.org/
[license]: http://www.apache.org/licenses/LICENSE-2.0
[distribute]: http://pypi.python.org/pypi/distribute
[etsy repo]: https://github.com/etsy/statsd
[etsy post]: http://codeascraft.etsy.com/2011/02/15/measure-anything-measure-everything/

6
debian/changelog vendored
View File

@ -1,3 +1,9 @@
gstatsd (0.3) lucid; urgency=low
* Release.
-- Patrick Hensley <spaceboy@indirect.com> Wed, 29 Jun 2011 11:59:01 -0400
gstatsd (0.2) lucid; urgency=low
* Initial release.

View File

@ -1,7 +1,12 @@
# standard
import random
import socket
import time
E_NOSTART = 'you must call start() before stop(). ignoring.'
class StatsClient(object):
@ -17,7 +22,7 @@ class StatsClient(object):
self._sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
def timer(self, key, timestamp, sample_rate=1):
self._send('%s:%d|ms' % (key, timestamp), sample_rate)
self._send('%s:%d|ms' % (key, round(timestamp)), sample_rate)
def increment(self, key, sample_rate=1):
return self.counter(key, 1, sample_rate)
@ -29,7 +34,7 @@ class StatsClient(object):
if not isinstance(keys, (list, tuple)):
keys = [keys]
for key in keys:
self._send('%s:%s|c' % (key, magnitude), sample_rate)
self._send('%s:%d|c' % (key, round(magnitude)), sample_rate)
def _send(self, data, sample_rate=1):
packet = None
@ -41,3 +46,53 @@ class StatsClient(object):
if packet:
self._sock.sendto(packet, self._hostport)
class StatsCounter(object):
def __init__(self, client, key, sample_rate=1):
self._client = client
self._key = key
self._sample_rate = sample_rate
def increment(self):
self._client.increment(self._key, self._sample_rate)
def decrement(self):
self._client.decrement(self._key, self._sample_rate)
def add(self, val):
self._client.counter(self._key, val, self._sample_rate)
class StatsTimer(object):
def __init__(self, client, key):
self._client = client
self._key = key
self._started = 0
self._timestamp = 0
def start(self):
self._started = 1
self._timestamp = time.time()
def stop(self):
if not self._started:
raise UserWarning(E_NOSTART)
return
elapsed = time.time() - self._timestamp
self._client.timer(self._key, int(elapsed / 1000.0))
self._started = 0
class Stats(object):
def __init__(self, client):
self._client = client
def get_counter(self, key, sample_rate=1):
return StatsCounter(self._client, key, sample_rate)
def get_timer(self, key):
return StatsTimer(self._client, key)

89
gstatsd/client_test.py Normal file
View File

@ -0,0 +1,89 @@
# standard
import unittest
# local
from gstatsd import client
class StatsDummyClient(client.StatsClient):
def __init__(self, hostport=None):
client.StatsClient.__init__(self, hostport)
self.packets = []
def _send(self, data, sample_rate=1):
self.packets.append((data, sample_rate))
class StatsClientTest(unittest.TestCase):
def setUp(self):
self._cli = StatsDummyClient()
def test_timer(self):
self._cli.timer('foo', 15, 1)
self.assertEquals(self._cli.packets[-1], ('foo:15|ms', 1))
self._cli.timer('bar.baz', 1.35, 1)
self.assertEquals(self._cli.packets[-1], ('bar.baz:1|ms', 1))
self._cli.timer('x', 1.99, 1)
self.assertEquals(self._cli.packets[-1], ('x:2|ms', 1))
self._cli.timer('x', 1, 0.5)
self.assertEquals(self._cli.packets[-1], ('x:1|ms', 0.5))
def test_increment(self):
self._cli.increment('foo')
self.assertEquals(self._cli.packets[-1], ('foo:1|c', 1))
self._cli.increment('x', 0.5)
self.assertEquals(self._cli.packets[-1], ('x:1|c', 0.5))
def test_decrement(self):
self._cli.decrement('foo')
self.assertEquals(self._cli.packets[-1], ('foo:-1|c', 1))
self._cli.decrement('x', 0.2)
self.assertEquals(self._cli.packets[-1], ('x:-1|c', 0.2))
def test_counter(self):
self._cli.counter('foo', 5)
self.assertEquals(self._cli.packets[-1], ('foo:5|c', 1))
self._cli.counter('foo', -50)
self.assertEquals(self._cli.packets[-1], ('foo:-50|c', 1))
self._cli.counter('foo', 5.9)
self.assertEquals(self._cli.packets[-1], ('foo:6|c', 1))
self._cli.counter('foo', 1, 0.2)
self.assertEquals(self._cli.packets[-1], ('foo:1|c', 0.2))
class StatsTest(unittest.TestCase):
def setUp(self):
self._cli = StatsDummyClient()
self._stat = client.Stats(self._cli)
def test_timer(self):
timer = self._stat.get_timer('foo')
timer.start()
timer.stop()
data, sr = self._cli.packets[-1]
pkt = data.split(':')
self.assertEquals(pkt[0], 'foo')
# ensure warning is raised for mismatched start/stop
timer = self._stat.get_timer('foo')
self.assertRaises(UserWarning, timer.stop)
def test_counter(self):
count = self._stat.get_counter('foo')
count.increment()
count.decrement()
count.add(5)
self.assertEquals(self._cli.packets[-1], ('foo:5|c', 1))
def main():
unittest.main()
if __name__ == '__main__':
main()

View File

@ -1,3 +1,3 @@
__version__ = '0.2'
__version__ = '0.3'

View File

@ -29,8 +29,10 @@ A statsd service in Python + gevent.
'''
# table to remove invalid characters from keys
KEY_VALIDCHARS = string.uppercase + string.lowercase + string.digits + '_-.'
KEY_SANITIZE = string.maketrans(KEY_VALIDCHARS + '/', KEY_VALIDCHARS + '_')
ALL_ASCII = set(chr(c) for c in range(256))
KEY_VALID = string.ascii_letters + string.digits + '_-.'
KEY_TABLE = string.maketrans(KEY_VALID + '/', KEY_VALID + '_')
KEY_DELETIONS = ''.join(ALL_ASCII.difference(KEY_VALID + '/'))
# error messages
E_BADADDR = 'invalid bind address specified %r'
@ -156,7 +158,7 @@ class StatsDaemon(object):
self._error('packet: %r' % data)
if not parts:
return
key = parts[0].translate(KEY_SANITIZE, string.whitespace)
key = parts[0].translate(KEY_TABLE, KEY_DELETIONS)
for part in parts[1:]:
srate = 1.0
fields = part.split('|')
@ -246,11 +248,12 @@ def main():
opts.add_option('-d', '--dest', dest='dest_addr', action='append',
default=[],
help="receiver [backend:]host:port (backend defaults to 'graphite')")
opts.add_option('-v', dest='verbose', action='count', default=0)
opts.add_option('-v', dest='verbose', action='count', default=0,
help="increase verbosity (currently used for debugging)")
opts.add_option('-f', '--flush', dest='interval', default=INTERVAL,
help="flush interval, in seconds")
help="flush interval, in seconds (default 10)")
opts.add_option('-p', '--percent', dest='percent', default=PERCENT,
help="percent threshold")
help="percent threshold (default 90)")
opts.add_option('-l', '--list', dest='list_backends', action='store_true',
help="list supported backends")
opts.add_option('-D', '--daemonize', dest='daemonize', action='store_true',

69
gstatsd/service_test.py Normal file
View File

@ -0,0 +1,69 @@
# standard
import unittest
# local
from gstatsd import service
class StatsServiceTest(unittest.TestCase):
def setUp(self):
args = (':8125', [':9100'], 5, 90, 0)
self.svc = service.StatsDaemon(*args)
def test_construct(self):
svc = service.StatsDaemon('8125', ['9100'], 5, 90, 0)
self.assertEquals(svc._bindaddr, ('', 8125))
self.assertEquals(svc._interval, 5.0)
self.assertEquals(svc._percent, 90.0)
self.assertEquals(svc._debug, 0)
self.assertEquals(svc._targets[0], (svc._send_graphite, ('', 9100)))
svc = service.StatsDaemon('bar:8125', ['foo:9100'], 5, 90, 1)
self.assertEquals(svc._bindaddr, ('bar', 8125))
self.assertEquals(svc._targets[0], (svc._send_graphite, ('foo', 9100)))
self.assertEquals(svc._debug, 1)
def test_backend(self):
service.StatsDaemon._send_foo = lambda self, x, y: None
svc = service.StatsDaemon('8125', ['foo:bar:9100'], 5, 90, 0)
self.assertEquals(svc._targets[0], (svc._send_foo, ('bar', 9100)))
def test_counters(self):
pkt = 'foo:1|c'
self.svc._process(pkt, None)
self.assertEquals(self.svc._counts, {'foo': 1})
self.svc._process(pkt, None)
self.assertEquals(self.svc._counts, {'foo': 2})
pkt = 'foo:-1|c'
self.svc._process(pkt, None)
self.assertEquals(self.svc._counts, {'foo': 1})
def test_counters_sampled(self):
pkt = 'foo:1|c|@.5'
self.svc._process(pkt, None)
self.assertEquals(self.svc._counts, {'foo': 2})
def test_timers(self):
pkt = 'foo:20|ms'
self.svc._process(pkt, None)
self.assertEquals(self.svc._timers, {'foo': [20.0]})
pkt = 'foo:10|ms'
self.svc._process(pkt, None)
self.assertEquals(self.svc._timers, {'foo': [20.0, 10.0]})
def test_key_sanitize(self):
pkt = '\t\n#! foo . bar \0 ^:1|c'
self.svc._process(pkt, None)
self.assertEquals(self.svc._counts, {'foo.bar': 1})
def main():
unittest.main()
if __name__ == '__main__':
main()

View File

@ -1,22 +0,0 @@
from client import StatsClient
def main():
cli = StatsClient()
for num in range(1, 11):
cli.timer('foo', num)
return
cli.increment('baz', 0.5)
cli.increment('baz', 0.5)
cli.timer('t3', 100, 0.5)
return
cli.increment('foo')
cli.counter(['foo', 'bar'], 2)
cli.timer('t1', 12)
cli.timer('t2', 30)
if __name__ == '__main__':
main()

View File

@ -24,7 +24,7 @@ def main():
'console_scripts': ['gstatsd=gstatsd.service:main']
},
classifiers = [
"Development Status :: 2 - Pre-Alpha",
"Development Status :: 3 - Alpha",
"Intended Audience :: Developers",
"License :: OSI Approved :: Apache Software License",
"Operating System :: MacOS :: MacOS X",