diff --git a/README.md b/README.md index 88b70b9..5e258fb 100644 --- a/README.md +++ b/README.md @@ -1,15 +1,48 @@ gstatsd - A statsd service implementation in Python + gevent. -License: Apache 2.0 +If you are unfamiliar with statsd, you can read [why statsd exists][etsy post], +or look at the [NodeJS statsd implementation][etsy repo]. -Usage: ------- +License: [Apache 2.0][license] + +Requirements +------------ + + * [Python][python] - I'm testing on 2.6/2.7 at the moment. + * [gevent][gevent] - A libevent wrapper. + * [distribute][distribute] - (or setuptools) for builds. + + +Using gstatsd +------------- Show gstatsd help: % gstatsd -h +Options: + + Usage: gstatsd [options] + + A statsd service in Python + gevent. + + Options: + --version show program's version number and exit + -h, --help show this help message and exit + -b BIND_ADDR, --bind=BIND_ADDR + bind [host]:port (host defaults to '') + -d DEST_ADDR, --dest=DEST_ADDR + receiver [backend:]host:port (backend defaults to + 'graphite') + -v increase verbosity (currently used for debugging) + -f INTERVAL, --flush=INTERVAL + flush interval, in seconds (default 10) + -p PERCENT, --percent=PERCENT + percent threshold (default 90) + -l, --list list supported backends + -D, --daemonize daemonize the service + Start gstatsd and send stats to port 9100 every 5 seconds: % gstatsd -d :9100 -f 5 @@ -18,3 +51,48 @@ Bind listener to host 'hostname' port 8126: % gstatsd -b hostname:8126 -d :9100 -f 5 +To send the stats to multiple graphite servers, specify multiple destinations: + + % gstatsd -b :8125 -d stats1:9100 stats2:9100 + + +Using the client +---------------- + +The code example below demonstrates using the low-level client interface: + + from gstatsd import client + + # location of the statsd server + hostport = ('', 8125) + + raw = client.StatsClient(hostport) + + # add 1 to the 'foo' bucket + raw.increment('foo') + + # timer 'bar' took 25ms to complete + raw.timer('bar', 25) + + +You may prefer to use the stateful client: + + # wraps the raw client + cli = client.Stats(raw) + + timer = cli.get_timer('foo') + timer.start() + + ... do some work .. + + # when .stop() is called, the stat is sent to the server + timer.stop() + + +[python]: http://www.python.org/ +[gevent]: http://www.gevent.org/ +[license]: http://www.apache.org/licenses/LICENSE-2.0 +[distribute]: http://pypi.python.org/pypi/distribute +[etsy repo]: https://github.com/etsy/statsd +[etsy post]: http://codeascraft.etsy.com/2011/02/15/measure-anything-measure-everything/ + diff --git a/debian/changelog b/debian/changelog index 7825ee0..fba9623 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,9 @@ +gstatsd (0.3) lucid; urgency=low + + * Release. + + -- Patrick Hensley Wed, 29 Jun 2011 11:59:01 -0400 + gstatsd (0.2) lucid; urgency=low * Initial release. diff --git a/gstatsd/client.py b/gstatsd/client.py index 5d5f34f..0c6647e 100644 --- a/gstatsd/client.py +++ b/gstatsd/client.py @@ -1,7 +1,12 @@ +# standard import random import socket +import time + + +E_NOSTART = 'you must call start() before stop(). ignoring.' class StatsClient(object): @@ -17,7 +22,7 @@ class StatsClient(object): self._sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) def timer(self, key, timestamp, sample_rate=1): - self._send('%s:%d|ms' % (key, timestamp), sample_rate) + self._send('%s:%d|ms' % (key, round(timestamp)), sample_rate) def increment(self, key, sample_rate=1): return self.counter(key, 1, sample_rate) @@ -29,7 +34,7 @@ class StatsClient(object): if not isinstance(keys, (list, tuple)): keys = [keys] for key in keys: - self._send('%s:%s|c' % (key, magnitude), sample_rate) + self._send('%s:%d|c' % (key, round(magnitude)), sample_rate) def _send(self, data, sample_rate=1): packet = None @@ -41,3 +46,53 @@ class StatsClient(object): if packet: self._sock.sendto(packet, self._hostport) + +class StatsCounter(object): + + def __init__(self, client, key, sample_rate=1): + self._client = client + self._key = key + self._sample_rate = sample_rate + + def increment(self): + self._client.increment(self._key, self._sample_rate) + + def decrement(self): + self._client.decrement(self._key, self._sample_rate) + + def add(self, val): + self._client.counter(self._key, val, self._sample_rate) + + +class StatsTimer(object): + + def __init__(self, client, key): + self._client = client + self._key = key + self._started = 0 + self._timestamp = 0 + + def start(self): + self._started = 1 + self._timestamp = time.time() + + def stop(self): + if not self._started: + raise UserWarning(E_NOSTART) + return + elapsed = time.time() - self._timestamp + self._client.timer(self._key, int(elapsed / 1000.0)) + self._started = 0 + + +class Stats(object): + + def __init__(self, client): + self._client = client + + def get_counter(self, key, sample_rate=1): + return StatsCounter(self._client, key, sample_rate) + + def get_timer(self, key): + return StatsTimer(self._client, key) + diff --git a/gstatsd/client_test.py b/gstatsd/client_test.py new file mode 100644 index 0000000..70958dd --- /dev/null +++ b/gstatsd/client_test.py @@ -0,0 +1,89 @@ + +# standard +import unittest + +# local +from gstatsd import client + + +class StatsDummyClient(client.StatsClient): + + def __init__(self, hostport=None): + client.StatsClient.__init__(self, hostport) + self.packets = [] + + def _send(self, data, sample_rate=1): + self.packets.append((data, sample_rate)) + + +class StatsClientTest(unittest.TestCase): + + def setUp(self): + self._cli = StatsDummyClient() + + def test_timer(self): + self._cli.timer('foo', 15, 1) + self.assertEquals(self._cli.packets[-1], ('foo:15|ms', 1)) + self._cli.timer('bar.baz', 1.35, 1) + self.assertEquals(self._cli.packets[-1], ('bar.baz:1|ms', 1)) + self._cli.timer('x', 1.99, 1) + self.assertEquals(self._cli.packets[-1], ('x:2|ms', 1)) + self._cli.timer('x', 1, 0.5) + self.assertEquals(self._cli.packets[-1], ('x:1|ms', 0.5)) + + def test_increment(self): + self._cli.increment('foo') + self.assertEquals(self._cli.packets[-1], ('foo:1|c', 1)) + self._cli.increment('x', 0.5) + self.assertEquals(self._cli.packets[-1], ('x:1|c', 0.5)) + + def test_decrement(self): + self._cli.decrement('foo') + self.assertEquals(self._cli.packets[-1], ('foo:-1|c', 1)) + self._cli.decrement('x', 0.2) + self.assertEquals(self._cli.packets[-1], ('x:-1|c', 0.2)) + + def test_counter(self): + self._cli.counter('foo', 5) + self.assertEquals(self._cli.packets[-1], ('foo:5|c', 1)) + self._cli.counter('foo', -50) + self.assertEquals(self._cli.packets[-1], ('foo:-50|c', 1)) + self._cli.counter('foo', 5.9) + self.assertEquals(self._cli.packets[-1], ('foo:6|c', 1)) + self._cli.counter('foo', 1, 0.2) + self.assertEquals(self._cli.packets[-1], ('foo:1|c', 0.2)) + + +class StatsTest(unittest.TestCase): + + def setUp(self): + self._cli = StatsDummyClient() + self._stat = client.Stats(self._cli) + + def test_timer(self): + timer = self._stat.get_timer('foo') + timer.start() + timer.stop() + data, sr = self._cli.packets[-1] + pkt = data.split(':') + self.assertEquals(pkt[0], 'foo') + + # ensure warning is raised for mismatched start/stop + timer = self._stat.get_timer('foo') + self.assertRaises(UserWarning, timer.stop) + + def test_counter(self): + count = self._stat.get_counter('foo') + count.increment() + count.decrement() + count.add(5) + self.assertEquals(self._cli.packets[-1], ('foo:5|c', 1)) + + +def main(): + unittest.main() + + +if __name__ == '__main__': + main() + diff --git a/gstatsd/core.py b/gstatsd/core.py index 7e860d9..56d8ded 100644 --- a/gstatsd/core.py +++ b/gstatsd/core.py @@ -1,3 +1,3 @@ -__version__ = '0.2' +__version__ = '0.3' diff --git a/gstatsd/service.py b/gstatsd/service.py index c4ef565..ee9ef99 100644 --- a/gstatsd/service.py +++ b/gstatsd/service.py @@ -29,8 +29,10 @@ A statsd service in Python + gevent. ''' # table to remove invalid characters from keys -KEY_VALIDCHARS = string.uppercase + string.lowercase + string.digits + '_-.' -KEY_SANITIZE = string.maketrans(KEY_VALIDCHARS + '/', KEY_VALIDCHARS + '_') +ALL_ASCII = set(chr(c) for c in range(256)) +KEY_VALID = string.ascii_letters + string.digits + '_-.' +KEY_TABLE = string.maketrans(KEY_VALID + '/', KEY_VALID + '_') +KEY_DELETIONS = ''.join(ALL_ASCII.difference(KEY_VALID + '/')) # error messages E_BADADDR = 'invalid bind address specified %r' @@ -156,7 +158,7 @@ class StatsDaemon(object): self._error('packet: %r' % data) if not parts: return - key = parts[0].translate(KEY_SANITIZE, string.whitespace) + key = parts[0].translate(KEY_TABLE, KEY_DELETIONS) for part in parts[1:]: srate = 1.0 fields = part.split('|') @@ -246,11 +248,12 @@ def main(): opts.add_option('-d', '--dest', dest='dest_addr', action='append', default=[], help="receiver [backend:]host:port (backend defaults to 'graphite')") - opts.add_option('-v', dest='verbose', action='count', default=0) + opts.add_option('-v', dest='verbose', action='count', default=0, + help="increase verbosity (currently used for debugging)") opts.add_option('-f', '--flush', dest='interval', default=INTERVAL, - help="flush interval, in seconds") + help="flush interval, in seconds (default 10)") opts.add_option('-p', '--percent', dest='percent', default=PERCENT, - help="percent threshold") + help="percent threshold (default 90)") opts.add_option('-l', '--list', dest='list_backends', action='store_true', help="list supported backends") opts.add_option('-D', '--daemonize', dest='daemonize', action='store_true', diff --git a/gstatsd/service_test.py b/gstatsd/service_test.py new file mode 100644 index 0000000..1b6bc2d --- /dev/null +++ b/gstatsd/service_test.py @@ -0,0 +1,69 @@ + +# standard +import unittest + +# local +from gstatsd import service + + +class StatsServiceTest(unittest.TestCase): + + def setUp(self): + args = (':8125', [':9100'], 5, 90, 0) + self.svc = service.StatsDaemon(*args) + + def test_construct(self): + svc = service.StatsDaemon('8125', ['9100'], 5, 90, 0) + self.assertEquals(svc._bindaddr, ('', 8125)) + self.assertEquals(svc._interval, 5.0) + self.assertEquals(svc._percent, 90.0) + self.assertEquals(svc._debug, 0) + self.assertEquals(svc._targets[0], (svc._send_graphite, ('', 9100))) + + svc = service.StatsDaemon('bar:8125', ['foo:9100'], 5, 90, 1) + self.assertEquals(svc._bindaddr, ('bar', 8125)) + self.assertEquals(svc._targets[0], (svc._send_graphite, ('foo', 9100))) + self.assertEquals(svc._debug, 1) + + def test_backend(self): + service.StatsDaemon._send_foo = lambda self, x, y: None + svc = service.StatsDaemon('8125', ['foo:bar:9100'], 5, 90, 0) + self.assertEquals(svc._targets[0], (svc._send_foo, ('bar', 9100))) + + def test_counters(self): + pkt = 'foo:1|c' + self.svc._process(pkt, None) + self.assertEquals(self.svc._counts, {'foo': 1}) + self.svc._process(pkt, None) + self.assertEquals(self.svc._counts, {'foo': 2}) + pkt = 'foo:-1|c' + self.svc._process(pkt, None) + self.assertEquals(self.svc._counts, {'foo': 1}) + + def test_counters_sampled(self): + pkt = 'foo:1|c|@.5' + self.svc._process(pkt, None) + self.assertEquals(self.svc._counts, {'foo': 2}) + + def test_timers(self): + pkt = 'foo:20|ms' + self.svc._process(pkt, None) + self.assertEquals(self.svc._timers, {'foo': [20.0]}) + pkt = 'foo:10|ms' + self.svc._process(pkt, None) + self.assertEquals(self.svc._timers, {'foo': [20.0, 10.0]}) + + def test_key_sanitize(self): + pkt = '\t\n#! foo . bar \0 ^:1|c' + self.svc._process(pkt, None) + self.assertEquals(self.svc._counts, {'foo.bar': 1}) + + +def main(): + unittest.main() + + +if __name__ == '__main__': + main() + + diff --git a/gstatsd/test.py b/gstatsd/test.py deleted file mode 100644 index 824a323..0000000 --- a/gstatsd/test.py +++ /dev/null @@ -1,22 +0,0 @@ - -from client import StatsClient - - -def main(): - cli = StatsClient() - for num in range(1, 11): - cli.timer('foo', num) - return - cli.increment('baz', 0.5) - cli.increment('baz', 0.5) - cli.timer('t3', 100, 0.5) - return - - cli.increment('foo') - cli.counter(['foo', 'bar'], 2) - cli.timer('t1', 12) - cli.timer('t2', 30) - -if __name__ == '__main__': - main() - diff --git a/setup.py b/setup.py index dcd66c5..a0a5069 100644 --- a/setup.py +++ b/setup.py @@ -24,7 +24,7 @@ def main(): 'console_scripts': ['gstatsd=gstatsd.service:main'] }, classifiers = [ - "Development Status :: 2 - Pre-Alpha", + "Development Status :: 3 - Alpha", "Intended Audience :: Developers", "License :: OSI Approved :: Apache Software License", "Operating System :: MacOS :: MacOS X",