Release 0.3.
* Added initial unit tests for service, client. * Created stateful timer and counter client classes to simplify usage. * Finished key sanitization. * Bumped to alpha status. * Removed 'test.py', updated README with usage examples.
This commit is contained in:
parent
dcc4f2413d
commit
a0e07e48cd
84
README.md
84
README.md
|
@ -1,15 +1,48 @@
|
||||||
|
|
||||||
gstatsd - A statsd service implementation in Python + gevent.
|
gstatsd - A statsd service implementation in Python + gevent.
|
||||||
|
|
||||||
License: Apache 2.0
|
If you are unfamiliar with statsd, you can read [why statsd exists][etsy post],
|
||||||
|
or look at the [NodeJS statsd implementation][etsy repo].
|
||||||
|
|
||||||
Usage:
|
License: [Apache 2.0][license]
|
||||||
------
|
|
||||||
|
Requirements
|
||||||
|
------------
|
||||||
|
|
||||||
|
* [Python][python] - I'm testing on 2.6/2.7 at the moment.
|
||||||
|
* [gevent][gevent] - A libevent wrapper.
|
||||||
|
* [distribute][distribute] - (or setuptools) for builds.
|
||||||
|
|
||||||
|
|
||||||
|
Using gstatsd
|
||||||
|
-------------
|
||||||
|
|
||||||
Show gstatsd help:
|
Show gstatsd help:
|
||||||
|
|
||||||
% gstatsd -h
|
% gstatsd -h
|
||||||
|
|
||||||
|
Options:
|
||||||
|
|
||||||
|
Usage: gstatsd [options]
|
||||||
|
|
||||||
|
A statsd service in Python + gevent.
|
||||||
|
|
||||||
|
Options:
|
||||||
|
--version show program's version number and exit
|
||||||
|
-h, --help show this help message and exit
|
||||||
|
-b BIND_ADDR, --bind=BIND_ADDR
|
||||||
|
bind [host]:port (host defaults to '')
|
||||||
|
-d DEST_ADDR, --dest=DEST_ADDR
|
||||||
|
receiver [backend:]host:port (backend defaults to
|
||||||
|
'graphite')
|
||||||
|
-v increase verbosity (currently used for debugging)
|
||||||
|
-f INTERVAL, --flush=INTERVAL
|
||||||
|
flush interval, in seconds (default 10)
|
||||||
|
-p PERCENT, --percent=PERCENT
|
||||||
|
percent threshold (default 90)
|
||||||
|
-l, --list list supported backends
|
||||||
|
-D, --daemonize daemonize the service
|
||||||
|
|
||||||
Start gstatsd and send stats to port 9100 every 5 seconds:
|
Start gstatsd and send stats to port 9100 every 5 seconds:
|
||||||
|
|
||||||
% gstatsd -d :9100 -f 5
|
% gstatsd -d :9100 -f 5
|
||||||
|
@ -18,3 +51,48 @@ Bind listener to host 'hostname' port 8126:
|
||||||
|
|
||||||
% gstatsd -b hostname:8126 -d :9100 -f 5
|
% gstatsd -b hostname:8126 -d :9100 -f 5
|
||||||
|
|
||||||
|
To send the stats to multiple graphite servers, specify multiple destinations:
|
||||||
|
|
||||||
|
% gstatsd -b :8125 -d stats1:9100 stats2:9100
|
||||||
|
|
||||||
|
|
||||||
|
Using the client
|
||||||
|
----------------
|
||||||
|
|
||||||
|
The code example below demonstrates using the low-level client interface:
|
||||||
|
|
||||||
|
from gstatsd import client
|
||||||
|
|
||||||
|
# location of the statsd server
|
||||||
|
hostport = ('', 8125)
|
||||||
|
|
||||||
|
raw = client.StatsClient(hostport)
|
||||||
|
|
||||||
|
# add 1 to the 'foo' bucket
|
||||||
|
raw.increment('foo')
|
||||||
|
|
||||||
|
# timer 'bar' took 25ms to complete
|
||||||
|
raw.timer('bar', 25)
|
||||||
|
|
||||||
|
|
||||||
|
You may prefer to use the stateful client:
|
||||||
|
|
||||||
|
# wraps the raw client
|
||||||
|
cli = client.Stats(raw)
|
||||||
|
|
||||||
|
timer = cli.get_timer('foo')
|
||||||
|
timer.start()
|
||||||
|
|
||||||
|
... do some work ..
|
||||||
|
|
||||||
|
# when .stop() is called, the stat is sent to the server
|
||||||
|
timer.stop()
|
||||||
|
|
||||||
|
|
||||||
|
[python]: http://www.python.org/
|
||||||
|
[gevent]: http://www.gevent.org/
|
||||||
|
[license]: http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
[distribute]: http://pypi.python.org/pypi/distribute
|
||||||
|
[etsy repo]: https://github.com/etsy/statsd
|
||||||
|
[etsy post]: http://codeascraft.etsy.com/2011/02/15/measure-anything-measure-everything/
|
||||||
|
|
||||||
|
|
|
@ -1,3 +1,9 @@
|
||||||
|
gstatsd (0.3) lucid; urgency=low
|
||||||
|
|
||||||
|
* Release.
|
||||||
|
|
||||||
|
-- Patrick Hensley <spaceboy@indirect.com> Wed, 29 Jun 2011 11:59:01 -0400
|
||||||
|
|
||||||
gstatsd (0.2) lucid; urgency=low
|
gstatsd (0.2) lucid; urgency=low
|
||||||
|
|
||||||
* Initial release.
|
* Initial release.
|
||||||
|
|
|
@ -1,7 +1,12 @@
|
||||||
|
|
||||||
|
|
||||||
|
# standard
|
||||||
import random
|
import random
|
||||||
import socket
|
import socket
|
||||||
|
import time
|
||||||
|
|
||||||
|
|
||||||
|
E_NOSTART = 'you must call start() before stop(). ignoring.'
|
||||||
|
|
||||||
|
|
||||||
class StatsClient(object):
|
class StatsClient(object):
|
||||||
|
@ -17,7 +22,7 @@ class StatsClient(object):
|
||||||
self._sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
|
self._sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
|
||||||
|
|
||||||
def timer(self, key, timestamp, sample_rate=1):
|
def timer(self, key, timestamp, sample_rate=1):
|
||||||
self._send('%s:%d|ms' % (key, timestamp), sample_rate)
|
self._send('%s:%d|ms' % (key, round(timestamp)), sample_rate)
|
||||||
|
|
||||||
def increment(self, key, sample_rate=1):
|
def increment(self, key, sample_rate=1):
|
||||||
return self.counter(key, 1, sample_rate)
|
return self.counter(key, 1, sample_rate)
|
||||||
|
@ -29,7 +34,7 @@ class StatsClient(object):
|
||||||
if not isinstance(keys, (list, tuple)):
|
if not isinstance(keys, (list, tuple)):
|
||||||
keys = [keys]
|
keys = [keys]
|
||||||
for key in keys:
|
for key in keys:
|
||||||
self._send('%s:%s|c' % (key, magnitude), sample_rate)
|
self._send('%s:%d|c' % (key, round(magnitude)), sample_rate)
|
||||||
|
|
||||||
def _send(self, data, sample_rate=1):
|
def _send(self, data, sample_rate=1):
|
||||||
packet = None
|
packet = None
|
||||||
|
@ -41,3 +46,53 @@ class StatsClient(object):
|
||||||
if packet:
|
if packet:
|
||||||
self._sock.sendto(packet, self._hostport)
|
self._sock.sendto(packet, self._hostport)
|
||||||
|
|
||||||
|
|
||||||
|
class StatsCounter(object):
|
||||||
|
|
||||||
|
def __init__(self, client, key, sample_rate=1):
|
||||||
|
self._client = client
|
||||||
|
self._key = key
|
||||||
|
self._sample_rate = sample_rate
|
||||||
|
|
||||||
|
def increment(self):
|
||||||
|
self._client.increment(self._key, self._sample_rate)
|
||||||
|
|
||||||
|
def decrement(self):
|
||||||
|
self._client.decrement(self._key, self._sample_rate)
|
||||||
|
|
||||||
|
def add(self, val):
|
||||||
|
self._client.counter(self._key, val, self._sample_rate)
|
||||||
|
|
||||||
|
|
||||||
|
class StatsTimer(object):
|
||||||
|
|
||||||
|
def __init__(self, client, key):
|
||||||
|
self._client = client
|
||||||
|
self._key = key
|
||||||
|
self._started = 0
|
||||||
|
self._timestamp = 0
|
||||||
|
|
||||||
|
def start(self):
|
||||||
|
self._started = 1
|
||||||
|
self._timestamp = time.time()
|
||||||
|
|
||||||
|
def stop(self):
|
||||||
|
if not self._started:
|
||||||
|
raise UserWarning(E_NOSTART)
|
||||||
|
return
|
||||||
|
elapsed = time.time() - self._timestamp
|
||||||
|
self._client.timer(self._key, int(elapsed / 1000.0))
|
||||||
|
self._started = 0
|
||||||
|
|
||||||
|
|
||||||
|
class Stats(object):
|
||||||
|
|
||||||
|
def __init__(self, client):
|
||||||
|
self._client = client
|
||||||
|
|
||||||
|
def get_counter(self, key, sample_rate=1):
|
||||||
|
return StatsCounter(self._client, key, sample_rate)
|
||||||
|
|
||||||
|
def get_timer(self, key):
|
||||||
|
return StatsTimer(self._client, key)
|
||||||
|
|
||||||
|
|
|
@ -0,0 +1,89 @@
|
||||||
|
|
||||||
|
# standard
|
||||||
|
import unittest
|
||||||
|
|
||||||
|
# local
|
||||||
|
from gstatsd import client
|
||||||
|
|
||||||
|
|
||||||
|
class StatsDummyClient(client.StatsClient):
|
||||||
|
|
||||||
|
def __init__(self, hostport=None):
|
||||||
|
client.StatsClient.__init__(self, hostport)
|
||||||
|
self.packets = []
|
||||||
|
|
||||||
|
def _send(self, data, sample_rate=1):
|
||||||
|
self.packets.append((data, sample_rate))
|
||||||
|
|
||||||
|
|
||||||
|
class StatsClientTest(unittest.TestCase):
|
||||||
|
|
||||||
|
def setUp(self):
|
||||||
|
self._cli = StatsDummyClient()
|
||||||
|
|
||||||
|
def test_timer(self):
|
||||||
|
self._cli.timer('foo', 15, 1)
|
||||||
|
self.assertEquals(self._cli.packets[-1], ('foo:15|ms', 1))
|
||||||
|
self._cli.timer('bar.baz', 1.35, 1)
|
||||||
|
self.assertEquals(self._cli.packets[-1], ('bar.baz:1|ms', 1))
|
||||||
|
self._cli.timer('x', 1.99, 1)
|
||||||
|
self.assertEquals(self._cli.packets[-1], ('x:2|ms', 1))
|
||||||
|
self._cli.timer('x', 1, 0.5)
|
||||||
|
self.assertEquals(self._cli.packets[-1], ('x:1|ms', 0.5))
|
||||||
|
|
||||||
|
def test_increment(self):
|
||||||
|
self._cli.increment('foo')
|
||||||
|
self.assertEquals(self._cli.packets[-1], ('foo:1|c', 1))
|
||||||
|
self._cli.increment('x', 0.5)
|
||||||
|
self.assertEquals(self._cli.packets[-1], ('x:1|c', 0.5))
|
||||||
|
|
||||||
|
def test_decrement(self):
|
||||||
|
self._cli.decrement('foo')
|
||||||
|
self.assertEquals(self._cli.packets[-1], ('foo:-1|c', 1))
|
||||||
|
self._cli.decrement('x', 0.2)
|
||||||
|
self.assertEquals(self._cli.packets[-1], ('x:-1|c', 0.2))
|
||||||
|
|
||||||
|
def test_counter(self):
|
||||||
|
self._cli.counter('foo', 5)
|
||||||
|
self.assertEquals(self._cli.packets[-1], ('foo:5|c', 1))
|
||||||
|
self._cli.counter('foo', -50)
|
||||||
|
self.assertEquals(self._cli.packets[-1], ('foo:-50|c', 1))
|
||||||
|
self._cli.counter('foo', 5.9)
|
||||||
|
self.assertEquals(self._cli.packets[-1], ('foo:6|c', 1))
|
||||||
|
self._cli.counter('foo', 1, 0.2)
|
||||||
|
self.assertEquals(self._cli.packets[-1], ('foo:1|c', 0.2))
|
||||||
|
|
||||||
|
|
||||||
|
class StatsTest(unittest.TestCase):
|
||||||
|
|
||||||
|
def setUp(self):
|
||||||
|
self._cli = StatsDummyClient()
|
||||||
|
self._stat = client.Stats(self._cli)
|
||||||
|
|
||||||
|
def test_timer(self):
|
||||||
|
timer = self._stat.get_timer('foo')
|
||||||
|
timer.start()
|
||||||
|
timer.stop()
|
||||||
|
data, sr = self._cli.packets[-1]
|
||||||
|
pkt = data.split(':')
|
||||||
|
self.assertEquals(pkt[0], 'foo')
|
||||||
|
|
||||||
|
# ensure warning is raised for mismatched start/stop
|
||||||
|
timer = self._stat.get_timer('foo')
|
||||||
|
self.assertRaises(UserWarning, timer.stop)
|
||||||
|
|
||||||
|
def test_counter(self):
|
||||||
|
count = self._stat.get_counter('foo')
|
||||||
|
count.increment()
|
||||||
|
count.decrement()
|
||||||
|
count.add(5)
|
||||||
|
self.assertEquals(self._cli.packets[-1], ('foo:5|c', 1))
|
||||||
|
|
||||||
|
|
||||||
|
def main():
|
||||||
|
unittest.main()
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == '__main__':
|
||||||
|
main()
|
||||||
|
|
|
@ -1,3 +1,3 @@
|
||||||
|
|
||||||
__version__ = '0.2'
|
__version__ = '0.3'
|
||||||
|
|
||||||
|
|
|
@ -29,8 +29,10 @@ A statsd service in Python + gevent.
|
||||||
'''
|
'''
|
||||||
|
|
||||||
# table to remove invalid characters from keys
|
# table to remove invalid characters from keys
|
||||||
KEY_VALIDCHARS = string.uppercase + string.lowercase + string.digits + '_-.'
|
ALL_ASCII = set(chr(c) for c in range(256))
|
||||||
KEY_SANITIZE = string.maketrans(KEY_VALIDCHARS + '/', KEY_VALIDCHARS + '_')
|
KEY_VALID = string.ascii_letters + string.digits + '_-.'
|
||||||
|
KEY_TABLE = string.maketrans(KEY_VALID + '/', KEY_VALID + '_')
|
||||||
|
KEY_DELETIONS = ''.join(ALL_ASCII.difference(KEY_VALID + '/'))
|
||||||
|
|
||||||
# error messages
|
# error messages
|
||||||
E_BADADDR = 'invalid bind address specified %r'
|
E_BADADDR = 'invalid bind address specified %r'
|
||||||
|
@ -156,7 +158,7 @@ class StatsDaemon(object):
|
||||||
self._error('packet: %r' % data)
|
self._error('packet: %r' % data)
|
||||||
if not parts:
|
if not parts:
|
||||||
return
|
return
|
||||||
key = parts[0].translate(KEY_SANITIZE, string.whitespace)
|
key = parts[0].translate(KEY_TABLE, KEY_DELETIONS)
|
||||||
for part in parts[1:]:
|
for part in parts[1:]:
|
||||||
srate = 1.0
|
srate = 1.0
|
||||||
fields = part.split('|')
|
fields = part.split('|')
|
||||||
|
@ -246,11 +248,12 @@ def main():
|
||||||
opts.add_option('-d', '--dest', dest='dest_addr', action='append',
|
opts.add_option('-d', '--dest', dest='dest_addr', action='append',
|
||||||
default=[],
|
default=[],
|
||||||
help="receiver [backend:]host:port (backend defaults to 'graphite')")
|
help="receiver [backend:]host:port (backend defaults to 'graphite')")
|
||||||
opts.add_option('-v', dest='verbose', action='count', default=0)
|
opts.add_option('-v', dest='verbose', action='count', default=0,
|
||||||
|
help="increase verbosity (currently used for debugging)")
|
||||||
opts.add_option('-f', '--flush', dest='interval', default=INTERVAL,
|
opts.add_option('-f', '--flush', dest='interval', default=INTERVAL,
|
||||||
help="flush interval, in seconds")
|
help="flush interval, in seconds (default 10)")
|
||||||
opts.add_option('-p', '--percent', dest='percent', default=PERCENT,
|
opts.add_option('-p', '--percent', dest='percent', default=PERCENT,
|
||||||
help="percent threshold")
|
help="percent threshold (default 90)")
|
||||||
opts.add_option('-l', '--list', dest='list_backends', action='store_true',
|
opts.add_option('-l', '--list', dest='list_backends', action='store_true',
|
||||||
help="list supported backends")
|
help="list supported backends")
|
||||||
opts.add_option('-D', '--daemonize', dest='daemonize', action='store_true',
|
opts.add_option('-D', '--daemonize', dest='daemonize', action='store_true',
|
||||||
|
|
|
@ -0,0 +1,69 @@
|
||||||
|
|
||||||
|
# standard
|
||||||
|
import unittest
|
||||||
|
|
||||||
|
# local
|
||||||
|
from gstatsd import service
|
||||||
|
|
||||||
|
|
||||||
|
class StatsServiceTest(unittest.TestCase):
|
||||||
|
|
||||||
|
def setUp(self):
|
||||||
|
args = (':8125', [':9100'], 5, 90, 0)
|
||||||
|
self.svc = service.StatsDaemon(*args)
|
||||||
|
|
||||||
|
def test_construct(self):
|
||||||
|
svc = service.StatsDaemon('8125', ['9100'], 5, 90, 0)
|
||||||
|
self.assertEquals(svc._bindaddr, ('', 8125))
|
||||||
|
self.assertEquals(svc._interval, 5.0)
|
||||||
|
self.assertEquals(svc._percent, 90.0)
|
||||||
|
self.assertEquals(svc._debug, 0)
|
||||||
|
self.assertEquals(svc._targets[0], (svc._send_graphite, ('', 9100)))
|
||||||
|
|
||||||
|
svc = service.StatsDaemon('bar:8125', ['foo:9100'], 5, 90, 1)
|
||||||
|
self.assertEquals(svc._bindaddr, ('bar', 8125))
|
||||||
|
self.assertEquals(svc._targets[0], (svc._send_graphite, ('foo', 9100)))
|
||||||
|
self.assertEquals(svc._debug, 1)
|
||||||
|
|
||||||
|
def test_backend(self):
|
||||||
|
service.StatsDaemon._send_foo = lambda self, x, y: None
|
||||||
|
svc = service.StatsDaemon('8125', ['foo:bar:9100'], 5, 90, 0)
|
||||||
|
self.assertEquals(svc._targets[0], (svc._send_foo, ('bar', 9100)))
|
||||||
|
|
||||||
|
def test_counters(self):
|
||||||
|
pkt = 'foo:1|c'
|
||||||
|
self.svc._process(pkt, None)
|
||||||
|
self.assertEquals(self.svc._counts, {'foo': 1})
|
||||||
|
self.svc._process(pkt, None)
|
||||||
|
self.assertEquals(self.svc._counts, {'foo': 2})
|
||||||
|
pkt = 'foo:-1|c'
|
||||||
|
self.svc._process(pkt, None)
|
||||||
|
self.assertEquals(self.svc._counts, {'foo': 1})
|
||||||
|
|
||||||
|
def test_counters_sampled(self):
|
||||||
|
pkt = 'foo:1|c|@.5'
|
||||||
|
self.svc._process(pkt, None)
|
||||||
|
self.assertEquals(self.svc._counts, {'foo': 2})
|
||||||
|
|
||||||
|
def test_timers(self):
|
||||||
|
pkt = 'foo:20|ms'
|
||||||
|
self.svc._process(pkt, None)
|
||||||
|
self.assertEquals(self.svc._timers, {'foo': [20.0]})
|
||||||
|
pkt = 'foo:10|ms'
|
||||||
|
self.svc._process(pkt, None)
|
||||||
|
self.assertEquals(self.svc._timers, {'foo': [20.0, 10.0]})
|
||||||
|
|
||||||
|
def test_key_sanitize(self):
|
||||||
|
pkt = '\t\n#! foo . bar \0 ^:1|c'
|
||||||
|
self.svc._process(pkt, None)
|
||||||
|
self.assertEquals(self.svc._counts, {'foo.bar': 1})
|
||||||
|
|
||||||
|
|
||||||
|
def main():
|
||||||
|
unittest.main()
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == '__main__':
|
||||||
|
main()
|
||||||
|
|
||||||
|
|
|
@ -1,22 +0,0 @@
|
||||||
|
|
||||||
from client import StatsClient
|
|
||||||
|
|
||||||
|
|
||||||
def main():
|
|
||||||
cli = StatsClient()
|
|
||||||
for num in range(1, 11):
|
|
||||||
cli.timer('foo', num)
|
|
||||||
return
|
|
||||||
cli.increment('baz', 0.5)
|
|
||||||
cli.increment('baz', 0.5)
|
|
||||||
cli.timer('t3', 100, 0.5)
|
|
||||||
return
|
|
||||||
|
|
||||||
cli.increment('foo')
|
|
||||||
cli.counter(['foo', 'bar'], 2)
|
|
||||||
cli.timer('t1', 12)
|
|
||||||
cli.timer('t2', 30)
|
|
||||||
|
|
||||||
if __name__ == '__main__':
|
|
||||||
main()
|
|
||||||
|
|
2
setup.py
2
setup.py
|
@ -24,7 +24,7 @@ def main():
|
||||||
'console_scripts': ['gstatsd=gstatsd.service:main']
|
'console_scripts': ['gstatsd=gstatsd.service:main']
|
||||||
},
|
},
|
||||||
classifiers = [
|
classifiers = [
|
||||||
"Development Status :: 2 - Pre-Alpha",
|
"Development Status :: 3 - Alpha",
|
||||||
"Intended Audience :: Developers",
|
"Intended Audience :: Developers",
|
||||||
"License :: OSI Approved :: Apache Software License",
|
"License :: OSI Approved :: Apache Software License",
|
||||||
"Operating System :: MacOS :: MacOS X",
|
"Operating System :: MacOS :: MacOS X",
|
||||||
|
|
Loading…
Reference in New Issue