diff --git a/.gitreview b/.gitreview new file mode 100644 index 0000000..add0bf0 --- /dev/null +++ b/.gitreview @@ -0,0 +1,4 @@ +[gerrit] +host=review.openstack.org +port=29418 +project=stackforge/tomograph.git diff --git a/doc/source/conf.py b/doc/source/conf.py new file mode 100644 index 0000000..0c3686f --- /dev/null +++ b/doc/source/conf.py @@ -0,0 +1,75 @@ +import os +import sys + +# If extensions (or modules to document with autodoc) are in another directory, +# add these directories to sys.path here. If the directory is relative to the +# documentation root, use os.path.abspath to make it absolute, like shown here. +sys.path.insert(0, os.path.abspath('../../')) +sys.path.insert(0, os.path.abspath('../')) +sys.path.insert(0, os.path.abspath('./')) +sys.path.insert(0, os.path.abspath('.')) + +from tomograph import version as tomograph_version + +# Supress warnings for docs that aren't used yet +#unused_docs = [ +#] + +# -- General configuration ----------------------------------------------------- + +# If your documentation needs a minimal Sphinx version, state it here. +#needs_sphinx = '1.0' + +# Add any Sphinx extension module names here, as strings. They can be extensions +# coming with Sphinx (named 'sphinx.ext.*') or your custom ones. +extensions = [ + 'sphinx.ext.intersphinx', +] + +intersphinx_mapping = { + 'sphinx': ('http://sphinx.pocoo.org', None) +} + +# The suffix of source filenames. +source_suffix = '.rst' + +# The master toctree document. +master_doc = 'index' + +# General information about the project. +project = 'TOMOGRAPH' + +# The version info for the project you're documenting, acts as replacement for +# |version| and |release|, also used in various other places throughout the +# built documents. +release = tomograph_version.version_string() +version = tomograph_version.canonical_version_string() + +# Set the default Pygments syntax +highlight_language = 'python' + +# List of patterns, relative to source directory, that match files and +# directories to ignore when looking for source files. +exclude_patterns = [] + +# If true, sectionauthor and moduleauthor directives will be shown in the +# output. They are ignored by default. +show_authors = False + +# -- Options for HTML output --------------------------------------------------- + +# The theme to use for HTML and HTML Help pages. See the documentation for +# a list of builtin themes. +html_theme = 'default' + +# Theme options are theme-specific and customize the look and feel of a theme +# further. For a list of options available for each theme, see the +# documentation. +html_theme_options = { + "bodyfont": "Arial, sans-serif", + "headfont": "Arial, sans-serif" +} + +# The name of an image file (relative to this directory) to place at the top +# of the sidebar. +#html_logo = 'img/tomograph-tiny.png' diff --git a/doc/source/index.rst b/doc/source/index.rst new file mode 100644 index 0000000..12de5b0 --- /dev/null +++ b/doc/source/index.rst @@ -0,0 +1,11 @@ +.. _index: + +===================== +Tomograph +===================== + +.. rubric:: A library to help distributed applications send trace information to +metrics backends like [Zipkin][zipkin] and [Statsd][statsd]. + + +---- diff --git a/py33-requirements.txt b/py33-requirements.txt new file mode 100644 index 0000000..2f4e3a1 --- /dev/null +++ b/py33-requirements.txt @@ -0,0 +1,3 @@ +webob +statsd + diff --git a/py33-test-requirements.txt b/py33-test-requirements.txt new file mode 100644 index 0000000..edb54da --- /dev/null +++ b/py33-test-requirements.txt @@ -0,0 +1,9 @@ +# Install bounded pep8/pyflakes first, then let flake8 install +pep8==1.4.5 +pyflakes>=0.7.2,<0.7.4 +flake8==2.0 +hacking>=0.5.6,<0.8 + +nose +nose-exclude +sphinx>=1.1.2 diff --git a/tools/pip-requires b/requirements.txt similarity index 96% rename from tools/pip-requires rename to requirements.txt index 20332ef..c53d103 100644 --- a/tools/pip-requires +++ b/requirements.txt @@ -1,4 +1,5 @@ -eventlet webob statsd +eventlet thrift + diff --git a/setup.cfg b/setup.cfg new file mode 100644 index 0000000..12d346f --- /dev/null +++ b/setup.cfg @@ -0,0 +1,39 @@ +[metadata] +name = tomograph +summary = Tiny tims tracing tomograph +description-file = + README.md +author = Tomograph Developers +author-email = timjr@yahoo-inc.com +classifier = + Development Status :: 3 - Alpha Development Status + Environment :: OpenStack + Intended Audience :: Information Technology + Intended Audience :: Developers + License :: OSI Approved :: Apache Software License + Operating System :: POSIX :: Linux + Programming Language :: Python + Programming Language :: Python :: 2 + Programming Language :: Python :: 2.6 + Programming Language :: Python :: 2.7 + Programming Language :: Python :: 3.3 + +[global] +setup-hooks = + pbr.hooks.setup_hook + +[files] +packages = + tomograph + +[nosetests] +cover-erase = true +verbosity = 2 + +[build_sphinx] +source-dir = doc/source +build-dir = doc/build +all_files = 1 + +[upload_sphinx] +upload-dir = docs/build/html diff --git a/setup.py b/setup.py index 97815d4..1139d8f 100755 --- a/setup.py +++ b/setup.py @@ -1,34 +1,23 @@ #!/usr/bin/env python +# Copyright (c) 2013 Hewlett-Packard Development Company, L.P. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# THIS FILE IS MANAGED BY THE GLOBAL REQUIREMENTS REPO - DO NOT EDIT import setuptools - -def read_requires(): - requires = [] - with open('tools/pip-requires', 'r') as fh: - contents = fh.read() - for line in contents.splitlines(): - line = line.strip() - if line.startswith("#") or not line: - continue - try: - (line, after) = line.split("#", 1) - except ValueError: - pass - if not line: - continue - requires.append(line) - return requires - - setuptools.setup( - name='tomograph', - version="0.0.1", - description='Tiny tims tracing tomograph', - author="Y! OpenStack Team", - author_email='timjr@yahoo-inc.com', - license='Apache License, Version 2.0', - packages=setuptools.find_packages(), - long_description=open('README.md').read(), - install_requires=read_requires(), -) \ No newline at end of file + setup_requires=['pbr'], + pbr=True) diff --git a/test-requirements.txt b/test-requirements.txt new file mode 100644 index 0000000..9d57fa1 --- /dev/null +++ b/test-requirements.txt @@ -0,0 +1,11 @@ +# Install bounded pep8/pyflakes first, then let flake8 install +pep8==1.4.5 +pyflakes>=0.7.2,<0.7.4 +flake8==2.0 +hacking>=0.5.6,<0.8 + +nose +nose-exclude +openstack.nose_plugin>=0.7 +pylint==0.25.2 +sphinx>=1.1.2 diff --git a/tests/basic.py b/tests/basic.py index cd9fdf3..182a9f7 100755 --- a/tests/basic.py +++ b/tests/basic.py @@ -1,6 +1,6 @@ #!/usr/bin/env python -# Copyright (c) 2012 Yahoo! Inc. All rights reserved. +# Copyright (c) 2012 Yahoo! Inc. All rights reserved. # Licensed under the Apache License, Version 2.0 (the "License"); you # may not use this file except in compliance with the License. You may # obtain a copy of the License at @@ -16,6 +16,7 @@ import tomograph import sys import time + @tomograph.traced('test server', 'server response', port=80) def server(latency): tomograph.annotate('this is an annotation') @@ -24,11 +25,13 @@ def server(latency): tomograph.tag('this is a string', 'foo') tomograph.tag('this is an int', 42) + @tomograph.traced('test client', 'client request') def client(client_overhead, server_latency): time.sleep(client_overhead) server(server_latency) + if __name__ == '__main__': if len(sys.argv) > 1: tomograph.config.set_backends(sys.argv[1:]) diff --git a/tests/bench.py b/tests/bench.py index 0de020f..1c9a64c 100644 --- a/tests/bench.py +++ b/tests/bench.py @@ -1,6 +1,6 @@ #!/usr/bin/env python -# Copyright (c) 2012 Yahoo! Inc. All rights reserved. +# Copyright (c) 2012 Yahoo! Inc. All rights reserved. # Licensed under the Apache License, Version 2.0 (the "License"); you # may not use this file except in compliance with the License. You may # obtain a copy of the License at @@ -11,11 +11,12 @@ # License for the specific language governing permissions and # limitations under the License. See accompanying LICENSE file. -import tomograph -import cProfile import sys import time +import tomograph + + @tomograph.traced('test server', 'server response', port=80) def server(latency): time.sleep(latency) @@ -26,14 +27,14 @@ def client(client_overhead, server_latency): time.sleep(client_overhead) server(server_latency) + def clientloop(): for i in xrange(10000): client(0, 0) + if __name__ == '__main__': if len(sys.argv) > 1: tomograph.config.set_backends(sys.argv[1:]) #cProfile.run('clientloop()', 'tomo-bench') clientloop() - - diff --git a/tomograph/__init__.py b/tomograph/__init__.py index 63a70c8..4e7bd36 100644 --- a/tomograph/__init__.py +++ b/tomograph/__init__.py @@ -1,4 +1,4 @@ -# Copyright (c) 2012 Yahoo! Inc. All rights reserved. +# Copyright (c) 2012 Yahoo! Inc. All rights reserved. # Licensed under the Apache License, Version 2.0 (the "License"); you # may not use this file except in compliance with the License. You may # obtain a copy of the License at @@ -12,28 +12,32 @@ ### Initialize logging in case it hasn't been done. We need two ### versions of this, one for the eventlet logging module and one for ### the non-eventlet one... + +from __future__ import absolute_import + import logging import sys + import eventlet eventlet_logging = eventlet.import_patched('logging') eventlet_sys = eventlet.import_patched('sys') + def _initLogging(logging, sys): - """ - set up some default stuff, in case nobody configured logging yet - """ + """Set up some default stuff, in case nobody configured logging yet.""" logger = logging.getLogger('tomograph') if logger.level == logging.NOTSET: logger.setLevel(logging.INFO) if not logger.handlers: handler = logging.StreamHandler(sys.stdout) - handler.setFormatter(logging.Formatter( - '%(asctime)s %(levelname)s %(name)s %(message)s')) + handler.setFormatter(logging.Formatter('%(asctime)s %(levelname)s ' + '%(name)s %(message)s')) logger.addHandler(handler) + _initLogging(logging, sys) _initLogging(eventlet_logging, eventlet_sys) -import config -from tomograph import * + +from tomograph.tomograph import * diff --git a/tomograph/backends/__init__.py b/tomograph/backends/__init__.py index 81a0a31..acd7346 100644 --- a/tomograph/backends/__init__.py +++ b/tomograph/backends/__init__.py @@ -1,4 +1,4 @@ -# Copyright (c) 2012 Yahoo! Inc. All rights reserved. +# Copyright (c) 2012 Yahoo! Inc. All rights reserved. # Licensed under the Apache License, Version 2.0 (the "License"); you # may not use this file except in compliance with the License. You may # obtain a copy of the License at @@ -8,4 +8,3 @@ # OR CONDITIONS OF ANY KIND, either express or implied. See the # License for the specific language governing permissions and # limitations under the License. See accompanying LICENSE file. - diff --git a/tomograph/backends/log/__init__.py b/tomograph/backends/log/__init__.py index 887367f..d2cf4b1 100644 --- a/tomograph/backends/log/__init__.py +++ b/tomograph/backends/log/__init__.py @@ -1,4 +1,4 @@ -# Copyright (c) 2012 Yahoo! Inc. All rights reserved. +# Copyright (c) 2012 Yahoo! Inc. All rights reserved. # Licensed under the Apache License, Version 2.0 (the "License"); you # may not use this file except in compliance with the License. You may # obtain a copy of the License at @@ -14,5 +14,6 @@ import sys logger = logging.getLogger(__name__) + def send(span): logger.info(span) diff --git a/tomograph/backends/statsd/__init__.py b/tomograph/backends/statsd/__init__.py index 083acf3..ff5fec4 100644 --- a/tomograph/backends/statsd/__init__.py +++ b/tomograph/backends/statsd/__init__.py @@ -1,4 +1,4 @@ -# Copyright (c) 2012 Yahoo! Inc. All rights reserved. +# Copyright (c) 2012 Yahoo! Inc. All rights reserved. # Licensed under the Apache License, Version 2.0 (the "License"); you # may not use this file except in compliance with the License. You may # obtain a copy of the License at @@ -9,8 +9,5 @@ # License for the specific language governing permissions and # limitations under the License. See accompanying LICENSE file. + from statsd import * - - - - diff --git a/tomograph/backends/statsd/statsd.py b/tomograph/backends/statsd/statsd.py index 6b31684..45db3a9 100644 --- a/tomograph/backends/statsd/statsd.py +++ b/tomograph/backends/statsd/statsd.py @@ -1,4 +1,4 @@ -# Copyright (c) 2012 Yahoo! Inc. All rights reserved. +# Copyright (c) 2012 Yahoo! Inc. All rights reserved. # Licensed under the Apache License, Version 2.0 (the "License"); you # may not use this file except in compliance with the License. You may # obtain a copy of the License at @@ -10,8 +10,8 @@ # limitations under the License. See accompanying LICENSE file. import eventlet -from tomograph import config from tomograph import cache +from tomograph import config logging = eventlet.import_patched('logging') socket = eventlet.import_patched('socket') @@ -25,17 +25,22 @@ hostname_cache = cache.Cache(socket.gethostbyname) lock = threading.Lock() + def send(span): def statsd_send(name, value, units): - stat = str(name).replace(' ', '-') + ':' + str(int(value)) + '|' + str(units) + stat = (str(name).replace(' ', '-') + ':' + str(int(value)) + + '|' + str(units)) with lock: try: - udp_socket.sendto(stat, (hostname_cache.get(config.statsd_host), config.statsd_port)) + udp_socket.sendto(stat, + (hostname_cache.get(config.statsd_host), + config.statsd_port)) except Exception: if config.debug: - logger.warning("Error sending metric to statsd.", exc_info=True) - + logger.warning("Error sending metric to statsd.", + exc_info=True) + def server_name(note): address = note.address.replace('.', '-') return note.service_name + ' ' + address + ' ' + str(note.port) diff --git a/tomograph/backends/zipkin/__init__.py b/tomograph/backends/zipkin/__init__.py index 2da7a37..0a383b4 100644 --- a/tomograph/backends/zipkin/__init__.py +++ b/tomograph/backends/zipkin/__init__.py @@ -1,4 +1,4 @@ -# Copyright (c) 2012 Yahoo! Inc. All rights reserved. +# Copyright (c) 2012 Yahoo! Inc. All rights reserved. # Licensed under the Apache License, Version 2.0 (the "License"); you # may not use this file except in compliance with the License. You may # obtain a copy of the License at diff --git a/tomograph/backends/zipkin/sender.py b/tomograph/backends/zipkin/sender.py index a788397..0f7750a 100644 --- a/tomograph/backends/zipkin/sender.py +++ b/tomograph/backends/zipkin/sender.py @@ -1,8 +1,20 @@ +# Copyright (c) 2012 Yahoo! Inc. All rights reserved. +# Licensed under the Apache License, Version 2.0 (the "License"); you +# may not use this file except in compliance with the License. You may +# obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 Unless required by +# applicable law or agreed to in writing, software distributed under +# the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES +# OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and +# limitations under the License. See accompanying LICENSE file. + import eventlet socket = eventlet.import_patched('socket') time = eventlet.import_patched('time') -scribe = eventlet.import_patched('tomograph.backends.zipkin.generated.scribe.scribe') +scribe = eventlet.import_patched('tomograph.backends.zipkin.' + 'generated.scribe.scribe') TTransport = eventlet.import_patched('thrift.transport.TTransport') TSocket = eventlet.import_patched('thrift.transport.TSocket') collections = eventlet.import_patched('collections') @@ -11,9 +23,10 @@ threading = eventlet.import_patched('threading') from thrift.protocol import TBinaryProtocol + class ScribeSender(object): - def __init__(self, host='127.0.0.1', port=1463,debug=False, - target_write_size=1000, max_write_interval=1.0, + def __init__(self, host='127.0.0.1', port=1463, debug=False, + target_write_size=1000, max_write_interval=1.0, socket_timeout=5.0, max_queue_length=50000, must_yield=True): self.dropped = 0 self._remote_host = host @@ -35,13 +48,11 @@ class ScribeSender(object): self.flush() def send(self, category, msg): - """ - Send one record to scribe. - """ + """Send one record to scribe.""" log_entry = scribe.LogEntry(category=category, message=msg) self._log_buffer.append(log_entry) self._dropMsgs() - + now = time.time() if len(self._log_buffer) >= self._target_write_size or \ now - self._last_write > self._max_write_interval: @@ -66,22 +77,24 @@ class ScribeSender(object): buf.append(self._log_buffer.popleft()) if buf: if self._debug: - print "ScribeSender: flushing {0} msgs".format(len(buf)) + print("ScribeSender: flushing {0} msgs".format(len(buf))) try: client = self._getClient() result = client.Log(messages=buf) if result == scribe.ResultCode.TRY_LATER: dropped += len(buf) - except: + except Exception: if self._debug: - print "ScribeSender: caught exception writing log message:" + print("ScribeSender: caught exception writing " + "log message:") traceback.print_exc() dropped += len(buf) finally: self._lock.release() self.dropped += dropped if self._debug and dropped: - print "ScribeSender: dropped {0} messages for communication problem.".format(dropped) + print("ScribeSender: dropped {0} messages for " + "communication problem.".format(dropped)) def _dropMsgs(self): dropped = 0 @@ -90,7 +103,8 @@ class ScribeSender(object): dropped += 1 self.dropped += dropped if self._debug and dropped: - print "ScribeSender: dropped {0} messages for queue length.".format(dropped) + print("ScribeSender: dropped {0} messages for queue " + "length.".format(dropped)) def _getClient(self): # We can't just keep a connection because the app might fork @@ -102,4 +116,3 @@ class ScribeSender(object): transport.open() protocol = TBinaryProtocol.TBinaryProtocolAccelerated(transport) return scribe.Client(protocol) - diff --git a/tomograph/backends/zipkin/zipkin.py b/tomograph/backends/zipkin/zipkin.py index 17c8499..3388af5 100644 --- a/tomograph/backends/zipkin/zipkin.py +++ b/tomograph/backends/zipkin/zipkin.py @@ -1,4 +1,4 @@ -# Copyright (c) 2012 Yahoo! Inc. All rights reserved. +# Copyright (c) 2012 Yahoo! Inc. All rights reserved. # Licensed under the Apache License, Version 2.0 (the "License"); you # may not use this file except in compliance with the License. You may # obtain a copy of the License at @@ -9,54 +9,64 @@ # License for the specific language governing permissions and # limitations under the License. See accompanying LICENSE file. -from generated.scribe import scribe -import sender -import zipkin_thrift -from thrift.transport import TTransport -from thrift.transport import TSocket +from __future__ import print_function + from thrift.protocol import TBinaryProtocol +from thrift.transport import TSocket +from thrift.transport import TTransport + +from tomograph.backends.zipkin.generated.scribe import scribe +from tomograph.backends.zipkin import sender +from tomograph.backends.zipkin import zipkin_thrift -from tomograph import config from tomograph import cache +from tomograph import config -import base64 import StringIO -import time + +import atexit +import base64 import random import socket import struct import sys +import time import traceback -import atexit -scribe_sender = sender.ScribeSender(host=config.zipkin_host, - port=config.zipkin_port, - socket_timeout=config.zipkin_socket_timeout, - target_write_size=config.zipkin_target_write_size, - max_queue_length=config.zipkin_max_queue_length, - must_yield=config.zipkin_must_yield, - max_write_interval=config.zipkin_max_write_interval, - debug=config.zipkin_debug_scribe_sender) +scribe_config = { + 'host': config.zipkin_host, + 'port': config.zipkin_port, + 'socket_timeout': config.zipkin_socket_timeout, + 'target_write_size': config.zipkin_target_write_size, + 'max_queue_length': config.zipkin_max_queue_length, + 'must_yield': config.zipkin_must_yield, + 'max_write_interval': config.zipkin_max_write_interval, + 'debug': config.zipkin_debug_scribe_sender, +} +scribe_sender = sender.ScribeSender(**scribe_config) atexit.register(scribe_sender.close) hostname_cache = cache.Cache(socket.gethostbyname) + def send(span): def endpoint(note): try: ip = hostname_cache.get(note.address) - except: - print >>sys.stderr, 'host resolution error: ', traceback.format_exc() + except Exception: + print('host resolution error: %s' % traceback.format_exc(), + file=sys.stderr) ip = '0.0.0.0' - return zipkin_thrift.Endpoint(ipv4 = ip_to_i32(ip), - port = port_to_i16(note.port), - service_name = note.service_name) + return zipkin_thrift.Endpoint(ipv4=ip_to_i32(ip), + port=port_to_i16(note.port), + service_name=note.service_name) + def annotation(note): - return zipkin_thrift.Annotation(timestamp = int(note.time * 1e6), - value = note.value, - duration = note.duration, - host = endpoint(note)) + return zipkin_thrift.Annotation(timestamp=int(note.time * 1e6), + value=note.value, + duration=note.duration, + host=endpoint(note)) def binary_annotation(dimension): if isinstance(dimension.value, str): @@ -70,18 +80,18 @@ def send(span): val = struct.pack('>q', dimension.value) else: raise RuntimeError("unsupported tag type") - return zipkin_thrift.BinaryAnnotation(key = dimension.key, - value = val, - annotation_type = tag_type, - host = endpoint(dimension)) + return zipkin_thrift.BinaryAnnotation(key=dimension.key, + value=val, + annotation_type=tag_type, + host=endpoint(dimension)) - zspan = zipkin_thrift.Span(trace_id = span.trace_id, - id = span.id, - name = span.name, - parent_id = span.parent_id, - annotations = [annotation(n) for n in span.notes], - binary_annotations = \ - [binary_annotation(d) for d in span.dimensions]) + binary_annotations = [binary_annotation(d) for d in span.dimensions] + zspan = zipkin_thrift.Span(trace_id=span.trace_id, + id=span.id, + name=span.name, + parent_id=span.parent_id, + annotations=[annotation(n) for n in span.notes], + binary_annotations=binary_annotations) out = StringIO.StringIO() #raw = TBinaryProtocol.TBinaryProtocolAccelerated(out) raw = TBinaryProtocol.TBinaryProtocol(out) @@ -91,12 +101,14 @@ def send(span): traceback.print_exc() scribe_sender.send('zipkin', base64.b64encode(out.getvalue())) + def ip_to_i32(ip_str): """convert an ip address from a string to a signed 32-bit number""" return struct.unpack('!i', socket.inet_aton(ip_str))[0] + def port_to_i16(port_num): """conver a port number to a signed 16-bit int""" - if port_num > 2**15: - port_num -= 2**16 + if port_num > 2 ** 15: + port_num -= 2 ** 16 return port_num diff --git a/tomograph/backends/zipkin/zipkin_thrift.py b/tomograph/backends/zipkin/zipkin_thrift.py index 9607539..41dd607 100644 --- a/tomograph/backends/zipkin/zipkin_thrift.py +++ b/tomograph/backends/zipkin/zipkin_thrift.py @@ -1,4 +1,4 @@ -# Copyright (c) 2012 Yahoo! Inc. All rights reserved. +# Copyright (c) 2012 Yahoo! Inc. All rights reserved. # Licensed under the Apache License, Version 2.0 (the "License"); you # may not use this file except in compliance with the License. You may # obtain a copy of the License at diff --git a/tomograph/cache.py b/tomograph/cache.py index 62d86fd..df6342a 100644 --- a/tomograph/cache.py +++ b/tomograph/cache.py @@ -1,4 +1,4 @@ -# Copyright (c) 2012 Yahoo! Inc. All rights reserved. +# Copyright (c) 2012 Yahoo! Inc. All rights reserved. # Licensed under the Apache License, Version 2.0 (the "License"); you # may not use this file except in compliance with the License. You may # obtain a copy of the License at @@ -11,6 +11,7 @@ import threading + class Cache(object): def __init__(self, thunk, size_limit=1000): self._map = {} @@ -20,7 +21,7 @@ class Cache(object): def get(self, k): with self._lock: - if self._map.has_key(k): + if k in self._map: return self._map[k] else: while len(self._map) >= self._size_limit: @@ -28,4 +29,3 @@ class Cache(object): v = self._thunk(k) self._map[k] = v return v - diff --git a/tomograph/config.py b/tomograph/config.py index c261d51..f815d73 100644 --- a/tomograph/config.py +++ b/tomograph/config.py @@ -1,4 +1,4 @@ -# Copyright (c) 2012 Yahoo! Inc. All rights reserved. +# Copyright (c) 2012 Yahoo! Inc. All rights reserved. # Licensed under the Apache License, Version 2.0 (the "License"); you # may not use this file except in compliance with the License. You may # obtain a copy of the License at @@ -29,17 +29,16 @@ zipkin_max_queue_length = 50000 zipkin_target_write_size = 1000 zipkin_max_write_interval = 1 zipkin_must_yield = True -zipkin_debug_scribe_sender=False +zipkin_debug_scribe_sender = False debug = False db_tracing_enabled = True db_trace_as_spans = False + def set_backends(backends): - """ - Set the list of enabled backends. Backend name should be the full - module name of the backend. All backends must support a - send(span) method. + """Set the list of enabled backends. Backend name should be the full + module name of the backend. All backends must support a send(span) method. """ global enabled_backends global backend_modules @@ -53,11 +52,11 @@ def set_backends(backends): module = getattr(module, submodule) backend_modules.append(module) except (ImportError, AttributeError, ValueError) as err: - raise RuntimeError('Could not load tomograph backend {0}: {1}'.format( - backend, err)) + raise RuntimeError('Could not load tomograph backend ' + '{0}: {1}'.format(backend, err)) + def get_backends(): if not backend_modules: set_backends(enabled_backends) return backend_modules - diff --git a/tomograph/tomograph.py b/tomograph/tomograph.py index bafed92..8566fc0 100644 --- a/tomograph/tomograph.py +++ b/tomograph/tomograph.py @@ -1,4 +1,4 @@ -# Copyright (c) 2012 Yahoo! Inc. All rights reserved. +# Copyright (c) 2012 Yahoo! Inc. All rights reserved. # Licensed under the Apache License, Version 2.0 (the "License"); you # may not use this file except in compliance with the License. You may # obtain a copy of the License at @@ -9,21 +9,27 @@ # License for the specific language governing permissions and # limitations under the License. See accompanying LICENSE file. -import config -from types import Span, Note, Tag +from __future__ import absolute_import -import random -import sys -import time -from eventlet import corolocal -import socket -import pickle import base64 import logging +import pickle +import random +import socket +import sys +import time + +from eventlet import corolocal + +from tomograph import config +from tomograph import types + import webob.dec + span_stack = corolocal.local() + def start(service_name, name, address, port, trace_info=None): parent_id = None if tracing_started(): @@ -37,35 +43,43 @@ def start(service_name, name, address, port, trace_info=None): parent_id = trace_info[1] span_stack.spans = [] - span = Span(trace_id, parent_id, getId(), name, [], []) + span = types.Span(trace_id, parent_id, getId(), name, [], []) span_stack.spans.append(span) annotate('start', service_name, address, port) + def tracing_started(): return hasattr(span_stack, 'trace_id') + def cur_span(): if not tracing_started(): start('orphan', 'orphan', '127.0.0.1', '1') return span_stack.spans[-1] + def get_trace_info(): if tracing_started(): return (span_stack.trace_id, cur_span().id) else: return None + def stop(name): annotate('stop') span = span_stack.spans.pop() - assert span.name == name, 'start span name {0} not equal to end span name {1}'.format(span.name, name) + assert span.name == name, ('start span name {0} not equal ' + 'to end span name {1}'.format(span.name, name)) if not span_stack.spans: del(span_stack.trace_id) for backend in config.get_backends(): backend.send(span) + def annotate(value, service_name=None, address=None, port=None, duration=None): - """add an annotation at a particular point in time (with an optional duration)""" + """Add an annotation at a particular point in time, (with an optional + duration). + """ # attempt to default some values if service_name is None: service_name = cur_span().notes[0].service_name @@ -75,32 +89,39 @@ def annotate(value, service_name=None, address=None, port=None, duration=None): port = cur_span().notes[0].port if duration is None: duration = 0 - note = Note(time.time(), str(value), service_name, address, int(port), - int(duration)) + note = types.Note(time.time(), str(value), service_name, address, + int(port), int(duration)) cur_span().notes.append(note) + def tag(key, value, service_name=None, address=None, port=None): - """add a key/value tag to the current span. values can be int, - float, or string.""" - assert isinstance(value, str) or isinstance(value, int) or isinstance(value, float) + """Add a key/value tag to the current span. values can be int, + float, or string. + """ + assert (isinstance(value, str) or isinstance(value, int) + or isinstance(value, float)) if service_name is None: service_name = cur_span().notes[0].service_name if address is None: address = cur_span().notes[0].address if port is None: port = cur_span().notes[0].port - tag = Tag(str(key), value, service_name, address, port) + tag = types.Tag(str(key), value, service_name, address, port) cur_span().dimensions.append(tag) - + + def getId(): return random.randrange(sys.maxint >> 10) + ## wrapper/decorators def tracewrap(func, service_name, name, host='0.0.0.0', port=0): if host == '0.0.0.0': host = socket.gethostname() + def trace_and_call(*args, **kwargs): - if service_name is None and len(args) > 0 and isinstance(args[0], object): + if service_name is None and len(args) > 0 \ + and isinstance(args[0], object): s = args[0].__class__.__name__ else: s = service_name @@ -108,24 +129,29 @@ def tracewrap(func, service_name, name, host='0.0.0.0', port=0): ret = func(*args, **kwargs) stop(name) return ret + return trace_and_call + def traced(service_name, name, host='0.0.0.0', port=0): + def t1(func): return tracewrap(func, service_name, name, host, port) + return t1 -## sqlalchemy event listeners +## sqlalchemy event listeners def before_execute(name): + def handler(conn, clauseelement, multiparams, params): if not config.db_tracing_enabled: return h = str(conn.connection.connection) a = h.find("'") - b = h.find("'", a+1) + b = h.find("'", a + 1) if b > a: - h = h[a+1:b] + h = h[a + 1:b] else: h = 'unknown' port = conn.connection.connection.port @@ -134,8 +160,10 @@ def before_execute(name): if config.db_trace_as_spans: start(str(name) + 'db client', 'execute', h, port) annotate(clauseelement) + return handler + def after_execute(name): # name isn't used, at least not yet... def handler(conn, clauseelement, multiparams, params, result): @@ -145,13 +173,16 @@ def after_execute(name): # fix up the duration on the annotation for the sql query start_time = cur_span().notes[0].time last_note = cur_span().notes.pop() - cur_span().notes.append(Note(last_note.time, last_note.value, - last_note.service_name, last_note.address, - last_note.port, time.time() - start_time)) + cur_span().notes.append(types.Note(last_note.time, last_note.value, + last_note.service_name, + last_note.address, + last_note.port, + time.time() - start_time)) if config.db_trace_as_spans: stop('execute') return handler + def dbapi_error(name): def handler(conn, cursor, statement, parameters, context, exception): if not config.db_tracing_enabled: @@ -160,6 +191,7 @@ def dbapi_error(name): stop('execute') return handler + ## http helpers def start_http(service_name, name, request): trace_info_enc = request.headers.get('X-Trace-Info') @@ -170,6 +202,7 @@ def start_http(service_name, name, request): trace_info = None start(service_name, name, host, port, trace_info) + def add_trace_info_header(headers): trace_info = get_trace_info() if trace_info: @@ -178,15 +211,13 @@ def add_trace_info_header(headers): ## WSGI middleware class Middleware(object): - """ - WSGI Middleware that enables tomograph tracing for an application. - """ + """WSGI Middleware that enables tomograph tracing for an application.""" def __init__(self, application, service_name='Server', name='WSGI'): self.application = application self.service_name = service_name self.name = name - + @classmethod def factory(cls, global_conf, **local_conf): def filter(app): @@ -199,4 +230,3 @@ class Middleware(object): response = req.get_response(self.application) stop(self.name) return response - diff --git a/tomograph/types.py b/tomograph/types.py index 05bed38..7ed9a2b 100644 --- a/tomograph/types.py +++ b/tomograph/types.py @@ -1,4 +1,4 @@ -# Copyright (c) 2012 Yahoo! Inc. All rights reserved. +# Copyright (c) 2012 Yahoo! Inc. All rights reserved. # Licensed under the Apache License, Version 2.0 (the "License"); you # may not use this file except in compliance with the License. You may # obtain a copy of the License at @@ -9,8 +9,10 @@ # License for the specific language governing permissions and # limitations under the License. See accompanying LICENSE file. -from collections import namedtuple +import collections -Span = namedtuple('Span', 'trace_id parent_id id name notes dimensions') -Note = namedtuple('Note', 'time value service_name address port duration') -Tag = namedtuple('Tag', 'key value service_name address port') +Span = collections.namedtuple('Span', 'trace_id parent_id id name notes' + ' dimensions') +Note = collections.namedtuple('Note', 'time value service_name' + ' address port duration') +Tag = collections.namedtuple('Tag', 'key value service_name address port') diff --git a/tomograph/version.py b/tomograph/version.py new file mode 100644 index 0000000..69fe974 --- /dev/null +++ b/tomograph/version.py @@ -0,0 +1,25 @@ +# Copyright (c) 2013 Yahoo! Inc. All rights reserved. +# Licensed under the Apache License, Version 2.0 (the "License"); you +# may not use this file except in compliance with the License. You may +# obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 Unless required by +# applicable law or agreed to in writing, software distributed under +# the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES +# OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and +# limitations under the License. See accompanying LICENSE file. + +TOMOGRAPH_VERSION = ['2013', '1', None] +YEAR, COUNT, REVISION = TOMOGRAPH_VERSION +FINAL = False # May never be final ;) + + +def canonical_version_string(): + return '.'.join(filter(None, TOMOGRAPH_VERSION)) + + +def version_string(): + if FINAL: + return canonical_version_string() + else: + return '%s-dev' % (canonical_version_string()) diff --git a/tox.ini b/tox.ini new file mode 100644 index 0000000..0c45174 --- /dev/null +++ b/tox.ini @@ -0,0 +1,50 @@ +[tox] +minversion = 1.6 +skipsdist = True +envlist = py26,py27,py33,pep8 + +[testenv] +usedevelop = True +install_command = pip install {opts} {packages} +setenv = VIRTUAL_ENV={envdir} + LANG=en_US.UTF-8 + LANGUAGE=en_US:en + LC_ALL=C + NOSE_WITH_OPENSTACK=1 + NOSE_OPENSTACK_COLOR=1 + NOSE_OPENSTACK_RED=0.05 + NOSE_OPENSTACK_YELLOW=0.025 + NOSE_OPENSTACK_SHOW_ELAPSED=1 + NOSE_OPENSTACK_STDOUT=1 +deps = -r{toxinidir}/requirements.txt + -r{toxinidir}/test-requirements.txt +commands = nosetests {posargs} + +[testenv:py33] +deps = -r{toxinidir}/py33-requirements.txt + -r{toxinidir}/py33-test-requirements.txt +commands = true + +[tox:jenkins] +downloadcache = ~/cache/pip + +[testenv:pep8] +commands = + flake8 {posargs} + +[testenv:pylint] +setenv = VIRTUAL_ENV={envdir} +deps = -r{toxinidir}/requirements.txt + pylint==0.26.0 +commands = pylint + +[testenv:cover] +setenv = NOSE_WITH_COVERAGE=1 + +[testenv:venv] +commands = {posargs} + +[flake8] +ignore = H202,H402,F401,F403,H303 +builtins = _ +exclude = .venv,.tox,dist,doc,*egg,.git,build,tools,./tomograph/backends/zipkin/generated