stackforge/tomograph
Code Issues Proposed changes

Add supporting files

Browse Source 
add rudimentary documentation skeleton

dummy out the py33 check until we figure out how to handle it with thrift & eventlet

Change-Id: I16cb41e72ed6ae296e574ff4973a5ab28d49a100
This commit is contained in:
Joshua Harlow 2013-11-15 09:43:08 -08:00 committed by Tim Daly, Jr
parent a8d619c524
commit def962c912
26 changed files with 444 additions and 161 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
.gitreview Normal file
View File

@ -0,0 +1,4 @@
[gerrit]
host=review.openstack.org
port=29418
project=stackforge/tomograph.git

75
doc/source/conf.py Normal file
View File

@ -0,0 +1,75 @@
import os
import sys
# If extensions (or modules to document with autodoc) are in another directory,
# add these directories to sys.path here. If the directory is relative to the
# documentation root, use os.path.abspath to make it absolute, like shown here.
sys.path.insert(0, os.path.abspath('../../'))
sys.path.insert(0, os.path.abspath('../'))
sys.path.insert(0, os.path.abspath('./'))
sys.path.insert(0, os.path.abspath('.'))
from tomograph import version as tomograph_version
# Supress warnings for docs that aren't used yet
#unused_docs = [
#]
# -- General configuration -----------------------------------------------------
# If your documentation needs a minimal Sphinx version, state it here.
#needs_sphinx = '1.0'
# Add any Sphinx extension module names here, as strings. They can be extensions
# coming with Sphinx (named 'sphinx.ext.*') or your custom ones.
extensions = [
'sphinx.ext.intersphinx',
]
intersphinx_mapping = {
'sphinx': ('http://sphinx.pocoo.org', None)
}
# The suffix of source filenames.
source_suffix = '.rst'
# The master toctree document.
master_doc = 'index'
# General information about the project.
project = 'TOMOGRAPH'
# The version info for the project you're documenting, acts as replacement for
# |version| and |release|, also used in various other places throughout the
# built documents.
release = tomograph_version.version_string()
version = tomograph_version.canonical_version_string()
# Set the default Pygments syntax
highlight_language = 'python'
# List of patterns, relative to source directory, that match files and
# directories to ignore when looking for source files.
exclude_patterns = []
# If true, sectionauthor and moduleauthor directives will be shown in the
# output. They are ignored by default.
show_authors = False
# -- Options for HTML output ---------------------------------------------------
# The theme to use for HTML and HTML Help pages. See the documentation for
# a list of builtin themes.
html_theme = 'default'
# Theme options are theme-specific and customize the look and feel of a theme
# further. For a list of options available for each theme, see the
# documentation.
html_theme_options = {
"bodyfont": "Arial, sans-serif",
"headfont": "Arial, sans-serif"
}
# The name of an image file (relative to this directory) to place at the top
# of the sidebar.
#html_logo = 'img/tomograph-tiny.png'

11
doc/source/index.rst Normal file
View File

@ -0,0 +1,11 @@
.. _index:
=====================
Tomograph
=====================
.. rubric:: A library to help distributed applications send trace information to
metrics backends like [Zipkin][zipkin] and [Statsd][statsd].
----

3
py33-requirements.txt Normal file
View File

@ -0,0 +1,3 @@
webob
statsd

9
py33-test-requirements.txt Normal file
View File

@ -0,0 +1,9 @@
# Install bounded pep8/pyflakes first, then let flake8 install
pep8==1.4.5
pyflakes>=0.7.2,<0.7.4
flake8==2.0
hacking>=0.5.6,<0.8
nose
nose-exclude
sphinx>=1.1.2

3
tools/pip-requires → requirements.txt
View File

@ -1,4 +1,5 @@
eventlet
webob
statsd
eventlet
thrift

39
setup.cfg Normal file
View File

@ -0,0 +1,39 @@
[metadata]
name = tomograph
summary = Tiny tims tracing tomograph
description-file =
README.md
author = Tomograph Developers
author-email = timjr@yahoo-inc.com
classifier =
Development Status :: 3 - Alpha Development Status
Environment :: OpenStack
Intended Audience :: Information Technology
Intended Audience :: Developers
License :: OSI Approved :: Apache Software License
Operating System :: POSIX :: Linux
Programming Language :: Python
Programming Language :: Python :: 2
Programming Language :: Python :: 2.6
Programming Language :: Python :: 2.7
Programming Language :: Python :: 3.3
[global]
setup-hooks =
pbr.hooks.setup_hook
[files]
packages =
tomograph
[nosetests]
cover-erase = true
verbosity = 2
[build_sphinx]
source-dir = doc/source
build-dir = doc/build
all_files = 1
[upload_sphinx]
upload-dir = docs/build/html

47
setup.py
View File

@ -1,34 +1,23 @@
#!/usr/bin/env python
# Copyright (c) 2013 Hewlett-Packard Development Company, L.P.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# THIS FILE IS MANAGED BY THE GLOBAL REQUIREMENTS REPO - DO NOT EDIT
import setuptools
def read_requires():
requires = []
with open('tools/pip-requires', 'r') as fh:
contents = fh.read()
for line in contents.splitlines():
line = line.strip()
if line.startswith("#") or not line:
continue
try:
(line, after) = line.split("#", 1)
except ValueError:
pass
if not line:
continue
requires.append(line)
return requires
setuptools.setup(
name='tomograph',
version="0.0.1",
description='Tiny tims tracing tomograph',
author="Y! OpenStack Team",
author_email='timjr@yahoo-inc.com',
license='Apache License, Version 2.0',
packages=setuptools.find_packages(),
long_description=open('README.md').read(),
install_requires=read_requires(),
)
setup_requires=['pbr'],
pbr=True)

11
test-requirements.txt Normal file
View File

@ -0,0 +1,11 @@
# Install bounded pep8/pyflakes first, then let flake8 install
pep8==1.4.5
pyflakes>=0.7.2,<0.7.4
flake8==2.0
hacking>=0.5.6,<0.8
nose
nose-exclude
openstack.nose_plugin>=0.7
pylint==0.25.2
sphinx>=1.1.2

5
tests/basic.py
View File

@ -1,6 +1,6 @@
#!/usr/bin/env python
# Copyright (c) 2012 Yahoo! Inc. All rights reserved.
# Copyright (c) 2012 Yahoo! Inc. All rights reserved.
# Licensed under the Apache License, Version 2.0 (the "License"); you
# may not use this file except in compliance with the License. You may
# obtain a copy of the License at
@ -16,6 +16,7 @@ import tomograph
import sys
import time
@tomograph.traced('test server', 'server response', port=80)
def server(latency):
tomograph.annotate('this is an annotation')
@ -24,11 +25,13 @@ def server(latency):
tomograph.tag('this is a string', 'foo')
tomograph.tag('this is an int', 42)
@tomograph.traced('test client', 'client request')
def client(client_overhead, server_latency):
time.sleep(client_overhead)
server(server_latency)
if __name__ == '__main__':
if len(sys.argv) > 1:
tomograph.config.set_backends(sys.argv[1:])

11
tests/bench.py
View File

@ -1,6 +1,6 @@
#!/usr/bin/env python
# Copyright (c) 2012 Yahoo! Inc. All rights reserved.
# Copyright (c) 2012 Yahoo! Inc. All rights reserved.
# Licensed under the Apache License, Version 2.0 (the "License"); you
# may not use this file except in compliance with the License. You may
# obtain a copy of the License at
@ -11,11 +11,12 @@
# License for the specific language governing permissions and
# limitations under the License. See accompanying LICENSE file.
import tomograph
import cProfile
import sys
import time
import tomograph
@tomograph.traced('test server', 'server response', port=80)
def server(latency):
time.sleep(latency)
@ -26,14 +27,14 @@ def client(client_overhead, server_latency):
time.sleep(client_overhead)
server(server_latency)
def clientloop():
for i in xrange(10000):
client(0, 0)
if __name__ == '__main__':
if len(sys.argv) > 1:
tomograph.config.set_backends(sys.argv[1:])
#cProfile.run('clientloop()', 'tomo-bench')
clientloop()

20
tomograph/__init__.py
View File

@ -1,4 +1,4 @@
# Copyright (c) 2012 Yahoo! Inc. All rights reserved.
# Copyright (c) 2012 Yahoo! Inc. All rights reserved.
# Licensed under the Apache License, Version 2.0 (the "License"); you
# may not use this file except in compliance with the License. You may
# obtain a copy of the License at
@ -12,28 +12,32 @@
### Initialize logging in case it hasn't been done. We need two
### versions of this, one for the eventlet logging module and one for
### the non-eventlet one...
from __future__ import absolute_import
import logging
import sys
import eventlet
eventlet_logging = eventlet.import_patched('logging')
eventlet_sys = eventlet.import_patched('sys')
def _initLogging(logging, sys):
"""
set up some default stuff, in case nobody configured logging yet
"""
"""Set up some default stuff, in case nobody configured logging yet."""
logger = logging.getLogger('tomograph')
if logger.level == logging.NOTSET:
logger.setLevel(logging.INFO)
if not logger.handlers:
handler = logging.StreamHandler(sys.stdout)
handler.setFormatter(logging.Formatter(
'%(asctime)s %(levelname)s %(name)s %(message)s'))
handler.setFormatter(logging.Formatter('%(asctime)s %(levelname)s '
'%(name)s %(message)s'))
logger.addHandler(handler)
_initLogging(logging, sys)
_initLogging(eventlet_logging, eventlet_sys)
import config
from tomograph import *
from tomograph.tomograph import *

3
tomograph/backends/__init__.py
View File

@ -1,4 +1,4 @@
# Copyright (c) 2012 Yahoo! Inc. All rights reserved.
# Copyright (c) 2012 Yahoo! Inc. All rights reserved.
# Licensed under the Apache License, Version 2.0 (the "License"); you
# may not use this file except in compliance with the License. You may
# obtain a copy of the License at
@ -8,4 +8,3 @@
# OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and
# limitations under the License. See accompanying LICENSE file.

3
tomograph/backends/log/__init__.py
View File

@ -1,4 +1,4 @@
# Copyright (c) 2012 Yahoo! Inc. All rights reserved.
# Copyright (c) 2012 Yahoo! Inc. All rights reserved.
# Licensed under the Apache License, Version 2.0 (the "License"); you
# may not use this file except in compliance with the License. You may
# obtain a copy of the License at
@ -14,5 +14,6 @@ import sys
logger = logging.getLogger(__name__)
def send(span):
logger.info(span)

7
tomograph/backends/statsd/__init__.py
View File

@ -1,4 +1,4 @@
# Copyright (c) 2012 Yahoo! Inc. All rights reserved.
# Copyright (c) 2012 Yahoo! Inc. All rights reserved.
# Licensed under the Apache License, Version 2.0 (the "License"); you
# may not use this file except in compliance with the License. You may
# obtain a copy of the License at
@ -9,8 +9,5 @@
# License for the specific language governing permissions and
# limitations under the License. See accompanying LICENSE file.
from statsd import *

17
tomograph/backends/statsd/statsd.py
View File

@ -1,4 +1,4 @@
# Copyright (c) 2012 Yahoo! Inc. All rights reserved.
# Copyright (c) 2012 Yahoo! Inc. All rights reserved.
# Licensed under the Apache License, Version 2.0 (the "License"); you
# may not use this file except in compliance with the License. You may
# obtain a copy of the License at
@ -10,8 +10,8 @@
# limitations under the License. See accompanying LICENSE file.
import eventlet
from tomograph import config
from tomograph import cache
from tomograph import config
logging = eventlet.import_patched('logging')
socket = eventlet.import_patched('socket')
@ -25,17 +25,22 @@ hostname_cache = cache.Cache(socket.gethostbyname)
lock = threading.Lock()
def send(span):
def statsd_send(name, value, units):
stat = str(name).replace(' ', '-') + ':' + str(int(value)) + '|' + str(units)
stat = (str(name).replace(' ', '-') + ':' + str(int(value)) +
'|' + str(units))
with lock:
try:
udp_socket.sendto(stat, (hostname_cache.get(config.statsd_host), config.statsd_port))
udp_socket.sendto(stat,
(hostname_cache.get(config.statsd_host),
config.statsd_port))
except Exception:
if config.debug:
logger.warning("Error sending metric to statsd.", exc_info=True)
logger.warning("Error sending metric to statsd.",
exc_info=True)
def server_name(note):
address = note.address.replace('.', '-')
return note.service_name + ' ' + address + ' ' + str(note.port)

2
tomograph/backends/zipkin/__init__.py
View File

@ -1,4 +1,4 @@
# Copyright (c) 2012 Yahoo! Inc. All rights reserved.
# Copyright (c) 2012 Yahoo! Inc. All rights reserved.
# Licensed under the Apache License, Version 2.0 (the "License"); you
# may not use this file except in compliance with the License. You may
# obtain a copy of the License at

39
tomograph/backends/zipkin/sender.py
View File

@ -1,8 +1,20 @@
# Copyright (c) 2012 Yahoo! Inc. All rights reserved.
# Licensed under the Apache License, Version 2.0 (the "License"); you
# may not use this file except in compliance with the License. You may
# obtain a copy of the License at
# http://www.apache.org/licenses/LICENSE-2.0 Unless required by
# applicable law or agreed to in writing, software distributed under
# the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES
# OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and
# limitations under the License. See accompanying LICENSE file.
import eventlet
socket = eventlet.import_patched('socket')
time = eventlet.import_patched('time')
scribe = eventlet.import_patched('tomograph.backends.zipkin.generated.scribe.scribe')
scribe = eventlet.import_patched('tomograph.backends.zipkin.'
'generated.scribe.scribe')
TTransport = eventlet.import_patched('thrift.transport.TTransport')
TSocket = eventlet.import_patched('thrift.transport.TSocket')
collections = eventlet.import_patched('collections')
@ -11,9 +23,10 @@ threading = eventlet.import_patched('threading')
from thrift.protocol import TBinaryProtocol
class ScribeSender(object):
def __init__(self, host='127.0.0.1', port=1463,debug=False,
target_write_size=1000, max_write_interval=1.0,
def __init__(self, host='127.0.0.1', port=1463, debug=False,
target_write_size=1000, max_write_interval=1.0,
socket_timeout=5.0, max_queue_length=50000, must_yield=True):
self.dropped = 0
self._remote_host = host
@ -35,13 +48,11 @@ class ScribeSender(object):
self.flush()
def send(self, category, msg):
"""
Send one record to scribe.
"""
"""Send one record to scribe."""
log_entry = scribe.LogEntry(category=category, message=msg)
self._log_buffer.append(log_entry)
self._dropMsgs()
now = time.time()
if len(self._log_buffer) >= self._target_write_size or \
now - self._last_write > self._max_write_interval:
@ -66,22 +77,24 @@ class ScribeSender(object):
buf.append(self._log_buffer.popleft())
if buf:
if self._debug:
print "ScribeSender: flushing {0} msgs".format(len(buf))
print("ScribeSender: flushing {0} msgs".format(len(buf)))
try:
client = self._getClient()
result = client.Log(messages=buf)
if result == scribe.ResultCode.TRY_LATER:
dropped += len(buf)
except:
except Exception:
if self._debug:
print "ScribeSender: caught exception writing log message:"
print("ScribeSender: caught exception writing "
"log message:")
traceback.print_exc()
dropped += len(buf)
finally:
self._lock.release()
self.dropped += dropped
if self._debug and dropped:
print "ScribeSender: dropped {0} messages for communication problem.".format(dropped)
print("ScribeSender: dropped {0} messages for "
"communication problem.".format(dropped))
def _dropMsgs(self):
dropped = 0
@ -90,7 +103,8 @@ class ScribeSender(object):
dropped += 1
self.dropped += dropped
if self._debug and dropped:
print "ScribeSender: dropped {0} messages for queue length.".format(dropped)
print("ScribeSender: dropped {0} messages for queue "
"length.".format(dropped))
def _getClient(self):
# We can't just keep a connection because the app might fork
@ -102,4 +116,3 @@ class ScribeSender(object):
transport.open()
protocol = TBinaryProtocol.TBinaryProtocolAccelerated(transport)
return scribe.Client(protocol)

92
tomograph/backends/zipkin/zipkin.py
View File

@ -1,4 +1,4 @@
# Copyright (c) 2012 Yahoo! Inc. All rights reserved.
# Copyright (c) 2012 Yahoo! Inc. All rights reserved.
# Licensed under the Apache License, Version 2.0 (the "License"); you
# may not use this file except in compliance with the License. You may
# obtain a copy of the License at
@ -9,54 +9,64 @@
# License for the specific language governing permissions and
# limitations under the License. See accompanying LICENSE file.
from generated.scribe import scribe
import sender
import zipkin_thrift
from thrift.transport import TTransport
from thrift.transport import TSocket
from __future__ import print_function
from thrift.protocol import TBinaryProtocol
from thrift.transport import TSocket
from thrift.transport import TTransport
from tomograph.backends.zipkin.generated.scribe import scribe
from tomograph.backends.zipkin import sender
from tomograph.backends.zipkin import zipkin_thrift
from tomograph import config
from tomograph import cache
from tomograph import config
import base64
import StringIO
import time
import atexit
import base64
import random
import socket
import struct
import sys
import time
import traceback
import atexit
scribe_sender = sender.ScribeSender(host=config.zipkin_host,
port=config.zipkin_port,
socket_timeout=config.zipkin_socket_timeout,
target_write_size=config.zipkin_target_write_size,
max_queue_length=config.zipkin_max_queue_length,
must_yield=config.zipkin_must_yield,
max_write_interval=config.zipkin_max_write_interval,
debug=config.zipkin_debug_scribe_sender)
scribe_config = {
'host': config.zipkin_host,
'port': config.zipkin_port,
'socket_timeout': config.zipkin_socket_timeout,
'target_write_size': config.zipkin_target_write_size,
'max_queue_length': config.zipkin_max_queue_length,
'must_yield': config.zipkin_must_yield,
'max_write_interval': config.zipkin_max_write_interval,
'debug': config.zipkin_debug_scribe_sender,
}
scribe_sender = sender.ScribeSender(**scribe_config)
atexit.register(scribe_sender.close)
hostname_cache = cache.Cache(socket.gethostbyname)
def send(span):
def endpoint(note):
try:
ip = hostname_cache.get(note.address)
except:
print >>sys.stderr, 'host resolution error: ', traceback.format_exc()
except Exception:
print('host resolution error: %s' % traceback.format_exc(),
file=sys.stderr)
ip = '0.0.0.0'
return zipkin_thrift.Endpoint(ipv4 = ip_to_i32(ip),
port = port_to_i16(note.port),
service_name = note.service_name)
return zipkin_thrift.Endpoint(ipv4=ip_to_i32(ip),
port=port_to_i16(note.port),
service_name=note.service_name)
def annotation(note):
return zipkin_thrift.Annotation(timestamp = int(note.time * 1e6),
value = note.value,
duration = note.duration,
host = endpoint(note))
return zipkin_thrift.Annotation(timestamp=int(note.time * 1e6),
value=note.value,
duration=note.duration,
host=endpoint(note))
def binary_annotation(dimension):
if isinstance(dimension.value, str):
@ -70,18 +80,18 @@ def send(span):
val = struct.pack('>q', dimension.value)
else:
raise RuntimeError("unsupported tag type")
return zipkin_thrift.BinaryAnnotation(key = dimension.key,
value = val,
annotation_type = tag_type,
host = endpoint(dimension))
return zipkin_thrift.BinaryAnnotation(key=dimension.key,
value=val,
annotation_type=tag_type,
host=endpoint(dimension))
zspan = zipkin_thrift.Span(trace_id = span.trace_id,
id = span.id,
name = span.name,
parent_id = span.parent_id,
annotations = [annotation(n) for n in span.notes],
binary_annotations = \
[binary_annotation(d) for d in span.dimensions])
binary_annotations = [binary_annotation(d) for d in span.dimensions]
zspan = zipkin_thrift.Span(trace_id=span.trace_id,
id=span.id,
name=span.name,
parent_id=span.parent_id,
annotations=[annotation(n) for n in span.notes],
binary_annotations=binary_annotations)
out = StringIO.StringIO()
#raw = TBinaryProtocol.TBinaryProtocolAccelerated(out)
raw = TBinaryProtocol.TBinaryProtocol(out)
@ -91,12 +101,14 @@ def send(span):
traceback.print_exc()
scribe_sender.send('zipkin', base64.b64encode(out.getvalue()))
def ip_to_i32(ip_str):
"""convert an ip address from a string to a signed 32-bit number"""
return struct.unpack('!i', socket.inet_aton(ip_str))[0]
def port_to_i16(port_num):
"""conver a port number to a signed 16-bit int"""
if port_num > 2**15:
port_num -= 2**16
if port_num > 2 ** 15:
port_num -= 2 ** 16
return port_num

2
tomograph/backends/zipkin/zipkin_thrift.py
View File

@ -1,4 +1,4 @@
# Copyright (c) 2012 Yahoo! Inc. All rights reserved.
# Copyright (c) 2012 Yahoo! Inc. All rights reserved.
# Licensed under the Apache License, Version 2.0 (the "License"); you
# may not use this file except in compliance with the License. You may
# obtain a copy of the License at

6
tomograph/cache.py
View File

@ -1,4 +1,4 @@
# Copyright (c) 2012 Yahoo! Inc. All rights reserved.
# Copyright (c) 2012 Yahoo! Inc. All rights reserved.
# Licensed under the Apache License, Version 2.0 (the "License"); you
# may not use this file except in compliance with the License. You may
# obtain a copy of the License at
@ -11,6 +11,7 @@
import threading
class Cache(object):
def __init__(self, thunk, size_limit=1000):
self._map = {}
@ -20,7 +21,7 @@ class Cache(object):
def get(self, k):
with self._lock:
if self._map.has_key(k):
if k in self._map:
return self._map[k]
else:
while len(self._map) >= self._size_limit:
@ -28,4 +29,3 @@ class Cache(object):
v = self._thunk(k)
self._map[k] = v
return v

17
tomograph/config.py
View File

@ -1,4 +1,4 @@
# Copyright (c) 2012 Yahoo! Inc. All rights reserved.
# Copyright (c) 2012 Yahoo! Inc. All rights reserved.
# Licensed under the Apache License, Version 2.0 (the "License"); you
# may not use this file except in compliance with the License. You may
# obtain a copy of the License at
@ -29,17 +29,16 @@ zipkin_max_queue_length = 50000
zipkin_target_write_size = 1000
zipkin_max_write_interval = 1
zipkin_must_yield = True
zipkin_debug_scribe_sender=False
zipkin_debug_scribe_sender = False
debug = False
db_tracing_enabled = True
db_trace_as_spans = False
def set_backends(backends):
"""
Set the list of enabled backends. Backend name should be the full
module name of the backend. All backends must support a
send(span) method.
"""Set the list of enabled backends. Backend name should be the full
module name of the backend. All backends must support a send(span) method.
"""
global enabled_backends
global backend_modules
@ -53,11 +52,11 @@ def set_backends(backends):
module = getattr(module, submodule)
backend_modules.append(module)
except (ImportError, AttributeError, ValueError) as err:
raise RuntimeError('Could not load tomograph backend {0}: {1}'.format(
backend, err))
raise RuntimeError('Could not load tomograph backend '
'{0}: {1}'.format(backend, err))
def get_backends():
if not backend_modules:
set_backends(enabled_backends)
return backend_modules

92
tomograph/tomograph.py
View File

@ -1,4 +1,4 @@
# Copyright (c) 2012 Yahoo! Inc. All rights reserved.
# Copyright (c) 2012 Yahoo! Inc. All rights reserved.
# Licensed under the Apache License, Version 2.0 (the "License"); you
# may not use this file except in compliance with the License. You may
# obtain a copy of the License at
@ -9,21 +9,27 @@
# License for the specific language governing permissions and
# limitations under the License. See accompanying LICENSE file.
import config
from types import Span, Note, Tag
from __future__ import absolute_import
import random
import sys
import time
from eventlet import corolocal
import socket
import pickle
import base64
import logging
import pickle
import random
import socket
import sys
import time
from eventlet import corolocal
from tomograph import config
from tomograph import types
import webob.dec
span_stack = corolocal.local()
def start(service_name, name, address, port, trace_info=None):
parent_id = None
if tracing_started():
@ -37,35 +43,43 @@ def start(service_name, name, address, port, trace_info=None):
parent_id = trace_info[1]
span_stack.spans = []
span = Span(trace_id, parent_id, getId(), name, [], [])
span = types.Span(trace_id, parent_id, getId(), name, [], [])
span_stack.spans.append(span)
annotate('start', service_name, address, port)
def tracing_started():
return hasattr(span_stack, 'trace_id')
def cur_span():
if not tracing_started():
start('orphan', 'orphan', '127.0.0.1', '1')
return span_stack.spans[-1]
def get_trace_info():
if tracing_started():
return (span_stack.trace_id, cur_span().id)
else:
return None
def stop(name):
annotate('stop')
span = span_stack.spans.pop()
assert span.name == name, 'start span name {0} not equal to end span name {1}'.format(span.name, name)
assert span.name == name, ('start span name {0} not equal '
'to end span name {1}'.format(span.name, name))
if not span_stack.spans:
del(span_stack.trace_id)
for backend in config.get_backends():
backend.send(span)
def annotate(value, service_name=None, address=None, port=None, duration=None):
"""add an annotation at a particular point in time (with an optional duration)"""
"""Add an annotation at a particular point in time, (with an optional
duration).
"""
# attempt to default some values
if service_name is None:
service_name = cur_span().notes[0].service_name
@ -75,32 +89,39 @@ def annotate(value, service_name=None, address=None, port=None, duration=None):
port = cur_span().notes[0].port
if duration is None:
duration = 0
note = Note(time.time(), str(value), service_name, address, int(port),
int(duration))
note = types.Note(time.time(), str(value), service_name, address,
int(port), int(duration))
cur_span().notes.append(note)
def tag(key, value, service_name=None, address=None, port=None):
"""add a key/value tag to the current span. values can be int,
float, or string."""
assert isinstance(value, str) or isinstance(value, int) or isinstance(value, float)
"""Add a key/value tag to the current span. values can be int,
float, or string.
"""
assert (isinstance(value, str) or isinstance(value, int)
or isinstance(value, float))
if service_name is None:
service_name = cur_span().notes[0].service_name
if address is None:
address = cur_span().notes[0].address
if port is None:
port = cur_span().notes[0].port
tag = Tag(str(key), value, service_name, address, port)
tag = types.Tag(str(key), value, service_name, address, port)
cur_span().dimensions.append(tag)
def getId():
return random.randrange(sys.maxint >> 10)
## wrapper/decorators
def tracewrap(func, service_name, name, host='0.0.0.0', port=0):
if host == '0.0.0.0':
host = socket.gethostname()
def trace_and_call(*args, **kwargs):
if service_name is None and len(args) > 0 and isinstance(args[0], object):
if service_name is None and len(args) > 0 \
and isinstance(args[0], object):
s = args[0].__class__.__name__
else:
s = service_name
@ -108,24 +129,29 @@ def tracewrap(func, service_name, name, host='0.0.0.0', port=0):
ret = func(*args, **kwargs)
stop(name)
return ret
return trace_and_call
def traced(service_name, name, host='0.0.0.0', port=0):
def t1(func):
return tracewrap(func, service_name, name, host, port)
return t1
## sqlalchemy event listeners
## sqlalchemy event listeners
def before_execute(name):
def handler(conn, clauseelement, multiparams, params):
if not config.db_tracing_enabled:
return
h = str(conn.connection.connection)
a = h.find("'")
b = h.find("'", a+1)
b = h.find("'", a + 1)
if b > a:
h = h[a+1:b]
h = h[a + 1:b]
else:
h = 'unknown'
port = conn.connection.connection.port
@ -134,8 +160,10 @@ def before_execute(name):
if config.db_trace_as_spans:
start(str(name) + 'db client', 'execute', h, port)
annotate(clauseelement)
return handler
def after_execute(name):
# name isn't used, at least not yet...
def handler(conn, clauseelement, multiparams, params, result):
@ -145,13 +173,16 @@ def after_execute(name):
# fix up the duration on the annotation for the sql query
start_time = cur_span().notes[0].time
last_note = cur_span().notes.pop()
cur_span().notes.append(Note(last_note.time, last_note.value,
last_note.service_name, last_note.address,
last_note.port, time.time() - start_time))
cur_span().notes.append(types.Note(last_note.time, last_note.value,
last_note.service_name,
last_note.address,
last_note.port,
time.time() - start_time))
if config.db_trace_as_spans:
stop('execute')
return handler
def dbapi_error(name):
def handler(conn, cursor, statement, parameters, context, exception):
if not config.db_tracing_enabled:
@ -160,6 +191,7 @@ def dbapi_error(name):
stop('execute')
return handler
## http helpers
def start_http(service_name, name, request):
trace_info_enc = request.headers.get('X-Trace-Info')
@ -170,6 +202,7 @@ def start_http(service_name, name, request):
trace_info = None
start(service_name, name, host, port, trace_info)
def add_trace_info_header(headers):
trace_info = get_trace_info()
if trace_info:
@ -178,15 +211,13 @@ def add_trace_info_header(headers):
## WSGI middleware
class Middleware(object):
"""
WSGI Middleware that enables tomograph tracing for an application.
"""
"""WSGI Middleware that enables tomograph tracing for an application."""
def __init__(self, application, service_name='Server', name='WSGI'):
self.application = application
self.service_name = service_name
self.name = name
@classmethod
def factory(cls, global_conf, **local_conf):
def filter(app):
@ -199,4 +230,3 @@ class Middleware(object):
response = req.get_response(self.application)
stop(self.name)
return response

12
tomograph/types.py
View File

@ -1,4 +1,4 @@
# Copyright (c) 2012 Yahoo! Inc. All rights reserved.
# Copyright (c) 2012 Yahoo! Inc. All rights reserved.
# Licensed under the Apache License, Version 2.0 (the "License"); you
# may not use this file except in compliance with the License. You may
# obtain a copy of the License at
@ -9,8 +9,10 @@
# License for the specific language governing permissions and
# limitations under the License. See accompanying LICENSE file.
from collections import namedtuple
import collections
Span = namedtuple('Span', 'trace_id parent_id id name notes dimensions')
Note = namedtuple('Note', 'time value service_name address port duration')
Tag = namedtuple('Tag', 'key value service_name address port')
Span = collections.namedtuple('Span', 'trace_id parent_id id name notes'
' dimensions')
Note = collections.namedtuple('Note', 'time value service_name'
' address port duration')
Tag = collections.namedtuple('Tag', 'key value service_name address port')

25
tomograph/version.py Normal file
View File

@ -0,0 +1,25 @@
# Copyright (c) 2013 Yahoo! Inc. All rights reserved.
# Licensed under the Apache License, Version 2.0 (the "License"); you
# may not use this file except in compliance with the License. You may
# obtain a copy of the License at
# http://www.apache.org/licenses/LICENSE-2.0 Unless required by
# applicable law or agreed to in writing, software distributed under
# the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES
# OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and
# limitations under the License. See accompanying LICENSE file.
TOMOGRAPH_VERSION = ['2013', '1', None]
YEAR, COUNT, REVISION = TOMOGRAPH_VERSION
FINAL = False # May never be final ;)
def canonical_version_string():
return '.'.join(filter(None, TOMOGRAPH_VERSION))
def version_string():
if FINAL:
return canonical_version_string()
else:
return '%s-dev' % (canonical_version_string())

50
tox.ini Normal file
View File

@ -0,0 +1,50 @@
[tox]
minversion = 1.6
skipsdist = True
envlist = py26,py27,py33,pep8
[testenv]
usedevelop = True
install_command = pip install {opts} {packages}
setenv = VIRTUAL_ENV={envdir}
LANG=en_US.UTF-8
LANGUAGE=en_US:en
LC_ALL=C
NOSE_WITH_OPENSTACK=1
NOSE_OPENSTACK_COLOR=1
NOSE_OPENSTACK_RED=0.05
NOSE_OPENSTACK_YELLOW=0.025
NOSE_OPENSTACK_SHOW_ELAPSED=1
NOSE_OPENSTACK_STDOUT=1
deps = -r{toxinidir}/requirements.txt
-r{toxinidir}/test-requirements.txt
commands = nosetests {posargs}
[testenv:py33]
deps = -r{toxinidir}/py33-requirements.txt
-r{toxinidir}/py33-test-requirements.txt
commands = true
[tox:jenkins]
downloadcache = ~/cache/pip
[testenv:pep8]
commands =
flake8 {posargs}
[testenv:pylint]
setenv = VIRTUAL_ENV={envdir}
deps = -r{toxinidir}/requirements.txt
pylint==0.26.0
commands = pylint
[testenv:cover]
setenv = NOSE_WITH_COVERAGE=1
[testenv:venv]
commands = {posargs}
[flake8]
ignore = H202,H402,F401,F403,H303
builtins = _
exclude = .venv,.tox,dist,doc,*egg,.git,build,tools,./tomograph/backends/zipkin/generated