commit 609841ee4b958345f88bc7ac1ade21ebc639ea76 Author: timjr Date: Tue Oct 30 21:32:17 2012 +0000 Initial commit. Has basic tracing API with zipkin, statsd, and log backends. diff --git a/LICENSE.txt b/LICENSE.txt new file mode 100644 index 0000000..d5c4984 --- /dev/null +++ b/LICENSE.txt @@ -0,0 +1,209 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + + +THIRD-PARTY DEPENDENCIES +======================== +Convenience copies of some third-party dependencies are distributed with +Apache Cassandra as Java jar files in lib/. Licensing information for +these files can be found in the lib/licenses directory. diff --git a/README.md b/README.md new file mode 100644 index 0000000..e06291a --- /dev/null +++ b/README.md @@ -0,0 +1,120 @@ +tomograph +========= + +A library for sending trace information to metrics backends like +[Zipkin][zipkin] and [Statsd][statsd]. + +Data Model +---------- + +A request to a distributed application is modeled as a trace. Each +trace consists of a set of spans, and a span is a set of notes. + +Each span's extent is defined by its first and last notes. Any number +of additional notes can be added in between -- for example in a +handler for ERROR-level logging. + +The tomograph data model is basically the Dapper/Zipkin data model. +For translation to statsd, we emit the length of the span as a timer +metric, and each note gets emitted individually as a counter metric. + +For example, here is a basic client/server interaction. It is one +trace, with two spans, each with two notes -- their beginning and end: + +![zipkin client server](tomograph/raw/master/doc/screenshots/client-server-zipkin.png) + +This is the same data as it would be viewed in using the statsd +backend with graphite: + +![graphite client server](tomograph/raw/master/doc/screenshots/client-server-graphite.png) + +Tracing Your Application +------------------------ + +There are a few basic ways to add tracing to your application. The +lowest level one is to call start, stop, and annotate yourself: + + import tomograph + + tomograph.start('my service', 'a query', '127.0.0.1', 80) + (...) + tomograph.annotate('something happened') + (...) + tomograph.stop('a query') + +Each start/stop pair defines a span. Spans can be arbitrarily nested +using this interface as long they stay on a single thread: tomograph +keeps the current span stack in thread local storage. + +When continuing a trace from one thread to another, you must grab the +trace token from tomograph and pass it: + + token = tomograph.get_trace_info() + (...) + tomograph.start('my service', 'a query', '127.0.0.1', 80, token) + (...) + +That will enable tomograph to connect all of the spans into one trace. + +Helpers +------- + +There are some slightly higher level interfaces to help you add +tracing. For HTTP, add_trace_info_header() will add an X-Trace-Info +header to a dict on the client side, and start_http() will consume +that header on the server side: + + def traced_http_client(url, body, headers): + tomograph.start('client', 'http request', socket.gethostname(), 0) + tomograph.add_trace_info_header(headers) + http_request(url, body, headers) + tomograph.stop('http request') + + + def traced_http_server(request): + tomograph.start_http('server', 'http response', request) + (...) + tomograph.stop('http response') + +There's no need to call start and stop yourself -- you can use the +@tomograph.traced decorator: + + @tomograph.traced('My Server', 'myfunc') + def myfunc(yadda): + dosomething() + +For WSGI pipelines, there's the class tomograph.Middleware that will +consume the X-Trace-Info header. It can be added to a paste pipeline +like so: + + [pipeline:foo] + pipeline = tomo foo bar baz... + + [filter:tomo] + paste.filter_factory = tomograph:Middleware.factory + service_name = glance-registry + +If you use [SQL Alchemy][sql alchemy] in your application, there are +some event listeners available that will trace SQL statement +execution: + + _ENGINE = sqlalchemy.create_engine(FLAGS.sql_connection, **engine_args) + + sqlalchemy.event.listen(_ENGINE, 'before_execute', tomograph.before_execute('my app')) + sqlalchemy.event.listen(_ENGINE, 'after_execute', tomograph.after_execute('my app')) + + +Screenshots +----------- + +Here is a slightly more involved example -- a glance image list +command in [Openstack][openstack]. It uses SQL statement tracing and +the tomograph middleware: + +![zipkin glance image list](tomograph/raw/master/doc/screenshots/zipkin-glance-image-list.png) + + +[openstack]: http://www.openstack.org/ +[statsd]: https://github.com/etsy/statsd +[zipkin]: http://twitter.github.com/zipkin/ +[sql alchemy]: http://www.sqlalchemy.org/ diff --git a/doc/screenshots/client-server-graphite.png b/doc/screenshots/client-server-graphite.png new file mode 100644 index 0000000..592967c Binary files /dev/null and b/doc/screenshots/client-server-graphite.png differ diff --git a/doc/screenshots/client-server-zipkin.png b/doc/screenshots/client-server-zipkin.png new file mode 100644 index 0000000..8f654ab Binary files /dev/null and b/doc/screenshots/client-server-zipkin.png differ diff --git a/doc/screenshots/zipkin-glance-image-list.png b/doc/screenshots/zipkin-glance-image-list.png new file mode 100644 index 0000000..a730c1d Binary files /dev/null and b/doc/screenshots/zipkin-glance-image-list.png differ diff --git a/tests/basic.py b/tests/basic.py new file mode 100755 index 0000000..0de4ed2 --- /dev/null +++ b/tests/basic.py @@ -0,0 +1,32 @@ +#!/usr/bin/env python + +# Copyright (c) 2012 Yahoo! Inc. All rights reserved. +# Licensed under the Apache License, Version 2.0 (the "License"); you +# may not use this file except in compliance with the License. You may +# obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 Unless required by +# applicable law or agreed to in writing, software distributed under +# the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES +# OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and +# limitations under the License. See accompanying LICENSE file. + +import tomograph + +import sys +import time + +@tomograph.traced('test server', 'server response', port=80) +def server(latency): + time.sleep(latency) + + +@tomograph.traced('test client', 'client request') +def client(client_overhead, server_latency): + time.sleep(client_overhead) + server(server_latency) + +if __name__ == '__main__': + if len(sys.argv) > 1: + tomograph.config.set_backends(sys.argv[1:]) + client(0.1, 2.4) diff --git a/tomograph/__init__.py b/tomograph/__init__.py new file mode 100644 index 0000000..576e1d5 --- /dev/null +++ b/tomograph/__init__.py @@ -0,0 +1,30 @@ +# Copyright (c) 2012 Yahoo! Inc. All rights reserved. +# Licensed under the Apache License, Version 2.0 (the "License"); you +# may not use this file except in compliance with the License. You may +# obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 Unless required by +# applicable law or agreed to in writing, software distributed under +# the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES +# OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and +# limitations under the License. See accompanying LICENSE file. + +from tomograph import * +import config +import logging + +def _initLogging(): + """ + set up some default stuff, in case nobody configured logging yet + """ + logger = logging.getLogger(__name__) + + if logger.level == logging.NOTSET: + logger.setLevel(logging.INFO) + if not logger.handlers: + handler = logging.StreamHandler(sys.stdout) + handler.setFormatter(logging.Formatter( + '%(asctime)s %(levelname)s %(name)s %(message)s')) + logger.addHandler(handler) + +_initLogging() diff --git a/tomograph/backends/__init__.py b/tomograph/backends/__init__.py new file mode 100644 index 0000000..81a0a31 --- /dev/null +++ b/tomograph/backends/__init__.py @@ -0,0 +1,11 @@ +# Copyright (c) 2012 Yahoo! Inc. All rights reserved. +# Licensed under the Apache License, Version 2.0 (the "License"); you +# may not use this file except in compliance with the License. You may +# obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 Unless required by +# applicable law or agreed to in writing, software distributed under +# the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES +# OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and +# limitations under the License. See accompanying LICENSE file. + diff --git a/tomograph/backends/log/__init__.py b/tomograph/backends/log/__init__.py new file mode 100644 index 0000000..887367f --- /dev/null +++ b/tomograph/backends/log/__init__.py @@ -0,0 +1,18 @@ +# Copyright (c) 2012 Yahoo! Inc. All rights reserved. +# Licensed under the Apache License, Version 2.0 (the "License"); you +# may not use this file except in compliance with the License. You may +# obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 Unless required by +# applicable law or agreed to in writing, software distributed under +# the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES +# OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and +# limitations under the License. See accompanying LICENSE file. + +import logging +import sys + +logger = logging.getLogger(__name__) + +def send(span): + logger.info(span) diff --git a/tomograph/backends/statsd/__init__.py b/tomograph/backends/statsd/__init__.py new file mode 100644 index 0000000..083acf3 --- /dev/null +++ b/tomograph/backends/statsd/__init__.py @@ -0,0 +1,16 @@ +# Copyright (c) 2012 Yahoo! Inc. All rights reserved. +# Licensed under the Apache License, Version 2.0 (the "License"); you +# may not use this file except in compliance with the License. You may +# obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 Unless required by +# applicable law or agreed to in writing, software distributed under +# the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES +# OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and +# limitations under the License. See accompanying LICENSE file. + +from statsd import * + + + + diff --git a/tomograph/backends/statsd/statsd.py b/tomograph/backends/statsd/statsd.py new file mode 100644 index 0000000..b6d4cae --- /dev/null +++ b/tomograph/backends/statsd/statsd.py @@ -0,0 +1,41 @@ +# Copyright (c) 2012 Yahoo! Inc. All rights reserved. +# Licensed under the Apache License, Version 2.0 (the "License"); you +# may not use this file except in compliance with the License. You may +# obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 Unless required by +# applicable law or agreed to in writing, software distributed under +# the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES +# OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and +# limitations under the License. See accompanying LICENSE file. + +from tomograph import config + +import logging +import socket + +logger = logging.getLogger(__name__) + +udp_socket = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) + +def send(span): + + def statsd_send(name, value, units): + stat = str(name).replace(' ', '-') + ':' + str(int(value)) + '|' + str(units) + logger.info('sending stat {0}'.format(stat)) + udp_socket.sendto(stat, (config.statsd_host, config.statsd_port)) + + def server_name(note): + address = note.address.replace('.', '-') + return note.service_name + ' ' + address + ' ' + str(note.port) + + # the timing stat: + delta = span.notes[-1].time - span.notes[0].time + deltams = delta * 1000 + time_stat_name = server_name(span.notes[0]) + '.' + span.name + statsd_send(time_stat_name, deltams, 'ms') + + # a count stat for each note + for note in span.notes: + stat_name = server_name(note) + '.' + span.name + '.' + str(note.value) + statsd_send(stat_name, 1, 'c') diff --git a/tomograph/backends/zipkin/__init__.py b/tomograph/backends/zipkin/__init__.py new file mode 100644 index 0000000..2da7a37 --- /dev/null +++ b/tomograph/backends/zipkin/__init__.py @@ -0,0 +1,12 @@ +# Copyright (c) 2012 Yahoo! Inc. All rights reserved. +# Licensed under the Apache License, Version 2.0 (the "License"); you +# may not use this file except in compliance with the License. You may +# obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 Unless required by +# applicable law or agreed to in writing, software distributed under +# the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES +# OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and +# limitations under the License. See accompanying LICENSE file. + +from zipkin import * diff --git a/tomograph/backends/zipkin/generated/__init__.py b/tomograph/backends/zipkin/generated/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tomograph/backends/zipkin/generated/scribe/__init__.py b/tomograph/backends/zipkin/generated/scribe/__init__.py new file mode 100644 index 0000000..5561d0f --- /dev/null +++ b/tomograph/backends/zipkin/generated/scribe/__init__.py @@ -0,0 +1 @@ +__all__ = ['ttypes', 'constants', 'scribe'] diff --git a/tomograph/backends/zipkin/generated/scribe/constants.py b/tomograph/backends/zipkin/generated/scribe/constants.py new file mode 100644 index 0000000..9102045 --- /dev/null +++ b/tomograph/backends/zipkin/generated/scribe/constants.py @@ -0,0 +1,11 @@ +# +# Autogenerated by Thrift Compiler (0.9.0) +# +# DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING +# +# options string: py:new_style=true +# + +from thrift.Thrift import TType, TMessageType, TException, TApplicationException +from ttypes import * + diff --git a/tomograph/backends/zipkin/generated/scribe/scribe-remote b/tomograph/backends/zipkin/generated/scribe/scribe-remote new file mode 100755 index 0000000..6a4fdb6 --- /dev/null +++ b/tomograph/backends/zipkin/generated/scribe/scribe-remote @@ -0,0 +1,88 @@ +#!/usr/bin/env python +# +# Autogenerated by Thrift Compiler (0.9.0) +# +# DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING +# +# options string: py:new_style=true +# + +import sys +import pprint +from urlparse import urlparse +from thrift.transport import TTransport +from thrift.transport import TSocket +from thrift.transport import THttpClient +from thrift.protocol import TBinaryProtocol + +import scribe +from ttypes import * + +if len(sys.argv) <= 1 or sys.argv[1] == '--help': + print '' + print 'Usage: ' + sys.argv[0] + ' [-h host[:port]] [-u url] [-f[ramed]] function [arg1 [arg2...]]' + print '' + print 'Functions:' + print ' ResultCode Log( messages)' + print '' + sys.exit(0) + +pp = pprint.PrettyPrinter(indent = 2) +host = 'localhost' +port = 9090 +uri = '' +framed = False +http = False +argi = 1 + +if sys.argv[argi] == '-h': + parts = sys.argv[argi+1].split(':') + host = parts[0] + if len(parts) > 1: + port = int(parts[1]) + argi += 2 + +if sys.argv[argi] == '-u': + url = urlparse(sys.argv[argi+1]) + parts = url[1].split(':') + host = parts[0] + if len(parts) > 1: + port = int(parts[1]) + else: + port = 80 + uri = url[2] + if url[4]: + uri += '?%s' % url[4] + http = True + argi += 2 + +if sys.argv[argi] == '-f' or sys.argv[argi] == '-framed': + framed = True + argi += 1 + +cmd = sys.argv[argi] +args = sys.argv[argi+1:] + +if http: + transport = THttpClient.THttpClient(host, port, uri) +else: + socket = TSocket.TSocket(host, port) + if framed: + transport = TTransport.TFramedTransport(socket) + else: + transport = TTransport.TBufferedTransport(socket) +protocol = TBinaryProtocol.TBinaryProtocol(transport) +client = scribe.Client(protocol) +transport.open() + +if cmd == 'Log': + if len(args) != 1: + print 'Log requires 1 args' + sys.exit(1) + pp.pprint(client.Log(eval(args[0]),)) + +else: + print 'Unrecognized method %s' % cmd + sys.exit(1) + +transport.close() diff --git a/tomograph/backends/zipkin/generated/scribe/scribe.py b/tomograph/backends/zipkin/generated/scribe/scribe.py new file mode 100644 index 0000000..7d41194 --- /dev/null +++ b/tomograph/backends/zipkin/generated/scribe/scribe.py @@ -0,0 +1,228 @@ +# +# Autogenerated by Thrift Compiler (0.9.0) +# +# DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING +# +# options string: py:new_style=true +# + +from thrift.Thrift import TType, TMessageType, TException, TApplicationException +from ttypes import * +from thrift.Thrift import TProcessor +from thrift.transport import TTransport +from thrift.protocol import TBinaryProtocol, TProtocol +try: + from thrift.protocol import fastbinary +except: + fastbinary = None + + +class Iface(object): + def Log(self, messages): + """ + Parameters: + - messages + """ + pass + + +class Client(Iface): + def __init__(self, iprot, oprot=None): + self._iprot = self._oprot = iprot + if oprot is not None: + self._oprot = oprot + self._seqid = 0 + + def Log(self, messages): + """ + Parameters: + - messages + """ + self.send_Log(messages) + return self.recv_Log() + + def send_Log(self, messages): + self._oprot.writeMessageBegin('Log', TMessageType.CALL, self._seqid) + args = Log_args() + args.messages = messages + args.write(self._oprot) + self._oprot.writeMessageEnd() + self._oprot.trans.flush() + + def recv_Log(self, ): + (fname, mtype, rseqid) = self._iprot.readMessageBegin() + if mtype == TMessageType.EXCEPTION: + x = TApplicationException() + x.read(self._iprot) + self._iprot.readMessageEnd() + raise x + result = Log_result() + result.read(self._iprot) + self._iprot.readMessageEnd() + if result.success is not None: + return result.success + raise TApplicationException(TApplicationException.MISSING_RESULT, "Log failed: unknown result"); + + +class Processor(Iface, TProcessor): + def __init__(self, handler): + self._handler = handler + self._processMap = {} + self._processMap["Log"] = Processor.process_Log + + def process(self, iprot, oprot): + (name, type, seqid) = iprot.readMessageBegin() + if name not in self._processMap: + iprot.skip(TType.STRUCT) + iprot.readMessageEnd() + x = TApplicationException(TApplicationException.UNKNOWN_METHOD, 'Unknown function %s' % (name)) + oprot.writeMessageBegin(name, TMessageType.EXCEPTION, seqid) + x.write(oprot) + oprot.writeMessageEnd() + oprot.trans.flush() + return + else: + self._processMap[name](self, seqid, iprot, oprot) + return True + + def process_Log(self, seqid, iprot, oprot): + args = Log_args() + args.read(iprot) + iprot.readMessageEnd() + result = Log_result() + result.success = self._handler.Log(args.messages) + oprot.writeMessageBegin("Log", TMessageType.REPLY, seqid) + result.write(oprot) + oprot.writeMessageEnd() + oprot.trans.flush() + + +# HELPER FUNCTIONS AND STRUCTURES + +class Log_args(object): + """ + Attributes: + - messages + """ + + thrift_spec = ( + None, # 0 + (1, TType.LIST, 'messages', (TType.STRUCT,(LogEntry, LogEntry.thrift_spec)), None, ), # 1 + ) + + def __init__(self, messages=None,): + self.messages = messages + + def read(self, iprot): + if iprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None and fastbinary is not None: + fastbinary.decode_binary(self, iprot.trans, (self.__class__, self.thrift_spec)) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.LIST: + self.messages = [] + (_etype3, _size0) = iprot.readListBegin() + for _i4 in xrange(_size0): + _elem5 = LogEntry() + _elem5.read(iprot) + self.messages.append(_elem5) + iprot.readListEnd() + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and self.thrift_spec is not None and fastbinary is not None: + oprot.trans.write(fastbinary.encode_binary(self, (self.__class__, self.thrift_spec))) + return + oprot.writeStructBegin('Log_args') + if self.messages is not None: + oprot.writeFieldBegin('messages', TType.LIST, 1) + oprot.writeListBegin(TType.STRUCT, len(self.messages)) + for iter6 in self.messages: + iter6.write(oprot) + oprot.writeListEnd() + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + + def __repr__(self): + L = ['%s=%r' % (key, value) + for key, value in self.__dict__.iteritems()] + return '%s(%s)' % (self.__class__.__name__, ', '.join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + +class Log_result(object): + """ + Attributes: + - success + """ + + thrift_spec = ( + (0, TType.I32, 'success', None, None, ), # 0 + ) + + def __init__(self, success=None,): + self.success = success + + def read(self, iprot): + if iprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None and fastbinary is not None: + fastbinary.decode_binary(self, iprot.trans, (self.__class__, self.thrift_spec)) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 0: + if ftype == TType.I32: + self.success = iprot.readI32(); + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and self.thrift_spec is not None and fastbinary is not None: + oprot.trans.write(fastbinary.encode_binary(self, (self.__class__, self.thrift_spec))) + return + oprot.writeStructBegin('Log_result') + if self.success is not None: + oprot.writeFieldBegin('success', TType.I32, 0) + oprot.writeI32(self.success) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + + def __repr__(self): + L = ['%s=%r' % (key, value) + for key, value in self.__dict__.iteritems()] + return '%s(%s)' % (self.__class__.__name__, ', '.join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) diff --git a/tomograph/backends/zipkin/generated/scribe/ttypes.py b/tomograph/backends/zipkin/generated/scribe/ttypes.py new file mode 100644 index 0000000..480e046 --- /dev/null +++ b/tomograph/backends/zipkin/generated/scribe/ttypes.py @@ -0,0 +1,104 @@ +# +# Autogenerated by Thrift Compiler (0.9.0) +# +# DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING +# +# options string: py:new_style=true +# + +from thrift.Thrift import TType, TMessageType, TException, TApplicationException + +from thrift.transport import TTransport +from thrift.protocol import TBinaryProtocol, TProtocol +try: + from thrift.protocol import fastbinary +except: + fastbinary = None + + +class ResultCode(object): + OK = 0 + TRY_LATER = 1 + + _VALUES_TO_NAMES = { + 0: "OK", + 1: "TRY_LATER", + } + + _NAMES_TO_VALUES = { + "OK": 0, + "TRY_LATER": 1, + } + + +class LogEntry(object): + """ + Attributes: + - category + - message + """ + + thrift_spec = ( + None, # 0 + (1, TType.STRING, 'category', None, None, ), # 1 + (2, TType.STRING, 'message', None, None, ), # 2 + ) + + def __init__(self, category=None, message=None,): + self.category = category + self.message = message + + def read(self, iprot): + if iprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None and fastbinary is not None: + fastbinary.decode_binary(self, iprot.trans, (self.__class__, self.thrift_spec)) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRING: + self.category = iprot.readString(); + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRING: + self.message = iprot.readString(); + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and self.thrift_spec is not None and fastbinary is not None: + oprot.trans.write(fastbinary.encode_binary(self, (self.__class__, self.thrift_spec))) + return + oprot.writeStructBegin('LogEntry') + if self.category is not None: + oprot.writeFieldBegin('category', TType.STRING, 1) + oprot.writeString(self.category) + oprot.writeFieldEnd() + if self.message is not None: + oprot.writeFieldBegin('message', TType.STRING, 2) + oprot.writeString(self.message) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + + def __repr__(self): + L = ['%s=%r' % (key, value) + for key, value in self.__dict__.iteritems()] + return '%s(%s)' % (self.__class__.__name__, ', '.join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) diff --git a/tomograph/backends/zipkin/generated/zipkinCore/__init__.py b/tomograph/backends/zipkin/generated/zipkinCore/__init__.py new file mode 100644 index 0000000..adefd8e --- /dev/null +++ b/tomograph/backends/zipkin/generated/zipkinCore/__init__.py @@ -0,0 +1 @@ +__all__ = ['ttypes', 'constants'] diff --git a/tomograph/backends/zipkin/generated/zipkinCore/constants.py b/tomograph/backends/zipkin/generated/zipkinCore/constants.py new file mode 100644 index 0000000..8a7859c --- /dev/null +++ b/tomograph/backends/zipkin/generated/zipkinCore/constants.py @@ -0,0 +1,15 @@ +# +# Autogenerated by Thrift Compiler (0.9.0) +# +# DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING +# +# options string: py:new_style=true +# + +from thrift.Thrift import TType, TMessageType, TException, TApplicationException +from ttypes import * + +CLIENT_SEND = "cs" +CLIENT_RECV = "cr" +SERVER_SEND = "ss" +SERVER_RECV = "sr" diff --git a/tomograph/backends/zipkin/generated/zipkinCore/ttypes.py b/tomograph/backends/zipkin/generated/zipkinCore/ttypes.py new file mode 100644 index 0000000..7cf09f9 --- /dev/null +++ b/tomograph/backends/zipkin/generated/zipkinCore/ttypes.py @@ -0,0 +1,477 @@ +# +# Autogenerated by Thrift Compiler (0.9.0) +# +# DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING +# +# options string: py:new_style=true +# + +from thrift.Thrift import TType, TMessageType, TException, TApplicationException + +from thrift.transport import TTransport +from thrift.protocol import TBinaryProtocol, TProtocol +try: + from thrift.protocol import fastbinary +except: + fastbinary = None + + +class AnnotationType(object): + BOOL = 0 + BYTES = 1 + I16 = 2 + I32 = 3 + I64 = 4 + DOUBLE = 5 + STRING = 6 + + _VALUES_TO_NAMES = { + 0: "BOOL", + 1: "BYTES", + 2: "I16", + 3: "I32", + 4: "I64", + 5: "DOUBLE", + 6: "STRING", + } + + _NAMES_TO_VALUES = { + "BOOL": 0, + "BYTES": 1, + "I16": 2, + "I32": 3, + "I64": 4, + "DOUBLE": 5, + "STRING": 6, + } + + +class Endpoint(object): + """ + Attributes: + - ipv4 + - port + - service_name + """ + + thrift_spec = ( + None, # 0 + (1, TType.I32, 'ipv4', None, None, ), # 1 + (2, TType.I16, 'port', None, None, ), # 2 + (3, TType.STRING, 'service_name', None, None, ), # 3 + ) + + def __init__(self, ipv4=None, port=None, service_name=None,): + self.ipv4 = ipv4 + self.port = port + self.service_name = service_name + + def read(self, iprot): + if iprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None and fastbinary is not None: + fastbinary.decode_binary(self, iprot.trans, (self.__class__, self.thrift_spec)) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.I32: + self.ipv4 = iprot.readI32(); + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.I16: + self.port = iprot.readI16(); + else: + iprot.skip(ftype) + elif fid == 3: + if ftype == TType.STRING: + self.service_name = iprot.readString(); + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and self.thrift_spec is not None and fastbinary is not None: + oprot.trans.write(fastbinary.encode_binary(self, (self.__class__, self.thrift_spec))) + return + oprot.writeStructBegin('Endpoint') + if self.ipv4 is not None: + oprot.writeFieldBegin('ipv4', TType.I32, 1) + oprot.writeI32(self.ipv4) + oprot.writeFieldEnd() + if self.port is not None: + oprot.writeFieldBegin('port', TType.I16, 2) + oprot.writeI16(self.port) + oprot.writeFieldEnd() + if self.service_name is not None: + oprot.writeFieldBegin('service_name', TType.STRING, 3) + oprot.writeString(self.service_name) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + + def __repr__(self): + L = ['%s=%r' % (key, value) + for key, value in self.__dict__.iteritems()] + return '%s(%s)' % (self.__class__.__name__, ', '.join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + +class Annotation(object): + """ + Attributes: + - timestamp + - value + - host + - duration + """ + + thrift_spec = ( + None, # 0 + (1, TType.I64, 'timestamp', None, None, ), # 1 + (2, TType.STRING, 'value', None, None, ), # 2 + (3, TType.STRUCT, 'host', (Endpoint, Endpoint.thrift_spec), None, ), # 3 + (4, TType.I32, 'duration', None, None, ), # 4 + ) + + def __init__(self, timestamp=None, value=None, host=None, duration=None,): + self.timestamp = timestamp + self.value = value + self.host = host + self.duration = duration + + def read(self, iprot): + if iprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None and fastbinary is not None: + fastbinary.decode_binary(self, iprot.trans, (self.__class__, self.thrift_spec)) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.I64: + self.timestamp = iprot.readI64(); + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRING: + self.value = iprot.readString(); + else: + iprot.skip(ftype) + elif fid == 3: + if ftype == TType.STRUCT: + self.host = Endpoint() + self.host.read(iprot) + else: + iprot.skip(ftype) + elif fid == 4: + if ftype == TType.I32: + self.duration = iprot.readI32(); + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and self.thrift_spec is not None and fastbinary is not None: + oprot.trans.write(fastbinary.encode_binary(self, (self.__class__, self.thrift_spec))) + return + oprot.writeStructBegin('Annotation') + if self.timestamp is not None: + oprot.writeFieldBegin('timestamp', TType.I64, 1) + oprot.writeI64(self.timestamp) + oprot.writeFieldEnd() + if self.value is not None: + oprot.writeFieldBegin('value', TType.STRING, 2) + oprot.writeString(self.value) + oprot.writeFieldEnd() + if self.host is not None: + oprot.writeFieldBegin('host', TType.STRUCT, 3) + self.host.write(oprot) + oprot.writeFieldEnd() + if self.duration is not None: + oprot.writeFieldBegin('duration', TType.I32, 4) + oprot.writeI32(self.duration) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + + def __repr__(self): + L = ['%s=%r' % (key, value) + for key, value in self.__dict__.iteritems()] + return '%s(%s)' % (self.__class__.__name__, ', '.join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + +class BinaryAnnotation(object): + """ + Attributes: + - key + - value + - annotation_type + - host + """ + + thrift_spec = ( + None, # 0 + (1, TType.STRING, 'key', None, None, ), # 1 + (2, TType.STRING, 'value', None, None, ), # 2 + (3, TType.I32, 'annotation_type', None, None, ), # 3 + (4, TType.STRUCT, 'host', (Endpoint, Endpoint.thrift_spec), None, ), # 4 + ) + + def __init__(self, key=None, value=None, annotation_type=None, host=None,): + self.key = key + self.value = value + self.annotation_type = annotation_type + self.host = host + + def read(self, iprot): + if iprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None and fastbinary is not None: + fastbinary.decode_binary(self, iprot.trans, (self.__class__, self.thrift_spec)) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRING: + self.key = iprot.readString(); + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRING: + self.value = iprot.readString(); + else: + iprot.skip(ftype) + elif fid == 3: + if ftype == TType.I32: + self.annotation_type = iprot.readI32(); + else: + iprot.skip(ftype) + elif fid == 4: + if ftype == TType.STRUCT: + self.host = Endpoint() + self.host.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and self.thrift_spec is not None and fastbinary is not None: + oprot.trans.write(fastbinary.encode_binary(self, (self.__class__, self.thrift_spec))) + return + oprot.writeStructBegin('BinaryAnnotation') + if self.key is not None: + oprot.writeFieldBegin('key', TType.STRING, 1) + oprot.writeString(self.key) + oprot.writeFieldEnd() + if self.value is not None: + oprot.writeFieldBegin('value', TType.STRING, 2) + oprot.writeString(self.value) + oprot.writeFieldEnd() + if self.annotation_type is not None: + oprot.writeFieldBegin('annotation_type', TType.I32, 3) + oprot.writeI32(self.annotation_type) + oprot.writeFieldEnd() + if self.host is not None: + oprot.writeFieldBegin('host', TType.STRUCT, 4) + self.host.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + + def __repr__(self): + L = ['%s=%r' % (key, value) + for key, value in self.__dict__.iteritems()] + return '%s(%s)' % (self.__class__.__name__, ', '.join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + +class Span(object): + """ + Attributes: + - trace_id + - name + - id + - parent_id + - annotations + - binary_annotations + - debug + """ + + thrift_spec = ( + None, # 0 + (1, TType.I64, 'trace_id', None, None, ), # 1 + None, # 2 + (3, TType.STRING, 'name', None, None, ), # 3 + (4, TType.I64, 'id', None, None, ), # 4 + (5, TType.I64, 'parent_id', None, None, ), # 5 + (6, TType.LIST, 'annotations', (TType.STRUCT,(Annotation, Annotation.thrift_spec)), None, ), # 6 + None, # 7 + (8, TType.LIST, 'binary_annotations', (TType.STRUCT,(BinaryAnnotation, BinaryAnnotation.thrift_spec)), None, ), # 8 + (9, TType.BOOL, 'debug', None, False, ), # 9 + ) + + def __init__(self, trace_id=None, name=None, id=None, parent_id=None, annotations=None, binary_annotations=None, debug=thrift_spec[9][4],): + self.trace_id = trace_id + self.name = name + self.id = id + self.parent_id = parent_id + self.annotations = annotations + self.binary_annotations = binary_annotations + self.debug = debug + + def read(self, iprot): + if iprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None and fastbinary is not None: + fastbinary.decode_binary(self, iprot.trans, (self.__class__, self.thrift_spec)) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.I64: + self.trace_id = iprot.readI64(); + else: + iprot.skip(ftype) + elif fid == 3: + if ftype == TType.STRING: + self.name = iprot.readString(); + else: + iprot.skip(ftype) + elif fid == 4: + if ftype == TType.I64: + self.id = iprot.readI64(); + else: + iprot.skip(ftype) + elif fid == 5: + if ftype == TType.I64: + self.parent_id = iprot.readI64(); + else: + iprot.skip(ftype) + elif fid == 6: + if ftype == TType.LIST: + self.annotations = [] + (_etype3, _size0) = iprot.readListBegin() + for _i4 in xrange(_size0): + _elem5 = Annotation() + _elem5.read(iprot) + self.annotations.append(_elem5) + iprot.readListEnd() + else: + iprot.skip(ftype) + elif fid == 8: + if ftype == TType.LIST: + self.binary_annotations = [] + (_etype9, _size6) = iprot.readListBegin() + for _i10 in xrange(_size6): + _elem11 = BinaryAnnotation() + _elem11.read(iprot) + self.binary_annotations.append(_elem11) + iprot.readListEnd() + else: + iprot.skip(ftype) + elif fid == 9: + if ftype == TType.BOOL: + self.debug = iprot.readBool(); + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and self.thrift_spec is not None and fastbinary is not None: + oprot.trans.write(fastbinary.encode_binary(self, (self.__class__, self.thrift_spec))) + return + oprot.writeStructBegin('Span') + if self.trace_id is not None: + oprot.writeFieldBegin('trace_id', TType.I64, 1) + oprot.writeI64(self.trace_id) + oprot.writeFieldEnd() + if self.name is not None: + oprot.writeFieldBegin('name', TType.STRING, 3) + oprot.writeString(self.name) + oprot.writeFieldEnd() + if self.id is not None: + oprot.writeFieldBegin('id', TType.I64, 4) + oprot.writeI64(self.id) + oprot.writeFieldEnd() + if self.parent_id is not None: + oprot.writeFieldBegin('parent_id', TType.I64, 5) + oprot.writeI64(self.parent_id) + oprot.writeFieldEnd() + if self.annotations is not None: + oprot.writeFieldBegin('annotations', TType.LIST, 6) + oprot.writeListBegin(TType.STRUCT, len(self.annotations)) + for iter12 in self.annotations: + iter12.write(oprot) + oprot.writeListEnd() + oprot.writeFieldEnd() + if self.binary_annotations is not None: + oprot.writeFieldBegin('binary_annotations', TType.LIST, 8) + oprot.writeListBegin(TType.STRUCT, len(self.binary_annotations)) + for iter13 in self.binary_annotations: + iter13.write(oprot) + oprot.writeListEnd() + oprot.writeFieldEnd() + if self.debug is not None: + oprot.writeFieldBegin('debug', TType.BOOL, 9) + oprot.writeBool(self.debug) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + + def __repr__(self): + L = ['%s=%r' % (key, value) + for key, value in self.__dict__.iteritems()] + return '%s(%s)' % (self.__class__.__name__, ', '.join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) diff --git a/tomograph/backends/zipkin/zipkin.py b/tomograph/backends/zipkin/zipkin.py new file mode 100644 index 0000000..8c1f8a2 --- /dev/null +++ b/tomograph/backends/zipkin/zipkin.py @@ -0,0 +1,63 @@ +# Copyright (c) 2012 Yahoo! Inc. All rights reserved. +# Licensed under the Apache License, Version 2.0 (the "License"); you +# may not use this file except in compliance with the License. You may +# obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 Unless required by +# applicable law or agreed to in writing, software distributed under +# the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES +# OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and +# limitations under the License. See accompanying LICENSE file. + +import zipkin_thrift +from generated.scribe import scribe +from thrift.transport import TTransport +from thrift.transport import TSocket +from thrift.protocol import TBinaryProtocol + +from tomograph import config + +import base64 +import StringIO +import IPy +import time +import random +import socket +import sys +import traceback + + + +def send(span): + tsocket = TSocket.TSocket(config.zipkin_host, config.zipkin_port) + transport = TTransport.TFramedTransport(tsocket) + transport.open() + protocol = TBinaryProtocol.TBinaryProtocol(transport) + client = scribe.Client(protocol) + + def endpoint(note): + try: + ip = socket.gethostbyname(note.address) + except: + print >>sys.stderr, 'host resolution error: ', traceback.format_exc() + ip = '0.0.0.0' + return zipkin_thrift.Endpoint(ipv4 = IPy.IP(ip).int(), + port = note.port, + service_name = note.service_name) + def annotation(note): + return zipkin_thrift.Annotation(timestamp = int(note.time * 1e6), + value = note.value, + host = endpoint(note)) + + zspan = zipkin_thrift.Span(trace_id = span.trace_id, + id = span.id, + name = span.name, + parent_id = span.parent_id, + annotations = [annotation(n) for n in span.notes]) + + out = StringIO.StringIO() + raw = TBinaryProtocol.TBinaryProtocol(out) + zspan.write(raw) + logentry = scribe.LogEntry('zipkin', base64.b64encode(out.getvalue())) + client.Log([logentry]) + transport.close() diff --git a/tomograph/backends/zipkin/zipkin_thrift.py b/tomograph/backends/zipkin/zipkin_thrift.py new file mode 100644 index 0000000..9607539 --- /dev/null +++ b/tomograph/backends/zipkin/zipkin_thrift.py @@ -0,0 +1,12 @@ +# Copyright (c) 2012 Yahoo! Inc. All rights reserved. +# Licensed under the Apache License, Version 2.0 (the "License"); you +# may not use this file except in compliance with the License. You may +# obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 Unless required by +# applicable law or agreed to in writing, software distributed under +# the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES +# OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and +# limitations under the License. See accompanying LICENSE file. + +from generated.zipkinCore.constants import * diff --git a/tomograph/config.py b/tomograph/config.py new file mode 100644 index 0000000..4dd27ce --- /dev/null +++ b/tomograph/config.py @@ -0,0 +1,52 @@ +# Copyright (c) 2012 Yahoo! Inc. All rights reserved. +# Licensed under the Apache License, Version 2.0 (the "License"); you +# may not use this file except in compliance with the License. You may +# obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 Unless required by +# applicable law or agreed to in writing, software distributed under +# the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES +# OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and +# limitations under the License. See accompanying LICENSE file. + +import logging + +logger = logging.getLogger(__name__) + +enabled_backends = ['tomograph.backends.zipkin', + 'tomograph.backends.statsd', + 'tomograph.backends.log'] +backend_modules = [] + +def set_backends(backends): + """ + Set the list of enabled backends. Backend name should be the full + module name of the backend. All backends must support a + send(span) method. + """ + global enabled_backends + global backend_modules + enabled_backends = backends[:] + backend_modules = [] + for backend in enabled_backends: + try: + logger.info('loading backend {0}'.format(backend)) + module = __import__(backend) + for submodule in backend.split('.')[1:]: + module = getattr(module, submodule) + backend_modules.append(module) + except (ImportError, AttributeError, ValueError) as err: + raise RuntimeError('Could not load tomograph backend {0}: {1}'.format( + backend, err)) + +def get_backends(): + if not backend_modules: + set_backends(enabled_backends) + return backend_modules + +zipkin_host = '172.16.77.141' +zipkin_port = 9410 + +statsd_host = '172.16.77.141' +statsd_port = 8125 + diff --git a/tomograph/tomograph.py b/tomograph/tomograph.py new file mode 100644 index 0000000..0afb747 --- /dev/null +++ b/tomograph/tomograph.py @@ -0,0 +1,152 @@ +# Copyright (c) 2012 Yahoo! Inc. All rights reserved. +# Licensed under the Apache License, Version 2.0 (the "License"); you +# may not use this file except in compliance with the License. You may +# obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 Unless required by +# applicable law or agreed to in writing, software distributed under +# the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES +# OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and +# limitations under the License. See accompanying LICENSE file. + +import config + +import random +import sys +import time +from eventlet import corolocal +from collections import namedtuple +import socket +import pickle +import base64 +import logging +import webob.dec + +span_stack = corolocal.local() + +Span = namedtuple('Span', 'trace_id parent_id id name notes') +Note = namedtuple('Note', 'time value service_name address port') + +def start(service_name, name, address, port, trace_info=None): + parent_id = None + if hasattr(span_stack, 'trace_id'): + trace_id = span_stack.trace_id + parent_id = span_stack.spans[-1].id + else: + if trace_info is None: + trace_id = span_stack.trace_id = getId() + else: + trace_id = span_stack.trace_id = trace_info[0] + parent_id = trace_info[1] + span_stack.spans = [] + + span = Span(trace_id, parent_id, getId(), name, []) + span_stack.spans.append(span) + annotate('start', service_name, address, port) + +def get_trace_info(): + return (span_stack.trace_id, span_stack.spans[-1].id) + +def stop(name): + annotate('stop') + span = span_stack.spans.pop() + assert span.name == name, 'start span name {0} not equal to end span name {1}'.format(span.name, name) + for backend in config.get_backends(): + backend.send(span) + if not span_stack.spans: + del(span_stack.trace_id) + +def annotate(value, service_name=None, address=None, port=None): + last_span = span_stack.spans[-1] + if service_name is None: + last_note = last_span.notes[-1] + service_name = last_note.service_name + address = last_note.address + port = last_note.port + note = Note(time.time(), value, service_name, address, int(port)) + span_stack.spans[-1].notes.append(note) + +def getId(): + return random.randrange(sys.maxint >> 10) + +## wrapper/decorators +def tracewrap(func, service_name, name, host='0.0.0.0', port=0): + if host == '0.0.0.0': + host = socket.gethostname() + def trace_and_call(*args, **kwargs): + if service_name is None and len(args) > 0 and isinstance(args[0], object): + s = args[0].__class__.__name__ + else: + s = service_name + start(s, name, host, port) + ret = func(*args, **kwargs) + stop(name) + return ret + return trace_and_call + +def traced(service_name, name, host='0.0.0.0', port=0): + def t1(func): + return tracewrap(func, service_name, name, host, port) + return t1 + + +## sqlalchemy event listeners +def before_execute(name): + def handler(conn, clauseelement, multiparams, params): + h = str(conn.connection.connection) + a = h.find("'") + b = h.find("'", a+1) + if b > a: + h = h[a+1:b] + else: + h = 'unknown' + port = conn.connection.connection.port + #print >>sys.stderr, 'connection is {0}:{1}'.format(h, port) + #print >>sys.stderr, 'sql statement is {0}'.format(clauseelement) + start(str(name) + 'db client', 'execute', h, port) + return handler + +def after_execute(name): + # name isn't used, at least not yet... + def handler(conn, clauseelement, multiparams, params, result): + stop('execute') + return handler + +## http helpers +def start_http(service_name, name, request): + trace_info_enc = request.headers.get('X-Trace-Info') + (host, port) = request.host.split(':') + if trace_info_enc: + trace_info = pickle.loads(base64.b64decode(trace_info_enc)) + else: + trace_info = None + start(service_name, name, host, port, trace_info) + +def add_trace_info_header(headers): + headers['X-Trace-Info'] = base64.b64encode(pickle.dumps(get_trace_info())) + + +## WSGI middleware +class Middleware(object): + """ + WSGI Middleware that enables tomograph tracing for an application. + """ + + def __init__(self, application, service_name='Server', name='WSGI'): + self.application = application + self.service_name = service_name + self.name = name + + @classmethod + def factory(cls, global_conf, **local_conf): + def filter(app): + return cls(app, **local_conf) + return filter + + @webob.dec.wsgify + def __call__(self, req): + start_http(self.service_name, self.name, req) + response = req.get_response(self.application) + stop(self.name) + return response +