From 3c41a33bb7fca32527b92556b22dc9c73debd2ab Mon Sep 17 00:00:00 2001 From: Sandy Walsh Date: Fri, 3 Apr 2015 07:12:32 -0700 Subject: [PATCH] More docs around debugging streams. And minor patch to status utility. Change-Id: I3eb9f8028b0140a9f4dae22137f137c37a80e6c9 --- bin/stv3_status.py | 13 +++++++++ docs/about.html | 69 ++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 82 insertions(+) diff --git a/bin/stv3_status.py b/bin/stv3_status.py index 4e021cf..c69f8f5 100644 --- a/bin/stv3_status.py +++ b/bin/stv3_status.py @@ -41,6 +41,7 @@ def ssh(host, cmds, user, password, port): return None outputs = [] for cmd in cmds: + print "Running:", cmd s.sendline(cmd) s.prompt() outputs.append(s.before) @@ -72,11 +73,16 @@ vhost = config.get('vhost', '/') lines = config.get('tail_lines', '100') queue_prefixes = config.get('queue_prefixes', ['monitor']) +num_cells = len(cell_names) for worker in worker_hostnames: commands = ["ps auxww | grep -E 'yagi-event|pipeline_worker'"] for cell in cell_names: commands.append("tail --lines %s /var/log/stv3/yagi-%s.log" % (lines, cell)) + commands.append("tail --lines %s /var/log/stv3/stv3.log" % lines) + archive_directories = ["/etc/stv3/%s/events" % cell for cell in cell_names] + commands.append("du -ch --time %s /etc/stv3/tarballs" % " ".join(archive_directories)) + commands.append("ls -lah /etc/stv3/tarballs") print "--- worker: %s" % (worker, ) ret = ssh(worker, commands, username, password, port) @@ -87,6 +93,13 @@ for worker in worker_hostnames: with open("%s-yagi-%s.log" % (worker, cell), "w") as o: o.write(ret[i+1]) + print "Writing pipeline worker: %s-stv3.log" % worker + with open("%s-stv3.log" % worker, "w") as o: + o.write(ret[-3]) + + print ret[-2] + print ret[-1] + for api in api_hostnames: commands = ["ps auxww | grep gunicorn", "tail --lines %s /var/log/stv3/gunicorn.log" % (lines, )] diff --git a/docs/about.html b/docs/about.html index d665841..08aa864 100644 --- a/docs/about.html +++ b/docs/about.html @@ -441,6 +441,75 @@ winchester.debugging[INFO line: 161] ----------------------------

The winchester.debugging lines will tell you how fire and matching criteria is progressing. In this case, it's saying that 397 firing criteria checks were made and only 2 passed. If your debug level is 2, you will get breakdown of the reasons the checks failed. You can use this information to review your trigger definitions and see if something could be wrong. Additionally, the matching criteria results are detailed. In this case we see that, of 207 events, 200 were acceptable. The details on the 7 rejected are listed below. Finally, some "counters" are supplied on the stream processing in general. 58 new streams were created on this pass, 100 new events added to various "test_trigger" streams, and 1 stream is ready to fire.

By selectively turning on per-stream debugging, you can quickly find processing problems and ignore a lot of log noise.

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
MessageExplanation
Pre-commit successfulThe handle_events() call was successful for this pipeline. +1 per pipeline, not per handler.
Pipeline errorA handle_events() call failed. Details in log files.
Commit successfulThe commit() calls for the pipeline succeeded. +1 per handler.
Commit errorA commit() call for the pipeline failed. +1 per handler.
Rollback successfulThe rollback() calls for the pipeline succeeded. +1 per handler.
Rollback failedA rollback() calls for the pipeline succeeded. +1 per handler.
LockedAttempted to fire/expire a stream that was locked in the db.
Unknown trigger def '[trigger]'Could not find the trigger definition in order to fire/expire this stream.
Unknown pipeline '[pipeline]'Could not find the fire/expire pipeline for this stream.
No fire pipeline for '[stream name]'A fire pipeline was not defined for this stream.
No expire pipeline for '[stream name]'An expire pipeline was not defined for this stream.
Streams firedThe number of streams fired during this cycle.
Streams expiredThe number of streams expired during this cycle.
New streamNew streams created for this trigger definition during this cycle.
Ready to fireNumber of streams for this trigger definition ready to fire this cycle.
Added eventsNumber of events added to this trigger definition during this cycle.
Match failed: not '[trait name]'Event did not match this trigger def because it was missing a needed distinguishing trait.
Match: PassedSome criteria matched for adding this event to a stream.
Match: No matching criteriaNo criteria matched to add this event to a stream.
Fire: Not enough matching criteriaNumber of fire criteria matches < criteria number (default 1).
Fire: PassedEnough criteria matched for adding this stream to fire.
Match/Fire: not [trait name]Criteria match failed because named trait not present in event.
Match/Fire: == failedCriteria match failed because named trait value != expected.
Match/Fire: > failedCriteria match failed because named trait value <= expected.
Match/Fire: < failedCriteria match failed because named trait value >= expected.
Match/Fire: Criterion match() fall-thruCriteria match failed because of undefined comparison operation.
Match/Fire: Time: not '[trait name]'Time criteria match failed because trait not present in event.
Match/Fire: Time: no referenced traitTime criteria match failed because time expression bad.
Match/Fire: Time: not in timerangeTime criteria match failed because time value was not in required range.
Match/Fire: Wrong event typeCriteria match failed because event is wrong type.
Match/Fire: Time: No timestamp traitCriteria match failed because referenced timestamp trait doesn't exist.
Match/Fire: Time: Not time yetCriteria match failed because 'timestamp' not in defined time ranges.
+ +

Winchester Pipeline Handlers

Winchester comes with a set of stock pipeline handlers for the most popular OpenStack operations.