From 01fb487df11e3606dec5e2ab21cb79ab01f34c34 Mon Sep 17 00:00:00 2001 From: Christian Berendt Date: Thu, 1 Oct 2015 20:24:44 +0200 Subject: [PATCH 01/11] Fix syntax of the README file Change-Id: I3a86e2db67c4e4ab4ea579b7b7a565e3a0582b15 --- README.rst | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/README.rst b/README.rst index 21f2842..c03be76 100644 --- a/README.rst +++ b/README.rst @@ -1,8 +1,8 @@ -=============================== +======== os-testr -=============================== +======== -A testr wrapper to provide functionality for OpenStack projects +A testr wrapper to provide functionality for OpenStack projects. * Free software: Apache license * Documentation: http://docs.openstack.org/developer/os-testr @@ -12,8 +12,8 @@ A testr wrapper to provide functionality for OpenStack projects Features -------- -* ostestr: a testr wrapper that uses subunit-trace for output and builds some - helpful extra functionality around testr -* subunit-trace: an output filter for a subunit stream which provides useful - information about the run -* subunit2html: generates a test results html page from a subunit stream +* ``ostestr``: a testr wrapper that uses subunit-trace for output and builds + some helpful extra functionality around testr +* ``subunit-trace``: an output filter for a subunit stream which provides + useful information about the run +* ``subunit2html``: generates a test results html page from a subunit stream From 2a22826c58d581c9b41a0a931394be9fb628ca8d Mon Sep 17 00:00:00 2001 From: Matthew Treinish Date: Wed, 18 Nov 2015 10:14:10 -0500 Subject: [PATCH 02/11] Fail if no tests were successfully executed This commit adds another sanity check to ensure we didn't have a test run with just skipped tests. Change-Id: Ie4bd185c209d9e9230ca8a3f54e8a433e82d9c14 --- os_testr/subunit_trace.py | 6 ++++++ .../tests/sample_streams/all_skips.subunit | Bin 0 -> 2453 bytes .../tests/sample_streams/successful.subunit | Bin 0 -> 12563 bytes os_testr/tests/test_subunit_trace.py | 20 ++++++++++++++++++ 4 files changed, 26 insertions(+) create mode 100644 os_testr/tests/sample_streams/all_skips.subunit create mode 100644 os_testr/tests/sample_streams/successful.subunit diff --git a/os_testr/subunit_trace.py b/os_testr/subunit_trace.py index 315d850..1e9b51d 100755 --- a/os_testr/subunit_trace.py +++ b/os_testr/subunit_trace.py @@ -354,6 +354,12 @@ def main(): print_fails(sys.stdout) if not args.no_summary: print_summary(sys.stdout, elapsed_time) + + # NOTE(mtreinish): Ideally this should live in testtools streamSummary + # this is just in place until the behavior lands there (if it ever does) + if count_tests('status', '^success$') == 0: + print("\nNo tests were successful during the run") + exit(1) exit(0 if summary.wasSuccessful() else 1) diff --git a/os_testr/tests/sample_streams/all_skips.subunit b/os_testr/tests/sample_streams/all_skips.subunit new file mode 100644 index 0000000000000000000000000000000000000000..54156b5945c9fe6f598e9a788f470e5ead04d8e9 GIT binary patch literal 2453 zcmdn2&Eyah=Dp_J?d1^-4xYuSC7}h*If=!^3K}J;xdo}kC3=YknR+RSC5iC`Mfu68 z#l@L<>3StViFmMJaeiqLP*l$as3I854gtz(8YV2Bv{}2rVHyq07B5MyDA6y-NzBZ% zPR>Xy0y^5Zv?R@fttd6II6qG+I1wnAs8C#5P>^3#qL5jvP?TC&npu>Z@;m$Ixy{_MyYgT?zv7T3c5;z=P=IVxk!m*7C zW|1Fm(bsdlLE#og1y>-#>HO@OCqUr@j*aKHR$XuiCMlqjb4rU#Qj6k22{k7Xm?ZU_ zAtE7A0dNSmOyrjXh2R2^+i7MC`C+@>w)`F_Yzt`WK6t1bPrhTJ2nzLVkbPfgX)`!P zkQC~`sRR e6q=2VP|{EU literal 0 HcmV?d00001 diff --git a/os_testr/tests/sample_streams/successful.subunit b/os_testr/tests/sample_streams/successful.subunit new file mode 100644 index 0000000000000000000000000000000000000000..6f3b6641252484c5f40d630d4a28ac08208ba2bf GIT binary patch literal 12563 zcmeI2d2ka|9LEczh!vb7j>s?!3uyhLq%s4ihwX2QcgvIVNkBY0$QLj98-<~kz?c%6@9;#G>cA$)nwC4GSf_w?*4x7 z_xt?5@BQ*#DkECc$)j3_XPyE5lizct$thTtguB&Z_&+g`~YVjxKdv1p(?I^J=(}h4l7B6)~Pa+ z%f?ZX%14UhsWGO!Ec|40xEW`@o0Apf;@m7XICW5bN@`NN9Su!NO~?wR{`SDvOKdJ% zMA;G&6EdkcD3eKKlspQqMQJuPB*~T#Z%?z0Cja6EQw}FPh1Xe$vL|4ZNki%}C`zoN zjIX=F^Nx&6RvjwODdaT_u%3Hmw@`?)TX(bfExAL?aXiSW7XY~vW8oM#xyvc=PF0dv z-Z`GiNVBCR*wX89tMUrxHn|witxBxNo5b;oi7j%v)f_e})RW^l3g=yr;wVPSRSQ^N z!Rhk&ilcZzp>PS{;pb3=98TeV*VI)<=!q{fUdH7?yMtUFf~tg^94L;cjF?W_1NPOb zZJ{9Lvy!QA;Vk=sN-UIN#OC7lWidWc{9B=r;@EUPuPw-=JF)ZIwoE_m=!<0ILy>p| zkELE$hYc>BtBRtcz*F*8TyRvW*G=&Tu=)HWHR*j`zpx97N93hByV48e-uQL z1Qt()NJX*#RV3bN@6P3@aGy#}z0zr|zlQW>vf-gjnt)8a+;v6by)N`dOz{9vNj|Bx zpcM$nWXq=u?t)Cx19he8Wa+RCyFn+ziOJlE%mKPOvEtsuF4&-B@$_WS;vk+WV9Qs> z#6v0sVoW;naUjNY^%62;)TZeTK>grZOsdB_MAf3~3r82>^7;+c@<9+3Xz&@I2< z;QuIOn2vhx9JmptqvI`wvZlzL+p=mjMamIn1{E+?nuav7Pe2t_@$=Es54cH zR-O~D8#To=T0r`W9C?@}t(IpJYVje1A`2#3NRQ`HWD5vcQ{?4G!{>k^=aR{0N?0ac zY{(jk*FL`#IYQuBd@FtkGrBGWX>BDVD>+osWJad1k&3Ezh0Ss7Q0z;Jvch2bS<>Y4 zeqYrxw#iVnyT}-4w=1L9`CGO4G!BtQu0gvx+|;2pKZI06kz61_lqVi-jzQ>aVRYKz z;uanz$cDWz?KH^d`{qnjQ`kFgug%sJHn+DC#o3@u4Q|p<_T_GewbJ_}a+oeisxrhfEktqr>PK0k2^Jc3rS4Jz*)^3dOXA z3~NmnYttB(z6!3)`v@ObG_!$2T_5VRdlA(2;--~xtx|t~Dy9l5^;JB;?3x<7f=;Qg zQuUAaJlmG|NYC@j49~{p%C9O)fZG%Z|6(u89Yd|_*Uh{gsjC~yKj2@+b=?E_^MVq; zk7Cr%`%hg1idjJM#P&`Vbh3XG>keqW#|Y${w!BwaEhKR=1aax$loBJ2>od&ybAS6r zat{cZj0NKWVQw&@_Yv&#W9iv4AeapVPm0s3=;8hm{HG4RCqWy+CoF?3PJ=Kmx0&>; z!S?kz-rO<||JI>Q4H+^WEuN?Mop;(0MoxB$nhEYF(euxAS`^H Date: Tue, 24 Nov 2015 14:02:04 +0900 Subject: [PATCH 03/11] Add *.egg* to .gitignore This commit adds "*.egg*" to .gitignore and removes unnecessary entries. Change-Id: I3b0985b3a11eccb3e16b8ee8eebcd2ae113500ec --- .gitignore | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.gitignore b/.gitignore index 8a3c704..8139c6d 100644 --- a/.gitignore +++ b/.gitignore @@ -4,8 +4,7 @@ *.so # Packages -*.egg -*.egg-info +*.egg* dist build eggs From 1a3d0d9ec0ea328e508eb21e2c1ab859ede42d46 Mon Sep 17 00:00:00 2001 From: Masayuki Igawa Date: Tue, 24 Nov 2015 13:46:46 +0900 Subject: [PATCH 04/11] Add colored output feature to subunit-trace This commit enables that subunit-trace shows colored output. Because there are some people who like colored output. While this doesn't work for the OpenStack gate for local testing it would be nice to appease those folks and have a --color flag that adds different colors on the console output from subunit-trace. Change-Id: Iac48591786d10c908d63e6df94b979ded7d8dbe1 Closes-Bug: #1519044 --- os_testr/subunit_trace.py | 40 +++++++++++---- os_testr/utils/__init__.py | 0 os_testr/utils/colorizer.py | 98 +++++++++++++++++++++++++++++++++++++ 3 files changed, 129 insertions(+), 9 deletions(-) create mode 100644 os_testr/utils/__init__.py create mode 100644 os_testr/utils/colorizer.py diff --git a/os_testr/subunit_trace.py b/os_testr/subunit_trace.py index 315d850..48f2416 100755 --- a/os_testr/subunit_trace.py +++ b/os_testr/subunit_trace.py @@ -17,6 +17,7 @@ # under the License. """Trace a subunit stream in reasonable detail and high accuracy.""" +from __future__ import absolute_import import argparse import datetime @@ -28,6 +29,8 @@ import sys import subunit import testtools +from os_testr.utils import colorizer + # NOTE(mtreinish) on python3 anydbm was renamed dbm and the python2 dbm module # was renamed to dbm.ndbm, this block takes that into account try: @@ -148,7 +151,8 @@ def find_test_run_time_diff(test_id, run_time): def show_outcome(stream, test, print_failures=False, failonly=False, - enable_diff=False, threshold='0', abbreviate=False): + enable_diff=False, threshold='0', abbreviate=False, + enable_color=False): global RESULTS status = test['status'] # TODO(sdague): ask lifeless why on this? @@ -167,19 +171,29 @@ def show_outcome(stream, test, print_failures=False, failonly=False, if name == 'process-returncode': return + for color in [colorizer.AnsiColorizer, colorizer.NullColorizer]: + if not enable_color: + color = colorizer.NullColorizer(stream) + break + if color.supported(): + color = color(stream) + break + if status == 'fail': FAILS.append(test) if abbreviate: - stream.write('F') + color.write('F', 'red') else: - stream.write('{%s} %s [%s] ... FAILED\n' % ( + stream.write('{%s} %s [%s] ... ' % ( worker, name, duration)) + color.write('FAILED', 'red') + stream.write('\n') if not print_failures: print_attachments(stream, test, all_channels=True) elif not failonly: if status == 'success': if abbreviate: - stream.write('.') + color.write('.', 'green') else: out_string = '{%s} %s [%s' % (worker, name, duration) perc_diff = find_test_run_time_diff(test['id'], duration) @@ -189,17 +203,22 @@ def show_outcome(stream, test, print_failures=False, failonly=False, out_string = out_string + ' +%.2f%%' % perc_diff else: out_string = out_string + ' %.2f%%' % perc_diff - stream.write(out_string + '] ... ok\n') + stream.write(out_string + '] ... ') + color.write('ok', 'green') + stream.write('\n') print_attachments(stream, test) elif status == 'skip': if abbreviate: - stream.write('S') + color.write('S', 'blue') else: reason = test['details'].get('reason', '') if reason: reason = ': ' + reason.as_text() - stream.write('{%s} %s ... SKIPPED%s\n' % ( - worker, name, reason)) + stream.write('{%s} %s ... ' % ( + worker, name)) + color.write('SKIPPED', 'blue') + stream.write('%s' % (reason)) + stream.write('\n') else: if abbreviate: stream.write('%s' % test['status'][0]) @@ -320,6 +339,8 @@ def parse_args(): parser.add_argument('--no-summary', action='store_true', help="Don't print the summary of the test run after " " completes") + parser.add_argument('--color', action='store_true', + help="Print results with colors") return parser.parse_args() @@ -332,7 +353,8 @@ def main(): print_failures=args.print_failures, failonly=args.failonly, enable_diff=args.enable_diff, - abbreviate=args.abbreviate)) + abbreviate=args.abbreviate, + enable_color=args.color)) summary = testtools.StreamSummary() result = testtools.CopyStreamResult([outcomes, summary]) result = testtools.StreamResultRouter(result) diff --git a/os_testr/utils/__init__.py b/os_testr/utils/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/os_testr/utils/colorizer.py b/os_testr/utils/colorizer.py new file mode 100644 index 0000000..8ecec35 --- /dev/null +++ b/os_testr/utils/colorizer.py @@ -0,0 +1,98 @@ +# Copyright 2015 NEC Corporation +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# +# Colorizer Code is borrowed from Twisted: +# Copyright (c) 2001-2010 Twisted Matrix Laboratories. +# +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to +# the following conditions: +# +# The above copyright notice and this permission notice shall be +# included in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +import sys + + +class AnsiColorizer(object): + """A colorizer is an object that loosely wraps around a stream + + allowing callers to write text to the stream in a particular color. + + Colorizer classes must implement C{supported()} and C{write(text, color)}. + """ + _colors = dict(black=30, red=31, green=32, yellow=33, + blue=34, magenta=35, cyan=36, white=37) + + def __init__(self, stream): + self.stream = stream + + @classmethod + def supported(cls, stream=sys.stdout): + """Check the current platform supports coloring terminal output + + A class method that returns True if the current platform supports + coloring terminal output using this method. Returns False otherwise. + """ + if not stream.isatty(): + return False # auto color only on TTYs + try: + import curses + except ImportError: + return False + else: + try: + try: + return curses.tigetnum("colors") > 2 + except curses.error: + curses.setupterm() + return curses.tigetnum("colors") > 2 + except Exception: + # guess false in case of error + return False + + def write(self, text, color): + """Write the given text to the stream in the given color. + + @param text: Text to be written to the stream. + + @param color: A string label for a color. e.g. 'red', 'white'. + """ + color = self._colors[color] + self.stream.write('\x1b[%s;1m%s\x1b[0m' % (color, text)) + + +class NullColorizer(object): + """See _AnsiColorizer docstring.""" + def __init__(self, stream): + self.stream = stream + + @classmethod + def supported(cls, stream=sys.stdout): + return True + + def write(self, text, color): + self.stream.write(text) From 2b5c0175fc0cf9eddf570b743bf3c89ad7e0fd3a Mon Sep 17 00:00:00 2001 From: step6829 Date: Tue, 17 Nov 2015 22:28:57 +0000 Subject: [PATCH 05/11] Change to always parsing classes from test_id Currently only v1 correctly parses out the class information from the test_id. This changes the default behavior to always parse the class and module from test_id. Closes-Bug: #1517229 Change-Id: I6340797064f9289a7e861f853c42a763c5f5e2c2 --- os_testr/subunit2html.py | 6 +++--- os_testr/tests/test_subunit2html.py | 31 +++++++++++++++++++++++++++++ 2 files changed, 34 insertions(+), 3 deletions(-) create mode 100644 os_testr/tests/test_subunit2html.py diff --git a/os_testr/subunit2html.py b/os_testr/subunit2html.py index fd93ac7..c40909e 100755 --- a/os_testr/subunit2html.py +++ b/os_testr/subunit2html.py @@ -633,10 +633,10 @@ class HtmlOutput(testtools.TestResult): test = test.test if test.__class__ == subunit.RemotedTestCase: cl = test._RemotedTestCase__description.rsplit('.', 1)[0] - mod = cl.rsplit('.', 1)[0] - cls = ClassInfoWrapper(cl, mod) else: - cls = ClassInfoWrapper(str(test.__class__), str(test.__module__)) + cl = test.id().rsplit('.', 1)[0] + mod = cl.rsplit('.', 1)[0] + cls = ClassInfoWrapper(cl, mod) if not str(cls) in rmap: rmap[str(cls)] = [] classes.append(cls) diff --git a/os_testr/tests/test_subunit2html.py b/os_testr/tests/test_subunit2html.py new file mode 100644 index 0000000..f3196a5 --- /dev/null +++ b/os_testr/tests/test_subunit2html.py @@ -0,0 +1,31 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from ddt import data +from ddt import ddt +from subunit import RemotedTestCase +from testtools import PlaceHolder + +from os_testr import subunit2html +from os_testr.tests import base + + +@ddt +class TestSubunit2html(base.TestCase): + @data(RemotedTestCase, PlaceHolder) + def test_class_parsing(self, test_cls): + """Tests that the class paths are parsed for v1 & v2 tests""" + test_ = test_cls("example.path.to.test.method") + obj_ = subunit2html.HtmlOutput() + cls_ = [] + obj_._add_cls({}, cls_, test_, ()) + self.assertEqual("example.path.to.test", cls_[0].name) From c797436528dce3f8d72cf006ee16dcde9cf0bac2 Mon Sep 17 00:00:00 2001 From: Masayuki Igawa Date: Tue, 1 Dec 2015 17:11:25 +0900 Subject: [PATCH 06/11] Add delete *.pyc command before executing ostestr This commit adds a command for deleting *.pyc before executing ostestr with tox. This patch would help that removing unnecessary *.pyc files when we rename, move or remove *.py ones. Change-Id: Ifa0eb18a10c7e7ee7e15ce7cc69a1b007a016a76 --- tox.ini | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index 537b2f2..c436cdb 100644 --- a/tox.ini +++ b/tox.ini @@ -8,9 +8,12 @@ usedevelop = True install_command = pip install -U {opts} {packages} setenv = VIRTUAL_ENV={envdir} +whitelist_externals = find deps = -r{toxinidir}/requirements.txt -r{toxinidir}/test-requirements.txt -commands = ostestr {posargs} +commands = + find . -type f -name "*.pyc" -delete + ostestr {posargs} [testenv:pep8] commands = flake8 From 2fb3b5ecff94e2fada7f356fc238485126f1a1e7 Mon Sep 17 00:00:00 2001 From: Masayuki Igawa Date: Fri, 4 Dec 2015 18:26:12 +0900 Subject: [PATCH 07/11] Fix coverage section in tox.ini This commit fixes the coverage section in tox.ini to get coverage and also changes .gitignore to ignore the cover directory and .coverage directories. Change-Id: Ia58b5d109ad045833d6c37e16fd0fe3705212ed0 --- .gitignore | 4 +++- tox.ini | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/.gitignore b/.gitignore index 8139c6d..d6ae633 100644 --- a/.gitignore +++ b/.gitignore @@ -21,7 +21,9 @@ lib64 pip-log.txt # Unit test / coverage reports -.coverage +cover/ +.coverage* +!.coveragerc .tox nosetests.xml .testrepository diff --git a/tox.ini b/tox.ini index c436cdb..931250f 100644 --- a/tox.ini +++ b/tox.ini @@ -22,7 +22,7 @@ commands = flake8 commands = {posargs} [testenv:cover] -commands = python setup.py testr --coverage --testr-args='{posargs}' +commands = python setup.py testr --coverage --coverage-package-name='os_testr' --testr-args='{posargs}' [testenv:docs] commands = python setup.py build_sphinx From 33119634b5216c63f41d09c231148f8eacea2e0d Mon Sep 17 00:00:00 2001 From: Masayuki Igawa Date: Sat, 5 Dec 2015 22:53:26 +0900 Subject: [PATCH 08/11] Fix documentation typos This commit fixes some typos in the documentation. Change-Id: I34ef7f863763bed2fb30421627968607bd787500 --- doc/source/ostestr.rst | 2 +- doc/source/subunit_trace.rst | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/doc/source/ostestr.rst b/doc/source/ostestr.rst index 6c36bac..84b920a 100644 --- a/doc/source/ostestr.rst +++ b/doc/source/ostestr.rst @@ -112,7 +112,7 @@ exposed via the --regex option. For example:: $ ostestr --regex 'magic\.regex' This will do a straight passthrough of the provided regex to testr. -Additionally, ostestr allows you to specify a a blacklist file to define a set +Additionally, ostestr allows you to specify a blacklist file to define a set of regexes to exclude. You can specify a blacklist file with the --blacklist-file/-b option, for example:: diff --git a/doc/source/subunit_trace.rst b/doc/source/subunit_trace.rst index 1338fe6..028f970 100644 --- a/doc/source/subunit_trace.rst +++ b/doc/source/subunit_trace.rst @@ -20,7 +20,7 @@ Options Disable printing failure debug information in realtime --fails, -f Print failure debug information after the stream is - proccesed + processed --failonly Don't print success items --perc-diff, -d @@ -58,7 +58,7 @@ disabled by using --no-failure-debug, -n. For example:: $ testr run --subunit | subunit-trace --no-failure-debug -Rhere is also the option to print all failures together at the end of the test +Here is also the option to print all failures together at the end of the test run before the summary view. This is done using the --fails/-f option. For example:: @@ -77,7 +77,7 @@ example:: $ testr run --subunit | subunit-trace --failonly -The last output option provided by subunit-trace is to diable the summary view +The last output option provided by subunit-trace is to disable the summary view of the test run which is normally displayed at the end of a run. You can do this using the --no-summary option. For example:: From 2cb71893d12af09b07048bcd086b1b35c20d5319 Mon Sep 17 00:00:00 2001 From: Matthew Treinish Date: Thu, 10 Dec 2015 18:15:37 -0500 Subject: [PATCH 09/11] Add support to ostestr to use subunit-trace color This commit adds a new flag to the ostestr cli, --color, which is a passthrough option to subunit-trace to enabled colorized output. Change-Id: Ic38e008982d8f5bca78c52f51c69b5333744ecbc --- os_testr/os_testr.py | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/os_testr/os_testr.py b/os_testr/os_testr.py index db02446..6c5b778 100755 --- a/os_testr/os_testr.py +++ b/os_testr/os_testr.py @@ -58,6 +58,8 @@ def get_parser(args): 'this is mutually exclusive with --pretty') parser.add_argument('--list', '-l', action='store_true', help='List all the tests which will be run.') + parser.add_argument('--color', action='store_true', + help='Use color in the pretty output') slowest = parser.add_mutually_exclusive_group() slowest.add_argument('--slowest', dest='slowest', action='store_true', help="after the test run print the slowest tests") @@ -166,7 +168,7 @@ def construct_regex(blacklist_file, whitelist_file, regex, print_exclude): def call_testr(regex, subunit, pretty, list_tests, slowest, parallel, concur, - until_failure): + until_failure, color): if parallel: cmd = ['testr', 'run', '--parallel'] if concur: @@ -181,6 +183,12 @@ def call_testr(regex, subunit, pretty, list_tests, slowest, parallel, concur, cmd.append('--until-failure') cmd.append(regex) env = copy.deepcopy(os.environ) + + if pretty: + subunit_trace_cmd = ['subunit-trace', '--no-failure-debug', '-f'] + if color: + subunit_trace_cmd.append('--color') + # This workaround is necessary because of lp bug 1411804 it's super hacky # and makes tons of unfounded assumptions, but it works for the most part if (subunit or pretty) and until_failure: @@ -198,9 +206,9 @@ def call_testr(regex, subunit, pretty, list_tests, slowest, parallel, concur, if pretty: cmd = ['python', '-m', 'subunit.run', test] ps = subprocess.Popen(cmd, env=env, stdout=subprocess.PIPE) - proc = subprocess.Popen(['subunit-trace', - '--no-failure-debug', '-f', - '--no-summary'], env=env, + subunit_trace_cmd.append('--no-summary') + proc = subprocess.Popen(subunit_trace_cmd, + env=env, stdin=ps.stdout) ps.stdout.close() proc.communicate() @@ -223,7 +231,7 @@ def call_testr(regex, subunit, pretty, list_tests, slowest, parallel, concur, # If not until-failure special case call testr like normal elif pretty and not list_tests: ps = subprocess.Popen(cmd, env=env, stdout=subprocess.PIPE) - proc = subprocess.Popen(['subunit-trace', '--no-failure-debug', '-f'], + proc = subprocess.Popen(subunit_trace_cmd, env=env, stdin=ps.stdout) ps.stdout.close() else: @@ -261,7 +269,7 @@ def _select_and_call_runner(opts, exclude_regex): if not opts.no_discover and not opts.pdb: ec = call_testr(exclude_regex, opts.subunit, opts.pretty, opts.list, opts.slowest, opts.parallel, opts.concurrency, - opts.until_failure) + opts.until_failure, opts.color) else: test_to_run = opts.no_discover or opts.pdb if test_to_run.find('/') != -1: From cc64a3a585dbc4df3e2c2a886db76def5e433a8c Mon Sep 17 00:00:00 2001 From: Matthew Treinish Date: Mon, 4 Jan 2016 17:14:09 -0500 Subject: [PATCH 10/11] Add tool to create a subunit stream This commit adds a generate_subunit.py script which is used to generate a subunit stream with a single result for a period of time. It takes 2 mandatory args and 2 optional to specify the start time, stop time, and optionally the status (it defaults to success) and an id for test (if one isn't provided 'devstack' is used) The resulting stream is written to STDOUT. There is some overlap with subunit-output from tool from python-subunit, but this is a much smaller scope to just just do a test_id, status, and timestamps. subunit-output doesn't support timestamps. Eventually it'll be good to add the missing pieces to subunit-output at which point we can likely deprecate and remove this. The intent here is to leverage this to inject 'test results' into the subunit2sql db to reflect failures that occur before tempest (or any other test suite) is run. This is necessary for the openstack-health dashboard. (otherwise it makes our failure rate look much better than it is) This is only needed until we get a zuul mysql reporter in place which can give us the higher level run information. Change-Id: Icc7df33e4d73ba6322af38fbdf3aea230f2fcf4d --- os_testr/generate_subunit.py | 50 ++++++++++++++++++++++++++++++++++++ setup.cfg | 1 + 2 files changed, 51 insertions(+) create mode 100755 os_testr/generate_subunit.py diff --git a/os_testr/generate_subunit.py b/os_testr/generate_subunit.py new file mode 100755 index 0000000..cc38c5d --- /dev/null +++ b/os_testr/generate_subunit.py @@ -0,0 +1,50 @@ +#!/usr/bin/env python2 +# Copyright 2015 Hewlett-Packard Development Company, L.P. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + + +import datetime +import sys + +import subunit +from subunit import iso8601 + + +def main(): + start_time = datetime.datetime.fromtimestamp(float(sys.argv[1])).replace( + tzinfo=iso8601.UTC) + elapsed_time = datetime.timedelta(seconds=int(sys.argv[2])) + stop_time = start_time + elapsed_time + + if len(sys.argv) > 3: + status = sys.argv[3] + else: + status = 'success' + + if len(sys.argv) > 4: + test_id = sys.argv[4] + else: + test_id = 'devstack' + + # Write the subunit test + output = subunit.v2.StreamResultToBytes(sys.stdout) + output.startTestRun() + output.status(timestamp=start_time, test_id=test_id) + # Write the end of the test + output.status(test_status=status, timestamp=stop_time, test_id=test_id) + output.stopTestRun() + + +if __name__ == '__main__': + main() diff --git a/setup.cfg b/setup.cfg index ab1151a..dc234ac 100644 --- a/setup.cfg +++ b/setup.cfg @@ -29,6 +29,7 @@ console_scripts = subunit-trace = os_testr.subunit_trace:main ostestr = os_testr.os_testr:main subunit2html = os_testr.subunit2html:main + generate-subunit = os_testr.generate_subunit:main [build_sphinx] source-dir = doc/source From b46734734b2129f1c0ee77a8de2553e4d639bddb Mon Sep 17 00:00:00 2001 From: Davanum Srinivas Date: Tue, 5 Jan 2016 11:43:28 -0500 Subject: [PATCH 11/11] Support comments in whitelist files We already support comments in blacklist files. we should do the same for whitelist files as well. Change-Id: I5ad0c113cdb04dd1bdaa4d8bfdd1a3ab169fb0af --- os_testr/os_testr.py | 9 ++++++++- os_testr/tests/test_os_testr.py | 12 ++++++++++++ 2 files changed, 20 insertions(+), 1 deletion(-) diff --git a/os_testr/os_testr.py b/os_testr/os_testr.py index 6c5b778..6222bed 100755 --- a/os_testr/os_testr.py +++ b/os_testr/os_testr.py @@ -132,7 +132,14 @@ def path_to_regex(path): def get_regex_from_whitelist_file(file_path): - return '|'.join(open(file_path).read().splitlines()) + lines = [] + for line in open(file_path).read().splitlines(): + split_line = line.strip().split('#') + # Before the # is the regex + line_regex = split_line[0].strip() + if line_regex: + lines.append(line_regex) + return '|'.join(lines) def construct_regex(blacklist_file, whitelist_file, regex, print_exclude): diff --git a/os_testr/tests/test_os_testr.py b/os_testr/tests/test_os_testr.py index 65fb0a9..8560757 100644 --- a/os_testr/tests/test_os_testr.py +++ b/os_testr/tests/test_os_testr.py @@ -144,6 +144,18 @@ class TestConstructRegex(base.TestCase): result, "^((?!fake_regex_3|fake_regex_2|fake_regex_1|fake_regex_0).)*$") + def test_whitelist_regex_with_comments(self): + whitelist_file = six.StringIO() + for i in range(4): + whitelist_file.write('fake_regex_%s # A Comment\n' % i) + whitelist_file.seek(0) + with mock.patch('six.moves.builtins.open', + return_value=whitelist_file): + result = os_testr.construct_regex(None, 'fake_path', None, False) + self.assertEqual( + result, + "fake_regex_0|fake_regex_1|fake_regex_2|fake_regex_3") + def test_blacklist_regex_without_comments(self): blacklist_file = six.StringIO() for i in range(4):