diff --git a/.gitignore b/.gitignore index 8a3c704..d6ae633 100644 --- a/.gitignore +++ b/.gitignore @@ -4,8 +4,7 @@ *.so # Packages -*.egg -*.egg-info +*.egg* dist build eggs @@ -22,7 +21,9 @@ lib64 pip-log.txt # Unit test / coverage reports -.coverage +cover/ +.coverage* +!.coveragerc .tox nosetests.xml .testrepository diff --git a/README.rst b/README.rst index 21f2842..c03be76 100644 --- a/README.rst +++ b/README.rst @@ -1,8 +1,8 @@ -=============================== +======== os-testr -=============================== +======== -A testr wrapper to provide functionality for OpenStack projects +A testr wrapper to provide functionality for OpenStack projects. * Free software: Apache license * Documentation: http://docs.openstack.org/developer/os-testr @@ -12,8 +12,8 @@ A testr wrapper to provide functionality for OpenStack projects Features -------- -* ostestr: a testr wrapper that uses subunit-trace for output and builds some - helpful extra functionality around testr -* subunit-trace: an output filter for a subunit stream which provides useful - information about the run -* subunit2html: generates a test results html page from a subunit stream +* ``ostestr``: a testr wrapper that uses subunit-trace for output and builds + some helpful extra functionality around testr +* ``subunit-trace``: an output filter for a subunit stream which provides + useful information about the run +* ``subunit2html``: generates a test results html page from a subunit stream diff --git a/doc/source/ostestr.rst b/doc/source/ostestr.rst index 6c36bac..84b920a 100644 --- a/doc/source/ostestr.rst +++ b/doc/source/ostestr.rst @@ -112,7 +112,7 @@ exposed via the --regex option. For example:: $ ostestr --regex 'magic\.regex' This will do a straight passthrough of the provided regex to testr. -Additionally, ostestr allows you to specify a a blacklist file to define a set +Additionally, ostestr allows you to specify a blacklist file to define a set of regexes to exclude. You can specify a blacklist file with the --blacklist-file/-b option, for example:: diff --git a/doc/source/subunit_trace.rst b/doc/source/subunit_trace.rst index 1338fe6..028f970 100644 --- a/doc/source/subunit_trace.rst +++ b/doc/source/subunit_trace.rst @@ -20,7 +20,7 @@ Options Disable printing failure debug information in realtime --fails, -f Print failure debug information after the stream is - proccesed + processed --failonly Don't print success items --perc-diff, -d @@ -58,7 +58,7 @@ disabled by using --no-failure-debug, -n. For example:: $ testr run --subunit | subunit-trace --no-failure-debug -Rhere is also the option to print all failures together at the end of the test +Here is also the option to print all failures together at the end of the test run before the summary view. This is done using the --fails/-f option. For example:: @@ -77,7 +77,7 @@ example:: $ testr run --subunit | subunit-trace --failonly -The last output option provided by subunit-trace is to diable the summary view +The last output option provided by subunit-trace is to disable the summary view of the test run which is normally displayed at the end of a run. You can do this using the --no-summary option. For example:: diff --git a/os_testr/generate_subunit.py b/os_testr/generate_subunit.py new file mode 100755 index 0000000..cc38c5d --- /dev/null +++ b/os_testr/generate_subunit.py @@ -0,0 +1,50 @@ +#!/usr/bin/env python2 +# Copyright 2015 Hewlett-Packard Development Company, L.P. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + + +import datetime +import sys + +import subunit +from subunit import iso8601 + + +def main(): + start_time = datetime.datetime.fromtimestamp(float(sys.argv[1])).replace( + tzinfo=iso8601.UTC) + elapsed_time = datetime.timedelta(seconds=int(sys.argv[2])) + stop_time = start_time + elapsed_time + + if len(sys.argv) > 3: + status = sys.argv[3] + else: + status = 'success' + + if len(sys.argv) > 4: + test_id = sys.argv[4] + else: + test_id = 'devstack' + + # Write the subunit test + output = subunit.v2.StreamResultToBytes(sys.stdout) + output.startTestRun() + output.status(timestamp=start_time, test_id=test_id) + # Write the end of the test + output.status(test_status=status, timestamp=stop_time, test_id=test_id) + output.stopTestRun() + + +if __name__ == '__main__': + main() diff --git a/os_testr/os_testr.py b/os_testr/os_testr.py index db02446..6222bed 100755 --- a/os_testr/os_testr.py +++ b/os_testr/os_testr.py @@ -58,6 +58,8 @@ def get_parser(args): 'this is mutually exclusive with --pretty') parser.add_argument('--list', '-l', action='store_true', help='List all the tests which will be run.') + parser.add_argument('--color', action='store_true', + help='Use color in the pretty output') slowest = parser.add_mutually_exclusive_group() slowest.add_argument('--slowest', dest='slowest', action='store_true', help="after the test run print the slowest tests") @@ -130,7 +132,14 @@ def path_to_regex(path): def get_regex_from_whitelist_file(file_path): - return '|'.join(open(file_path).read().splitlines()) + lines = [] + for line in open(file_path).read().splitlines(): + split_line = line.strip().split('#') + # Before the # is the regex + line_regex = split_line[0].strip() + if line_regex: + lines.append(line_regex) + return '|'.join(lines) def construct_regex(blacklist_file, whitelist_file, regex, print_exclude): @@ -166,7 +175,7 @@ def construct_regex(blacklist_file, whitelist_file, regex, print_exclude): def call_testr(regex, subunit, pretty, list_tests, slowest, parallel, concur, - until_failure): + until_failure, color): if parallel: cmd = ['testr', 'run', '--parallel'] if concur: @@ -181,6 +190,12 @@ def call_testr(regex, subunit, pretty, list_tests, slowest, parallel, concur, cmd.append('--until-failure') cmd.append(regex) env = copy.deepcopy(os.environ) + + if pretty: + subunit_trace_cmd = ['subunit-trace', '--no-failure-debug', '-f'] + if color: + subunit_trace_cmd.append('--color') + # This workaround is necessary because of lp bug 1411804 it's super hacky # and makes tons of unfounded assumptions, but it works for the most part if (subunit or pretty) and until_failure: @@ -198,9 +213,9 @@ def call_testr(regex, subunit, pretty, list_tests, slowest, parallel, concur, if pretty: cmd = ['python', '-m', 'subunit.run', test] ps = subprocess.Popen(cmd, env=env, stdout=subprocess.PIPE) - proc = subprocess.Popen(['subunit-trace', - '--no-failure-debug', '-f', - '--no-summary'], env=env, + subunit_trace_cmd.append('--no-summary') + proc = subprocess.Popen(subunit_trace_cmd, + env=env, stdin=ps.stdout) ps.stdout.close() proc.communicate() @@ -223,7 +238,7 @@ def call_testr(regex, subunit, pretty, list_tests, slowest, parallel, concur, # If not until-failure special case call testr like normal elif pretty and not list_tests: ps = subprocess.Popen(cmd, env=env, stdout=subprocess.PIPE) - proc = subprocess.Popen(['subunit-trace', '--no-failure-debug', '-f'], + proc = subprocess.Popen(subunit_trace_cmd, env=env, stdin=ps.stdout) ps.stdout.close() else: @@ -261,7 +276,7 @@ def _select_and_call_runner(opts, exclude_regex): if not opts.no_discover and not opts.pdb: ec = call_testr(exclude_regex, opts.subunit, opts.pretty, opts.list, opts.slowest, opts.parallel, opts.concurrency, - opts.until_failure) + opts.until_failure, opts.color) else: test_to_run = opts.no_discover or opts.pdb if test_to_run.find('/') != -1: diff --git a/os_testr/subunit2html.py b/os_testr/subunit2html.py index fd93ac7..c40909e 100755 --- a/os_testr/subunit2html.py +++ b/os_testr/subunit2html.py @@ -633,10 +633,10 @@ class HtmlOutput(testtools.TestResult): test = test.test if test.__class__ == subunit.RemotedTestCase: cl = test._RemotedTestCase__description.rsplit('.', 1)[0] - mod = cl.rsplit('.', 1)[0] - cls = ClassInfoWrapper(cl, mod) else: - cls = ClassInfoWrapper(str(test.__class__), str(test.__module__)) + cl = test.id().rsplit('.', 1)[0] + mod = cl.rsplit('.', 1)[0] + cls = ClassInfoWrapper(cl, mod) if not str(cls) in rmap: rmap[str(cls)] = [] classes.append(cls) diff --git a/os_testr/subunit_trace.py b/os_testr/subunit_trace.py index 315d850..f194c6e 100755 --- a/os_testr/subunit_trace.py +++ b/os_testr/subunit_trace.py @@ -17,6 +17,7 @@ # under the License. """Trace a subunit stream in reasonable detail and high accuracy.""" +from __future__ import absolute_import import argparse import datetime @@ -28,6 +29,8 @@ import sys import subunit import testtools +from os_testr.utils import colorizer + # NOTE(mtreinish) on python3 anydbm was renamed dbm and the python2 dbm module # was renamed to dbm.ndbm, this block takes that into account try: @@ -148,7 +151,8 @@ def find_test_run_time_diff(test_id, run_time): def show_outcome(stream, test, print_failures=False, failonly=False, - enable_diff=False, threshold='0', abbreviate=False): + enable_diff=False, threshold='0', abbreviate=False, + enable_color=False): global RESULTS status = test['status'] # TODO(sdague): ask lifeless why on this? @@ -167,19 +171,29 @@ def show_outcome(stream, test, print_failures=False, failonly=False, if name == 'process-returncode': return + for color in [colorizer.AnsiColorizer, colorizer.NullColorizer]: + if not enable_color: + color = colorizer.NullColorizer(stream) + break + if color.supported(): + color = color(stream) + break + if status == 'fail': FAILS.append(test) if abbreviate: - stream.write('F') + color.write('F', 'red') else: - stream.write('{%s} %s [%s] ... FAILED\n' % ( + stream.write('{%s} %s [%s] ... ' % ( worker, name, duration)) + color.write('FAILED', 'red') + stream.write('\n') if not print_failures: print_attachments(stream, test, all_channels=True) elif not failonly: if status == 'success': if abbreviate: - stream.write('.') + color.write('.', 'green') else: out_string = '{%s} %s [%s' % (worker, name, duration) perc_diff = find_test_run_time_diff(test['id'], duration) @@ -189,17 +203,22 @@ def show_outcome(stream, test, print_failures=False, failonly=False, out_string = out_string + ' +%.2f%%' % perc_diff else: out_string = out_string + ' %.2f%%' % perc_diff - stream.write(out_string + '] ... ok\n') + stream.write(out_string + '] ... ') + color.write('ok', 'green') + stream.write('\n') print_attachments(stream, test) elif status == 'skip': if abbreviate: - stream.write('S') + color.write('S', 'blue') else: reason = test['details'].get('reason', '') if reason: reason = ': ' + reason.as_text() - stream.write('{%s} %s ... SKIPPED%s\n' % ( - worker, name, reason)) + stream.write('{%s} %s ... ' % ( + worker, name)) + color.write('SKIPPED', 'blue') + stream.write('%s' % (reason)) + stream.write('\n') else: if abbreviate: stream.write('%s' % test['status'][0]) @@ -320,6 +339,8 @@ def parse_args(): parser.add_argument('--no-summary', action='store_true', help="Don't print the summary of the test run after " " completes") + parser.add_argument('--color', action='store_true', + help="Print results with colors") return parser.parse_args() @@ -332,7 +353,8 @@ def main(): print_failures=args.print_failures, failonly=args.failonly, enable_diff=args.enable_diff, - abbreviate=args.abbreviate)) + abbreviate=args.abbreviate, + enable_color=args.color)) summary = testtools.StreamSummary() result = testtools.CopyStreamResult([outcomes, summary]) result = testtools.StreamResultRouter(result) @@ -354,6 +376,12 @@ def main(): print_fails(sys.stdout) if not args.no_summary: print_summary(sys.stdout, elapsed_time) + + # NOTE(mtreinish): Ideally this should live in testtools streamSummary + # this is just in place until the behavior lands there (if it ever does) + if count_tests('status', '^success$') == 0: + print("\nNo tests were successful during the run") + exit(1) exit(0 if summary.wasSuccessful() else 1) diff --git a/os_testr/tests/sample_streams/all_skips.subunit b/os_testr/tests/sample_streams/all_skips.subunit new file mode 100644 index 0000000..54156b5 Binary files /dev/null and b/os_testr/tests/sample_streams/all_skips.subunit differ diff --git a/os_testr/tests/sample_streams/successful.subunit b/os_testr/tests/sample_streams/successful.subunit new file mode 100644 index 0000000..6f3b664 Binary files /dev/null and b/os_testr/tests/sample_streams/successful.subunit differ diff --git a/os_testr/tests/test_os_testr.py b/os_testr/tests/test_os_testr.py index 65fb0a9..8560757 100644 --- a/os_testr/tests/test_os_testr.py +++ b/os_testr/tests/test_os_testr.py @@ -144,6 +144,18 @@ class TestConstructRegex(base.TestCase): result, "^((?!fake_regex_3|fake_regex_2|fake_regex_1|fake_regex_0).)*$") + def test_whitelist_regex_with_comments(self): + whitelist_file = six.StringIO() + for i in range(4): + whitelist_file.write('fake_regex_%s # A Comment\n' % i) + whitelist_file.seek(0) + with mock.patch('six.moves.builtins.open', + return_value=whitelist_file): + result = os_testr.construct_regex(None, 'fake_path', None, False) + self.assertEqual( + result, + "fake_regex_0|fake_regex_1|fake_regex_2|fake_regex_3") + def test_blacklist_regex_without_comments(self): blacklist_file = six.StringIO() for i in range(4): diff --git a/os_testr/tests/test_subunit2html.py b/os_testr/tests/test_subunit2html.py new file mode 100644 index 0000000..f3196a5 --- /dev/null +++ b/os_testr/tests/test_subunit2html.py @@ -0,0 +1,31 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from ddt import data +from ddt import ddt +from subunit import RemotedTestCase +from testtools import PlaceHolder + +from os_testr import subunit2html +from os_testr.tests import base + + +@ddt +class TestSubunit2html(base.TestCase): + @data(RemotedTestCase, PlaceHolder) + def test_class_parsing(self, test_cls): + """Tests that the class paths are parsed for v1 & v2 tests""" + test_ = test_cls("example.path.to.test.method") + obj_ = subunit2html.HtmlOutput() + cls_ = [] + obj_._add_cls({}, cls_, test_, ()) + self.assertEqual("example.path.to.test", cls_[0].name) diff --git a/os_testr/tests/test_subunit_trace.py b/os_testr/tests/test_subunit_trace.py index 5544636..736dff9 100644 --- a/os_testr/tests/test_subunit_trace.py +++ b/os_testr/tests/test_subunit_trace.py @@ -14,6 +14,8 @@ # under the License. from datetime import datetime as dt +import os +import subprocess from ddt import data from ddt import ddt @@ -59,3 +61,21 @@ class TestSubunitTrace(base.TestCase): } with patch.dict(subunit_trace.RESULTS, patched_res, clear=True): self.assertEqual(subunit_trace.run_time(), expected_result) + + def test_return_code_all_skips(self): + skips_stream = os.path.join( + os.path.dirname(os.path.abspath(__file__)), + 'sample_streams/all_skips.subunit') + p = subprocess.Popen(['subunit-trace'], stdin=subprocess.PIPE) + with open(skips_stream, 'rb') as stream: + p.communicate(stream.read()) + self.assertEqual(1, p.returncode) + + def test_return_code_normal_run(self): + regular_stream = os.path.join( + os.path.dirname(os.path.abspath(__file__)), + 'sample_streams/successful.subunit') + p = subprocess.Popen(['subunit-trace'], stdin=subprocess.PIPE) + with open(regular_stream, 'rb') as stream: + p.communicate(stream.read()) + self.assertEqual(0, p.returncode) diff --git a/os_testr/utils/__init__.py b/os_testr/utils/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/os_testr/utils/colorizer.py b/os_testr/utils/colorizer.py new file mode 100644 index 0000000..8ecec35 --- /dev/null +++ b/os_testr/utils/colorizer.py @@ -0,0 +1,98 @@ +# Copyright 2015 NEC Corporation +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# +# Colorizer Code is borrowed from Twisted: +# Copyright (c) 2001-2010 Twisted Matrix Laboratories. +# +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to +# the following conditions: +# +# The above copyright notice and this permission notice shall be +# included in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +import sys + + +class AnsiColorizer(object): + """A colorizer is an object that loosely wraps around a stream + + allowing callers to write text to the stream in a particular color. + + Colorizer classes must implement C{supported()} and C{write(text, color)}. + """ + _colors = dict(black=30, red=31, green=32, yellow=33, + blue=34, magenta=35, cyan=36, white=37) + + def __init__(self, stream): + self.stream = stream + + @classmethod + def supported(cls, stream=sys.stdout): + """Check the current platform supports coloring terminal output + + A class method that returns True if the current platform supports + coloring terminal output using this method. Returns False otherwise. + """ + if not stream.isatty(): + return False # auto color only on TTYs + try: + import curses + except ImportError: + return False + else: + try: + try: + return curses.tigetnum("colors") > 2 + except curses.error: + curses.setupterm() + return curses.tigetnum("colors") > 2 + except Exception: + # guess false in case of error + return False + + def write(self, text, color): + """Write the given text to the stream in the given color. + + @param text: Text to be written to the stream. + + @param color: A string label for a color. e.g. 'red', 'white'. + """ + color = self._colors[color] + self.stream.write('\x1b[%s;1m%s\x1b[0m' % (color, text)) + + +class NullColorizer(object): + """See _AnsiColorizer docstring.""" + def __init__(self, stream): + self.stream = stream + + @classmethod + def supported(cls, stream=sys.stdout): + return True + + def write(self, text, color): + self.stream.write(text) diff --git a/setup.cfg b/setup.cfg index ab1151a..dc234ac 100644 --- a/setup.cfg +++ b/setup.cfg @@ -29,6 +29,7 @@ console_scripts = subunit-trace = os_testr.subunit_trace:main ostestr = os_testr.os_testr:main subunit2html = os_testr.subunit2html:main + generate-subunit = os_testr.generate_subunit:main [build_sphinx] source-dir = doc/source diff --git a/tox.ini b/tox.ini index 537b2f2..931250f 100644 --- a/tox.ini +++ b/tox.ini @@ -8,9 +8,12 @@ usedevelop = True install_command = pip install -U {opts} {packages} setenv = VIRTUAL_ENV={envdir} +whitelist_externals = find deps = -r{toxinidir}/requirements.txt -r{toxinidir}/test-requirements.txt -commands = ostestr {posargs} +commands = + find . -type f -name "*.pyc" -delete + ostestr {posargs} [testenv:pep8] commands = flake8 @@ -19,7 +22,7 @@ commands = flake8 commands = {posargs} [testenv:cover] -commands = python setup.py testr --coverage --testr-args='{posargs}' +commands = python setup.py testr --coverage --coverage-package-name='os_testr' --testr-args='{posargs}' [testenv:docs] commands = python setup.py build_sphinx