diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..8a3c704 --- /dev/null +++ b/.gitignore @@ -0,0 +1,53 @@ +*.py[cod] + +# C extensions +*.so + +# Packages +*.egg +*.egg-info +dist +build +eggs +parts +bin +var +sdist +develop-eggs +.installed.cfg +lib +lib64 + +# Installer logs +pip-log.txt + +# Unit test / coverage reports +.coverage +.tox +nosetests.xml +.testrepository +.venv + +# Translations +*.mo + +# Mr Developer +.mr.developer.cfg +.project +.pydevproject + +# Complexity +output/*.html +output/*/index.html + +# Sphinx +doc/build + +# pbr generates these +AUTHORS +ChangeLog + +# Editors +*~ +.*.swp +.*sw? diff --git a/.gitreview b/.gitreview new file mode 100644 index 0000000..fb67c90 --- /dev/null +++ b/.gitreview @@ -0,0 +1,4 @@ +[gerrit] +host=review.openstack.org +port=29418 +project=openstack/os-testr.git diff --git a/TODO.rst b/TODO.rst new file mode 100644 index 0000000..135bcd6 --- /dev/null +++ b/TODO.rst @@ -0,0 +1,17 @@ +Work Items for os-testr +======================= + +Short Term +---------- + * Expose all subunit-trace options through ostestr + * Add --html option to ostestr to run testr with subunit2html output + * Add unit tests + * For ostestr test selection api + * Response code validation on more argument permutations +Long Term +--------- + * Lock down test selection CLI + * When this is done it will become release 1.0.0 + * Add subunit-trace functional tests + ** Sample subunit streams and test output from subunit-trace + * Add testing for subunit2html diff --git a/doc/source/index.rst b/doc/source/index.rst index feb274b..448d655 100644 --- a/doc/source/index.rst +++ b/doc/source/index.rst @@ -15,6 +15,7 @@ Contents: installation usage contributing + todo Indices and tables ================== diff --git a/doc/source/todo.rst b/doc/source/todo.rst new file mode 100644 index 0000000..0a7409f --- /dev/null +++ b/doc/source/todo.rst @@ -0,0 +1 @@ +.. include:: ../../TODO.rst diff --git a/os_testr/os_testr.py b/os_testr/os_testr.py index 149d5ce..353868c 100755 --- a/os_testr/os_testr.py +++ b/os_testr/os_testr.py @@ -66,20 +66,69 @@ def parse_args(): 'encountered. Running with subunit or pretty' 'output enable will force the loop to run tests' 'serially') + parser.add_argument('--print-exclude', action='store_true', + help='If an exclude file is used this option will ' + 'prints the comment from the same line and all ' + 'skipped tests before the test run') parser.set_defaults(pretty=True, slowest=True, parallel=True) opts = parser.parse_args() return opts -def construct_regex(blacklist_file, regex): +def _get_test_list(regex, env=None): + env = env or copy.deepcopy(os.environ) + proc = subprocess.Popen(['testr', 'list-tests', regex], env=env, + stdout=subprocess.PIPE) + out = proc.communicate()[0] + raw_test_list = out.split('\n') + bad = False + test_list = [] + exclude_list = ['OS_', 'CAPTURE', 'TEST_TIMEOUT', 'PYTHON', + 'subunit.run discover'] + for line in raw_test_list: + for exclude in exclude_list: + if exclude in line: + bad = True + break + elif not line: + bad = True + break + if not bad: + test_list.append(line) + bad = False + return test_list + + +def print_skips(regex, message): + test_list = _get_test_list(regex) + if test_list: + if message: + print(message) + else: + print('Skipped because of regex %s:' % regex) + for test in test_list: + print(test) + # Extra whitespace to separate + print('\n') + + +def construct_regex(blacklist_file, regex, print_exclude): if not blacklist_file: exclude_regex = '' else: black_file = open(blacklist_file, 'r') exclude_regex = '' for line in black_file: - regex = line.strip() - exclude_regex = '|'.join([regex, exclude_regex]) + raw_line = line.strip() + split_line = raw_line.split('#') + # Before the # is the regex + regex = split_line[0].strip() + # After the # is a comment + comment = split_line[1].strip() + if regex: + if print_exclude: + print_skips(regex, comment) + exclude_regex = '|'.join([regex, exclude_regex]) if exclude_regex: exclude_regex = "'(?!.*" + exclude_regex + ")" if regex: @@ -106,25 +155,7 @@ def call_testr(regex, subunit, pretty, list_tests, slowest, parallel, concur, # This workaround is necessary because of lp bug 1411804 it's super hacky # and makes tons of unfounded assumptions, but it works for the most part if (subunit or pretty) and until_failure: - proc = subprocess.Popen(['testr', 'list-tests', regex], env=env, - stdout=subprocess.PIPE) - out = proc.communicate()[0] - raw_test_list = out.split('\n') - bad = False - test_list = [] - exclude_list = ['CAPTURE', 'TEST_TIMEOUT', 'PYTHON', - 'subunit.run discover'] - for line in raw_test_list: - for exclude in exclude_list: - if exclude in line: - bad = True - break - elif not line: - bad = True - break - if not bad: - test_list.append(line) - bad = False + test_list = _get_test_list(regex, env) count = 0 failed = False if not test_list: @@ -217,7 +248,8 @@ def main(): msg = "You can not use until_failure mode with pdb or no-discover" print(msg) exit(5) - exclude_regex = construct_regex(opts.blacklist_file, opts.regex) + exclude_regex = construct_regex(opts.blacklist_file, opts.regex, + opts.print_exclude) if not os.path.isdir('.testrepository'): subprocess.call(['testr', 'init']) if not opts.no_discover and not opts.pdb: diff --git a/os_testr/subunit_trace.py b/os_testr/subunit_trace.py index 91c2ff7..7c00825 100755 --- a/os_testr/subunit_trace.py +++ b/os_testr/subunit_trace.py @@ -131,7 +131,14 @@ def find_test_run_time_diff(test_id, run_time): test_times = dbm.open(times_db_path) except Exception: return False - avg_runtime = float(test_times.get(str(test_id), False)) + try: + avg_runtime = float(test_times.get(str(test_id), False)) + except Exception: + try: + avg_runtime = float(test_times[str(test_id)]) + except Exception: + avg_runtime = False + if avg_runtime and avg_runtime > 0: run_time = float(run_time.rstrip('s')) perc_diff = ((run_time - avg_runtime) / avg_runtime) * 100 @@ -140,7 +147,7 @@ def find_test_run_time_diff(test_id, run_time): def show_outcome(stream, test, print_failures=False, failonly=False, - threshold='0'): + enable_diff=False, threshold='0'): global RESULTS status = test['status'] # TODO(sdague): ask lifeless why on this? @@ -169,11 +176,12 @@ def show_outcome(stream, test, print_failures=False, failonly=False, if status == 'success': out_string = '{%s} %s [%s' % (worker, name, duration) perc_diff = find_test_run_time_diff(test['id'], duration) - if perc_diff and abs(perc_diff) >= abs(float(threshold)): - if perc_diff > 0: - out_string = out_string + ' +%.2f%%' % perc_diff - else: - out_string = out_string + ' %.2f%%' % perc_diff + if enable_diff: + if perc_diff and abs(perc_diff) >= abs(float(threshold)): + if perc_diff > 0: + out_string = out_string + ' +%.2f%%' % perc_diff + else: + out_string = out_string + ' %.2f%%' % perc_diff stream.write(out_string + '] ... ok\n') print_attachments(stream, test) elif status == 'skip': @@ -220,7 +228,11 @@ def run_time(): runtime = 0.0 for k, v in RESULTS.items(): for test in v: - runtime += float(get_duration(test['timestamps']).strip('s')) + test_dur = get_duration(test['timestamps']).strip('s') + # NOTE(toabctl): get_duration() can return an empty string + # which leads to a ValueError when casting to float + if test_dur: + runtime += float(test_dur) return runtime @@ -271,6 +283,9 @@ def parse_args(): default=( os.environ.get('TRACE_FAILONLY', False) is not False)) + parser.add_argument('--perc-diff', '-d', action='store_true', + dest='enable_diff', + help="Print percent change in run time on each test ") parser.add_argument('--diff-threshold', '-t', dest='threshold', help="Threshold to use for displaying percent change " "from the avg run time. If one is not specified " @@ -288,7 +303,8 @@ def main(): outcomes = testtools.StreamToDict( functools.partial(show_outcome, sys.stdout, print_failures=args.print_failures, - failonly=args.failonly)) + failonly=args.failonly, + enable_diff=args.enable_diff)) summary = testtools.StreamSummary() result = testtools.CopyStreamResult([outcomes, summary]) result = testtools.StreamResultRouter(result) diff --git a/os_testr/tests/test_return_codes.py b/os_testr/tests/test_return_codes.py index 591e4dd..082a74a 100644 --- a/os_testr/tests/test_return_codes.py +++ b/os_testr/tests/test_return_codes.py @@ -14,13 +14,13 @@ import os import shutil -import StringIO import subprocess import tempfile import testtools from os_testr.tests import base +from six import StringIO DEVNULL = open(os.devnull, 'wb') @@ -47,8 +47,8 @@ class TestReturnCodes(base.TestCase): shutil.copy('os_testr/tests/files/setup.cfg', self.setup_cfg_file) shutil.copy('os_testr/tests/files/__init__.py', self.init_file) - self.stdout = StringIO.StringIO() - self.stderr = StringIO.StringIO() + self.stdout = StringIO() + self.stderr = StringIO() # Change directory, run wrapper and check result self.addCleanup(os.chdir, os.path.abspath(os.curdir)) os.chdir(self.directory) diff --git a/os_testr/tests/test_subunit_trace.py b/os_testr/tests/test_subunit_trace.py new file mode 100644 index 0000000..5544636 --- /dev/null +++ b/os_testr/tests/test_subunit_trace.py @@ -0,0 +1,61 @@ +# Copyright 2015 SUSE Linux GmbH +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from datetime import datetime as dt + +from ddt import data +from ddt import ddt +from ddt import unpack +from mock import patch + +from os_testr import subunit_trace +from os_testr.tests import base + + +@ddt +class TestSubunitTrace(base.TestCase): + + @data(([dt(2015, 4, 17, 22, 23, 14, 111111), + dt(2015, 4, 17, 22, 23, 14, 111111)], + "0.000000s"), + ([dt(2015, 4, 17, 22, 23, 14, 111111), + dt(2015, 4, 17, 22, 23, 15, 111111)], + "1.000000s"), + ([dt(2015, 4, 17, 22, 23, 14, 111111), + None], + "")) + @unpack + def test_get_durating(self, timestamps, expected_result): + self.assertEqual(subunit_trace.get_duration(timestamps), + expected_result) + + @data(([dt(2015, 4, 17, 22, 23, 14, 111111), + dt(2015, 4, 17, 22, 23, 14, 111111)], + 0.0), + ([dt(2015, 4, 17, 22, 23, 14, 111111), + dt(2015, 4, 17, 22, 23, 15, 111111)], + 1.0), + ([dt(2015, 4, 17, 22, 23, 14, 111111), + None], + 0.0)) + @unpack + def test_run_time(self, timestamps, expected_result): + patched_res = { + 0: [ + {'timestamps': timestamps} + ] + } + with patch.dict(subunit_trace.RESULTS, patched_res, clear=True): + self.assertEqual(subunit_trace.run_time(), expected_result) diff --git a/setup.cfg b/setup.cfg index eaa4f28..ab1151a 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,34 +1,34 @@ [metadata] name = os-testr summary = A testr wrapper to provide functionality for OpenStack projects -description-file = - README.rst +description-file = + README.rst author = OpenStack author-email = openstack-dev@lists.openstack.org home-page = http://www.openstack.org/ -classifier = - Environment :: OpenStack - Intended Audience :: Information Technology - Intended Audience :: System Administrators - License :: OSI Approved :: Apache Software License - Operating System :: POSIX :: Linux - Programming Language :: Python - Programming Language :: Python :: 2 - Programming Language :: Python :: 2.7 - Programming Language :: Python :: 2.6 - Programming Language :: Python :: 3 - Programming Language :: Python :: 3.3 - Programming Language :: Python :: 3.4 +classifier = + Environment :: OpenStack + Intended Audience :: Information Technology + Intended Audience :: System Administrators + License :: OSI Approved :: Apache Software License + Operating System :: POSIX :: Linux + Programming Language :: Python + Programming Language :: Python :: 2 + Programming Language :: Python :: 2.7 + Programming Language :: Python :: 2.6 + Programming Language :: Python :: 3 + Programming Language :: Python :: 3.3 + Programming Language :: Python :: 3.4 [files] -packages = - os_testr +packages = + os_testr [entry_points] -console_scripts = - subunit-trace = os_testr.subunit_trace:main - ostestr = os_testr.os_testr:main - subunit2html = os_testr.subunit2html:main +console_scripts = + subunit-trace = os_testr.subunit_trace:main + ostestr = os_testr.os_testr:main + subunit2html = os_testr.subunit2html:main [build_sphinx] source-dir = doc/source @@ -51,9 +51,3 @@ input_file = os_testr/locale/os-testr.pot keywords = _ gettext ngettext l_ lazy_gettext mapping_file = babel.cfg output_file = os_testr/locale/os-testr.pot - -[egg_info] -tag_date = 0 -tag_svn_revision = 0 -tag_build = - diff --git a/test-requirements.txt b/test-requirements.txt index c7208f0..608d49e 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -10,3 +10,5 @@ sphinx>=1.1.2,!=1.2.0,!=1.3b1,<1.3 oslosphinx>=2.2.0 # Apache-2.0 oslotest>=1.2.0 # Apache-2.0 testscenarios>=0.4 +ddt>=0.4.0 +six>=1.9.0 diff --git a/tox.ini b/tox.ini index 4657203..537b2f2 100644 --- a/tox.ini +++ b/tox.ini @@ -10,7 +10,7 @@ setenv = VIRTUAL_ENV={envdir} deps = -r{toxinidir}/requirements.txt -r{toxinidir}/test-requirements.txt -commands = python setup.py testr --slowest --testr-args='{posargs}' +commands = ostestr {posargs} [testenv:pep8] commands = flake8