x/deb-os-testr
Code Issues Proposed changes

Release 0.2.0

Browse Source 
This release of os-testr includes:
 
  * Support for having comments in a blacklist file
  * A new option, --print-exclude to print comments along with exclude tests
    when using a blacklist file
  * Printing percent change in subunit-trace is disabled by default and a new
    CLI option is added to enable it
  * Several bugfixes
 -----BEGIN PGP SIGNATURE-----
 Version: GnuPG v2
 
 iQIcBAABCgAGBQJVkqrPAAoJEP0SoPIUyeF3fFcP/A+omiNzguvj2L1USyqNUmcG
 2yXeS1L5WzfDnhmJZYxpC0yKAopERxbJLWNxIAxhXOErsrQ+b0xDhW0hjxyaLdS+
 vQcpd/I9Nz7RTm/KuMEPIDTZnkc/s8GhH9ZoI/ljIxdaCA4GGe5cS7MO0rSFjc8z
 n88GQWjj0LPVIOQAsfWoIci0iVxEbYT4BP/GYoNxlXBOKUnSnLzmtDFk8BI2Zjlq
 UQV8UCBapo10sNz8OAmRV26blXvtf9orEGDfEDLMKEENrQQwFjlS4Voaj5l48y47
 Jl6QwhrMQzE27Qjk++ddDCYugzWFeOcyh4gnrEfzXmgRYZehR3oHW7wCD9d4Z1vf
 JBFll0L9EAt44HW8uK/rw0S3PwLfo1+SWBxZkMxoiOxTTaR4I7Q5FHYb9nb8Z/JK
 NDwTuOvoU3rFbqXvBjv6TQ1a9a12vixmvx/j4TE4e2+Wo8viu59qyNyRgztOLv0F
 D9YMpXMJU6o/1vKudlDhC4SCzPpOTVDSjkhzEb6/UC69T0i8TUMEyTz7MZMQjIMl
 qlkHUOat9veJHVE7xp78BQOYKwnyb7Z5TKwtbW6LL6aTlym5feRg8VpEdMo2tI3I
 +7QbovG0BzvPGQ1r1FjvaX8sKA/W68J8nyvIGxDvakfDlghi0sHFx7ZDhoXcooVR
 vwt6lst3cHDZFVQmF8G0
 =gg4k
 -----END PGP SIGNATURE-----

Merge tag '0.2.0' into debian/liberty

Release 0.2.0

This release of os-testr includes:

 * Support for having comments in a blacklist file
 * A new option, --print-exclude to print comments along with exclude tests
   when using a blacklist file
 * Printing percent change in subunit-trace is disabled by default and a new
   CLI option is added to enable it
 * Several bugfixes
This commit is contained in:
Thomas Goirand 2015-07-07 21:40:23 +02:00
commit 206f013161
12 changed files with 244 additions and 63 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

53
.gitignore vendored Normal file
View File

@ -0,0 +1,53 @@
*.py[cod]
# C extensions
*.so
# Packages
*.egg
*.egg-info
dist
build
eggs
parts
bin
var
sdist
develop-eggs
.installed.cfg
lib
lib64
# Installer logs
pip-log.txt
# Unit test / coverage reports
.coverage
.tox
nosetests.xml
.testrepository
.venv
# Translations
*.mo
# Mr Developer
.mr.developer.cfg
.project
.pydevproject
# Complexity
output/*.html
output/*/index.html
# Sphinx
doc/build
# pbr generates these
AUTHORS
ChangeLog
# Editors
*~
.*.swp
.*sw?

4
.gitreview Normal file
View File

@ -0,0 +1,4 @@
[gerrit]
host=review.openstack.org
port=29418
project=openstack/os-testr.git

17
TODO.rst Normal file
View File

@ -0,0 +1,17 @@
Work Items for os-testr
=======================
Short Term
----------
* Expose all subunit-trace options through ostestr
* Add --html option to ostestr to run testr with subunit2html output
* Add unit tests
* For ostestr test selection api
* Response code validation on more argument permutations
Long Term
---------
* Lock down test selection CLI
* When this is done it will become release 1.0.0
* Add subunit-trace functional tests
** Sample subunit streams and test output from subunit-trace
* Add testing for subunit2html

1
doc/source/index.rst
View File

@ -15,6 +15,7 @@ Contents:
installation
usage
contributing
todo
Indices and tables
==================

1
doc/source/todo.rst Normal file
View File

@ -0,0 +1 @@
.. include:: ../../TODO.rst

78
os_testr/os_testr.py
View File

@ -66,20 +66,69 @@ def parse_args():
'encountered. Running with subunit or pretty'
'output enable will force the loop to run tests'
'serially')
parser.add_argument('--print-exclude', action='store_true',
help='If an exclude file is used this option will '
'prints the comment from the same line and all '
'skipped tests before the test run')
parser.set_defaults(pretty=True, slowest=True, parallel=True)
opts = parser.parse_args()
return opts
def construct_regex(blacklist_file, regex):
def _get_test_list(regex, env=None):
env = env or copy.deepcopy(os.environ)
proc = subprocess.Popen(['testr', 'list-tests', regex], env=env,
stdout=subprocess.PIPE)
out = proc.communicate()[0]
raw_test_list = out.split('\n')
bad = False
test_list = []
exclude_list = ['OS_', 'CAPTURE', 'TEST_TIMEOUT', 'PYTHON',
'subunit.run discover']
for line in raw_test_list:
for exclude in exclude_list:
if exclude in line:
bad = True
break
elif not line:
bad = True
break
if not bad:
test_list.append(line)
bad = False
return test_list
def print_skips(regex, message):
test_list = _get_test_list(regex)
if test_list:
if message:
print(message)
else:
print('Skipped because of regex %s:' % regex)
for test in test_list:
print(test)
# Extra whitespace to separate
print('\n')
def construct_regex(blacklist_file, regex, print_exclude):
if not blacklist_file:
exclude_regex = ''
else:
black_file = open(blacklist_file, 'r')
exclude_regex = ''
for line in black_file:
regex = line.strip()
exclude_regex = '|'.join([regex, exclude_regex])
raw_line = line.strip()
split_line = raw_line.split('#')
# Before the # is the regex
regex = split_line[0].strip()
# After the # is a comment
comment = split_line[1].strip()
if regex:
if print_exclude:
print_skips(regex, comment)
exclude_regex = '|'.join([regex, exclude_regex])
if exclude_regex:
exclude_regex = "'(?!.*" + exclude_regex + ")"
if regex:
@ -106,25 +155,7 @@ def call_testr(regex, subunit, pretty, list_tests, slowest, parallel, concur,
# This workaround is necessary because of lp bug 1411804 it's super hacky
# and makes tons of unfounded assumptions, but it works for the most part
if (subunit or pretty) and until_failure:
proc = subprocess.Popen(['testr', 'list-tests', regex], env=env,
stdout=subprocess.PIPE)
out = proc.communicate()[0]
raw_test_list = out.split('\n')
bad = False
test_list = []
exclude_list = ['CAPTURE', 'TEST_TIMEOUT', 'PYTHON',
'subunit.run discover']
for line in raw_test_list:
for exclude in exclude_list:
if exclude in line:
bad = True
break
elif not line:
bad = True
break
if not bad:
test_list.append(line)
bad = False
test_list = _get_test_list(regex, env)
count = 0
failed = False
if not test_list:
@ -217,7 +248,8 @@ def main():
msg = "You can not use until_failure mode with pdb or no-discover"
print(msg)
exit(5)
exclude_regex = construct_regex(opts.blacklist_file, opts.regex)
exclude_regex = construct_regex(opts.blacklist_file, opts.regex,
opts.print_exclude)
if not os.path.isdir('.testrepository'):
subprocess.call(['testr', 'init'])
if not opts.no_discover and not opts.pdb:

34
os_testr/subunit_trace.py
View File

@ -131,7 +131,14 @@ def find_test_run_time_diff(test_id, run_time):
test_times = dbm.open(times_db_path)
except Exception:
return False
avg_runtime = float(test_times.get(str(test_id), False))
try:
avg_runtime = float(test_times.get(str(test_id), False))
except Exception:
try:
avg_runtime = float(test_times[str(test_id)])
except Exception:
avg_runtime = False
if avg_runtime and avg_runtime > 0:
run_time = float(run_time.rstrip('s'))
perc_diff = ((run_time - avg_runtime) / avg_runtime) * 100
@ -140,7 +147,7 @@ def find_test_run_time_diff(test_id, run_time):
def show_outcome(stream, test, print_failures=False, failonly=False,
threshold='0'):
enable_diff=False, threshold='0'):
global RESULTS
status = test['status']
# TODO(sdague): ask lifeless why on this?
@ -169,11 +176,12 @@ def show_outcome(stream, test, print_failures=False, failonly=False,
if status == 'success':
out_string = '{%s} %s [%s' % (worker, name, duration)
perc_diff = find_test_run_time_diff(test['id'], duration)
if perc_diff and abs(perc_diff) >= abs(float(threshold)):
if perc_diff > 0:
out_string = out_string + ' +%.2f%%' % perc_diff
else:
out_string = out_string + ' %.2f%%' % perc_diff
if enable_diff:
if perc_diff and abs(perc_diff) >= abs(float(threshold)):
if perc_diff > 0:
out_string = out_string + ' +%.2f%%' % perc_diff
else:
out_string = out_string + ' %.2f%%' % perc_diff
stream.write(out_string + '] ... ok\n')
print_attachments(stream, test)
elif status == 'skip':
@ -220,7 +228,11 @@ def run_time():
runtime = 0.0
for k, v in RESULTS.items():
for test in v:
runtime += float(get_duration(test['timestamps']).strip('s'))
test_dur = get_duration(test['timestamps']).strip('s')
# NOTE(toabctl): get_duration() can return an empty string
# which leads to a ValueError when casting to float
if test_dur:
runtime += float(test_dur)
return runtime
@ -271,6 +283,9 @@ def parse_args():
default=(
os.environ.get('TRACE_FAILONLY', False)
is not False))
parser.add_argument('--perc-diff', '-d', action='store_true',
dest='enable_diff',
help="Print percent change in run time on each test ")
parser.add_argument('--diff-threshold', '-t', dest='threshold',
help="Threshold to use for displaying percent change "
"from the avg run time. If one is not specified "
@ -288,7 +303,8 @@ def main():
outcomes = testtools.StreamToDict(
functools.partial(show_outcome, sys.stdout,
print_failures=args.print_failures,
failonly=args.failonly))
failonly=args.failonly,
enable_diff=args.enable_diff))
summary = testtools.StreamSummary()
result = testtools.CopyStreamResult([outcomes, summary])
result = testtools.StreamResultRouter(result)

6
os_testr/tests/test_return_codes.py
View File

@ -14,13 +14,13 @@
import os
import shutil
import StringIO
import subprocess
import tempfile
import testtools
from os_testr.tests import base
from six import StringIO
DEVNULL = open(os.devnull, 'wb')
@ -47,8 +47,8 @@ class TestReturnCodes(base.TestCase):
shutil.copy('os_testr/tests/files/setup.cfg', self.setup_cfg_file)
shutil.copy('os_testr/tests/files/__init__.py', self.init_file)
self.stdout = StringIO.StringIO()
self.stderr = StringIO.StringIO()
self.stdout = StringIO()
self.stderr = StringIO()
# Change directory, run wrapper and check result
self.addCleanup(os.chdir, os.path.abspath(os.curdir))
os.chdir(self.directory)

61
os_testr/tests/test_subunit_trace.py Normal file
View File

@ -0,0 +1,61 @@
# Copyright 2015 SUSE Linux GmbH
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
from datetime import datetime as dt
from ddt import data
from ddt import ddt
from ddt import unpack
from mock import patch
from os_testr import subunit_trace
from os_testr.tests import base
@ddt
class TestSubunitTrace(base.TestCase):
@data(([dt(2015, 4, 17, 22, 23, 14, 111111),
dt(2015, 4, 17, 22, 23, 14, 111111)],
"0.000000s"),
([dt(2015, 4, 17, 22, 23, 14, 111111),
dt(2015, 4, 17, 22, 23, 15, 111111)],
"1.000000s"),
([dt(2015, 4, 17, 22, 23, 14, 111111),
None],
""))
@unpack
def test_get_durating(self, timestamps, expected_result):
self.assertEqual(subunit_trace.get_duration(timestamps),
expected_result)
@data(([dt(2015, 4, 17, 22, 23, 14, 111111),
dt(2015, 4, 17, 22, 23, 14, 111111)],
0.0),
([dt(2015, 4, 17, 22, 23, 14, 111111),
dt(2015, 4, 17, 22, 23, 15, 111111)],
1.0),
([dt(2015, 4, 17, 22, 23, 14, 111111),
None],
0.0))
@unpack
def test_run_time(self, timestamps, expected_result):
patched_res = {
0: [
{'timestamps': timestamps}
]
}
with patch.dict(subunit_trace.RESULTS, patched_res, clear=True):
self.assertEqual(subunit_trace.run_time(), expected_result)

48
setup.cfg
View File

@ -1,34 +1,34 @@
[metadata]
name = os-testr
summary = A testr wrapper to provide functionality for OpenStack projects
description-file =
README.rst
description-file =
README.rst
author = OpenStack
author-email = openstack-dev@lists.openstack.org
home-page = http://www.openstack.org/
classifier =
Environment :: OpenStack
Intended Audience :: Information Technology
Intended Audience :: System Administrators
License :: OSI Approved :: Apache Software License
Operating System :: POSIX :: Linux
Programming Language :: Python
Programming Language :: Python :: 2
Programming Language :: Python :: 2.7
Programming Language :: Python :: 2.6
Programming Language :: Python :: 3
Programming Language :: Python :: 3.3
Programming Language :: Python :: 3.4
classifier =
Environment :: OpenStack
Intended Audience :: Information Technology
Intended Audience :: System Administrators
License :: OSI Approved :: Apache Software License
Operating System :: POSIX :: Linux
Programming Language :: Python
Programming Language :: Python :: 2
Programming Language :: Python :: 2.7
Programming Language :: Python :: 2.6
Programming Language :: Python :: 3
Programming Language :: Python :: 3.3
Programming Language :: Python :: 3.4
[files]
packages =
os_testr
packages =
os_testr
[entry_points]
console_scripts =
subunit-trace = os_testr.subunit_trace:main
ostestr = os_testr.os_testr:main
subunit2html = os_testr.subunit2html:main
console_scripts =
subunit-trace = os_testr.subunit_trace:main
ostestr = os_testr.os_testr:main
subunit2html = os_testr.subunit2html:main
[build_sphinx]
source-dir = doc/source
@ -51,9 +51,3 @@ input_file = os_testr/locale/os-testr.pot
keywords = _ gettext ngettext l_ lazy_gettext
mapping_file = babel.cfg
output_file = os_testr/locale/os-testr.pot
[egg_info]
tag_date = 0
tag_svn_revision = 0
tag_build =

2
test-requirements.txt
View File

@ -10,3 +10,5 @@ sphinx>=1.1.2,!=1.2.0,!=1.3b1,<1.3
oslosphinx>=2.2.0 # Apache-2.0
oslotest>=1.2.0 # Apache-2.0
testscenarios>=0.4
ddt>=0.4.0
six>=1.9.0

2
tox.ini
View File

@ -10,7 +10,7 @@ setenv =
VIRTUAL_ENV={envdir}
deps = -r{toxinidir}/requirements.txt
-r{toxinidir}/test-requirements.txt
commands = python setup.py testr --slowest --testr-args='{posargs}'
commands = ostestr {posargs}
[testenv:pep8]
commands = flake8