Release 0.2.0

This release of os-testr includes:
 
  * Support for having comments in a blacklist file
  * A new option, --print-exclude to print comments along with exclude tests
    when using a blacklist file
  * Printing percent change in subunit-trace is disabled by default and a new
    CLI option is added to enable it
  * Several bugfixes
 -----BEGIN PGP SIGNATURE-----
 Version: GnuPG v2
 
 iQIcBAABCgAGBQJVkqrPAAoJEP0SoPIUyeF3fFcP/A+omiNzguvj2L1USyqNUmcG
 2yXeS1L5WzfDnhmJZYxpC0yKAopERxbJLWNxIAxhXOErsrQ+b0xDhW0hjxyaLdS+
 vQcpd/I9Nz7RTm/KuMEPIDTZnkc/s8GhH9ZoI/ljIxdaCA4GGe5cS7MO0rSFjc8z
 n88GQWjj0LPVIOQAsfWoIci0iVxEbYT4BP/GYoNxlXBOKUnSnLzmtDFk8BI2Zjlq
 UQV8UCBapo10sNz8OAmRV26blXvtf9orEGDfEDLMKEENrQQwFjlS4Voaj5l48y47
 Jl6QwhrMQzE27Qjk++ddDCYugzWFeOcyh4gnrEfzXmgRYZehR3oHW7wCD9d4Z1vf
 JBFll0L9EAt44HW8uK/rw0S3PwLfo1+SWBxZkMxoiOxTTaR4I7Q5FHYb9nb8Z/JK
 NDwTuOvoU3rFbqXvBjv6TQ1a9a12vixmvx/j4TE4e2+Wo8viu59qyNyRgztOLv0F
 D9YMpXMJU6o/1vKudlDhC4SCzPpOTVDSjkhzEb6/UC69T0i8TUMEyTz7MZMQjIMl
 qlkHUOat9veJHVE7xp78BQOYKwnyb7Z5TKwtbW6LL6aTlym5feRg8VpEdMo2tI3I
 +7QbovG0BzvPGQ1r1FjvaX8sKA/W68J8nyvIGxDvakfDlghi0sHFx7ZDhoXcooVR
 vwt6lst3cHDZFVQmF8G0
 =gg4k
 -----END PGP SIGNATURE-----

Merge tag '0.2.0' into debian/liberty

Release 0.2.0

This release of os-testr includes:

 * Support for having comments in a blacklist file
 * A new option, --print-exclude to print comments along with exclude tests
   when using a blacklist file
 * Printing percent change in subunit-trace is disabled by default and a new
   CLI option is added to enable it
 * Several bugfixes

.gitignore

@@ -0,0 +1,53 @@
+*.py[cod]
+
+# C extensions
+*.so
+
+# Packages
+*.egg
+*.egg-info
+dist
+build
+eggs
+parts
+bin
+var
+sdist
+develop-eggs
+.installed.cfg
+lib
+lib64
+
+# Installer logs
+pip-log.txt
+
+# Unit test / coverage reports
+.coverage
+.tox
+nosetests.xml
+.testrepository
+.venv
+
+# Translations
+*.mo
+
+# Mr Developer
+.mr.developer.cfg
+.project
+.pydevproject
+
+# Complexity
+output/*.html
+output/*/index.html
+
+# Sphinx
+doc/build
+
+# pbr generates these
+AUTHORS
+ChangeLog
+
+# Editors
+*~
+.*.swp
+.*sw?

.gitreview

@@ -0,0 +1,4 @@
+[gerrit]
+host=review.openstack.org
+port=29418
+project=openstack/os-testr.git

TODO.rst

@@ -0,0 +1,17 @@
+Work Items for os-testr
+=======================
+
+Short Term
+----------
+ * Expose all subunit-trace options through ostestr
+ * Add --html option to ostestr to run testr with subunit2html output
+ * Add unit tests
+   * For ostestr test selection api
+   * Response code validation on more argument permutations
+Long Term
+---------
+ * Lock down test selection CLI
+   * When this is done it will become release 1.0.0
+ * Add subunit-trace functional tests
+   ** Sample subunit streams and test output from subunit-trace
+ * Add testing for subunit2html

doc/source/index.rst

@@ -15,6 +15,7 @@ Contents:
    installation
    usage
    contributing
+   todo
 
 Indices and tables
 ==================

doc/source/todo.rst

@@ -0,0 +1 @@
+.. include:: ../../TODO.rst

os_testr/os_testr.py

@@ -66,19 +66,68 @@ def parse_args():
                              'encountered. Running with subunit or pretty'
                              'output enable will force the loop to run tests'
                              'serially')
+    parser.add_argument('--print-exclude', action='store_true',
+                        help='If an exclude file is used this option will '
+                             'prints the comment from the same line and all '
+                             'skipped tests before the test run')
     parser.set_defaults(pretty=True, slowest=True, parallel=True)
     opts = parser.parse_args()
     return opts
 
 
-def construct_regex(blacklist_file, regex):
+def _get_test_list(regex, env=None):
+    env = env or copy.deepcopy(os.environ)
+    proc = subprocess.Popen(['testr', 'list-tests', regex], env=env,
+                            stdout=subprocess.PIPE)
+    out = proc.communicate()[0]
+    raw_test_list = out.split('\n')
+    bad = False
+    test_list = []
+    exclude_list = ['OS_', 'CAPTURE', 'TEST_TIMEOUT', 'PYTHON',
+                    'subunit.run discover']
+    for line in raw_test_list:
+        for exclude in exclude_list:
+            if exclude in line:
+                bad = True
+                break
+            elif not line:
+                bad = True
+                break
+        if not bad:
+            test_list.append(line)
+        bad = False
+    return test_list
+
+
+def print_skips(regex, message):
+    test_list = _get_test_list(regex)
+    if test_list:
+        if message:
+            print(message)
+        else:
+            print('Skipped because of regex %s:' % regex)
+        for test in test_list:
+            print(test)
+        # Extra whitespace to separate
+        print('\n')
+
+
+def construct_regex(blacklist_file, regex, print_exclude):
     if not blacklist_file:
         exclude_regex = ''
     else:
         black_file = open(blacklist_file, 'r')
         exclude_regex = ''
         for line in black_file:
-            regex = line.strip()
+            raw_line = line.strip()
+            split_line = raw_line.split('#')
+            # Before the # is the regex
+            regex = split_line[0].strip()
+            # After the # is a comment
+            comment = split_line[1].strip()
+            if regex:
+                if print_exclude:
+                    print_skips(regex, comment)
                 exclude_regex = '|'.join([regex, exclude_regex])
         if exclude_regex:
             exclude_regex = "'(?!.*" + exclude_regex + ")"
@@ -106,25 +155,7 @@ def call_testr(regex, subunit, pretty, list_tests, slowest, parallel, concur,
     # This workaround is necessary because of lp bug 1411804 it's super hacky
     # and makes tons of unfounded assumptions, but it works for the most part
     if (subunit or pretty) and until_failure:
-        proc = subprocess.Popen(['testr', 'list-tests', regex], env=env,
+        test_list = _get_test_list(regex, env)
-                                stdout=subprocess.PIPE)
-        out = proc.communicate()[0]
-        raw_test_list = out.split('\n')
-        bad = False
-        test_list = []
-        exclude_list = ['CAPTURE', 'TEST_TIMEOUT', 'PYTHON',
-                        'subunit.run discover']
-        for line in raw_test_list:
-            for exclude in exclude_list:
-                if exclude in line:
-                    bad = True
-                    break
-                elif not line:
-                    bad = True
-                    break
-            if not bad:
-                test_list.append(line)
-            bad = False
         count = 0
         failed = False
         if not test_list:
@@ -217,7 +248,8 @@ def main():
         msg = "You can not use until_failure mode with pdb or no-discover"
         print(msg)
         exit(5)
-    exclude_regex = construct_regex(opts.blacklist_file, opts.regex)
+    exclude_regex = construct_regex(opts.blacklist_file, opts.regex,
+                                    opts.print_exclude)
     if not os.path.isdir('.testrepository'):
         subprocess.call(['testr', 'init'])
     if not opts.no_discover and not opts.pdb:

os_testr/subunit_trace.py

@@ -131,7 +131,14 @@ def find_test_run_time_diff(test_id, run_time):
             test_times = dbm.open(times_db_path)
         except Exception:
             return False
+        try:
             avg_runtime = float(test_times.get(str(test_id), False))
+        except Exception:
+            try:
+                avg_runtime = float(test_times[str(test_id)])
+            except Exception:
+                avg_runtime = False
+
         if avg_runtime and avg_runtime > 0:
             run_time = float(run_time.rstrip('s'))
             perc_diff = ((run_time - avg_runtime) / avg_runtime) * 100
@@ -140,7 +147,7 @@ def find_test_run_time_diff(test_id, run_time):
 
 
 def show_outcome(stream, test, print_failures=False, failonly=False,
-                 threshold='0'):
+                 enable_diff=False, threshold='0'):
     global RESULTS
     status = test['status']
     # TODO(sdague): ask lifeless why on this?
@@ -169,6 +176,7 @@ def show_outcome(stream, test, print_failures=False, failonly=False,
         if status == 'success':
             out_string = '{%s} %s [%s' % (worker, name, duration)
             perc_diff = find_test_run_time_diff(test['id'], duration)
+            if enable_diff:
                 if perc_diff and abs(perc_diff) >= abs(float(threshold)):
                     if perc_diff > 0:
                         out_string = out_string + ' +%.2f%%' % perc_diff
@@ -220,7 +228,11 @@ def run_time():
     runtime = 0.0
     for k, v in RESULTS.items():
         for test in v:
-            runtime += float(get_duration(test['timestamps']).strip('s'))
+            test_dur = get_duration(test['timestamps']).strip('s')
+            # NOTE(toabctl): get_duration() can return an empty string
+            # which leads to a ValueError when casting to float
+            if test_dur:
+                runtime += float(test_dur)
     return runtime
 
 
@@ -271,6 +283,9 @@ def parse_args():
                         default=(
                             os.environ.get('TRACE_FAILONLY', False)
                             is not False))
+    parser.add_argument('--perc-diff', '-d', action='store_true',
+                        dest='enable_diff',
+                        help="Print percent change in run time on each test ")
     parser.add_argument('--diff-threshold', '-t', dest='threshold',
                         help="Threshold to use for displaying percent change "
                              "from the avg run time. If one is not specified "
@@ -288,7 +303,8 @@ def main():
     outcomes = testtools.StreamToDict(
         functools.partial(show_outcome, sys.stdout,
                           print_failures=args.print_failures,
-                          failonly=args.failonly))
+                          failonly=args.failonly,
+                          enable_diff=args.enable_diff))
     summary = testtools.StreamSummary()
     result = testtools.CopyStreamResult([outcomes, summary])
     result = testtools.StreamResultRouter(result)

os_testr/tests/test_return_codes.py

@@ -14,13 +14,13 @@
 
 import os
 import shutil
-import StringIO
 import subprocess
 import tempfile
 
 import testtools
 
 from os_testr.tests import base
+from six import StringIO
 
 DEVNULL = open(os.devnull, 'wb')
 
@@ -47,8 +47,8 @@ class TestReturnCodes(base.TestCase):
         shutil.copy('os_testr/tests/files/setup.cfg', self.setup_cfg_file)
         shutil.copy('os_testr/tests/files/__init__.py', self.init_file)
 
-        self.stdout = StringIO.StringIO()
+        self.stdout = StringIO()
-        self.stderr = StringIO.StringIO()
+        self.stderr = StringIO()
         # Change directory, run wrapper and check result
         self.addCleanup(os.chdir, os.path.abspath(os.curdir))
         os.chdir(self.directory)

os_testr/tests/test_subunit_trace.py

@@ -0,0 +1,61 @@
+# Copyright 2015 SUSE Linux GmbH
+# All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+from datetime import datetime as dt
+
+from ddt import data
+from ddt import ddt
+from ddt import unpack
+from mock import patch
+
+from os_testr import subunit_trace
+from os_testr.tests import base
+
+
+@ddt
+class TestSubunitTrace(base.TestCase):
+
+    @data(([dt(2015, 4, 17, 22, 23, 14, 111111),
+            dt(2015, 4, 17, 22, 23, 14, 111111)],
+           "0.000000s"),
+          ([dt(2015, 4, 17, 22, 23, 14, 111111),
+            dt(2015, 4, 17, 22, 23, 15, 111111)],
+           "1.000000s"),
+          ([dt(2015, 4, 17, 22, 23, 14, 111111),
+            None],
+           ""))
+    @unpack
+    def test_get_durating(self, timestamps, expected_result):
+        self.assertEqual(subunit_trace.get_duration(timestamps),
+                         expected_result)
+
+    @data(([dt(2015, 4, 17, 22, 23, 14, 111111),
+            dt(2015, 4, 17, 22, 23, 14, 111111)],
+           0.0),
+          ([dt(2015, 4, 17, 22, 23, 14, 111111),
+            dt(2015, 4, 17, 22, 23, 15, 111111)],
+           1.0),
+          ([dt(2015, 4, 17, 22, 23, 14, 111111),
+            None],
+           0.0))
+    @unpack
+    def test_run_time(self, timestamps, expected_result):
+        patched_res = {
+            0: [
+                {'timestamps': timestamps}
+            ]
+        }
+        with patch.dict(subunit_trace.RESULTS, patched_res, clear=True):
+            self.assertEqual(subunit_trace.run_time(), expected_result)

setup.cfg

@@ -51,9 +51,3 @@ input_file = os_testr/locale/os-testr.pot
 keywords = _ gettext ngettext l_ lazy_gettext
 mapping_file = babel.cfg
 output_file = os_testr/locale/os-testr.pot
-
-[egg_info]
-tag_date = 0
-tag_svn_revision = 0
-tag_build = 
-

test-requirements.txt

@@ -10,3 +10,5 @@ sphinx>=1.1.2,!=1.2.0,!=1.3b1,<1.3
 oslosphinx>=2.2.0  # Apache-2.0
 oslotest>=1.2.0  # Apache-2.0
 testscenarios>=0.4
+ddt>=0.4.0
+six>=1.9.0

tox.ini

@@ -10,7 +10,7 @@ setenv =
    VIRTUAL_ENV={envdir}
 deps = -r{toxinidir}/requirements.txt
        -r{toxinidir}/test-requirements.txt
-commands = python setup.py testr --slowest --testr-args='{posargs}'
+commands = ostestr {posargs}
 
 [testenv:pep8]
 commands = flake8