Release 0.6.0

This release of os-testr includes:
  * ostestr: support for comments in whitelist files
  * A new utility to create subunit streams
 -----BEGIN PGP SIGNATURE-----
 Version: GnuPG v2
 
 iQIcBAABCgAGBQJWi/9XAAoJEP0SoPIUyeF3rTIQAO0yW7eQ98HDwLTbTbz1V6zZ
 S9Gz6EMZC9Ps70mmJZ1i/vm7YmPoinCloPCFIjtt4EZKnQDG5Qf3XjHZsOjjyFHM
 H625DQJaRAOuD2q3TGCp35qPl4tqujq0GlJjlCIGozUhfXIpRAtLv865TpCaUE+g
 Ot4TK937lYE70yavqWas694AMqPuLRIp+rJrV9se0obFuKNyz3OyfiZrshOVmpwC
 kLuOv+b8TKF4GBEF/Nblc/50xE0UmlxSGybv+Qb8T7ednSPCIOF3pKzlWUd+RTZh
 a1ChFTX4epMqdPO99DkfYD8Bw0auk0Ax4On5iXXsryOBnl9DFmkKJHs8B91RgJB2
 D3mqB8gUDJBlx6uRLv9L071GDvc16jn8xTSNC0EEwFuJHp1nx8Hv6bLtfKyrmkNc
 8iBo2anCBXg/+rk9W1X8K21ZV+avwjUaSSjBUUPrQ1L+OAQl1buhIxCZqN979idB
 GBCJPGn1w8j6GtJF2b1J5xin4n+G7b81qvKyaJtezk0LdtG2N4rqMgWJyTxnCeMV
 A23nJ9xxvfZ7hb4w1zas82SxxTyjXZGzACYExThkt6lh20Jl5nwl1eGQLckyqfBN
 hB+TKcCiKoCPfHOG5GahtBNSd6nfVFYbP36JC35FkdD1DISooH16fsSMsz4OZDEm
 llSUiPP4Bii28cUhGier
 =eiEu
 -----END PGP SIGNATURE-----

Merge tag '0.6.0' into debian/mitaka

Release 0.6.0

This release of os-testr includes:
 * ostestr: support for comments in whitelist files
 * A new utility to create subunit streams

.gitignore

@@ -4,8 +4,7 @@
 *.so
 
 # Packages
-*.egg
+*.egg*
-*.egg-info
 dist
 build
 eggs
@@ -22,7 +21,9 @@ lib64
 pip-log.txt
 
 # Unit test / coverage reports
-.coverage
+cover/
+.coverage*
+!.coveragerc
 .tox
 nosetests.xml
 .testrepository

README.rst

@@ -1,8 +1,8 @@
-===============================
+========
 os-testr
-===============================
+========
 
-A testr wrapper to provide functionality for OpenStack projects
+A testr wrapper to provide functionality for OpenStack projects.
 
 * Free software: Apache license
 * Documentation: http://docs.openstack.org/developer/os-testr
@@ -12,8 +12,8 @@ A testr wrapper to provide functionality for OpenStack projects
 Features
 --------
 
-* ostestr: a testr wrapper that uses subunit-trace for output and builds some
+* ``ostestr``: a testr wrapper that uses subunit-trace for output and builds
-           helpful extra functionality around testr
+  some helpful extra functionality around testr
-* subunit-trace: an output filter for a subunit stream which provides useful
+* ``subunit-trace``: an output filter for a subunit stream which provides
-                 information about the run
+  useful information about the run
-* subunit2html: generates a test results html page from a subunit stream
+* ``subunit2html``: generates a test results html page from a subunit stream

doc/source/ostestr.rst

@@ -112,7 +112,7 @@ exposed via the --regex option. For example::
     $ ostestr --regex 'magic\.regex'
 
 This will do a straight passthrough of the provided regex to testr.
-Additionally, ostestr allows you to specify a a blacklist file to define a set
+Additionally, ostestr allows you to specify a blacklist file to define a set
 of regexes to exclude. You can specify a blacklist file with the
 --blacklist-file/-b option, for example::
 

doc/source/subunit_trace.rst

@@ -20,7 +20,7 @@ Options
                         Disable printing failure debug information in realtime
   --fails, -f
                         Print failure debug information after the stream is
-                        proccesed
+                        processed
   --failonly
                         Don't print success items
   --perc-diff, -d
@@ -58,7 +58,7 @@ disabled by using --no-failure-debug, -n. For example::
 
     $ testr run --subunit | subunit-trace --no-failure-debug
 
-Rhere is also the option to print all failures together at the end of the test
+Here is also the option to print all failures together at the end of the test
 run before the summary view. This is done using the --fails/-f option. For
 example::
 
@@ -77,7 +77,7 @@ example::
 
      $ testr run --subunit | subunit-trace --failonly
 
-The last output option provided by subunit-trace is to diable the summary view
+The last output option provided by subunit-trace is to disable the summary view
 of the test run which is normally displayed at the end of a run. You can do
 this using the --no-summary option. For example::
 

os_testr/generate_subunit.py

@@ -0,0 +1,50 @@
+#!/usr/bin/env python2
+# Copyright 2015 Hewlett-Packard Development Company, L.P.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+
+import datetime
+import sys
+
+import subunit
+from subunit import iso8601
+
+
+def main():
+    start_time = datetime.datetime.fromtimestamp(float(sys.argv[1])).replace(
+        tzinfo=iso8601.UTC)
+    elapsed_time = datetime.timedelta(seconds=int(sys.argv[2]))
+    stop_time = start_time + elapsed_time
+
+    if len(sys.argv) > 3:
+        status = sys.argv[3]
+    else:
+        status = 'success'
+
+    if len(sys.argv) > 4:
+        test_id = sys.argv[4]
+    else:
+        test_id = 'devstack'
+
+    # Write the subunit test
+    output = subunit.v2.StreamResultToBytes(sys.stdout)
+    output.startTestRun()
+    output.status(timestamp=start_time, test_id=test_id)
+    # Write the end of the test
+    output.status(test_status=status, timestamp=stop_time, test_id=test_id)
+    output.stopTestRun()
+
+
+if __name__ == '__main__':
+    main()

os_testr/os_testr.py

@@ -58,6 +58,8 @@ def get_parser(args):
                              'this is mutually exclusive with --pretty')
     parser.add_argument('--list', '-l', action='store_true',
                         help='List all the tests which will be run.')
+    parser.add_argument('--color', action='store_true',
+                        help='Use color in the pretty output')
     slowest = parser.add_mutually_exclusive_group()
     slowest.add_argument('--slowest', dest='slowest', action='store_true',
                          help="after the test run print the slowest tests")
@@ -130,7 +132,14 @@ def path_to_regex(path):
 
 
 def get_regex_from_whitelist_file(file_path):
-    return '|'.join(open(file_path).read().splitlines())
+    lines = []
+    for line in open(file_path).read().splitlines():
+        split_line = line.strip().split('#')
+        # Before the # is the regex
+        line_regex = split_line[0].strip()
+        if line_regex:
+            lines.append(line_regex)
+    return '|'.join(lines)
 
 
 def construct_regex(blacklist_file, whitelist_file, regex, print_exclude):
@@ -166,7 +175,7 @@ def construct_regex(blacklist_file, whitelist_file, regex, print_exclude):
 
 
 def call_testr(regex, subunit, pretty, list_tests, slowest, parallel, concur,
-               until_failure):
+               until_failure, color):
     if parallel:
         cmd = ['testr', 'run', '--parallel']
         if concur:
@@ -181,6 +190,12 @@ def call_testr(regex, subunit, pretty, list_tests, slowest, parallel, concur,
         cmd.append('--until-failure')
     cmd.append(regex)
     env = copy.deepcopy(os.environ)
+
+    if pretty:
+        subunit_trace_cmd = ['subunit-trace', '--no-failure-debug', '-f']
+        if color:
+            subunit_trace_cmd.append('--color')
+
     # This workaround is necessary because of lp bug 1411804 it's super hacky
     # and makes tons of unfounded assumptions, but it works for the most part
     if (subunit or pretty) and until_failure:
@@ -198,9 +213,9 @@ def call_testr(regex, subunit, pretty, list_tests, slowest, parallel, concur,
                 if pretty:
                     cmd = ['python', '-m', 'subunit.run', test]
                     ps = subprocess.Popen(cmd, env=env, stdout=subprocess.PIPE)
-                    proc = subprocess.Popen(['subunit-trace',
+                    subunit_trace_cmd.append('--no-summary')
-                                             '--no-failure-debug', '-f',
+                    proc = subprocess.Popen(subunit_trace_cmd,
-                                             '--no-summary'], env=env,
+                                            env=env,
                                             stdin=ps.stdout)
                     ps.stdout.close()
                     proc.communicate()
@@ -223,7 +238,7 @@ def call_testr(regex, subunit, pretty, list_tests, slowest, parallel, concur,
     # If not until-failure special case call testr like normal
     elif pretty and not list_tests:
         ps = subprocess.Popen(cmd, env=env, stdout=subprocess.PIPE)
-        proc = subprocess.Popen(['subunit-trace', '--no-failure-debug', '-f'],
+        proc = subprocess.Popen(subunit_trace_cmd,
                                 env=env, stdin=ps.stdout)
         ps.stdout.close()
     else:
@@ -261,7 +276,7 @@ def _select_and_call_runner(opts, exclude_regex):
     if not opts.no_discover and not opts.pdb:
         ec = call_testr(exclude_regex, opts.subunit, opts.pretty, opts.list,
                         opts.slowest, opts.parallel, opts.concurrency,
-                        opts.until_failure)
+                        opts.until_failure, opts.color)
     else:
         test_to_run = opts.no_discover or opts.pdb
         if test_to_run.find('/') != -1:

os_testr/subunit2html.py

@@ -633,10 +633,10 @@ class HtmlOutput(testtools.TestResult):
             test = test.test
         if test.__class__ == subunit.RemotedTestCase:
             cl = test._RemotedTestCase__description.rsplit('.', 1)[0]
+        else:
+            cl = test.id().rsplit('.', 1)[0]
         mod = cl.rsplit('.', 1)[0]
         cls = ClassInfoWrapper(cl, mod)
-        else:
-            cls = ClassInfoWrapper(str(test.__class__), str(test.__module__))
         if not str(cls) in rmap:
             rmap[str(cls)] = []
             classes.append(cls)

os_testr/subunit_trace.py

@@ -17,6 +17,7 @@
 # under the License.
 
 """Trace a subunit stream in reasonable detail and high accuracy."""
+from __future__ import absolute_import
 
 import argparse
 import datetime
@@ -28,6 +29,8 @@ import sys
 import subunit
 import testtools
 
+from os_testr.utils import colorizer
+
 # NOTE(mtreinish) on python3 anydbm was renamed dbm and the python2 dbm module
 # was renamed to dbm.ndbm, this block takes that into account
 try:
@@ -148,7 +151,8 @@ def find_test_run_time_diff(test_id, run_time):
 
 
 def show_outcome(stream, test, print_failures=False, failonly=False,
-                 enable_diff=False, threshold='0', abbreviate=False):
+                 enable_diff=False, threshold='0', abbreviate=False,
+                 enable_color=False):
     global RESULTS
     status = test['status']
     # TODO(sdague): ask lifeless why on this?
@@ -167,19 +171,29 @@ def show_outcome(stream, test, print_failures=False, failonly=False,
     if name == 'process-returncode':
         return
 
+    for color in [colorizer.AnsiColorizer, colorizer.NullColorizer]:
+        if not enable_color:
+            color = colorizer.NullColorizer(stream)
+            break
+        if color.supported():
+            color = color(stream)
+            break
+
     if status == 'fail':
         FAILS.append(test)
         if abbreviate:
-            stream.write('F')
+            color.write('F', 'red')
         else:
-            stream.write('{%s} %s [%s] ... FAILED\n' % (
+            stream.write('{%s} %s [%s] ... ' % (
                 worker, name, duration))
+            color.write('FAILED', 'red')
+            stream.write('\n')
             if not print_failures:
                 print_attachments(stream, test, all_channels=True)
     elif not failonly:
         if status == 'success':
             if abbreviate:
-                stream.write('.')
+                color.write('.', 'green')
             else:
                 out_string = '{%s} %s [%s' % (worker, name, duration)
                 perc_diff = find_test_run_time_diff(test['id'], duration)
@@ -189,17 +203,22 @@ def show_outcome(stream, test, print_failures=False, failonly=False,
                             out_string = out_string + ' +%.2f%%' % perc_diff
                         else:
                             out_string = out_string + ' %.2f%%' % perc_diff
-                stream.write(out_string + '] ... ok\n')
+                stream.write(out_string + '] ... ')
+                color.write('ok', 'green')
+                stream.write('\n')
                 print_attachments(stream, test)
         elif status == 'skip':
             if abbreviate:
-                stream.write('S')
+                color.write('S', 'blue')
             else:
                 reason = test['details'].get('reason', '')
                 if reason:
                     reason = ': ' + reason.as_text()
-                stream.write('{%s} %s ... SKIPPED%s\n' % (
+                stream.write('{%s} %s ... ' % (
-                    worker, name, reason))
+                    worker, name))
+                color.write('SKIPPED', 'blue')
+                stream.write('%s' % (reason))
+                stream.write('\n')
         else:
             if abbreviate:
                 stream.write('%s' % test['status'][0])
@@ -320,6 +339,8 @@ def parse_args():
     parser.add_argument('--no-summary', action='store_true',
                         help="Don't print the summary of the test run after "
                              " completes")
+    parser.add_argument('--color', action='store_true',
+                        help="Print results with colors")
     return parser.parse_args()
 
 
@@ -332,7 +353,8 @@ def main():
                           print_failures=args.print_failures,
                           failonly=args.failonly,
                           enable_diff=args.enable_diff,
-                          abbreviate=args.abbreviate))
+                          abbreviate=args.abbreviate,
+                          enable_color=args.color))
     summary = testtools.StreamSummary()
     result = testtools.CopyStreamResult([outcomes, summary])
     result = testtools.StreamResultRouter(result)
@@ -354,6 +376,12 @@ def main():
         print_fails(sys.stdout)
     if not args.no_summary:
         print_summary(sys.stdout, elapsed_time)
+
+    # NOTE(mtreinish): Ideally this should live in testtools streamSummary
+    # this is just in place until the behavior lands there (if it ever does)
+    if count_tests('status', '^success$') == 0:
+        print("\nNo tests were successful during the run")
+        exit(1)
     exit(0 if summary.wasSuccessful() else 1)
 
 

os_testr/tests/test_os_testr.py

@@ -144,6 +144,18 @@ class TestConstructRegex(base.TestCase):
             result,
             "^((?!fake_regex_3|fake_regex_2|fake_regex_1|fake_regex_0).)*$")
 
+    def test_whitelist_regex_with_comments(self):
+        whitelist_file = six.StringIO()
+        for i in range(4):
+            whitelist_file.write('fake_regex_%s # A Comment\n' % i)
+        whitelist_file.seek(0)
+        with mock.patch('six.moves.builtins.open',
+                        return_value=whitelist_file):
+            result = os_testr.construct_regex(None, 'fake_path', None, False)
+        self.assertEqual(
+            result,
+            "fake_regex_0|fake_regex_1|fake_regex_2|fake_regex_3")
+
     def test_blacklist_regex_without_comments(self):
         blacklist_file = six.StringIO()
         for i in range(4):

os_testr/tests/test_subunit2html.py

@@ -0,0 +1,31 @@
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+from ddt import data
+from ddt import ddt
+from subunit import RemotedTestCase
+from testtools import PlaceHolder
+
+from os_testr import subunit2html
+from os_testr.tests import base
+
+
+@ddt
+class TestSubunit2html(base.TestCase):
+    @data(RemotedTestCase, PlaceHolder)
+    def test_class_parsing(self, test_cls):
+        """Tests that the class paths are parsed for v1 & v2 tests"""
+        test_ = test_cls("example.path.to.test.method")
+        obj_ = subunit2html.HtmlOutput()
+        cls_ = []
+        obj_._add_cls({}, cls_, test_, ())
+        self.assertEqual("example.path.to.test", cls_[0].name)

os_testr/tests/test_subunit_trace.py

@@ -14,6 +14,8 @@
 # under the License.
 
 from datetime import datetime as dt
+import os
+import subprocess
 
 from ddt import data
 from ddt import ddt
@@ -59,3 +61,21 @@ class TestSubunitTrace(base.TestCase):
         }
         with patch.dict(subunit_trace.RESULTS, patched_res, clear=True):
             self.assertEqual(subunit_trace.run_time(), expected_result)
+
+    def test_return_code_all_skips(self):
+        skips_stream = os.path.join(
+            os.path.dirname(os.path.abspath(__file__)),
+            'sample_streams/all_skips.subunit')
+        p = subprocess.Popen(['subunit-trace'], stdin=subprocess.PIPE)
+        with open(skips_stream, 'rb') as stream:
+            p.communicate(stream.read())
+        self.assertEqual(1, p.returncode)
+
+    def test_return_code_normal_run(self):
+        regular_stream = os.path.join(
+            os.path.dirname(os.path.abspath(__file__)),
+            'sample_streams/successful.subunit')
+        p = subprocess.Popen(['subunit-trace'], stdin=subprocess.PIPE)
+        with open(regular_stream, 'rb') as stream:
+            p.communicate(stream.read())
+        self.assertEqual(0, p.returncode)

os_testr/utils/colorizer.py

@@ -0,0 +1,98 @@
+# Copyright 2015 NEC Corporation
+# All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+#
+# Colorizer Code is borrowed from Twisted:
+# Copyright (c) 2001-2010 Twisted Matrix Laboratories.
+#
+#    Permission is hereby granted, free of charge, to any person obtaining
+#    a copy of this software and associated documentation files (the
+#    "Software"), to deal in the Software without restriction, including
+#    without limitation the rights to use, copy, modify, merge, publish,
+#    distribute, sublicense, and/or sell copies of the Software, and to
+#    permit persons to whom the Software is furnished to do so, subject to
+#    the following conditions:
+#
+#    The above copyright notice and this permission notice shall be
+#    included in all copies or substantial portions of the Software.
+#
+#    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+#    EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+#    MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+#    NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+#    LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+#    OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+#    WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+import sys
+
+
+class AnsiColorizer(object):
+    """A colorizer is an object that loosely wraps around a stream
+
+    allowing callers to write text to the stream in a particular color.
+
+    Colorizer classes must implement C{supported()} and C{write(text, color)}.
+    """
+    _colors = dict(black=30, red=31, green=32, yellow=33,
+                   blue=34, magenta=35, cyan=36, white=37)
+
+    def __init__(self, stream):
+        self.stream = stream
+
+    @classmethod
+    def supported(cls, stream=sys.stdout):
+        """Check the current platform supports coloring terminal output
+
+        A class method that returns True if the current platform supports
+        coloring terminal output using this method. Returns False otherwise.
+        """
+        if not stream.isatty():
+            return False  # auto color only on TTYs
+        try:
+            import curses
+        except ImportError:
+            return False
+        else:
+            try:
+                try:
+                    return curses.tigetnum("colors") > 2
+                except curses.error:
+                    curses.setupterm()
+                    return curses.tigetnum("colors") > 2
+            except Exception:
+                # guess false in case of error
+                return False
+
+    def write(self, text, color):
+        """Write the given text to the stream in the given color.
+
+        @param text: Text to be written to the stream.
+
+        @param color: A string label for a color. e.g. 'red', 'white'.
+        """
+        color = self._colors[color]
+        self.stream.write('\x1b[%s;1m%s\x1b[0m' % (color, text))
+
+
+class NullColorizer(object):
+    """See _AnsiColorizer docstring."""
+    def __init__(self, stream):
+        self.stream = stream
+
+    @classmethod
+    def supported(cls, stream=sys.stdout):
+        return True
+
+    def write(self, text, color):
+        self.stream.write(text)

setup.cfg

@@ -29,6 +29,7 @@ console_scripts =
     subunit-trace = os_testr.subunit_trace:main
     ostestr = os_testr.os_testr:main
     subunit2html = os_testr.subunit2html:main
+    generate-subunit = os_testr.generate_subunit:main
 
 [build_sphinx]
 source-dir = doc/source

tox.ini

@@ -8,9 +8,12 @@ usedevelop = True
 install_command = pip install -U {opts} {packages}
 setenv =
    VIRTUAL_ENV={envdir}
+whitelist_externals = find
 deps = -r{toxinidir}/requirements.txt
        -r{toxinidir}/test-requirements.txt
-commands = ostestr {posargs}
+commands =
+         find . -type f -name "*.pyc" -delete
+         ostestr {posargs}
 
 [testenv:pep8]
 commands = flake8
@@ -19,7 +22,7 @@ commands = flake8
 commands = {posargs}
 
 [testenv:cover]
-commands = python setup.py testr --coverage --testr-args='{posargs}'
+commands = python setup.py testr --coverage --coverage-package-name='os_testr' --testr-args='{posargs}'
 
 [testenv:docs]
 commands = python setup.py build_sphinx