openstack/monasca-common
Code Issues Proposed changes

Migrate to ostestr framework and PY3

Browse Source 
Following commits does several things:

* migrates CI of monasca-common to ostestr
* enables PY35 compatybility

Also:

* marked one tests as excluded under PY35 because changing
it would require affecting embedded kafka library which
will be eventually removed in future

(cherry-picked from commit cabc2ddd5f239e5f44e090d35aaac0770e5849ba)

Change-Id: I432a466e2620bc8d305ef2630307b636461c8e81
This commit is contained in:
Tomasz Trębski 2017-01-13 11:01:47 +01:00
parent 383d1bed0a
commit c5a64fe22f
13 changed files with 242 additions and 84 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
.coveragerc Normal file
View File

@ -0,0 +1,7 @@
[run]
branch = True
source = monasca_common
omit = monasca_common/tests/*
[report]
ignore_errors = True

3
.gitignore vendored
View File

@ -16,3 +16,6 @@ build
dist
*.egg-info
*.egg
cover/
.coverage
.testrepository/

9
.testr.conf Normal file
View File

@ -0,0 +1,9 @@
[DEFAULT]
test_command=OS_STDOUT_CAPTURE=${OS_STDOUT_CAPTURE:-1} \
OS_STDERR_CAPTURE=${OS_STDERR_CAPTURE:-1} \
OS_TEST_TIMEOUT=${OS_TEST_TIMEOUT:-160} \
${PYTHON:-python} -m subunit.run discover -t ./ $OS_TEST_PATH $LISTOPT $IDOPTION
test_id_option=--load-list $IDFILE
test_list_option=--list
group_regex=monasca_common\.tests(?:\.|_)([^_]+)

25
monasca_common/tests/test_kafka.py
View File

@ -11,10 +11,11 @@
# under the License.
import mock
import unittest
from monasca_common.kafka.consumer import KafkaConsumer
from monasca_common.kafka.producer import KafkaProducer
from oslotest import base
from monasca_common.kafka import consumer
from monasca_common.kafka import producer
FAKE_KAFKA_URL = "kafka_url"
@ -24,18 +25,22 @@ FAKE_KAFKA_CONSUMER_GROUP = "group"
FAKE_KAFKA_TOPIC = "topic"
class TestKafkaProducer(unittest.TestCase):
class TestKafkaProducer(base.BaseTestCase):
def setUp(self):
super(TestKafkaProducer, self).setUp()
self.kafka_client_patcher = mock.patch('kafka.client')
self.kafka_producer_patcher = mock.patch('kafka.producer')
self.mock_kafka_client = self.kafka_client_patcher.start()
self.mock_kafka_producer = self.kafka_producer_patcher.start()
self.producer = self.mock_kafka_producer.KeyedProducer.return_value
self.client = self.mock_kafka_client.KafkaClient.return_value
self.monasca_kafka_producer = KafkaProducer(FAKE_KAFKA_URL)
self.monasca_kafka_producer = producer.KafkaProducer(FAKE_KAFKA_URL)
def tearDown(self):
super(TestKafkaProducer, self).tearDown()
self.kafka_producer_patcher.stop()
self.kafka_client_patcher.stop()
@ -83,9 +88,11 @@ class TestKafkaProducer(unittest.TestCase):
'Error publishing to {} topic.'. format(topic))
class TestKafkaConsumer(unittest.TestCase):
class TestKafkaConsumer(base.BaseTestCase):
def setUp(self):
super(TestKafkaConsumer, self).setUp()
self.kafka_client_patcher = mock.patch('kafka.client')
self.kafka_common_patcher = mock.patch('kafka.common')
self.kafka_consumer_patcher = mock.patch('kafka.consumer')
@ -100,11 +107,13 @@ class TestKafkaConsumer(unittest.TestCase):
self.client = self.mock_kafka_client.KafkaClient.return_value
self.consumer = self.mock_kafka_consumer.SimpleConsumer.return_value
self.monasca_kafka_consumer = KafkaConsumer(
self.monasca_kafka_consumer = consumer.KafkaConsumer(
FAKE_KAFKA_URL, FAKE_ZOOKEEPER_URL, FAKE_ZOOKEEPER_PATH,
FAKE_KAFKA_CONSUMER_GROUP, FAKE_KAFKA_TOPIC)
def tearDown(self):
super(TestKafkaConsumer, self).tearDown()
self.kafka_client_patcher.stop()
self.kafka_common_patcher.stop()
self.kafka_consumer_patcher.stop()
@ -143,7 +152,7 @@ class TestKafkaConsumer(unittest.TestCase):
try:
list(self.monasca_kafka_consumer)
except Exception as e:
self.assertEqual(e.message, "Failed to acquire partition")
self.assertEqual(str(e), "Failed to acquire partition")
@mock.patch('monasca_common.kafka.consumer.SetPartitioner')
def test_kafka_consumer_reset_when_offset_out_of_range(

8
monasca_common/tests/test_repositories.py
View File

@ -10,22 +10,24 @@
# See the License for the specific language governing permissions and
# limitations under the License.
import unittest
import mock
from oslotest import base
import monasca_common.repositories.exceptions as exceptions
from monasca_common.repositories.mysql import mysql_repository
class TestMySQLRepository(unittest.TestCase):
class TestMySQLRepository(base.BaseTestCase):
def setUp(self):
super(TestMySQLRepository, self).setUp()
self.cfg_patcher = mock.patch('oslo_config.cfg.CONF')
self.mock_cfg = self.cfg_patcher.start()
def tearDown(self):
super(TestMySQLRepository, self).tearDown()
self.cfg_patcher.stop()
def test_init(self):

7
monasca_common/tests/test_rest.py
View File

@ -11,7 +11,8 @@
# under the License.
import mock
import unittest
from oslotest import base
from monasca_common.rest.exceptions import DataConversionException
from monasca_common.rest.exceptions import UnreadableContentError
@ -19,13 +20,15 @@ from monasca_common.rest.exceptions import UnsupportedContentTypeException
from monasca_common.rest import utils
class TestRestUtils(unittest.TestCase):
class TestRestUtils(base.BaseTestCase):
def setUp(self):
super(TestRestUtils, self).setUp()
self.mock_json_patcher = mock.patch('monasca_common.rest.utils.json')
self.mock_json = self.mock_json_patcher.start()
def tearDown(self):
super(TestRestUtils, self).tearDown()
self.mock_json_patcher.stop()
def test_read_body_with_success(self):

130
monasca_common/tests/test_simport.py
View File

@ -13,8 +13,11 @@
# See the License for the specific language governing permissions and
# limitations under the License.
import os
import sys
import unittest
from oslotest import base
import six
import monasca_common.simport.simport as simport
@ -34,7 +37,11 @@ class LocalClass(object):
pass
class TestSimport(unittest.TestCase):
PWD = os.path.dirname(os.path.abspath(__file__))
class TestSimport(base.BaseTestCase):
def test_bad_targets(self):
self.assertRaises(simport.BadDirectory, simport._get_module,
"|foo.Class")
@ -51,56 +58,99 @@ class TestSimport(unittest.TestCase):
self.assertFalse("AnyModuleName" in sys.modules)
self.assertRaises(simport.MissingMethodOrFunction, simport._get_module,
"tests|AnyModuleName:")
PWD + "|AnyModuleName:")
self.assertFalse("AnyModuleName" in sys.modules)
def test_good_external_targets(self):
self.assertEqual(("localmodule", "Foo", "method_a"),
simport._get_module("tests|"
"localmodule:Foo.method_a"))
simport._get_module(PWD + "|localmodule:Foo.method_a"))
self.assertRaises(simport.ImportFailed, simport._get_module,
"tests|that_module:function_a")
PWD + "|that_module:function_a")
def test_bad_load(self):
self.assertRaises(AttributeError, simport.load,
self.assertRaises(simport.ImportFailed, simport.load,
"TestSimport:missing")
def test_good_load_internal(self):
self.assertEqual(dummy_function,
simport.load("TestSimport:dummy_function"))
self.assertEqual(DummyClass.method_a,
simport.load("TestSimport:DummyClass.method_a"))
def test_good_load_local(self):
method = simport.load("tests|"
"localmodule:Foo.method_a")
import localmodule
self.assertEqual(method, localmodule.Foo.method_a)
self.assertEqual(localmodule.function_a,
simport.load("localmodule:function_a"))
def test_good_load_external(self):
method = simport.load("tests/external|"
"external.externalmodule:Blah.method_b")
self.assertTrue('external.externalmodule' in sys.modules)
old = sys.modules['external.externalmodule']
import external.externalmodule
self.assertEqual(external.externalmodule,
sys.modules['external.externalmodule'])
self.assertEqual(old, external.externalmodule)
self.assertEqual(method, external.externalmodule.Blah.method_b)
def test_import_class(self):
klass = simport.load("tests/external|"
"external.externalmodule:Blah")
import external.externalmodule
self.assertEqual(klass, external.externalmodule.Blah)
self.assertEqual(six.get_function_code(dummy_function),
six.get_function_code(simport.load("test_simport:dummy_function")))
self.assertEqual(six.get_function_code(DummyClass.method_a),
six.get_function_code(simport.load("test_simport:DummyClass.method_a")))
def test_local_class(self):
klass = simport.load("LocalClass", __name__)
self.assertEqual(klass, LocalClass)
# Check python versions for importing modules.
# Python 2 import modules with full path to this module as a module name,
# for example:
# <module 'monasca_common.tests.external.externalmodule' from
# 'full_path/monasca-common/monasca_common/tests/external/externalmodule.py'>
#
# while Python 3:
# <module 'external.externalmodule' from 'full_path/monasca-common/monasca_common/tests/external/externalmodule.py'>
# , that's why we need to provide different module names for simport in Python 2 and 3
#
if six.PY2:
class TestSimportPY2(base.BaseTestCase):
def test_good_load_local(self):
method = simport.load(PWD + "|monasca_common.tests.localmodule:Foo.method_a")
import localmodule
self.assertEqual(method, localmodule.Foo.method_a)
self.assertEqual(localmodule.function_a,
simport.load("monasca_common.tests.localmodule:function_a"))
def test_good_load_external(self):
method = simport.load(PWD + "/external|monasca_common.tests.external.externalmodule:Blah.method_b")
self.assertTrue('monasca_common.tests.external.externalmodule' in sys.modules)
old = sys.modules['monasca_common.tests.external.externalmodule']
import external.externalmodule
self.assertEqual(external.externalmodule,
sys.modules['monasca_common.tests.external.externalmodule'])
self.assertEqual(old, external.externalmodule)
self.assertEqual(method, external.externalmodule.Blah.method_b)
def test_import_class(self):
klass = simport.load(PWD + "/external|monasca_common.tests.external.externalmodule:Blah")
import external.externalmodule
self.assertEqual(klass, external.externalmodule.Blah)
elif six.PY3:
class TestSimportPY3(base.BaseTestCase):
def test_good_load_local(self):
method = simport.load(PWD + "|localmodule:Foo.method_a")
import localmodule
self.assertEqual(method, localmodule.Foo.method_a)
self.assertEqual(localmodule.function_a,
simport.load("localmodule:function_a"))
def test_good_load_external(self):
method = simport.load(PWD + "/external|external.externalmodule:Blah.method_b")
self.assertTrue('external.externalmodule' in sys.modules)
old = sys.modules['external.externalmodule']
import external.externalmodule
self.assertEqual(external.externalmodule,
sys.modules['external.externalmodule'])
self.assertEqual(old, external.externalmodule)
self.assertEqual(method, external.externalmodule.Blah.method_b)
def test_import_class(self):
klass = simport.load(PWD + "/external|external.externalmodule:Blah")
import external.externalmodule
self.assertEqual(klass, external.externalmodule.Blah)

26
monasca_common/tests/validation/test_metric_validation.py
View File

@ -1,4 +1,4 @@
# (C) Copyright 2016 Hewlett Packard Enterprise Development LP
# (C) Copyright 2016-2017 Hewlett Packard Enterprise Development LP
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@ -13,8 +13,10 @@
# See the License for the specific language governing permissions and
# limitations under the License.
import monasca_common.validation.metrics as metric_validator
import unittest
from oslotest import base
import six
from monasca_common.validation import metrics as metric_validator
# a few valid characters to test
valid_name_chars = ".'_-"
@ -25,7 +27,7 @@ valid_dimension_chars = " .'_-"
invalid_dimension_chars = "<>={}(),\"\\\\;&"
class TestMetricValidation(unittest.TestCase):
class TestMetricValidation(base.BaseTestCase):
def test_valid_single_metric(self):
metric = {"name": "test_metric_name",
"dimensions": {"key1": "value1",
@ -54,19 +56,19 @@ class TestMetricValidation(unittest.TestCase):
def test_valid_metric_unicode_dimension_value(self):
metric = {"name": "test_metric_name",
"timestamp": 1405630174123,
"dimensions": {unichr(2440): 'B', 'B': 'C', 'D': 'E'},
"dimensions": {six.unichr(2440): 'B', 'B': 'C', 'D': 'E'},
"value": 5}
metric_validator.validate(metric)
def test_valid_metric_unicode_dimension_key(self):
metric = {"name": 'test_metric_name',
"dimensions": {'A': 'B', 'B': unichr(920), 'D': 'E'},
"dimensions": {'A': 'B', 'B': six.unichr(920), 'D': 'E'},
"timestamp": 1405630174123,
"value": 5}
metric_validator.validate(metric)
def test_valid_metric_unicode_metric_name(self):
metric = {"name": unichr(6021),
metric = {"name": six.unichr(6021),
"dimensions": {"key1": "value1",
"key2": "value2"},
"timestamp": 1405630174123,
@ -273,7 +275,7 @@ class TestMetricValidation(unittest.TestCase):
def test_invalid_too_many_value_meta(self):
value_meta = {}
for i in range(0, 17):
for i in six.moves.range(0, 17):
value_meta['key{}'.format(i)] = 'value{}'.format(i)
metric = {"name": "test_metric_name",
"dimensions": {"key1": "value1",
@ -300,7 +302,7 @@ class TestMetricValidation(unittest.TestCase):
def test_invalid_too_long_value_meta_key(self):
key = "K"
for i in range(0, metric_validator.VALUE_META_NAME_MAX_LENGTH):
for i in six.moves.range(0, metric_validator.VALUE_META_NAME_MAX_LENGTH):
key = "{}{}".format(key, "1")
value_meta = {key: 'BBB'}
metric = {"name": "test_metric_name",
@ -317,10 +319,10 @@ class TestMetricValidation(unittest.TestCase):
def test_invalid_too_large_value_meta(self):
value_meta_value = ""
num_value_meta = 10
for i in range(0, metric_validator.VALUE_META_VALUE_MAX_LENGTH / num_value_meta):
for i in six.moves.range(0, int(metric_validator.VALUE_META_VALUE_MAX_LENGTH / num_value_meta)):
value_meta_value = '{}{}'.format(value_meta_value, '1')
value_meta = {}
for i in range(0, num_value_meta):
for i in six.moves.range(0, num_value_meta):
value_meta['key{}'.format(i)] = value_meta_value
metric = {"name": "test_metric_name",
"dimensions": {"key1": "value1",
@ -359,7 +361,7 @@ class TestMetricValidation(unittest.TestCase):
"timestamp": 1405630174123,
"value": 2.0}
]
for i in xrange(len(metrics)):
for i in six.moves.range(len(metrics)):
metric_validator.validate_name(metrics[i]['name'])
metric_validator.validate_value(metrics[i]['value'])
metric_validator.validate_timestamp(metrics[i]['timestamp'])

26
monasca_common/validation/metrics.py
View File

@ -1,4 +1,4 @@
# (C) Copyright 2016 Hewlett Packard Enterprise Development LP
# (C) Copyright 2016-2017 Hewlett Packard Enterprise Development LP
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@ -13,7 +13,10 @@
# See the License for the specific language governing permissions and
# limitations under the License.
import math
import re
import six
import ujson
# This is used to ensure that metrics with a timestamp older than
@ -31,6 +34,13 @@ INVALID_CHARS = "<>={}(),\"\\\\;&"
RESTRICTED_DIMENSION_CHARS = re.compile('[' + INVALID_CHARS + ']')
RESTRICTED_NAME_CHARS = re.compile('[' + INVALID_CHARS + ' ' + ']')
NUMERIC_VALUES = [int, float]
if six.PY2:
# according to PEP537 long was renamed to int in PY3
# need to add long, as possible value, for PY2
NUMERIC_VALUES += [long]
NUMERIC_VALUES = tuple(NUMERIC_VALUES) # convert to tuple for instance call
class InvalidMetricName(Exception):
pass
@ -79,7 +89,7 @@ def validate_value_meta(value_meta):
msg = "Too many valueMeta entries {0}, limit is {1}: valueMeta {2}".\
format(len(value_meta), VALUE_META_MAX_NUMBER, value_meta)
raise InvalidValueMeta(msg)
for key, value in value_meta.iteritems():
for key, value in six.iteritems(value_meta):
if not key:
raise InvalidValueMeta("valueMeta name cannot be empty: key={}, "
"value={}".format(key, value))
@ -100,8 +110,8 @@ def validate_value_meta(value_meta):
def validate_dimensions(dimensions):
for k, v in dimensions.iteritems():
if not isinstance(k, (str, unicode)):
for k, v in six.iteritems(dimensions):
if not isinstance(k, (str, six.text_type)):
msg = "invalid dimension key type: " \
"{0} in {1} is not a string type".format(k, dimensions)
raise InvalidDimensionKey(msg)
@ -114,7 +124,7 @@ def validate_dimensions(dimensions):
format(k, dimensions)
raise InvalidDimensionKey(msg)
if not isinstance(v, (str, unicode)):
if not isinstance(v, (str, six.text_type)):
msg = "invalid dimension value type: {0} for key {1} must be a " \
"string: {2}".format(v, k, dimensions)
raise InvalidDimensionValue(msg)
@ -129,7 +139,7 @@ def validate_dimensions(dimensions):
def validate_name(name):
if not isinstance(name, (str, unicode)):
if not isinstance(name, (str, six.text_type)):
msg = "invalid metric name type: {0} is not a string type ".format(
name)
raise InvalidMetricName(msg)
@ -142,14 +152,14 @@ def validate_name(name):
def validate_value(value):
if not isinstance(value, (int, long, float)):
if not isinstance(value, NUMERIC_VALUES):
msg = "invalid value type: {0} is not a number type for metric".\
format(value)
raise InvalidValue(msg)
def validate_timestamp(timestamp):
if not isinstance(timestamp, (int, float)):
if not isinstance(timestamp, NUMERIC_VALUES):
msg = "invalid timestamp type: {0} is not a number type for " \
"metric".format(timestamp)
raise InvalidTimeStamp(msg)

4
setup.cfg
View File

@ -11,6 +11,10 @@ classifier =
Programming Language :: Python
Programming Language :: Python :: 2.7
[global]
setup-hooks =
pbr.hooks.setup_hook
[files]
packages =
monasca_common

9
test-blacklist-py3.txt Normal file
View File

@ -0,0 +1,9 @@
#################################################
# note(trebskit) Following tests fails under PY3
# reason for exclusion is written above test name
#################################################
# TypeError: catching classes that do not inherit from BaseException is not allowed
# not possible under PY3
monasca_common.tests.test_kafka.TestKafkaConsumer.test_kafka_consumer_process_messages

3
test-requirements.txt
View File

@ -2,6 +2,7 @@
# of appearance. Changing the order has an impact on the overall integration
# process, which may cause wedges in the gate later.
# Hacking already pins down pep8, pyflakes and flake8
bandit>=1.1.0 # Apache-2.0
hacking<0.12,>=0.11.0 # Apache-2.0
Babel>=2.3.4 # BSD
coverage>=3.6 # Apache-2.0
@ -9,8 +10,8 @@ fixtures>=3.0.0 # Apache-2.0/BSD
httplib2>=0.7.5 # MIT
mock>=2.0 # BSD
mox>=0.5.3 # Apache-2.0
nose # LGPL
oslotest>=1.10.0 # Apache-2.0
os-testr>=0.7.0 # Apache-2.0
python-subunit>=0.0.18 # Apache-2.0/BSD
testrepository>=0.0.18 # Apache-2.0/BSD
testscenarios>=0.4 # Apache-2.0/BSD

69
tox.ini
View File

@ -1,22 +1,67 @@
[tox]
minversion = 1.6
envlist = py{27,35},pep8,cover
minversion = 2.5
skipsdist = True
envlist = py27,pep8
[testenv]
setenv = VIRTUAL_ENV={envdir}
usedevelop = True
install_command =
{toxinidir}/tools/tox_install.sh {env:UPPER_CONSTRAINTS_FILE:https://git.openstack.org/cgit/openstack/requirements/plain/upper-constraints.txt} {opts} {packages}
setenv = VIRTUAL_ENV={envdir}
OS_TEST_PATH=monasca_common/tests
CLIENT_NAME=monasca-common
passenv = http_proxy
HTTP_PROXY
https_proxy
HTTPS_PROXY
no_proxy
NO_PROXY
whitelist_externals = bash
find
rm
install_command = {toxinidir}/tools/tox_install.sh {env:UPPER_CONSTRAINTS_FILE:https://git.openstack.org/cgit/openstack/requirements/plain/upper-constraints.txt} {opts} {packages}
deps = -r{toxinidir}/requirements.txt
-r{toxinidir}/test-requirements.txt
whitelist_externals = find
commands =
find . -type f -name "*.pyc" -delete
nosetests --with-coverage --cover-package=monasca_common/. --cover-erase
find {toxinidir} -type f -name '*.pyc' -delete
[testenv:py27]
basepython = python2.7
commands =
{[testenv]commands}
ostestr {posargs}
[testenv:py35]
basepython = python3.5
setenv =
{[testenv]setenv}
BLACKLIST_FILE={toxinidir}/test-blacklist-py3.txt
commands =
{[testenv]commands}
ostestr --blacklist-file {env:BLACKLIST_FILE} {posargs}
[testenv:cover]
basepython = python2.7
commands =
{[testenv]commands}
coverage erase
python setup.py test --coverage --testr-args='{posargs}' --coverage-package-name=monasca_common
coverage report
[testenv:debug]
commands =
{[testenv]commands}
oslo_debug_helper -t {env:OS_TEST_PATH} {posargs}
[testenv:bandit]
# B101(assert_ussed) - Validation uses asserts because of performance reasons
commands = bandit -r monasca_common -n5 -s B101 -x monasca_common/tests
[testenv:flake8]
commands = flake8 monasca_common
[testenv:pep8]
commands = flake8 monasca_common
commands =
{[testenv:flake8]commands}
{[testenv:bandit]commands}
[testenv:venv]
commands = {posargs}
@ -25,5 +70,9 @@ commands = {posargs}
max-complexity = 50
max-line-length = 120
builtins = _
exclude=.venv,.git,.tox,dist,*openstack/common*,*egg,build
exclude=.venv,.git,.tox,dist,*egg,build
show-source = True
# note: Due to the need to fork kafka-python, many pep8 violations occure.
# All of the below ignores are caused by the forked kafka-python library
# so when monasca migrates to pykafka, the below line can be removed.
ignore = E121,E126,E127,E128,E131,E221,E226,E241,E251,E261,E302,E303,E501,E701,F401,H101,H102,H301,H304,H306,H404,H405