From 6ffab589a4e783cdf86bf31f0f56098d7dbff95b Mon Sep 17 00:00:00 2001 From: wangzihao Date: Mon, 23 Nov 2020 11:19:54 +0800 Subject: [PATCH] Remove six Remove six Replace the following items with Python 3 style code. - six.PY3 - six.xrange - six.string_types - six.binary_type - six.iteritems - six.unichr - six.wraps - six.get_function_code - six.PY2 - six.b - six.moves.range - reraise - six.text_type Story: 2008305 Task: 41301 Change-Id: Idf64154a013b53c7db771a25f1c63c1295e354e6 --- lower-constraints.txt | 1 - monasca_common/kafka/producer.py | 3 +- monasca_common/kafka_lib/codec.py | 4 +- monasca_common/kafka_lib/conn.py | 4 +- monasca_common/kafka_lib/consumer/kafka.py | 14 ++-- monasca_common/kafka_lib/consumer/simple.py | 4 +- .../kafka_lib/partitioner/hashed.py | 4 +- monasca_common/kafka_lib/producer/base.py | 8 +- monasca_common/kafka_lib/producer/simple.py | 7 +- monasca_common/kafka_lib/protocol.py | 6 +- monasca_common/kafka_lib/util.py | 10 +-- .../monasca_query_language/aql_parser.py | 4 +- monasca_common/rest/utils.py | 4 +- monasca_common/tests/test_simport.py | 84 ++++++------------- .../validation/test_metric_validation.py | 13 ++- monasca_common/validation/metrics.py | 20 ++--- requirements.txt | 1 - 17 files changed, 62 insertions(+), 129 deletions(-) diff --git a/lower-constraints.txt b/lower-constraints.txt index fe67a73d..499fcbbc 100644 --- a/lower-constraints.txt +++ b/lower-constraints.txt @@ -43,7 +43,6 @@ requests-mock==1.2.0 requestsexceptions==1.2.0 rfc3986==0.3.1 simplejson==3.13.2 -six==1.10.0 smmap==0.9.0 stestr==1.0.0 stevedore==1.20.0 diff --git a/monasca_common/kafka/producer.py b/monasca_common/kafka/producer.py index 89e644e4..f9c70b05 100644 --- a/monasca_common/kafka/producer.py +++ b/monasca_common/kafka/producer.py @@ -17,7 +17,6 @@ import logging import time from oslo_utils import encodeutils -from six import PY3 import monasca_common.kafka_lib.client as kafka_client import monasca_common.kafka_lib.producer as kafka_producer @@ -57,7 +56,7 @@ class KafkaProducer(object): messages = [encodeutils.to_utf8(m) for m in messages] - key = bytes(str(key), 'utf-8') if PY3 else str(key) + key = bytes(str(key), 'utf-8') while not success: try: diff --git a/monasca_common/kafka_lib/codec.py b/monasca_common/kafka_lib/codec.py index 211e1562..f337a4c5 100644 --- a/monasca_common/kafka_lib/codec.py +++ b/monasca_common/kafka_lib/codec.py @@ -14,8 +14,6 @@ import gzip from io import BytesIO import struct -from six.moves import xrange - _XERIAL_V1_HEADER = (-126, b'S', b'N', b'A', b'P', b'P', b'Y', 0, 1, 1) _XERIAL_V1_FORMAT = 'bccccccBii' @@ -92,7 +90,7 @@ def snappy_encode(payload, xerial_compatible=False, xerial_blocksize=32 * 1024): if xerial_compatible: def _chunker(): - for i in xrange(0, len(payload), xerial_blocksize): + for i in range(0, len(payload), xerial_blocksize): yield payload[i:i + xerial_blocksize] out = BytesIO() diff --git a/monasca_common/kafka_lib/conn.py b/monasca_common/kafka_lib/conn.py index 0f39db2e..ecc3488e 100644 --- a/monasca_common/kafka_lib/conn.py +++ b/monasca_common/kafka_lib/conn.py @@ -17,8 +17,6 @@ import socket import struct from threading import local -import six - from monasca_common.kafka_lib.common import ConnectionError @@ -34,7 +32,7 @@ def collect_hosts(hosts, randomize=True): randomize the returned list. """ - if isinstance(hosts, six.string_types): + if isinstance(hosts, str): hosts = hosts.strip().split(',') result = [] diff --git a/monasca_common/kafka_lib/consumer/kafka.py b/monasca_common/kafka_lib/consumer/kafka.py index ed8fbca1..a3299f41 100644 --- a/monasca_common/kafka_lib/consumer/kafka.py +++ b/monasca_common/kafka_lib/consumer/kafka.py @@ -17,8 +17,6 @@ import random import sys import time -import six - from monasca_common.kafka_lib.client import KafkaClient from monasca_common.kafka_lib.common import ( OffsetFetchRequest, OffsetCommitRequest, OffsetRequest, FetchRequest, @@ -202,7 +200,7 @@ class KafkaConsumer(object): for arg in topics: # Topic name str -- all partitions - if isinstance(arg, (six.string_types, six.binary_type)): + if isinstance(arg, (str, bytes)): topic = kafka_bytestring(arg) for partition in self._client.get_partition_ids_for_topic(topic): @@ -219,10 +217,10 @@ class KafkaConsumer(object): # { topic: partitions, ... } dict elif isinstance(arg, dict): - for key, value in six.iteritems(arg): + for key, value in arg.items(): # key can be string (a topic) - if isinstance(key, (six.string_types, six.binary_type)): + if isinstance(key, (str, bytes)): topic = kafka_bytestring(key) # topic: partition @@ -304,7 +302,7 @@ class KafkaConsumer(object): while True: try: - return six.next(self._get_message_iterator()) + return next(self._get_message_iterator()) # Handle batch completion except StopIteration: @@ -540,7 +538,7 @@ class KafkaConsumer(object): offsets = self._offsets.task_done commits = [] - for topic_partition, task_done_offset in six.iteritems(offsets): + for topic_partition, task_done_offset in offsets.items(): # Skip if None if task_done_offset is None: @@ -776,7 +774,7 @@ class KafkaConsumer(object): # def _deprecate_configs(self, **configs): - for old, new in six.iteritems(DEPRECATED_CONFIG_KEYS): + for old, new in DEPRECATED_CONFIG_KEYS.items(): if old in configs: logger.warning('Deprecated Kafka Consumer configuration: %s. ' 'Please use %s instead.', old, new) diff --git a/monasca_common/kafka_lib/consumer/simple.py b/monasca_common/kafka_lib/consumer/simple.py index 51ecbb61..69c61c5e 100644 --- a/monasca_common/kafka_lib/consumer/simple.py +++ b/monasca_common/kafka_lib/consumer/simple.py @@ -22,8 +22,6 @@ except ImportError: import sys import time -import six - from .base import ( Consumer, FETCH_DEFAULT_BLOCK_TIMEOUT, @@ -379,7 +377,7 @@ class SimpleConsumer(Consumer): partitions = dict((p, self.buffer_size) for p in self.fetch_offsets.keys()) while partitions: requests = [] - for partition, buffer_size in six.iteritems(partitions): + for partition, buffer_size in partitions.items(): requests.append(FetchRequest(self.topic, partition, self.fetch_offsets[partition], buffer_size)) diff --git a/monasca_common/kafka_lib/partitioner/hashed.py b/monasca_common/kafka_lib/partitioner/hashed.py index 2c04ad86..1328ce3a 100644 --- a/monasca_common/kafka_lib/partitioner/hashed.py +++ b/monasca_common/kafka_lib/partitioner/hashed.py @@ -10,8 +10,6 @@ # License for the specific language governing permissions and limitations # under the License. -import six - from .base import Partitioner @@ -63,7 +61,7 @@ def murmur2(key): """ # Convert key to bytes or bytearray - if isinstance(key, bytearray) or (six.PY3 and isinstance(key, bytes)): + if isinstance(key, bytes): data = key else: data = bytearray(str(key).encode()) diff --git a/monasca_common/kafka_lib/producer/base.py b/monasca_common/kafka_lib/producer/base.py index 29acebef..ee5ce483 100644 --- a/monasca_common/kafka_lib/producer/base.py +++ b/monasca_common/kafka_lib/producer/base.py @@ -22,8 +22,6 @@ from collections import defaultdict from threading import Thread, Event -import six - from monasca_common.kafka_lib.common import ( ProduceRequest, ProduceResponse, TopicAndPartition, RetryOptions, kafka_errors, UnsupportedCodecError, FailedPayloadsError, @@ -388,15 +386,15 @@ class Producer(object): if key is None: raise TypeError("key and payload can't be null in one") # Raise TypeError if any non-null message is not encoded as bytes - elif not isinstance(m, six.binary_type): + elif not isinstance(m, bytes): raise TypeError("all produce message payloads must be null or type bytes") # Raise TypeError if topic is not encoded as bytes - if not isinstance(topic, six.binary_type): + if not isinstance(topic, bytes): raise TypeError("the topic must be type bytes") # Raise TypeError if the key is not encoded as bytes - if key is not None and not isinstance(key, six.binary_type): + if key is not None and not isinstance(key, bytes): raise TypeError("the key must be type bytes") if self.is_async: diff --git a/monasca_common/kafka_lib/producer/simple.py b/monasca_common/kafka_lib/producer/simple.py index 66e42826..9cec61aa 100644 --- a/monasca_common/kafka_lib/producer/simple.py +++ b/monasca_common/kafka_lib/producer/simple.py @@ -13,9 +13,6 @@ from itertools import cycle import logging import random -import six - -from six.moves import xrange from .base import Producer @@ -50,13 +47,13 @@ class SimpleProducer(Producer): # Randomize the initial partition that is returned if self.random_start: num_partitions = len(self.client.get_partition_ids_for_topic(topic)) - for _ in xrange(random.randint(0, num_partitions - 1)): + for _ in range(random.randint(0, num_partitions - 1)): next(self.partition_cycles[topic]) return next(self.partition_cycles[topic]) def send_messages(self, topic, *msg): - if not isinstance(topic, six.binary_type): + if not isinstance(topic, bytes): topic = topic.encode('utf-8') partition = self._next_partition(topic) diff --git a/monasca_common/kafka_lib/protocol.py b/monasca_common/kafka_lib/protocol.py index 49ca1c4f..8f764161 100644 --- a/monasca_common/kafka_lib/protocol.py +++ b/monasca_common/kafka_lib/protocol.py @@ -13,8 +13,6 @@ import logging import struct -from six.moves import xrange - from monasca_common.kafka_lib.codec import ( gzip_encode, gzip_decode, snappy_encode, snappy_decode ) @@ -516,11 +514,11 @@ class KafkaProtocol(object): ((correlation_id,), cur) = relative_unpack('>i', data, 0) ((num_topics,), cur) = relative_unpack('>i', data, cur) - for _ in xrange(num_topics): + for _ in range(num_topics): (topic, cur) = read_short_string(data, cur) ((num_partitions,), cur) = relative_unpack('>i', data, cur) - for _ in xrange(num_partitions): + for _ in range(num_partitions): ((partition, error), cur) = relative_unpack('>ih', data, cur) yield OffsetCommitResponse(topic, partition, error) diff --git a/monasca_common/kafka_lib/util.py b/monasca_common/kafka_lib/util.py index 0b324b1e..46ee5ec9 100644 --- a/monasca_common/kafka_lib/util.py +++ b/monasca_common/kafka_lib/util.py @@ -16,8 +16,6 @@ import struct import sys from threading import Thread, Event -import six - from monasca_common.kafka_lib.common import BufferUnderflowError @@ -26,7 +24,7 @@ def crc32(data): def write_int_string(s): - if s is not None and not isinstance(s, six.binary_type): + if s is not None and not isinstance(s, bytes): raise TypeError('Expected "%s" to be bytes\n' 'data=%s' % (type(s), repr(s))) if s is None: @@ -36,7 +34,7 @@ def write_int_string(s): def write_short_string(s): - if s is not None and not isinstance(s, six.binary_type): + if s is not None and not isinstance(s, bytes): raise TypeError('Expected "%s" to be bytes\n' 'data=%s' % (type(s), repr(s))) if s is None: @@ -105,9 +103,9 @@ def kafka_bytestring(s): Takes a string or bytes instance Returns bytes, encoding strings in utf-8 as necessary """ - if isinstance(s, six.binary_type): + if isinstance(s, bytes): return s - if isinstance(s, six.string_types): + if isinstance(s, bytes): return s.encode('utf-8') raise TypeError(s) diff --git a/monasca_common/monasca_query_language/aql_parser.py b/monasca_common/monasca_query_language/aql_parser.py index 19e0ed70..9be4013c 100644 --- a/monasca_common/monasca_query_language/aql_parser.py +++ b/monasca_common/monasca_query_language/aql_parser.py @@ -13,8 +13,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -import six - import pyparsing from monasca_common.monasca_query_language import query_structures @@ -36,7 +34,7 @@ decimal_number.setParseAction(lambda tokens: float("".join(tokens))) # Initialize non-ascii unicode code points in the Basic Multilingual Plane. unicode_printables = u''.join( - six.unichr(c) for c in range(128, 65536) if not six.unichr(c).isspace()) + chr(c) for c in range(128, 65536) if not chr(c).isspace()) # Does not like comma. No Literals from above allowed. valid_identifier_chars = ( diff --git a/monasca_common/rest/utils.py b/monasca_common/rest/utils.py index 4ca600cc..a122b701 100644 --- a/monasca_common/rest/utils.py +++ b/monasca_common/rest/utils.py @@ -12,8 +12,8 @@ # License for the specific language governing permissions and limitations # under the License. +import functools import simplejson as json -import six from monasca_common.rest import exceptions @@ -25,7 +25,7 @@ JSON_CONTENT_TYPE = 'application/json' def _try_catch(fun): - @six.wraps(fun) + @functools.wraps(fun) def wrapper(*args, **kwargs): try: return fun(*args, **kwargs) diff --git a/monasca_common/tests/test_simport.py b/monasca_common/tests/test_simport.py index 6c664458..6daab867 100755 --- a/monasca_common/tests/test_simport.py +++ b/monasca_common/tests/test_simport.py @@ -17,7 +17,6 @@ import os import sys from oslotest import base -import six import monasca_common.simport.simport as simport @@ -73,10 +72,10 @@ class TestSimport(base.BaseTestCase): "TestSimport:missing") def test_good_load_internal(self): - self.assertEqual(six.get_function_code(dummy_function), - six.get_function_code(simport.load("test_simport:dummy_function"))) - self.assertEqual(six.get_function_code(DummyClass.method_a), - six.get_function_code(simport.load("test_simport:DummyClass.method_a"))) + self.assertEqual(dummy_function.__code__, + simport.load("test_simport:dummy_function").__code__) + self.assertEqual(DummyClass.method_a.__code__, + simport.load("test_simport:DummyClass.method_a").__code__) def test_local_class(self): klass = simport.load("LocalClass", __name__) @@ -95,66 +94,31 @@ class TestSimport(base.BaseTestCase): # -if six.PY2: +class TestSimportPY3(base.BaseTestCase): - class TestSimportPY2(base.BaseTestCase): + def test_good_load_local(self): + method = simport.load(PWD + "|localmodule:Foo.method_a") + import localmodule - def test_good_load_local(self): - method = simport.load(PWD + "|monasca_common.tests.localmodule:Foo.method_a") - import localmodule + self.assertEqual(method, localmodule.Foo.method_a) + self.assertEqual(localmodule.function_a, + simport.load("localmodule:function_a")) - self.assertEqual(method, localmodule.Foo.method_a) - self.assertEqual(localmodule.function_a, - simport.load("monasca_common.tests.localmodule:function_a")) + def test_good_load_external(self): - def test_good_load_external(self): + method = simport.load(PWD + "/external|external.externalmodule:Blah.method_b") - method = simport.load( - PWD + "/external|monasca_common.tests.external.externalmodule:Blah.method_b") + self.assertTrue('external.externalmodule' in sys.modules) + old = sys.modules['external.externalmodule'] + import external.externalmodule - self.assertTrue('monasca_common.tests.external.externalmodule' in sys.modules) - old = sys.modules['monasca_common.tests.external.externalmodule'] - import external.externalmodule + self.assertEqual(external.externalmodule, + sys.modules['external.externalmodule']) + self.assertEqual(old, external.externalmodule) + self.assertEqual(method, external.externalmodule.Blah.method_b) - self.assertEqual(external.externalmodule, - sys.modules['monasca_common.tests.external.externalmodule']) - self.assertEqual(old, external.externalmodule) - self.assertEqual(method, external.externalmodule.Blah.method_b) + def test_import_class(self): + klass = simport.load(PWD + "/external|external.externalmodule:Blah") + import external.externalmodule - def test_import_class(self): - klass = simport.load( - PWD + "/external|monasca_common.tests.external.externalmodule:Blah") - import external.externalmodule - - self.assertEqual(klass, external.externalmodule.Blah) - -elif six.PY3: - - class TestSimportPY3(base.BaseTestCase): - - def test_good_load_local(self): - method = simport.load(PWD + "|localmodule:Foo.method_a") - import localmodule - - self.assertEqual(method, localmodule.Foo.method_a) - self.assertEqual(localmodule.function_a, - simport.load("localmodule:function_a")) - - def test_good_load_external(self): - - method = simport.load(PWD + "/external|external.externalmodule:Blah.method_b") - - self.assertTrue('external.externalmodule' in sys.modules) - old = sys.modules['external.externalmodule'] - import external.externalmodule - - self.assertEqual(external.externalmodule, - sys.modules['external.externalmodule']) - self.assertEqual(old, external.externalmodule) - self.assertEqual(method, external.externalmodule.Blah.method_b) - - def test_import_class(self): - klass = simport.load(PWD + "/external|external.externalmodule:Blah") - import external.externalmodule - - self.assertEqual(klass, external.externalmodule.Blah) + self.assertEqual(klass, external.externalmodule.Blah) diff --git a/monasca_common/tests/validation/test_metric_validation.py b/monasca_common/tests/validation/test_metric_validation.py index 13338d2f..98e44964 100644 --- a/monasca_common/tests/validation/test_metric_validation.py +++ b/monasca_common/tests/validation/test_metric_validation.py @@ -16,7 +16,6 @@ import codecs from oslotest import base -import six from monasca_common.validation import metrics as metric_validator @@ -30,7 +29,7 @@ invalid_dimension_chars = "<>={},\"\\\\;&" def _hex_to_unicode(hex_raw): - hex_raw = six.b(hex_raw.replace(' ', '')) + hex_raw = hex_raw.replace(' ', '').encode("latin-1") hex_str_raw = codecs.getdecoder('hex')(hex_raw)[0] hex_str = hex_str_raw.decode('utf-8', 'replace') return hex_str @@ -350,7 +349,7 @@ class TestMetricValidation(base.BaseTestCase): def test_invalid_too_many_value_meta(self): value_meta = {} - for i in six.moves.range(0, metric_validator.VALUE_META_MAX_NUMBER + 3): + for i in range(0, metric_validator.VALUE_META_MAX_NUMBER + 3): value_meta['key{}'.format(i)] = 'value{}'.format(i) metric = {"name": "test_metric_name", "dimensions": {"key1": "value1", @@ -377,7 +376,7 @@ class TestMetricValidation(base.BaseTestCase): def test_invalid_too_long_value_meta_key(self): key = "K" - for i in six.moves.range(0, metric_validator.VALUE_META_NAME_MAX_LENGTH): + for i in range(0, metric_validator.VALUE_META_NAME_MAX_LENGTH): key = "{}{}".format(key, "1") value_meta = {key: 'BBB'} metric = {"name": "test_metric_name", @@ -394,11 +393,11 @@ class TestMetricValidation(base.BaseTestCase): def test_invalid_too_large_value_meta(self): value_meta_value = "" num_value_meta = 10 - for i in six.moves.range( + for i in range( 0, int(metric_validator.VALUE_META_VALUE_MAX_LENGTH / num_value_meta)): value_meta_value = '{}{}'.format(value_meta_value, '1') value_meta = {} - for i in six.moves.range(0, num_value_meta): + for i in range(0, num_value_meta): value_meta['key{}'.format(i)] = value_meta_value metric = {"name": "test_metric_name", "dimensions": {"key1": "value1", @@ -438,7 +437,7 @@ class TestMetricValidation(base.BaseTestCase): "timestamp": 1405630174123, "value": 2.0} ] - for i in six.moves.range(len(metrics)): + for i in range(len(metrics)): metric_validator.validate_name(metrics[i]['name']) metric_validator.validate_value(metrics[i]['value']) metric_validator.validate_timestamp(metrics[i]['timestamp']) diff --git a/monasca_common/validation/metrics.py b/monasca_common/validation/metrics.py index f7aaf7ec..68f044c4 100644 --- a/monasca_common/validation/metrics.py +++ b/monasca_common/validation/metrics.py @@ -17,7 +17,6 @@ import math import re import simplejson as json -import six import sys # This is used to ensure that metrics with a timestamp older than @@ -36,10 +35,6 @@ RESTRICTED_DIMENSION_CHARS = re.compile('[' + INVALID_CHARS + ']') RESTRICTED_NAME_CHARS = re.compile('[' + INVALID_CHARS + '() ' + ']') NUMERIC_VALUES = [int, float] -if six.PY2: - # according to PEP537 long was renamed to int in PY3 - # need to add long, as possible value, for PY2 - NUMERIC_VALUES += [long] # noqa NUMERIC_VALUES = tuple(NUMERIC_VALUES) # convert to tuple for instance call @@ -86,9 +81,8 @@ def validate_metric(metric): if "value_meta" in metric: validate_value_meta(metric['value_meta']) except Exception as ex: - six.reraise(type(ex), - type(ex)(str(ex) + ": for metric %s" % metric['name']), - sys.exc_info()[2]) + raise type(ex)(str(ex) + ": for metric %s" % metric['name']).\ + with_traceback(sys.exc_info()[2]) def validate_value_meta(value_meta): @@ -98,7 +92,7 @@ def validate_value_meta(value_meta): msg = "Too many valueMeta entries {0}, limit is {1}: valueMeta {2}".\ format(len(value_meta), VALUE_META_MAX_NUMBER, value_meta) raise InvalidValueMeta(msg) - for key, value in six.iteritems(value_meta): + for key, value in value_meta.items(): if not key: raise InvalidValueMeta("valueMeta name cannot be empty: key={}, " "value={}".format(key, value)) @@ -119,7 +113,7 @@ def validate_value_meta(value_meta): def validate_dimension_key(k): - if not isinstance(k, (str, six.text_type)): + if not isinstance(k, str): msg = "invalid dimension key type: " \ "{0} is not a string type".format(k) raise InvalidDimensionKey(msg) @@ -134,7 +128,7 @@ def validate_dimension_key(k): def validate_dimension_value(k, v): - if not isinstance(v, (str, six.text_type)): + if not isinstance(v, str): msg = "invalid dimension value type: {0} must be a " \ "string (from key {1})".format(v, k) raise InvalidDimensionValue(msg) @@ -148,13 +142,13 @@ def validate_dimension_value(k, v): def validate_dimensions(dimensions): - for k, v in six.iteritems(dimensions): + for k, v in dimensions.items(): validate_dimension_key(k) validate_dimension_value(k, v) def validate_name(name): - if not isinstance(name, (str, six.text_type)): + if not isinstance(name, str): msg = "invalid metric name type: {0} is not a string type ".format( name) raise InvalidMetricName(msg) diff --git a/requirements.txt b/requirements.txt index a6a9b537..0d95e568 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,7 +1,6 @@ # The order of packages is significant, because pip processes them in the order # of appearance. Changing the order has an impact on the overall integration # process, which may cause wedges in the gate later. -six>=1.10.0 # MIT kazoo>=2.8.0 # Apache-2.0 PyMySQL>=0.8.0 # MIT License oslo.config>=5.2.0 # Apache-2.0