From 24ddd12e8d0b2f933c0f6bfdfe47453fa2eaebfa Mon Sep 17 00:00:00 2001 From: Jose Idar Date: Mon, 25 Aug 2014 22:23:25 -0500 Subject: [PATCH] Moved model-based dataset generator tools to common * Adds datasets.py in cloudcafe/common * Adds methods for helping create dataset generators from model lists. * Adds compute dataset generators for images, flavors, and images_by_flavor Change-Id: I6480f0c61b04183514e49081fe96abbfbc57f82c --- cloudcafe/common/datasets.py | 106 ++++++++++++++++++++++++++++ cloudcafe/compute/datasets.py | 126 ++++++++++++++++++++++++++++++++++ 2 files changed, 232 insertions(+) create mode 100644 cloudcafe/common/datasets.py create mode 100644 cloudcafe/compute/datasets.py diff --git a/cloudcafe/common/datasets.py b/cloudcafe/common/datasets.py new file mode 100644 index 00000000..227ca43e --- /dev/null +++ b/cloudcafe/common/datasets.py @@ -0,0 +1,106 @@ +""" +Copyright 2014 Rackspace + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" +from random import shuffle + +from cafe.drivers.unittest.datasets import DatasetList + + +class DatasetGeneratorError(Exception): + pass + + +class ModelBasedDatasetToolkit(object): + """Collection of dataset generators and helper methods for + developing data driven tests + """ + INCLUSION_MODE = 'inclusion' + EXCLUSION_MODE = 'exclusion' + + @classmethod + def _get_model_list( + cls, get_model_list_method, model_type_name, + *get_method_args, **get_method_kwargs): + """Gets list of all models in the environment.""" + + resp = get_model_list_method(*get_method_args, **get_method_kwargs) + if not resp.ok: + raise DatasetGeneratorError( + "Request for list of {0} during data-driven-test setup failed " + "with an HTTP {1} ERROR".format( + model_type_name, resp.status_code)) + + if resp.entity is None: + raise DatasetGeneratorError( + "Unable to deserialize list of {0} during data-driven-test " + "setup. API responded with an HTTP {1}".format( + model_type_name, resp.status_code)) + + return resp.entity + + @classmethod + def _filter_model_list( + cls, model_list, model_filter=None, filter_mode=None): + """Filters should be dictionaries with model attributes as keys and + lists of attributes as key values. + example: {"id": ["12345", "42"]} + Include only those models who match at least one criteria in the + model_filter dictionary. + filter_mode can be 'inclusion' or 'exclusion'. + inclusion mode will include models that match any attributes in + the model_filter in the final model_list. + exclusion mode will exclude any models that match attributes in + the model-filer from the final model_list. + """ + + if not model_filter: + return model_list + + if filter_mode not in [cls.INCLUSION_MODE, cls.EXCLUSION_MODE]: + raise Exception( + "Invalid filter_mode {0}. _filter_model_list must be called " + "with a mode set to either 'inclusion' or 'exclusion'.".format( + filter_mode)) + + final_list = [] + for model in model_list: + excluded = False + for k in model_filter: + if filter_mode == cls.INCLUSION_MODE: + if str(getattr(model, k)) in model_filter[k]: + final_list.append(model) + break + elif filter_mode == cls.EXCLUSION_MODE: + model_value = str(getattr(model, k)) + filter_value = model_filter[k] + if not excluded and model_value not in filter_value: + final_list.append(model) + break + else: + excluded = True + return final_list + + @classmethod + def _modify_dataset_list( + cls, dataset_list, max_datasets=None, randomize=False): + """Aggregates common modifiers for dataset lists""" + + if randomize: + shuffle(dataset_list) + + if max_datasets: + dataset_list = DatasetList(dataset_list[:max_datasets]) + + return dataset_list diff --git a/cloudcafe/compute/datasets.py b/cloudcafe/compute/datasets.py new file mode 100644 index 00000000..ced51d6e --- /dev/null +++ b/cloudcafe/compute/datasets.py @@ -0,0 +1,126 @@ +""" +Copyright 2014 Rackspace + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" + +from cafe.drivers.unittest.datasets import DatasetList +from cafe.drivers.unittest.decorators import memoized + +from cloudcafe.common.datasets import ModelBasedDatasetToolkit +from cloudcafe.compute.composites import \ + _ComputeAuthComposite, ImagesComposite, FlavorsComposite + + +class ComputeDatasets(ModelBasedDatasetToolkit): + """Collection of dataset generators for compute and compute-integration + data driven tests + """ + + _images = ImagesComposite(_ComputeAuthComposite()) + _flavors = FlavorsComposite(_ComputeAuthComposite()) + + @classmethod + @memoized + def _get_images(cls): + """Gets list of all Images in the environment, and caches it for + future calls""" + return cls._get_model_list( + cls._images.client.list_images_with_detail, 'images') + + @classmethod + @memoized + def _get_flavors(cls): + """Gets list of all Flavors in the environment, and caches it for + future calls""" + return cls._get_model_list( + cls._flavors.client.list_flavors_with_detail, 'flavors') + + @classmethod + def images( + cls, max_datasets=None, randomize=False, model_filter=None, + filter_mode=ModelBasedDatasetToolkit.INCLUSION_MODE): + """Returns a DatasetList of all Images. + Filters should be dictionaries with model attributes as keys and + lists of attributes as key values + """ + image_list = cls._get_images() + image_list = cls._filter_model_list( + image_list, model_filter=model_filter, filter_mode=filter_mode) + + dataset_list = DatasetList() + for img in image_list: + data = {'image': img} + dataset_list.append_new_dataset( + str(img.name).replace(" ", "_"), data) + + # Apply modifiers + return cls._modify_dataset_list( + dataset_list, max_datasets=max_datasets, randomize=randomize) + + @classmethod + def flavors( + cls, max_datasets=None, randomize=False, model_filter=None, + filter_mode=ModelBasedDatasetToolkit.INCLUSION_MODE): + """Returns a DatasetList of all Flavors + Filters should be dictionaries with model attributes as keys and + lists of attributes as key values + """ + flavor_list = cls._get_flavors() + flavor_list = cls._filter_model_list( + flavor_list, model_filter=model_filter, filter_mode=filter_mode) + + dataset_list = DatasetList() + for flavor in flavor_list: + data = {'flavor': flavor} + dataset_list.append_new_dataset( + str(flavor.name).replace(" ", "_"), data) + + # Apply modifiers + return cls._modify_dataset_list( + dataset_list, max_datasets=max_datasets, randomize=randomize) + + @classmethod + def images_by_flavor( + cls, max_datasets=None, randomize=False, + image_filter=None, flavor_filter=None, + image_filter_mode=ModelBasedDatasetToolkit.INCLUSION_MODE, + flavor_filter_mode=ModelBasedDatasetToolkit.INCLUSION_MODE): + """Returns a DatasetList of all combinations of Flavors and Images. + Filters should be dictionaries with model attributes as keys and + lists of attributes as key values + """ + image_list = cls._get_images() + image_list = cls._filter_model_list( + image_list, model_filter=image_filter, + filter_mode=image_filter_mode) + + flavor_list = cls._get_flavors() + flavor_list = cls._filter_model_list( + flavor_list, model_filter=flavor_filter, + filter_mode=flavor_filter_mode) + + dataset_list = DatasetList() + for image in image_list: + for flavor in flavor_list: + data = {'flavor': flavor, + 'image': image} + testname = \ + "image_{0}_and_flavor_{1}".format( + str(image.name).replace(" ", "_"), + str(flavor.name).replace(" ", "_")) + dataset_list.append_new_dataset(testname, data) + + # Apply modifiers + return cls._modify_dataset_list( + dataset_list, max_datasets=max_datasets, randomize=randomize)