x/distil
Code Issues Proposed changes

Add /quotations rest api

Browse Source 
Allow user get current month estimated cost. The output data
structure is the same with /invoices api.

Also remove the unused /costs api and rater module.

Change-Id: I582afa6cf7e5b86cc54db58ead501793e727bfd6
This commit is contained in:
Lingxian Kong 2017-04-28 15:11:35 +12:00
parent a0f8ebf3da
commit 2ff5b1518c
12 changed files with 377 additions and 359 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

27
distil/api/v2.py
View File

@ -26,6 +26,7 @@ from distil.service.api.v2 import costs
from distil.service.api.v2 import health
from distil.service.api.v2 import invoices
from distil.service.api.v2 import products
from distil.service.api.v2 import quotations
LOG = log.getLogger(__name__)
@ -84,20 +85,6 @@ def products_get():
return api.render(products=products.get_products(regions))
@rest.get('/costs')
@acl.enforce("rating:costs:get")
def costs_get():
params = _get_request_args()
# NOTE(flwang): Here using 'usage' instead of 'costs' for backward
# compatibility.
return api.render(
usage=costs.get_costs(
params['project_id'], params['start'], params['end']
)
)
@rest.get('/measurements')
@acl.enforce("rating:measurements:get")
def measurements_get():
@ -123,3 +110,15 @@ def invoices_get():
detailed=params['detailed']
)
)
@rest.get('/quotations')
@acl.enforce("rating:quotations:get")
def quotations_get():
params = _get_request_args()
return api.render(
quotations.get_quotations(
params['project_id'], detailed=params['detailed']
)
)

16
distil/erp/driver.py
View File

@ -62,3 +62,19 @@ class BaseDriver(object):
:return: The history invoices information for each month.
"""
raise NotImplementedError()
def get_quotations(self, region, project_id, measurements=[], resources=[],
detailed=False):
"""Get usage cost for current month.
It depends on ERP system to decide how to get current month cost.
:param region: Region name.
:param project_id: Project ID.
:param measurements: Current month usage.
:param resources: List of resources.
:param detailed: If get detailed information or not.
:return: Current month quotation.
"""
raise NotImplementedError()

133
distil/erp/drivers/odoo.py
View File

@ -12,12 +12,16 @@
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import collections
from decimal import Decimal
import json
import odoorpc
from oslo_log import log
from distil.common import cache
from distil.common import general
from distil.common import openstack
from distil.erp import driver
from distil import exceptions
@ -290,3 +294,132 @@ class OdooDriver(driver.BaseDriver):
)
return result
@cache.memoize
def _get_service_mapping(self, products):
"""Gets mapping from service name to service type.
:param products: Product dict in a region returned from odoo.
"""
srv_mapping = {}
for category, p_list in products.items():
for p in p_list:
srv_mapping[p['name']] = category.title()
return srv_mapping
@cache.memoize
def _get_service_price(self, service_name, service_type, products):
"""Get service price information from price definitions."""
price = {'service_name': service_name}
if service_type in products:
for s in products[service_type]:
if s['name'] == service_name:
price.update({'rate': s['price'], 'unit': s['unit']})
break
else:
found = False
for category, services in products.items():
for s in services:
if s['name'] == service_name:
price.update({'rate': s['price'], 'unit': s['unit']})
found = True
break
if not found:
raise exceptions.NotFoundException(
'Price not found, service name: %s, service type: %s' %
(service_name, service_type)
)
return price
def get_quotations(self, region, project_id, measurements=[], resources=[],
detailed=False):
"""Get current month quotation.
Return value is in the following format:
{
'<current_date>': {
'total_cost': 100,
'details': {
'Compute': {
'total_cost': xxx,
'breakdown': {}
}
}
}
}
:param region: Region name.
:param project_id: Project ID.
:param measurements: Current month usage collection.
:param resources: List of resources.
:param detailed: If get detailed information or not.
:return: Current month quotation.
"""
total_cost = 0
price_mapping = {}
cost_details = {}
odoo_region = self.region_mapping.get(region, region).upper()
resources = {row.id: json.loads(row.info) for row in resources}
products = self.get_products([region])[region]
service_mapping = self._get_service_mapping(products)
for entry in measurements:
service_name = entry.get('service')
volume = entry.get('volume')
unit = entry.get('unit')
res_id = entry.get('resource_id')
# resource_type is the type defined in meter_mappings.yml.
resource_type = resources[res_id]['type']
service_type = service_mapping.get(service_name, resource_type)
if service_type not in cost_details:
cost_details[service_type] = {
'total_cost': 0,
'breakdown': collections.defaultdict(list)
}
if service_name not in price_mapping:
price_spec = self._get_service_price(
service_name, service_type, products
)
price_mapping[service_name] = price_spec
price_spec = price_mapping[service_name]
# Convert volume according to unit in price definition.
volume = general.convert_to(volume, unit, price_spec['unit'])
cost = (round(volume * Decimal(price_spec['rate']), 2)
if price_spec['rate'] else 0)
total_cost += cost
if detailed:
odoo_service_name = "%s.%s" % (odoo_region, service_name)
cost_details[service_type]['total_cost'] += cost
cost_details[service_type]['breakdown'][
odoo_service_name
].append(
{
"resource_name": resources[res_id].get('name', ''),
"resource_id": res_id,
"cost": cost,
"quantity": round(volume, 4),
"rate": price_spec['rate'],
"unit": price_spec['unit'],
}
)
result = {'total_cost': round(total_cost, 2)}
if detailed:
result.update({'details': cost_details})
return result

40
distil/rater/__init__.py
View File

@ -1,40 +0,0 @@
# Copyright 2014 Catalyst IT Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from oslo_config import cfg
from stevedore import driver
CONF = cfg.CONF
RATER = None
class BaseRater(object):
def __init__(self, conf=None):
self.conf = conf
def rate(self, name, region=None):
raise NotImplementedError("Not implemented in base class")
def get_rater():
global RATER
if RATER is None:
RATER = driver.DriverManager('distil.rater',
CONF.rater.rater_type,
invoke_on_load=True,
invoke_kwds={}).driver
return RATER

39
distil/rater/odoo.py
View File

@ -1,39 +0,0 @@
# Copyright 2014 Catalyst IT Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from distil import rater
from distil.rater import rate_file
from distil.service.api.v2 import products
class OdooRater(rater.BaseRater):
def __init__(self):
self.prices = products.get_products()
def rate(self, name, region=None):
if not self.prices:
return rate_file.FileRater().rate(name, region)
region_prices = (self.prices[region] if region else
self.prices.values[0])
for category in region_prices:
for product in region_prices[category]:
if product['resource'] == name:
return {'rate': product['price'],
'unit': product['unit']
}
return rate_file.FileRater().rate(name, region)

51
distil/rater/rate_file.py
View File

@ -1,51 +0,0 @@
# Copyright 2014 Catalyst IT Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import csv
from decimal import Decimal
from oslo_config import cfg
import oslo_log as log
from distil import rater
from distil import exceptions
CONF = cfg.CONF
class FileRater(rater.BaseRater):
def __init__(self):
try:
with open(CONF.rater.rate_file_path) as fh:
# Makes no opinions on the file structure
reader = csv.reader(fh, delimiter="|")
self.__rates = {
row[1].strip(): {
'rate': Decimal(row[3].strip()),
'region': row[0].strip(),
'unit': row[2].strip()
} for row in reader
}
except Exception as e:
msg = 'Failed to load rates file: `%s`' % e
log.critical(msg)
raise exceptions.InvalidConfig(msg)
def rate(self, name, region=None):
return {
'rate': self.__rates[name]['rate'],
'unit': self.__rates[name]['unit']
}

177
distil/service/api/v2/costs.py
View File

@ -1,177 +0,0 @@
# Copyright (c) 2016 Catalyst IT Ltd.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import json
from datetime import datetime
from decimal import Decimal
from oslo_config import cfg
from oslo_log import log as logging
from distil import exceptions
from distil import rater
from distil.common import constants
from distil.db import api as db_api
from distil.common import general
LOG = logging.getLogger(__name__)
CONF = cfg.CONF
def _validate_project_and_range(project_id, start, end):
try:
if start is not None:
try:
start = datetime.strptime(start, constants.iso_date)
except ValueError:
start = datetime.strptime(start, constants.iso_time)
else:
raise exceptions.DateTimeException(
message=(
"Missing parameter:" +
"'start' in format: y-m-d or y-m-dTH:M:S"))
if not end:
end = datetime.utcnow()
else:
try:
end = datetime.strptime(end, constants.iso_date)
except ValueError:
end = datetime.strptime(end, constants.iso_time)
except ValueError:
raise exceptions.DateTimeException(
message=(
"Missing parameter: " +
"'end' in format: y-m-d or y-m-dTH:M:S"))
if end <= start:
raise exceptions.DateTimeException(
message="End date must be greater than start.")
if not project_id:
raise exceptions.NotFoundException("Missing parameter: project_id")
valid_project = db_api.project_get(project_id)
return valid_project, start, end
def get_usage(project_id, start, end):
cleaned = _validate_project_and_range(project_id, start, end)
try:
valid_project, start, end = cleaned
except ValueError:
return cleaned
LOG.debug("Calculating unrated data for %s in range: %s - %s" %
(valid_project.id, start, end))
usage = db_api.usage_get(valid_project.id, start, end)
project_dict = _build_project_dict(valid_project, usage)
# add range:
project_dict['start'] = str(start)
project_dict['end'] = str(end)
return project_dict
def get_costs(project_id, start, end):
valid_project, start, end = _validate_project_and_range(
project_id, start, end)
LOG.debug("Calculating rated data for %s in range: %s - %s" %
(valid_project.id, start, end))
costs = _calculate_cost(valid_project, start, end)
return costs
def _calculate_cost(project, start, end):
"""Calculate a rated data dict from the given range."""
usage = db_api.usage_get(project.id, start, end)
# Transform the query result into a billable dict.
project_dict = _build_project_dict(project, usage)
project_dict = _add_costs_for_project(project_dict)
# add sales order range:
project_dict['start'] = str(start)
project_dict['end'] = str(end)
return project_dict
def _build_project_dict(project, usage):
"""Builds a dict structure for a given project."""
project_dict = {'name': project.name, 'tenant_id': project.id}
all_resource_ids = [entry.get('resource_id') for entry in usage]
res_list = db_api.resource_get_by_ids(project.id, all_resource_ids)
project_dict['resources'] = {row.id: json.loads(row.info)
for row in res_list}
for entry in usage:
service = {'name': entry.get('service'),
'volume': entry.get('volume'),
'unit': entry.get('unit')}
resource = project_dict['resources'][entry.get('resource_id')]
service_list = resource.setdefault('services', [])
service_list.append(service)
return project_dict
def _add_costs_for_project(project):
"""Adds cost values to services using the given rates manager."""
current_rater = rater.get_rater()
project_total = 0
for resource in project['resources'].values():
resource_total = 0
for service in resource['services']:
try:
rate = current_rater.rate(service['name'])
except KeyError:
# no rate exists for this service
service['cost'] = "0"
service['volume'] = "unknown unit conversion"
service['unit'] = "unknown"
service['rate'] = "missing rate"
continue
volume = general.convert_to(service['volume'],
service['unit'],
rate['unit'])
# round to 2dp so in dollars.
cost = round(volume * Decimal(rate['rate']), 2)
service['cost'] = str(cost)
service['volume'] = str(volume)
service['unit'] = rate['unit']
service['rate'] = str(rate['rate'])
resource_total += cost
resource['total_cost'] = str(resource_total)
project_total += resource_total
project['total_cost'] = str(project_total)
return project

67
distil/service/api/v2/quotations.py Normal file
View File

@ -0,0 +1,67 @@
# Copyright (c) 2017 Catalyst IT Ltd.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from datetime import date
from datetime import datetime
from oslo_config import cfg
from oslo_log import log as logging
from distil.db import api as db_api
from distil.erp import utils as erp_utils
from distil.service.api.v2 import products
LOG = logging.getLogger(__name__)
CONF = cfg.CONF
def get_quotations(project_id, detailed=False):
"""Get real time cost of current month."""
today = date.today()
start = datetime(today.year, today.month, 1)
end = datetime(today.year, today.month, today.day)
region_name = CONF.keystone_authtoken.region_name
project = db_api.project_get(project_id)
LOG.info(
'Get quotations for %s(%s) from %s to %s for current region: %s',
project.id, project.name, start, end, region_name
)
# Same format with get_invoices output.
output = {
'start': str(start),
'end': str(end),
'project_id': project.id,
'project_name': project.name,
}
usage = db_api.usage_get(project_id, start, end)
all_resource_ids = set([entry.get('resource_id') for entry in usage])
res_list = db_api.resource_get_by_ids(project_id, all_resource_ids)
erp_driver = erp_utils.load_erp_driver(CONF)
quotations = erp_driver.get_quotations(
region_name,
project_id,
measurements=usage,
resources=res_list,
detailed=detailed
)
output['quotations'] = {str(end.date()): quotations}
return output

147
distil/tests/unit/erp/drivers/test_odoo.py
View File

@ -189,3 +189,150 @@ class TestOdooDriver(base.DistilTestCase):
},
invoices
)
@mock.patch('odoorpc.ODOO')
@mock.patch('distil.erp.drivers.odoo.OdooDriver.get_products')
def test_get_quotations_without_details(self, mock_get_products,
mock_odoo):
mock_get_products.return_value = {
'nz_1': {
'Compute': [
{
'name': 'c1.c2r16', 'description': 'c1.c2r16',
'price': 0.01, 'unit': 'hour'
}
],
'Block Storage': [
{
'name': 'b1.standard', 'description': 'b1.standard',
'price': 0.02, 'unit': 'gigabyte'
}
]
}
}
class Resource(object):
def __init__(self, id, info):
self.id = id
self.info = info
resources = [
Resource(1, '{"name": "", "type": "Volume"}'),
Resource(2, '{"name": "", "type": "Virtual Machine"}')
]
usage = [
{
'service': 'b1.standard',
'resource_id': 1,
'volume': 1024 * 1024 * 1024,
'unit': 'byte',
},
{
'service': 'c1.c2r16',
'resource_id': 2,
'volume': 3600,
'unit': 'second',
}
]
odoodriver = odoo.OdooDriver(self.conf)
quotations = odoodriver.get_quotations(
'nz_1', 'fake_id', measurements=usage, resources=resources
)
self.assertEqual(
{'total_cost': 0.03},
quotations
)
@mock.patch('odoorpc.ODOO')
@mock.patch('distil.erp.drivers.odoo.OdooDriver.get_products')
def test_get_quotations_with_details(self, mock_get_products,
mock_odoo):
mock_get_products.return_value = {
'nz_1': {
'Compute': [
{
'name': 'c1.c2r16', 'description': 'c1.c2r16',
'price': 0.01, 'unit': 'hour'
}
],
'Block Storage': [
{
'name': 'b1.standard', 'description': 'b1.standard',
'price': 0.02, 'unit': 'gigabyte'
}
]
}
}
class Resource(object):
def __init__(self, id, info):
self.id = id
self.info = info
resources = [
Resource(1, '{"name": "volume1", "type": "Volume"}'),
Resource(2, '{"name": "instance2", "type": "Virtual Machine"}')
]
usage = [
{
'service': 'b1.standard',
'resource_id': 1,
'volume': 1024 * 1024 * 1024,
'unit': 'byte',
},
{
'service': 'c1.c2r16',
'resource_id': 2,
'volume': 3600,
'unit': 'second',
}
]
odoodriver = odoo.OdooDriver(self.conf)
quotations = odoodriver.get_quotations(
'nz_1', 'fake_id', measurements=usage, resources=resources,
detailed=True
)
self.assertEqual(
{
'total_cost': 0.03,
'details': {
'Compute': {
'total_cost': 0.01,
'breakdown': {
'NZ-1.c1.c2r16': [
{
"resource_name": "instance2",
"resource_id": 2,
"cost": 0.01,
"quantity": 1.0,
"rate": 0.01,
"unit": "hour",
}
],
}
},
'Block Storage': {
'total_cost': 0.02,
'breakdown': {
'NZ-1.b1.standard': [
{
"resource_name": "volume1",
"resource_id": 1,
"cost": 0.02,
"quantity": 1.0,
"rate": 0.02,
"unit": "gigabyte",
}
]
}
}
}
},
quotations
)

2
etc/policy.json.sample
View File

@ -3,7 +3,7 @@
"admin_or_owner": "is_admin:True or project_id:%(project_id)s",
"default": "rule:admin_or_owner",
"rating:costs:get": "rule:context_is_admin",
"rating:measurements:get": "rule:context_is_admin",
"rating:invoices:get": "rule:context_is_admin",
"rating:quotations:get": "rule:context_is_admin",
}

33
etc/rates.csv
View File

@ -1,33 +0,0 @@
region | m1.tiny | hour | 0.048
region | m1.small | hour | 0.096
region | m1.medium | hour | 0.191
region | c1.small | hour | 0.191
region | m1.large | hour | 0.382
region | m1.xlarge | hour | 0.76
region | c1.large | hour | 0.347
region | c1.xlarge | hour | 0.594
region | c1.xxlarge | hour | 1.040
region | m1.2xlarge | hour | 1.040
region | c1.c1r1 | hour | 0.044
region | c1.c1r2 | hour | 0.062
region | c1.c1r4 | hour | 0.098
region | c1.c2r1 | hour | 0.070
region | c1.c2r2 | hour | 0.088
region | c1.c2r4 | hour | 0.124
region | c1.c2r8 | hour | 0.196
region | c1.c2r16 | hour | 0.339
region | c1.c4r2 | hour | 0.140
region | c1.c4r4 | hour | 0.176
region | c1.c4r8 | hour | 0.248
region | c1.c4r16 | hour | 0.391
region | c1.c4r32 | hour | 0.678
region | c1.c8r4 | hour | 0.280
region | c1.c8r8 | hour | 0.352
region | c1.c8r16 | hour | 0.496
region | c1.c8r32 | hour | 0.783
region | b1.standard | gigabyte | 0.0005
region | o1.standard | gigabyte | 0.0005
region | n1.ipv4 | hour | 0.006
region | n1.network | hour | 0.016
region | n1.router | hour | 0.017
region | n1.vpn | hour | 0.017
1 region m1.tiny hour 0.048
2 region m1.small hour 0.096
3 region m1.medium hour 0.191
4 region c1.small hour 0.191
5 region m1.large hour 0.382
6 region m1.xlarge hour 0.76
7 region c1.large hour 0.347
8 region c1.xlarge hour 0.594
9 region c1.xxlarge hour 1.040
10 region m1.2xlarge hour 1.040
11 region c1.c1r1 hour 0.044
12 region c1.c1r2 hour 0.062
13 region c1.c1r4 hour 0.098
14 region c1.c2r1 hour 0.070
15 region c1.c2r2 hour 0.088
16 region c1.c2r4 hour 0.124
17 region c1.c2r8 hour 0.196
18 region c1.c2r16 hour 0.339
19 region c1.c4r2 hour 0.140
20 region c1.c4r4 hour 0.176
21 region c1.c4r8 hour 0.248
22 region c1.c4r16 hour 0.391
23 region c1.c4r32 hour 0.678
24 region c1.c8r4 hour 0.280
25 region c1.c8r8 hour 0.352
26 region c1.c8r16 hour 0.496
27 region c1.c8r32 hour 0.783
28 region b1.standard gigabyte 0.0005
29 region o1.standard gigabyte 0.0005
30 region n1.ipv4 hour 0.006
31 region n1.network hour 0.016
32 region n1.router hour 0.017
33 region n1.vpn hour 0.017

4
setup.cfg
View File

@ -37,10 +37,6 @@ oslo.config.opts =
distil.collector =
ceilometer = distil.collector.ceilometer:CeilometerCollector
distil.rater =
file = distil.rater.file:FileRater
odoo = distil.rater.odoo:OdooRater
distil.transformer =
max = distil.transformer.arithmetic:MaxTransformer
storagemax = distil.transformer.arithmetic:StorageMaxTransformer