x/distil
Code Issues Proposed changes

Merge "Add /invoices rest api"

Browse Source
This commit is contained in:
Jenkins 2017-05-10 20:35:40 +00:00 committed by Gerrit Code Review
commit a0f8ebf3da
10 changed files with 479 additions and 54 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

74
distil/api/v2.py
View File

@ -14,8 +14,8 @@
# limitations under the License.
from dateutil import parser
from oslo_log import log
from oslo_utils import strutils
from distil import exceptions
from distil.api import acl
@ -24,6 +24,7 @@ from distil.common import constants
from distil.common import openstack
from distil.service.api.v2 import costs
from distil.service.api.v2 import health
from distil.service.api.v2 import invoices
from distil.service.api.v2 import products
LOG = log.getLogger(__name__)
@ -31,6 +32,33 @@ LOG = log.getLogger(__name__)
rest = api.Rest('v2', __name__)
def _get_request_args():
cur_project_id = api.context.current().project_id
project_id = api.get_request_args().get('project_id', cur_project_id)
if not api.context.current().is_admin and cur_project_id != project_id:
raise exceptions.Forbidden()
start = api.get_request_args().get('start', None)
end = api.get_request_args().get('end', None)
detailed = strutils.bool_from_string(
api.get_request_args().get('detailed', False)
)
regions = api.get_request_args().get('regions', None)
params = {
'start': start,
'end': end,
'project_id': project_id,
'detailed': detailed,
'regions': regions
}
return params
@rest.get('/health')
def health_get():
return api.render(health=health.get_health())
@ -38,7 +66,9 @@ def health_get():
@rest.get('/products')
def products_get():
os_regions = api.get_request_args().get('regions', None)
params = _get_request_args()
os_regions = params.get('regions')
regions = os_regions.split(',') if os_regions else []
if regions:
@ -54,28 +84,42 @@ def products_get():
return api.render(products=products.get_products(regions))
def _get_usage_args():
# NOTE(flwang): Get 'tenant' first for backward compatibility.
tenant_id = api.get_request_args().get('tenant', None)
project_id = api.get_request_args().get('project_id', tenant_id)
start = api.get_request_args().get('start', None)
end = api.get_request_args().get('end', None)
return project_id, start, end
@rest.get('/costs')
@acl.enforce("rating:costs:get")
def costs_get():
project_id, start, end = _get_usage_args()
params = _get_request_args()
# NOTE(flwang): Here using 'usage' instead of 'costs' for backward
# compatibility.
return api.render(usage=costs.get_costs(project_id, start, end))
return api.render(
usage=costs.get_costs(
params['project_id'], params['start'], params['end']
)
)
@rest.get('/measurements')
@acl.enforce("rating:measurements:get")
def measurements_get():
project_id, start, end = _get_usage_args()
params = _get_request_args()
return api.render(measurements=costs.get_usage(project_id, start, end))
return api.render(
measurements=costs.get_usage(
params['project_id'], params['start'], params['end']
)
)
@rest.get('/invoices')
@acl.enforce("rating:invoices:get")
def invoices_get():
params = _get_request_args()
return api.render(
invoices.get_invoices(
params['project_id'],
params['start'],
params['end'],
detailed=params['detailed']
)
)

46
distil/common/general.py
View File

@ -24,6 +24,10 @@ import yaml
from oslo_config import cfg
from oslo_log import log as logging
from distil.common import constants
from distil.db import api as db_api
from distil import exceptions
CONF = cfg.CONF
LOG = logging.getLogger(__name__)
_TRANS_CONFIG = None
@ -107,3 +111,45 @@ def convert_to(value, from_unit, to_unit):
def get_process_identifier():
"""Gets current running process identifier."""
return "%s_%s" % (socket.gethostname(), CONF.collector.partitioning_suffix)
def convert_project_and_range(project_id, start, end):
now = datetime.utcnow()
try:
if start is not None:
try:
start = datetime.strptime(start, constants.iso_date)
except ValueError:
start = datetime.strptime(start, constants.iso_time)
else:
raise exceptions.DateTimeException(
message=(
"Missing parameter:" +
"'start' in format: y-m-d or y-m-dTH:M:S"))
if not end:
end = now
else:
try:
end = datetime.strptime(end, constants.iso_date)
except ValueError:
end = datetime.strptime(end, constants.iso_time)
if end > now:
end = now
except ValueError:
raise exceptions.DateTimeException(
message=(
"Missing parameter: " +
"'end' in format: y-m-d or y-m-dTH:M:S"))
if end <= start:
raise exceptions.DateTimeException(
message="End date must be greater than start.")
if not project_id:
raise exceptions.NotFoundException("Missing parameter: project_id")
valid_project = db_api.project_get(project_id)
return valid_project, start, end

24
distil/erp/driver.py
View File

@ -22,19 +22,6 @@ class BaseDriver(object):
def __init__(self, conf):
self.conf = conf
def get_salesOrders(self, project, start_at, end_at):
"""List sales orders based on the given project and time range
:param project: project id
:param start_at: start time
:param end_at: end time
:returns List of sales order, if the time range only cover one month,
then, the list will only contain 1 sale orders. Otherwise,
the length of the list depends on the months number of the
time range.
"""
raise NotImplementedError()
def get_products(self, regions=[]):
"""List products based o given regions
@ -64,3 +51,14 @@ class BaseDriver(object):
:param project: project
"""
raise NotImplementedError()
def get_invoices(self, start, end, project_id, detailed=False):
"""Get history invoices from ERP service given a time range.
:param start: Start time, a datetime object.
:param end: End time, a datetime object.
:param project_id: project ID.
:param detailed: If get detailed information or not.
:return: The history invoices information for each month.
"""
raise NotImplementedError()

150
distil/erp/drivers/odoo.py
View File

@ -17,8 +17,10 @@ import collections
import odoorpc
from oslo_log import log
from distil.common import cache
from distil.common import openstack
from distil.erp import driver
from distil import exceptions
LOG = log.getLogger(__name__)
@ -60,8 +62,14 @@ class OdooDriver(driver.BaseDriver):
self.pricelist = self.odoo.env['product.pricelist']
self.product = self.odoo.env['product.product']
self.category = self.odoo.env['product.category']
self.invoice = self.odoo.env['account.invoice']
self.invoice_line = self.odoo.env['account.invoice.line']
self.product_catagory_mapping = {}
@cache.memoize
def get_products(self, regions=[]):
self.product_catagory_mapping.clear()
odoo_regions = []
if not regions:
@ -96,6 +104,9 @@ class OdooDriver(driver.BaseDriver):
continue
category = product['categ_id'][1].split('/')[-1].strip()
self.product_catagory_mapping[product['id']] = category
price = round(product['lst_price'], 5)
# NOTE(flwang): default_code is Internal Reference on
# Odoo GUI
@ -140,3 +151,142 @@ class OdooDriver(driver.BaseDriver):
return {}
return prices
def _get_invoice_detail(self, invoice_id):
"""Get invoice details.
Return details in the following format:
{
'catagory': {
'total_cost': xxx,
'breakdown': {
'<product_name>': [
{
'resource_name': '',
'quantity': '',
'unit': '',
'rate': '',
'cost': ''
}
],
'<product_name>': [
{
'resource_name': '',
'quantity': '',
'unit': '',
'rate': '',
'cost': ''
}
]
}
}
}
"""
detail_dict = {}
invoice_lines_ids = self.invoice_line.search(
[('invoice_id', '=', invoice_id)]
)
invoice_lines = self.invoice_line.read(invoice_lines_ids)
for line in invoice_lines:
line_info = {
'resource_name': line['name'],
'quantity': line['quantity'],
'rate': line['price_unit'],
'unit': line['uos_id'][1],
'cost': round(line['price_subtotal'], 2)
}
# Original product is a string like "[hour] NZ-POR-1.c1.c2r8"
product = line['product_id'][1].split(']')[1].strip()
catagory = self.product_catagory_mapping[line['product_id'][0]]
if catagory not in detail_dict:
detail_dict[catagory] = {
'total_cost': 0,
'breakdown': collections.defaultdict(list)
}
detail_dict[catagory]['total_cost'] += line_info['cost']
detail_dict[catagory]['breakdown'][product].append(line_info)
return detail_dict
def get_invoices(self, start, end, project_id, detailed=False):
"""Get history invoices from Odoo given a time range.
Return value is in the following format:
{
'<billing_date1>': {
'total_cost': 100,
'details': {
...
}
},
'<billing_date2>': {
'total_cost': 200,
'details': {
...
}
}
}
:param start: Start time, a datetime object.
:param end: End time, a datetime object.
:param project_id: project ID.
:param detailed: Get detailed information.
:return: The history invoices information for each month.
"""
# Get invoices in time ascending order.
result = collections.OrderedDict()
try:
invoice_ids = self.invoice.search(
[
('date_invoice', '>=', str(start.date())),
('date_invoice', '<=', str(end.date())),
('comment', 'like', project_id)
],
order='date_invoice'
)
if not len(invoice_ids):
LOG.debug('No history invoices returned from Odoo.')
return result
LOG.debug('Found invoices: %s' % invoice_ids)
# Convert ids from string to int.
ids = [int(i) for i in invoice_ids]
invoices = self.odoo.execute(
'account.invoice',
'read',
ids,
['date_invoice', 'amount_total']
)
for v in invoices:
result[v['date_invoice']] = {
'total_cost': round(v['amount_total'], 2)
}
if detailed:
# Populate product catagory mapping first. This should be
# quick since we cached get_products()
if not self.product_catagory_mapping:
self.get_products()
details = self._get_invoice_detail(v['id'])
result[v['date_invoice']].update({'details': details})
except Exception as e:
LOG.exception(
'Error occured when getting invoices from Odoo, '
'error: %s' % str(e)
)
raise exceptions.ERPException(
'Failed to get invoices from ERP server.'
)
return result

29
distil/erp/utils.py
View File

@ -22,6 +22,7 @@ from stevedore import driver
from distil import exceptions
LOG = log.getLogger(__name__)
_ERP_DRIVER = None
def load_erp_driver(conf):
@ -31,17 +32,23 @@ def load_erp_driver(conf):
driver. Must include a 'drivers' group.
"""
_invoke_args = [conf]
global _ERP_DRIVER
try:
mgr = driver.DriverManager('distil.erp',
conf.erp_driver,
invoke_on_load=True,
invoke_args=_invoke_args)
if not _ERP_DRIVER:
_invoke_args = [conf]
return mgr.driver
try:
mgr = driver.DriverManager('distil.erp',
conf.erp_driver,
invoke_on_load=True,
invoke_args=_invoke_args)
except Exception as exc:
LOG.exception(exc)
raise exceptions.InvalidDriver('Failed to load ERP driver'
' for {0}'.format(conf.erp_driver))
_ERP_DRIVER = mgr.driver
except Exception as exc:
LOG.exception(exc)
raise exceptions.InvalidDriver(
'Failed to load ERP driver for {0}'.format(conf.erp_driver)
)
return _ERP_DRIVER

5
distil/exceptions.py
View File

@ -80,3 +80,8 @@ class Forbidden(DistilException):
class InvalidDriver(DistilException):
"""A driver was not found or loaded."""
message = _("Failed to load driver")
class ERPException(DistilException):
code = 500
message = _("ERP server error.")

54
distil/service/api/v2/invoices.py Normal file
View File

@ -0,0 +1,54 @@
# Copyright (c) 2017 Catalyst IT Ltd.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from oslo_config import cfg
from oslo_log import log as logging
from distil.erp import utils as erp_utils
from distil.service.api.v2 import utils
LOG = logging.getLogger(__name__)
CONF = cfg.CONF
def get_invoices(project_id, start, end, detailed=False):
project, start, end = utils.convert_project_and_range(
project_id, start, end)
LOG.info(
"Get invoices for %s(%s) in range: %s - %s" %
(project.id, project.name, start, end)
)
output = {
'start': str(start),
'end': str(end),
'project_name': project.name,
'project_id': project.id,
'invoices': {}
}
# Query from ERP.
erp_driver = erp_utils.load_erp_driver(CONF)
erp_invoices = erp_driver.get_invoices(
start,
end,
project.id,
detailed=detailed
)
output['invoices'] = erp_invoices
return output

3
distil/tests/unit/base.py
View File

@ -20,6 +20,7 @@ from oslotest import base
from oslo_config import cfg
from oslo_log import log
from distil.common import cache
from distil import context
from distil import config
from distil.db import api as db_api
@ -37,6 +38,8 @@ class DistilTestCase(base.BaseTestCase):
else:
self.conf = cfg.CONF
cache.setup_cache(self.conf)
self.conf.register_opts(config.DEFAULT_OPTIONS)
self.conf.register_opts(config.ODOO_OPTS, group=config.ODOO_GROUP)

147
distil/tests/unit/erp/drivers/test_odoo.py
View File

@ -14,6 +14,7 @@
# limitations under the License.
from collections import namedtuple
from datetime import datetime
import mock
@ -23,21 +24,30 @@ from distil.tests.unit import base
REGION = namedtuple('Region', ['id'])
PRODUCTS = [
{'categ_id': [1, 'All products (.NET) / nz_1 / Compute'],
'name_template': 'NZ-1.c1.c1r1',
'lst_price': 0.00015,
'default_code': 'hour',
'description': '1 CPU, 1GB RAM'},
{'categ_id': [2, 'All products (.NET) / nz_1 / Network'],
'name_template': 'NZ-1.n1.router',
'lst_price': 0.00025,
'default_code': 'hour',
'description': 'Router'},
{'categ_id': [1, 'All products (.NET) / nz_1 / Block Storage'],
'name_template': 'NZ-1.b1.volume',
'lst_price': 0.00035,
'default_code': 'hour',
'description': 'Block storage'}
{
'id': 1,
'categ_id': [1, 'All products (.NET) / nz_1 / Compute'],
'name_template': 'NZ-1.c1.c1r1',
'lst_price': 0.00015,
'default_code': 'hour',
'description': '1 CPU, 1GB RAM'
},
{
'id': 2,
'categ_id': [2, 'All products (.NET) / nz_1 / Network'],
'name_template': 'NZ-1.n1.router',
'lst_price': 0.00025,
'default_code': 'hour',
'description': 'Router'
},
{
'id': 3,
'categ_id': [1, 'All products (.NET) / nz_1 / Block Storage'],
'name_template': 'NZ-1.b1.volume',
'lst_price': 0.00035,
'default_code': 'hour',
'description': 'Block storage'
}
]
@ -72,3 +82,110 @@ class TestOdooDriver(base.DistilTestCase):
},
products
)
@mock.patch('odoorpc.ODOO')
def test_get_invoices_without_details(self, mock_odoo):
start = datetime(2017, 3, 1)
end = datetime(2017, 5, 1)
fake_project = '123'
odoodriver = odoo.OdooDriver(self.conf)
odoodriver.invoice.search.return_value = ['1', '2']
odoodriver.odoo.execute.return_value = [
{'date_invoice': '2017-03-31', 'amount_total': 10},
{'date_invoice': '2017-04-30', 'amount_total': 20}
]
invoices = odoodriver.get_invoices(start, end, fake_project)
self.assertEqual(
{
'2017-03-31': {'total_cost': 10},
'2017-04-30': {'total_cost': 20}
},
invoices
)
@mock.patch('odoorpc.ODOO')
@mock.patch('distil.erp.drivers.odoo.OdooDriver.get_products')
def test_get_invoices_with_details(self, mock_get_products, mock_odoo):
start = datetime(2017, 3, 1)
end = datetime(2017, 5, 1)
fake_project = '123'
odoodriver = odoo.OdooDriver(self.conf)
odoodriver.invoice.search.return_value = ['1', '2']
odoodriver.invoice_line.read.side_effect = [
[
{
'name': 'resource1',
'quantity': 100,
'price_unit': 0.01,
'uos_id': [1, 'Gigabyte-hour(s)'],
'price_subtotal': 10,
'product_id': [1, '[hour] NZ-POR-1.c1.c2r8']
}
],
[
{
'name': 'resource2',
'quantity': 200,
'price_unit': 0.01,
'uos_id': [1, 'Gigabyte-hour(s)'],
'price_subtotal': 20,
'product_id': [1, '[hour] NZ-POR-1.c1.c2r8']
}
]
]
odoodriver.odoo.execute.return_value = [
{'id': 1, 'date_invoice': '2017-03-31', 'amount_total': 10},
{'id': 2, 'date_invoice': '2017-04-30', 'amount_total': 20}
]
odoodriver.product_catagory_mapping = {
1: 'Compute'
}
invoices = odoodriver.get_invoices(
start, end, fake_project, detailed=True
)
self.assertEqual(
{
'2017-03-31': {
'total_cost': 10,
'details': {
'Compute': {
'total_cost': 10,
'breakdown': {
'NZ-POR-1.c1.c2r8': [{
"cost": 10,
"quantity": 100,
"rate": 0.01,
"resource_name": "resource1",
"unit": "Gigabyte-hour(s)"
}]
}
}
}
},
'2017-04-30': {
'total_cost': 20,
'details': {
'Compute': {
'total_cost': 20,
'breakdown': {
'NZ-POR-1.c1.c2r8': [{
"cost": 20,
"quantity": 200,
"rate": 0.01,
"resource_name": "resource2",
"unit": "Gigabyte-hour(s)"
}]
}
}
}
}
},
invoices
)

1
etc/policy.json.sample
View File

@ -5,4 +5,5 @@
"rating:costs:get": "rule:context_is_admin",
"rating:measurements:get": "rule:context_is_admin",
"rating:invoices:get": "rule:context_is_admin",
}