From 7285c5a109b9d357771e16b1558296d361e2e090 Mon Sep 17 00:00:00 2001 From: Lingxian Kong Date: Fri, 28 Apr 2017 14:59:36 +1200 Subject: [PATCH] Add /invoices rest api Allow end user query history invoices from ERP server. Change-Id: I06776b26c8565bb3e9735ffceeab0bd399ca487a --- distil/api/v2.py | 74 +++++++--- distil/common/general.py | 46 +++++++ distil/erp/driver.py | 24 ++-- distil/erp/drivers/odoo.py | 150 +++++++++++++++++++++ distil/erp/utils.py | 29 ++-- distil/exceptions.py | 5 + distil/service/api/v2/invoices.py | 54 ++++++++ distil/tests/unit/base.py | 3 + distil/tests/unit/erp/drivers/test_odoo.py | 147 +++++++++++++++++--- etc/policy.json.sample | 1 + 10 files changed, 479 insertions(+), 54 deletions(-) create mode 100644 distil/service/api/v2/invoices.py diff --git a/distil/api/v2.py b/distil/api/v2.py index f5bfa68..315a3f0 100644 --- a/distil/api/v2.py +++ b/distil/api/v2.py @@ -14,8 +14,8 @@ # limitations under the License. from dateutil import parser - from oslo_log import log +from oslo_utils import strutils from distil import exceptions from distil.api import acl @@ -24,6 +24,7 @@ from distil.common import constants from distil.common import openstack from distil.service.api.v2 import costs from distil.service.api.v2 import health +from distil.service.api.v2 import invoices from distil.service.api.v2 import products LOG = log.getLogger(__name__) @@ -31,6 +32,33 @@ LOG = log.getLogger(__name__) rest = api.Rest('v2', __name__) +def _get_request_args(): + cur_project_id = api.context.current().project_id + project_id = api.get_request_args().get('project_id', cur_project_id) + + if not api.context.current().is_admin and cur_project_id != project_id: + raise exceptions.Forbidden() + + start = api.get_request_args().get('start', None) + end = api.get_request_args().get('end', None) + + detailed = strutils.bool_from_string( + api.get_request_args().get('detailed', False) + ) + + regions = api.get_request_args().get('regions', None) + + params = { + 'start': start, + 'end': end, + 'project_id': project_id, + 'detailed': detailed, + 'regions': regions + } + + return params + + @rest.get('/health') def health_get(): return api.render(health=health.get_health()) @@ -38,7 +66,9 @@ def health_get(): @rest.get('/products') def products_get(): - os_regions = api.get_request_args().get('regions', None) + params = _get_request_args() + + os_regions = params.get('regions') regions = os_regions.split(',') if os_regions else [] if regions: @@ -54,28 +84,42 @@ def products_get(): return api.render(products=products.get_products(regions)) -def _get_usage_args(): - # NOTE(flwang): Get 'tenant' first for backward compatibility. - tenant_id = api.get_request_args().get('tenant', None) - project_id = api.get_request_args().get('project_id', tenant_id) - start = api.get_request_args().get('start', None) - end = api.get_request_args().get('end', None) - return project_id, start, end - - @rest.get('/costs') @acl.enforce("rating:costs:get") def costs_get(): - project_id, start, end = _get_usage_args() + params = _get_request_args() # NOTE(flwang): Here using 'usage' instead of 'costs' for backward # compatibility. - return api.render(usage=costs.get_costs(project_id, start, end)) + return api.render( + usage=costs.get_costs( + params['project_id'], params['start'], params['end'] + ) + ) @rest.get('/measurements') @acl.enforce("rating:measurements:get") def measurements_get(): - project_id, start, end = _get_usage_args() + params = _get_request_args() - return api.render(measurements=costs.get_usage(project_id, start, end)) + return api.render( + measurements=costs.get_usage( + params['project_id'], params['start'], params['end'] + ) + ) + + +@rest.get('/invoices') +@acl.enforce("rating:invoices:get") +def invoices_get(): + params = _get_request_args() + + return api.render( + invoices.get_invoices( + params['project_id'], + params['start'], + params['end'], + detailed=params['detailed'] + ) + ) diff --git a/distil/common/general.py b/distil/common/general.py index d093d1c..68a926e 100644 --- a/distil/common/general.py +++ b/distil/common/general.py @@ -24,6 +24,10 @@ import yaml from oslo_config import cfg from oslo_log import log as logging +from distil.common import constants +from distil.db import api as db_api +from distil import exceptions + CONF = cfg.CONF LOG = logging.getLogger(__name__) _TRANS_CONFIG = None @@ -107,3 +111,45 @@ def convert_to(value, from_unit, to_unit): def get_process_identifier(): """Gets current running process identifier.""" return "%s_%s" % (socket.gethostname(), CONF.collector.partitioning_suffix) + + +def convert_project_and_range(project_id, start, end): + now = datetime.utcnow() + + try: + if start is not None: + try: + start = datetime.strptime(start, constants.iso_date) + except ValueError: + start = datetime.strptime(start, constants.iso_time) + else: + raise exceptions.DateTimeException( + message=( + "Missing parameter:" + + "'start' in format: y-m-d or y-m-dTH:M:S")) + if not end: + end = now + else: + try: + end = datetime.strptime(end, constants.iso_date) + except ValueError: + end = datetime.strptime(end, constants.iso_time) + + if end > now: + end = now + except ValueError: + raise exceptions.DateTimeException( + message=( + "Missing parameter: " + + "'end' in format: y-m-d or y-m-dTH:M:S")) + + if end <= start: + raise exceptions.DateTimeException( + message="End date must be greater than start.") + + if not project_id: + raise exceptions.NotFoundException("Missing parameter: project_id") + + valid_project = db_api.project_get(project_id) + + return valid_project, start, end diff --git a/distil/erp/driver.py b/distil/erp/driver.py index a2d5d2c..ad76423 100644 --- a/distil/erp/driver.py +++ b/distil/erp/driver.py @@ -22,19 +22,6 @@ class BaseDriver(object): def __init__(self, conf): self.conf = conf - def get_salesOrders(self, project, start_at, end_at): - """List sales orders based on the given project and time range - - :param project: project id - :param start_at: start time - :param end_at: end time - :returns List of sales order, if the time range only cover one month, - then, the list will only contain 1 sale orders. Otherwise, - the length of the list depends on the months number of the - time range. - """ - raise NotImplementedError() - def get_products(self, regions=[]): """List products based o given regions @@ -64,3 +51,14 @@ class BaseDriver(object): :param project: project """ raise NotImplementedError() + + def get_invoices(self, start, end, project_id, detailed=False): + """Get history invoices from ERP service given a time range. + + :param start: Start time, a datetime object. + :param end: End time, a datetime object. + :param project_id: project ID. + :param detailed: If get detailed information or not. + :return: The history invoices information for each month. + """ + raise NotImplementedError() diff --git a/distil/erp/drivers/odoo.py b/distil/erp/drivers/odoo.py index 9d6df0a..e47c805 100644 --- a/distil/erp/drivers/odoo.py +++ b/distil/erp/drivers/odoo.py @@ -17,8 +17,10 @@ import collections import odoorpc from oslo_log import log +from distil.common import cache from distil.common import openstack from distil.erp import driver +from distil import exceptions LOG = log.getLogger(__name__) @@ -60,8 +62,14 @@ class OdooDriver(driver.BaseDriver): self.pricelist = self.odoo.env['product.pricelist'] self.product = self.odoo.env['product.product'] self.category = self.odoo.env['product.category'] + self.invoice = self.odoo.env['account.invoice'] + self.invoice_line = self.odoo.env['account.invoice.line'] + self.product_catagory_mapping = {} + + @cache.memoize def get_products(self, regions=[]): + self.product_catagory_mapping.clear() odoo_regions = [] if not regions: @@ -96,6 +104,9 @@ class OdooDriver(driver.BaseDriver): continue category = product['categ_id'][1].split('/')[-1].strip() + + self.product_catagory_mapping[product['id']] = category + price = round(product['lst_price'], 5) # NOTE(flwang): default_code is Internal Reference on # Odoo GUI @@ -140,3 +151,142 @@ class OdooDriver(driver.BaseDriver): return {} return prices + + def _get_invoice_detail(self, invoice_id): + """Get invoice details. + + Return details in the following format: + { + 'catagory': { + 'total_cost': xxx, + 'breakdown': { + '': [ + { + 'resource_name': '', + 'quantity': '', + 'unit': '', + 'rate': '', + 'cost': '' + } + ], + '': [ + { + 'resource_name': '', + 'quantity': '', + 'unit': '', + 'rate': '', + 'cost': '' + } + ] + } + } + } + """ + detail_dict = {} + + invoice_lines_ids = self.invoice_line.search( + [('invoice_id', '=', invoice_id)] + ) + invoice_lines = self.invoice_line.read(invoice_lines_ids) + + for line in invoice_lines: + line_info = { + 'resource_name': line['name'], + 'quantity': line['quantity'], + 'rate': line['price_unit'], + 'unit': line['uos_id'][1], + 'cost': round(line['price_subtotal'], 2) + } + + # Original product is a string like "[hour] NZ-POR-1.c1.c2r8" + product = line['product_id'][1].split(']')[1].strip() + catagory = self.product_catagory_mapping[line['product_id'][0]] + + if catagory not in detail_dict: + detail_dict[catagory] = { + 'total_cost': 0, + 'breakdown': collections.defaultdict(list) + } + + detail_dict[catagory]['total_cost'] += line_info['cost'] + detail_dict[catagory]['breakdown'][product].append(line_info) + + return detail_dict + + def get_invoices(self, start, end, project_id, detailed=False): + """Get history invoices from Odoo given a time range. + + Return value is in the following format: + { + '': { + 'total_cost': 100, + 'details': { + ... + } + }, + '': { + 'total_cost': 200, + 'details': { + ... + } + } + } + + :param start: Start time, a datetime object. + :param end: End time, a datetime object. + :param project_id: project ID. + :param detailed: Get detailed information. + :return: The history invoices information for each month. + """ + # Get invoices in time ascending order. + result = collections.OrderedDict() + + try: + invoice_ids = self.invoice.search( + [ + ('date_invoice', '>=', str(start.date())), + ('date_invoice', '<=', str(end.date())), + ('comment', 'like', project_id) + ], + order='date_invoice' + ) + + if not len(invoice_ids): + LOG.debug('No history invoices returned from Odoo.') + return result + + LOG.debug('Found invoices: %s' % invoice_ids) + + # Convert ids from string to int. + ids = [int(i) for i in invoice_ids] + + invoices = self.odoo.execute( + 'account.invoice', + 'read', + ids, + ['date_invoice', 'amount_total'] + ) + for v in invoices: + result[v['date_invoice']] = { + 'total_cost': round(v['amount_total'], 2) + } + + if detailed: + # Populate product catagory mapping first. This should be + # quick since we cached get_products() + if not self.product_catagory_mapping: + self.get_products() + + details = self._get_invoice_detail(v['id']) + result[v['date_invoice']].update({'details': details}) + except Exception as e: + LOG.exception( + 'Error occured when getting invoices from Odoo, ' + 'error: %s' % str(e) + ) + + raise exceptions.ERPException( + 'Failed to get invoices from ERP server.' + ) + + return result diff --git a/distil/erp/utils.py b/distil/erp/utils.py index fea2f44..b976a90 100644 --- a/distil/erp/utils.py +++ b/distil/erp/utils.py @@ -22,6 +22,7 @@ from stevedore import driver from distil import exceptions LOG = log.getLogger(__name__) +_ERP_DRIVER = None def load_erp_driver(conf): @@ -31,17 +32,23 @@ def load_erp_driver(conf): driver. Must include a 'drivers' group. """ - _invoke_args = [conf] + global _ERP_DRIVER - try: - mgr = driver.DriverManager('distil.erp', - conf.erp_driver, - invoke_on_load=True, - invoke_args=_invoke_args) + if not _ERP_DRIVER: + _invoke_args = [conf] - return mgr.driver + try: + mgr = driver.DriverManager('distil.erp', + conf.erp_driver, + invoke_on_load=True, + invoke_args=_invoke_args) - except Exception as exc: - LOG.exception(exc) - raise exceptions.InvalidDriver('Failed to load ERP driver' - ' for {0}'.format(conf.erp_driver)) + _ERP_DRIVER = mgr.driver + + except Exception as exc: + LOG.exception(exc) + raise exceptions.InvalidDriver( + 'Failed to load ERP driver for {0}'.format(conf.erp_driver) + ) + + return _ERP_DRIVER diff --git a/distil/exceptions.py b/distil/exceptions.py index 6626698..945a0ee 100644 --- a/distil/exceptions.py +++ b/distil/exceptions.py @@ -80,3 +80,8 @@ class Forbidden(DistilException): class InvalidDriver(DistilException): """A driver was not found or loaded.""" message = _("Failed to load driver") + + +class ERPException(DistilException): + code = 500 + message = _("ERP server error.") diff --git a/distil/service/api/v2/invoices.py b/distil/service/api/v2/invoices.py new file mode 100644 index 0000000..c60cec8 --- /dev/null +++ b/distil/service/api/v2/invoices.py @@ -0,0 +1,54 @@ +# Copyright (c) 2017 Catalyst IT Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from oslo_config import cfg +from oslo_log import log as logging + +from distil.erp import utils as erp_utils +from distil.service.api.v2 import utils + +LOG = logging.getLogger(__name__) +CONF = cfg.CONF + + +def get_invoices(project_id, start, end, detailed=False): + project, start, end = utils.convert_project_and_range( + project_id, start, end) + + LOG.info( + "Get invoices for %s(%s) in range: %s - %s" % + (project.id, project.name, start, end) + ) + + output = { + 'start': str(start), + 'end': str(end), + 'project_name': project.name, + 'project_id': project.id, + 'invoices': {} + } + + # Query from ERP. + erp_driver = erp_utils.load_erp_driver(CONF) + erp_invoices = erp_driver.get_invoices( + start, + end, + project.id, + detailed=detailed + ) + + output['invoices'] = erp_invoices + + return output diff --git a/distil/tests/unit/base.py b/distil/tests/unit/base.py index 456ea83..446d9b3 100644 --- a/distil/tests/unit/base.py +++ b/distil/tests/unit/base.py @@ -20,6 +20,7 @@ from oslotest import base from oslo_config import cfg from oslo_log import log +from distil.common import cache from distil import context from distil import config from distil.db import api as db_api @@ -37,6 +38,8 @@ class DistilTestCase(base.BaseTestCase): else: self.conf = cfg.CONF + cache.setup_cache(self.conf) + self.conf.register_opts(config.DEFAULT_OPTIONS) self.conf.register_opts(config.ODOO_OPTS, group=config.ODOO_GROUP) diff --git a/distil/tests/unit/erp/drivers/test_odoo.py b/distil/tests/unit/erp/drivers/test_odoo.py index 1b9a1f3..12cd420 100644 --- a/distil/tests/unit/erp/drivers/test_odoo.py +++ b/distil/tests/unit/erp/drivers/test_odoo.py @@ -14,6 +14,7 @@ # limitations under the License. from collections import namedtuple +from datetime import datetime import mock @@ -23,21 +24,30 @@ from distil.tests.unit import base REGION = namedtuple('Region', ['id']) PRODUCTS = [ - {'categ_id': [1, 'All products (.NET) / nz_1 / Compute'], - 'name_template': 'NZ-1.c1.c1r1', - 'lst_price': 0.00015, - 'default_code': 'hour', - 'description': '1 CPU, 1GB RAM'}, - {'categ_id': [2, 'All products (.NET) / nz_1 / Network'], - 'name_template': 'NZ-1.n1.router', - 'lst_price': 0.00025, - 'default_code': 'hour', - 'description': 'Router'}, - {'categ_id': [1, 'All products (.NET) / nz_1 / Block Storage'], - 'name_template': 'NZ-1.b1.volume', - 'lst_price': 0.00035, - 'default_code': 'hour', - 'description': 'Block storage'} + { + 'id': 1, + 'categ_id': [1, 'All products (.NET) / nz_1 / Compute'], + 'name_template': 'NZ-1.c1.c1r1', + 'lst_price': 0.00015, + 'default_code': 'hour', + 'description': '1 CPU, 1GB RAM' + }, + { + 'id': 2, + 'categ_id': [2, 'All products (.NET) / nz_1 / Network'], + 'name_template': 'NZ-1.n1.router', + 'lst_price': 0.00025, + 'default_code': 'hour', + 'description': 'Router' + }, + { + 'id': 3, + 'categ_id': [1, 'All products (.NET) / nz_1 / Block Storage'], + 'name_template': 'NZ-1.b1.volume', + 'lst_price': 0.00035, + 'default_code': 'hour', + 'description': 'Block storage' + } ] @@ -72,3 +82,110 @@ class TestOdooDriver(base.DistilTestCase): }, products ) + + @mock.patch('odoorpc.ODOO') + def test_get_invoices_without_details(self, mock_odoo): + start = datetime(2017, 3, 1) + end = datetime(2017, 5, 1) + fake_project = '123' + + odoodriver = odoo.OdooDriver(self.conf) + odoodriver.invoice.search.return_value = ['1', '2'] + odoodriver.odoo.execute.return_value = [ + {'date_invoice': '2017-03-31', 'amount_total': 10}, + {'date_invoice': '2017-04-30', 'amount_total': 20} + ] + + invoices = odoodriver.get_invoices(start, end, fake_project) + + self.assertEqual( + { + '2017-03-31': {'total_cost': 10}, + '2017-04-30': {'total_cost': 20} + }, + invoices + ) + + @mock.patch('odoorpc.ODOO') + @mock.patch('distil.erp.drivers.odoo.OdooDriver.get_products') + def test_get_invoices_with_details(self, mock_get_products, mock_odoo): + start = datetime(2017, 3, 1) + end = datetime(2017, 5, 1) + fake_project = '123' + + odoodriver = odoo.OdooDriver(self.conf) + odoodriver.invoice.search.return_value = ['1', '2'] + odoodriver.invoice_line.read.side_effect = [ + [ + { + 'name': 'resource1', + 'quantity': 100, + 'price_unit': 0.01, + 'uos_id': [1, 'Gigabyte-hour(s)'], + 'price_subtotal': 10, + 'product_id': [1, '[hour] NZ-POR-1.c1.c2r8'] + } + ], + [ + { + 'name': 'resource2', + 'quantity': 200, + 'price_unit': 0.01, + 'uos_id': [1, 'Gigabyte-hour(s)'], + 'price_subtotal': 20, + 'product_id': [1, '[hour] NZ-POR-1.c1.c2r8'] + } + ] + ] + odoodriver.odoo.execute.return_value = [ + {'id': 1, 'date_invoice': '2017-03-31', 'amount_total': 10}, + {'id': 2, 'date_invoice': '2017-04-30', 'amount_total': 20} + ] + + odoodriver.product_catagory_mapping = { + 1: 'Compute' + } + + invoices = odoodriver.get_invoices( + start, end, fake_project, detailed=True + ) + + self.assertEqual( + { + '2017-03-31': { + 'total_cost': 10, + 'details': { + 'Compute': { + 'total_cost': 10, + 'breakdown': { + 'NZ-POR-1.c1.c2r8': [{ + "cost": 10, + "quantity": 100, + "rate": 0.01, + "resource_name": "resource1", + "unit": "Gigabyte-hour(s)" + }] + } + } + } + }, + '2017-04-30': { + 'total_cost': 20, + 'details': { + 'Compute': { + 'total_cost': 20, + 'breakdown': { + 'NZ-POR-1.c1.c2r8': [{ + "cost": 20, + "quantity": 200, + "rate": 0.01, + "resource_name": "resource2", + "unit": "Gigabyte-hour(s)" + }] + } + } + } + } + }, + invoices + ) diff --git a/etc/policy.json.sample b/etc/policy.json.sample index 8da5daa..c6ab9a3 100644 --- a/etc/policy.json.sample +++ b/etc/policy.json.sample @@ -5,4 +5,5 @@ "rating:costs:get": "rule:context_is_admin", "rating:measurements:get": "rule:context_is_admin", + "rating:invoices:get": "rule:context_is_admin", }