Merge "Add /invoices rest api"

distil/api/v2.py

@@ -14,8 +14,8 @@
 # limitations under the License.
 
 from dateutil import parser
-
 from oslo_log import log
+from oslo_utils import strutils
 
 from distil import exceptions
 from distil.api import acl
@@ -24,6 +24,7 @@ from distil.common import constants
 from distil.common import openstack
 from distil.service.api.v2 import costs
 from distil.service.api.v2 import health
+from distil.service.api.v2 import invoices
 from distil.service.api.v2 import products
 
 LOG = log.getLogger(__name__)
@@ -31,6 +32,33 @@ LOG = log.getLogger(__name__)
 rest = api.Rest('v2', __name__)
 
 
+def _get_request_args():
+    cur_project_id = api.context.current().project_id
+    project_id = api.get_request_args().get('project_id', cur_project_id)
+
+    if not api.context.current().is_admin and cur_project_id != project_id:
+        raise exceptions.Forbidden()
+
+    start = api.get_request_args().get('start', None)
+    end = api.get_request_args().get('end', None)
+
+    detailed = strutils.bool_from_string(
+        api.get_request_args().get('detailed', False)
+    )
+
+    regions = api.get_request_args().get('regions', None)
+
+    params = {
+        'start': start,
+        'end': end,
+        'project_id': project_id,
+        'detailed': detailed,
+        'regions': regions
+    }
+
+    return params
+
+
 @rest.get('/health')
 def health_get():
     return api.render(health=health.get_health())
@@ -38,7 +66,9 @@ def health_get():
 
 @rest.get('/products')
 def products_get():
-    os_regions = api.get_request_args().get('regions', None)
+    params = _get_request_args()
+
+    os_regions = params.get('regions')
     regions = os_regions.split(',') if os_regions else []
 
     if regions:
@@ -54,28 +84,42 @@ def products_get():
     return api.render(products=products.get_products(regions))
 
 
-def _get_usage_args():
-    # NOTE(flwang): Get 'tenant' first for backward compatibility.
-    tenant_id = api.get_request_args().get('tenant', None)
-    project_id = api.get_request_args().get('project_id', tenant_id)
-    start = api.get_request_args().get('start', None)
-    end = api.get_request_args().get('end', None)
-    return project_id, start, end
-
-
 @rest.get('/costs')
 @acl.enforce("rating:costs:get")
 def costs_get():
-    project_id, start, end = _get_usage_args()
+    params = _get_request_args()
 
     # NOTE(flwang): Here using 'usage' instead of 'costs' for backward
     # compatibility.
-    return api.render(usage=costs.get_costs(project_id, start, end))
+    return api.render(
+        usage=costs.get_costs(
+            params['project_id'], params['start'], params['end']
+        )
+    )
 
 
 @rest.get('/measurements')
 @acl.enforce("rating:measurements:get")
 def measurements_get():
-    project_id, start, end = _get_usage_args()
+    params = _get_request_args()
 
-    return api.render(measurements=costs.get_usage(project_id, start, end))
+    return api.render(
+        measurements=costs.get_usage(
+            params['project_id'], params['start'], params['end']
+        )
+    )
+
+
+@rest.get('/invoices')
+@acl.enforce("rating:invoices:get")
+def invoices_get():
+    params = _get_request_args()
+
+    return api.render(
+        invoices.get_invoices(
+            params['project_id'],
+            params['start'],
+            params['end'],
+            detailed=params['detailed']
+        )
+    )

distil/common/general.py

@@ -24,6 +24,10 @@ import yaml
 from oslo_config import cfg
 from oslo_log import log as logging
 
+from distil.common import constants
+from distil.db import api as db_api
+from distil import exceptions
+
 CONF = cfg.CONF
 LOG = logging.getLogger(__name__)
 _TRANS_CONFIG = None
@@ -107,3 +111,45 @@ def convert_to(value, from_unit, to_unit):
 def get_process_identifier():
     """Gets current running process identifier."""
     return "%s_%s" % (socket.gethostname(), CONF.collector.partitioning_suffix)
+
+
+def convert_project_and_range(project_id, start, end):
+    now = datetime.utcnow()
+
+    try:
+        if start is not None:
+            try:
+                start = datetime.strptime(start, constants.iso_date)
+            except ValueError:
+                start = datetime.strptime(start, constants.iso_time)
+        else:
+            raise exceptions.DateTimeException(
+                message=(
+                    "Missing parameter:" +
+                    "'start' in format: y-m-d or y-m-dTH:M:S"))
+        if not end:
+            end = now
+        else:
+            try:
+                end = datetime.strptime(end, constants.iso_date)
+            except ValueError:
+                end = datetime.strptime(end, constants.iso_time)
+
+            if end > now:
+                end = now
+    except ValueError:
+        raise exceptions.DateTimeException(
+            message=(
+                "Missing parameter: " +
+                "'end' in format: y-m-d or y-m-dTH:M:S"))
+
+    if end <= start:
+        raise exceptions.DateTimeException(
+            message="End date must be greater than start.")
+
+    if not project_id:
+        raise exceptions.NotFoundException("Missing parameter: project_id")
+
+    valid_project = db_api.project_get(project_id)
+
+    return valid_project, start, end

distil/erp/driver.py

@@ -22,19 +22,6 @@ class BaseDriver(object):
     def __init__(self, conf):
         self.conf = conf
 
-    def get_salesOrders(self, project, start_at, end_at):
-        """List sales orders based on the given project and time range
-
-        :param project: project id
-        :param start_at: start time
-        :param end_at: end time
-        :returns List of sales order, if the time range only cover one month,
-                 then, the list will only contain 1 sale orders. Otherwise,
-                 the length of the list depends on the months number of the
-                 time range.
-        """
-        raise NotImplementedError()
-
     def get_products(self, regions=[]):
         """List products based o given regions
 
@@ -64,3 +51,14 @@ class BaseDriver(object):
         :param project: project
         """
         raise NotImplementedError()
+
+    def get_invoices(self, start, end, project_id, detailed=False):
+        """Get history invoices from ERP service given a time range.
+
+        :param start: Start time, a datetime object.
+        :param end: End time, a datetime object.
+        :param project_id: project ID.
+        :param detailed: If get detailed information or not.
+        :return: The history invoices information for each month.
+        """
+        raise NotImplementedError()

distil/erp/drivers/odoo.py

@@ -17,8 +17,10 @@ import collections
 import odoorpc
 from oslo_log import log
 
+from distil.common import cache
 from distil.common import openstack
 from distil.erp import driver
+from distil import exceptions
 
 LOG = log.getLogger(__name__)
 
@@ -60,8 +62,14 @@ class OdooDriver(driver.BaseDriver):
         self.pricelist = self.odoo.env['product.pricelist']
         self.product = self.odoo.env['product.product']
         self.category = self.odoo.env['product.category']
+        self.invoice = self.odoo.env['account.invoice']
+        self.invoice_line = self.odoo.env['account.invoice.line']
 
+        self.product_catagory_mapping = {}
+
+    @cache.memoize
     def get_products(self, regions=[]):
+        self.product_catagory_mapping.clear()
         odoo_regions = []
 
         if not regions:
@@ -96,6 +104,9 @@ class OdooDriver(driver.BaseDriver):
                         continue
 
                     category = product['categ_id'][1].split('/')[-1].strip()
+
+                    self.product_catagory_mapping[product['id']] = category
+
                     price = round(product['lst_price'], 5)
                     # NOTE(flwang): default_code is Internal Reference on
                     # Odoo GUI
@@ -140,3 +151,142 @@ class OdooDriver(driver.BaseDriver):
             return {}
 
         return prices
+
+    def _get_invoice_detail(self, invoice_id):
+        """Get invoice details.
+
+        Return details in the following format:
+        {
+          'catagory': {
+            'total_cost': xxx,
+            'breakdown': {
+              '<product_name>': [
+                {
+                  'resource_name': '',
+                  'quantity': '',
+                  'unit': '',
+                  'rate': '',
+                  'cost': ''
+                }
+              ],
+              '<product_name>': [
+                {
+                  'resource_name': '',
+                  'quantity': '',
+                  'unit': '',
+                  'rate': '',
+                  'cost': ''
+                }
+              ]
+            }
+          }
+        }
+        """
+        detail_dict = {}
+
+        invoice_lines_ids = self.invoice_line.search(
+            [('invoice_id', '=', invoice_id)]
+        )
+        invoice_lines = self.invoice_line.read(invoice_lines_ids)
+
+        for line in invoice_lines:
+            line_info = {
+                'resource_name': line['name'],
+                'quantity': line['quantity'],
+                'rate': line['price_unit'],
+                'unit': line['uos_id'][1],
+                'cost': round(line['price_subtotal'], 2)
+            }
+
+            # Original product is a string like "[hour] NZ-POR-1.c1.c2r8"
+            product = line['product_id'][1].split(']')[1].strip()
+            catagory = self.product_catagory_mapping[line['product_id'][0]]
+
+            if catagory not in detail_dict:
+                detail_dict[catagory] = {
+                    'total_cost': 0,
+                    'breakdown': collections.defaultdict(list)
+                }
+
+            detail_dict[catagory]['total_cost'] += line_info['cost']
+            detail_dict[catagory]['breakdown'][product].append(line_info)
+
+        return detail_dict
+
+    def get_invoices(self, start, end, project_id, detailed=False):
+        """Get history invoices from Odoo given a time range.
+
+        Return value is in the following format:
+        {
+          '<billing_date1>': {
+            'total_cost': 100,
+            'details': {
+                ...
+            }
+          },
+          '<billing_date2>': {
+            'total_cost': 200,
+            'details': {
+                ...
+            }
+          }
+        }
+
+        :param start: Start time, a datetime object.
+        :param end: End time, a datetime object.
+        :param project_id: project ID.
+        :param detailed: Get detailed information.
+        :return: The history invoices information for each month.
+        """
+        # Get invoices in time ascending order.
+        result = collections.OrderedDict()
+
+        try:
+            invoice_ids = self.invoice.search(
+                [
+                    ('date_invoice', '>=', str(start.date())),
+                    ('date_invoice', '<=', str(end.date())),
+                    ('comment', 'like', project_id)
+                ],
+                order='date_invoice'
+            )
+
+            if not len(invoice_ids):
+                LOG.debug('No history invoices returned from Odoo.')
+                return result
+
+            LOG.debug('Found invoices: %s' % invoice_ids)
+
+            # Convert ids from string to int.
+            ids = [int(i) for i in invoice_ids]
+
+            invoices = self.odoo.execute(
+                'account.invoice',
+                'read',
+                ids,
+                ['date_invoice', 'amount_total']
+            )
+            for v in invoices:
+                result[v['date_invoice']] = {
+                    'total_cost': round(v['amount_total'], 2)
+                }
+
+                if detailed:
+                    # Populate product catagory mapping first. This should be
+                    # quick since we cached get_products()
+                    if not self.product_catagory_mapping:
+                        self.get_products()
+
+                    details = self._get_invoice_detail(v['id'])
+                    result[v['date_invoice']].update({'details': details})
+        except Exception as e:
+            LOG.exception(
+                'Error occured when getting invoices from Odoo, '
+                'error: %s' % str(e)
+            )
+
+            raise exceptions.ERPException(
+                'Failed to get invoices from ERP server.'
+            )
+
+        return result

distil/erp/utils.py

@@ -22,6 +22,7 @@ from stevedore import driver
 from distil import exceptions
 
 LOG = log.getLogger(__name__)
+_ERP_DRIVER = None
 
 
 def load_erp_driver(conf):
@@ -31,17 +32,23 @@ def load_erp_driver(conf):
         driver. Must include a 'drivers' group.
     """
 
-    _invoke_args = [conf]
+    global _ERP_DRIVER
 
-    try:
+    if not _ERP_DRIVER:
-        mgr = driver.DriverManager('distil.erp',
+        _invoke_args = [conf]
-                                   conf.erp_driver,
-                                   invoke_on_load=True,
-                                   invoke_args=_invoke_args)
 
-        return mgr.driver
+        try:
+            mgr = driver.DriverManager('distil.erp',
+                                       conf.erp_driver,
+                                       invoke_on_load=True,
+                                       invoke_args=_invoke_args)
 
-    except Exception as exc:
+            _ERP_DRIVER = mgr.driver
-        LOG.exception(exc)
+
-        raise exceptions.InvalidDriver('Failed to load ERP driver'
+        except Exception as exc:
-                                       ' for {0}'.format(conf.erp_driver))
+            LOG.exception(exc)
+            raise exceptions.InvalidDriver(
+                'Failed to load ERP driver for {0}'.format(conf.erp_driver)
+            )
+
+    return _ERP_DRIVER

distil/exceptions.py

@@ -80,3 +80,8 @@ class Forbidden(DistilException):
 class InvalidDriver(DistilException):
     """A driver was not found or loaded."""
     message = _("Failed to load driver")
+
+
+class ERPException(DistilException):
+    code = 500
+    message = _("ERP server error.")

distil/service/api/v2/invoices.py

@@ -0,0 +1,54 @@
+# Copyright (c) 2017 Catalyst IT Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from oslo_config import cfg
+from oslo_log import log as logging
+
+from distil.erp import utils as erp_utils
+from distil.service.api.v2 import utils
+
+LOG = logging.getLogger(__name__)
+CONF = cfg.CONF
+
+
+def get_invoices(project_id, start, end, detailed=False):
+    project, start, end = utils.convert_project_and_range(
+        project_id, start, end)
+
+    LOG.info(
+        "Get invoices for %s(%s) in range: %s - %s" %
+        (project.id, project.name, start, end)
+    )
+
+    output = {
+        'start': str(start),
+        'end': str(end),
+        'project_name': project.name,
+        'project_id': project.id,
+        'invoices': {}
+    }
+
+    # Query from ERP.
+    erp_driver = erp_utils.load_erp_driver(CONF)
+    erp_invoices = erp_driver.get_invoices(
+        start,
+        end,
+        project.id,
+        detailed=detailed
+    )
+
+    output['invoices'] = erp_invoices
+
+    return output

distil/tests/unit/base.py

@@ -20,6 +20,7 @@ from oslotest import base
 from oslo_config import cfg
 from oslo_log import log
 
+from distil.common import cache
 from distil import context
 from distil import config
 from distil.db import api as db_api
@@ -37,6 +38,8 @@ class DistilTestCase(base.BaseTestCase):
         else:
             self.conf = cfg.CONF
 
+        cache.setup_cache(self.conf)
+
         self.conf.register_opts(config.DEFAULT_OPTIONS)
         self.conf.register_opts(config.ODOO_OPTS, group=config.ODOO_GROUP)
 

distil/tests/unit/erp/drivers/test_odoo.py

@@ -14,6 +14,7 @@
 # limitations under the License.
 
 from collections import namedtuple
+from datetime import datetime
 
 import mock
 
@@ -23,21 +24,30 @@ from distil.tests.unit import base
 REGION = namedtuple('Region', ['id'])
 
 PRODUCTS = [
-    {'categ_id': [1, 'All products (.NET) / nz_1 / Compute'],
+    {
-     'name_template': 'NZ-1.c1.c1r1',
+        'id': 1,
-     'lst_price': 0.00015,
+        'categ_id': [1, 'All products (.NET) / nz_1 / Compute'],
-     'default_code': 'hour',
+        'name_template': 'NZ-1.c1.c1r1',
-     'description': '1 CPU, 1GB RAM'},
+        'lst_price': 0.00015,
-    {'categ_id': [2, 'All products (.NET) / nz_1 / Network'],
+        'default_code': 'hour',
-     'name_template': 'NZ-1.n1.router',
+        'description': '1 CPU, 1GB RAM'
-     'lst_price': 0.00025,
+    },
-     'default_code': 'hour',
+    {
-     'description': 'Router'},
+        'id': 2,
-    {'categ_id': [1, 'All products (.NET) / nz_1 / Block Storage'],
+        'categ_id': [2, 'All products (.NET) / nz_1 / Network'],
-     'name_template': 'NZ-1.b1.volume',
+        'name_template': 'NZ-1.n1.router',
-     'lst_price': 0.00035,
+        'lst_price': 0.00025,
-     'default_code': 'hour',
+        'default_code': 'hour',
-     'description': 'Block storage'}
+        'description': 'Router'
+    },
+    {
+        'id': 3,
+        'categ_id': [1, 'All products (.NET) / nz_1 / Block Storage'],
+        'name_template': 'NZ-1.b1.volume',
+        'lst_price': 0.00035,
+        'default_code': 'hour',
+        'description': 'Block storage'
+    }
 ]
 
 
@@ -72,3 +82,110 @@ class TestOdooDriver(base.DistilTestCase):
             },
             products
         )
+
+    @mock.patch('odoorpc.ODOO')
+    def test_get_invoices_without_details(self, mock_odoo):
+        start = datetime(2017, 3, 1)
+        end = datetime(2017, 5, 1)
+        fake_project = '123'
+
+        odoodriver = odoo.OdooDriver(self.conf)
+        odoodriver.invoice.search.return_value = ['1', '2']
+        odoodriver.odoo.execute.return_value = [
+            {'date_invoice': '2017-03-31', 'amount_total': 10},
+            {'date_invoice': '2017-04-30', 'amount_total': 20}
+        ]
+
+        invoices = odoodriver.get_invoices(start, end, fake_project)
+
+        self.assertEqual(
+            {
+                '2017-03-31': {'total_cost': 10},
+                '2017-04-30': {'total_cost': 20}
+            },
+            invoices
+        )
+
+    @mock.patch('odoorpc.ODOO')
+    @mock.patch('distil.erp.drivers.odoo.OdooDriver.get_products')
+    def test_get_invoices_with_details(self, mock_get_products, mock_odoo):
+        start = datetime(2017, 3, 1)
+        end = datetime(2017, 5, 1)
+        fake_project = '123'
+
+        odoodriver = odoo.OdooDriver(self.conf)
+        odoodriver.invoice.search.return_value = ['1', '2']
+        odoodriver.invoice_line.read.side_effect = [
+            [
+                {
+                    'name': 'resource1',
+                    'quantity': 100,
+                    'price_unit': 0.01,
+                    'uos_id': [1, 'Gigabyte-hour(s)'],
+                    'price_subtotal': 10,
+                    'product_id': [1, '[hour] NZ-POR-1.c1.c2r8']
+                }
+            ],
+            [
+                {
+                    'name': 'resource2',
+                    'quantity': 200,
+                    'price_unit': 0.01,
+                    'uos_id': [1, 'Gigabyte-hour(s)'],
+                    'price_subtotal': 20,
+                    'product_id': [1, '[hour] NZ-POR-1.c1.c2r8']
+                }
+            ]
+        ]
+        odoodriver.odoo.execute.return_value = [
+            {'id': 1, 'date_invoice': '2017-03-31', 'amount_total': 10},
+            {'id': 2, 'date_invoice': '2017-04-30', 'amount_total': 20}
+        ]
+
+        odoodriver.product_catagory_mapping = {
+            1: 'Compute'
+        }
+
+        invoices = odoodriver.get_invoices(
+            start, end, fake_project, detailed=True
+        )
+
+        self.assertEqual(
+            {
+                '2017-03-31': {
+                    'total_cost': 10,
+                    'details': {
+                        'Compute': {
+                            'total_cost': 10,
+                            'breakdown': {
+                                'NZ-POR-1.c1.c2r8': [{
+                                    "cost": 10,
+                                    "quantity": 100,
+                                    "rate": 0.01,
+                                    "resource_name": "resource1",
+                                    "unit": "Gigabyte-hour(s)"
+                                }]
+                            }
+                        }
+                    }
+                },
+                '2017-04-30': {
+                    'total_cost': 20,
+                    'details': {
+                        'Compute': {
+                            'total_cost': 20,
+                            'breakdown': {
+                                'NZ-POR-1.c1.c2r8': [{
+                                    "cost": 20,
+                                    "quantity": 200,
+                                    "rate": 0.01,
+                                    "resource_name": "resource2",
+                                    "unit": "Gigabyte-hour(s)"
+                                }]
+                            }
+                        }
+                    }
+                }
+            },
+            invoices
+        )

etc/policy.json.sample

@@ -5,4 +5,5 @@
 
     "rating:costs:get": "rule:context_is_admin",
     "rating:measurements:get": "rule:context_is_admin",
+    "rating:invoices:get": "rule:context_is_admin",
 }