add cpu_util based on sample and revise nic metrics

Change-Id: Idcd46ce2d50c8d3d135e52153f9da4250c5b5ae7

fix ci errors

Change-Id: I2c75fdb1698a6e87f04951dcbe7b0dc5e2bd95e5

update unit test to create test directory

Change-Id: Ie650de50421be2a5ffc12114a72184b2d88c3972

fix use user home if cannot write to /var/lib/ceilometer

Change-Id: I3083c8004a325506e00fe6d57c3be47f589126f2

revise path handling for mock

Change-Id: I9f072383c954b962ce6ca9daec879832f37672be

ceilometer_zvm/compute/virt/zvm/inspector.py

@@ -1,4 +1,4 @@
-# Copyright 2015 IBM Corp.
+# Copyright 2015, 2020 IBM Corp.
 #
 #    Licensed under the Apache License, Version 2.0 (the "License"); you may
 #    not use this file except in compliance with the License. You may obtain
@@ -14,6 +14,7 @@
 
 
 import six
+import time
 
 from ceilometer.compute.virt import inspector as virt_inspector
 from ceilometer.i18n import _
@@ -21,7 +22,9 @@ from ceilometer_zvm.compute.virt.zvm import exception
 from ceilometer_zvm.compute.virt.zvm import utils as zvmutils
 from oslo_config import cfg
 from oslo_utils import units
+from oslo_log import log
 
+LOG = log.getLogger(__name__)
 
 zvm_opts = [
     cfg.URIOpt('zvm_cloud_connector_url',
@@ -50,6 +53,7 @@ class ZVMInspector(virt_inspector.Inspector):
 
     def __init__(self, conf):
         super(ZVMInspector, self).__init__(conf)
+        self.cache = {}
         self.conf.register_opts(zvm_opts)
         self._reqh = zvmutils.zVMConnectorRequestHandler(
             self.conf)
@@ -64,31 +68,65 @@ class ZVMInspector(virt_inspector.Inspector):
                 fref=None,
                 parameters=None,
                 rx_bytes=nic['nic_rx'],
-                rx_bytes_delta=nic['nic_rx_delta'],
                 rx_packets=nic['nic_fr_rx'],
-                rx_errors=None,
+                rx_bytes_delta=nic['nic_rx_delta'],
+                rx_errors=nic['nic_fr_rx_err'],
                 rx_drop=None,
                 tx_bytes=nic['nic_tx'],
-                tx_bytes_delta=nic['nic_tx_delta'],
                 tx_packets=nic['nic_fr_tx'],
-                tx_errors=None,
+                tx_bytes_delta=nic['nic_tx_delta'],
+                tx_errors=nic['nic_fr_tx_err'],
                 tx_drop=None
             )
 
     def inspect_instance(self, instance, duration):
         inst_stats = self._inspect_inst_data(instance, 'stats')
+        LOG.debug("instance stats:%s", str(inst_stats))
         cpu_number = inst_stats['guest_cpus']
         used_cpu_time = (inst_stats['used_cpu_time_us'] * units.k)
-        used_mem_mb = inst_stats['used_mem_kb'] // units.Ki
+        cpu_util = self._calc_cpu_util(instance, inst_stats)
+        used_mem_mb = inst_stats['used_mem_kb'] / units.Ki
         # Construct the final result
         return virt_inspector.InstanceStats(cpu_number=cpu_number,
                                             cpu_time=used_cpu_time,
-                                            memory_usage=used_mem_mb
+                                            memory_usage=used_mem_mb,
+                                            cpu_util=cpu_util
                                             )
 
+    def _save_inst_data(self, instance, inst_stats):
+        inst_name = zvmutils.get_inst_name(instance).upper()
+        data = inst_stats
+        data['time_stamp'] = time.time()
+        self.cache[inst_name] = data
+
+    def _load_inst_data(self, instance):
+        inst_name = zvmutils.get_inst_name(instance).upper()
+        try:
+            return self.cache[inst_name]
+        except KeyError:
+            LOG.info('no cache for instance %s', inst_name)
+            return None
+
+    def _calc_cpu_util(self, instance, inst_stats):
+        previous_stats = self._load_inst_data(instance)
+        if previous_stats:
+            current_time = time.time()
+            previous_time = previous_stats['time_stamp']
+            cpu_util = (inst_stats['used_cpu_time_us']/inst_stats['guest_cpus'] - \
+                previous_stats['used_cpu_time_us']/previous_stats['guest_cpus']) / \
+                ((current_time-previous_time)*units.k)
+            self._save_inst_data(instance, inst_stats)
+            return cpu_util
+        else:
+            cpu_util = inst_stats['used_cpu_time_us']/ \
+                inst_stats['elapsed_cpu_time_us']
+            self._save_inst_data(instance, inst_stats)
+            return cpu_util
+
     def _inspect_inst_data(self, instance, inspect_type):
         inspect_data = {}
-        inst_name = zvmutils.get_inst_name(instance)
+        # Make sure the inst name in upper case
+        inst_name = zvmutils.get_inst_name(instance).upper()
         msg_shutdown = _("Can not get vm info in shutdown state "
                     "for %s") % inst_name
         msg_notexist = _("Can not get vm info for %s, vm not exist"
@@ -109,7 +147,7 @@ class ZVMInspector(virt_inspector.Inspector):
             raise virt_inspector.NoDataException(msg_nodata)
 
         # Check the inst data is in the returned result
-        index_key = inst_name.upper()
+        index_key = inst_name
         if index_key not in inspect_data:
             # Check the reason: shutdown or not exist or other error
             power_stat = ''

ceilometer_zvm/tests/unit/compute/virt/zvm/test_inspector.py

@@ -1,4 +1,4 @@
-# Copyright 2015 IBM Corp.
+# Copyright 2015, 2020 IBM Corp.
 #
 #    Licensed under the Apache License, Version 2.0 (the "License"); you may
 #    not use this file except in compliance with the License. You may obtain
@@ -75,7 +75,7 @@ class TestZVMInspector(unittest.TestCase):
         self.assertIsInstance(rdata, virt_inspector.InstanceStats)
         self.assertEqual(rdata.cpu_number, 1)
         self.assertEqual(rdata.cpu_time, 7185838000)
-        self.assertEqual(rdata.memory_usage, 381)
+        self.assertEqual(rdata.memory_usage, 381.0859375)
 
     @mock.patch("ceilometer_zvm.compute.virt.zvm.inspector.ZVMInspector."
                 "_inspect_inst_data")

requirements.txt

@@ -4,6 +4,6 @@ oslo.config>=2.1.0                     # Apache-2.0
 oslo.i18n>=1.5.0                        # Apache-2.0
 oslo.log>=1.8.0                         # Apache-2.0
 oslo.service>=0.6.0                     # Apache-2.0
-pbr<2.0,>=1.4
+pbr==2.0.0
 six>=1.9.0
 zVMCloudConnector>=0.3.5  # Apache 2.0 License