VMware vSphere support: Memory Usage

1) Added vSphere Hypervisor inspector.
2) Implemented memory usage pollster to generate memory usage samples
for vSphere VMs.

Change-Id: I2aab2fdbc9d0ff4ccf8553bac19c44e463e6595a
Implements: blueprint vmware-vcenter-server

ceilometer/compute/pollsters/memory.py

@@ -0,0 +1,52 @@
+# Copyright (c) 2014 VMware, Inc.
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+from ceilometer.compute import plugin
+from ceilometer.compute.pollsters import util
+from ceilometer.compute.virt import inspector as virt_inspector
+from ceilometer.openstack.common.gettextutils import _  # noqa
+from ceilometer.openstack.common import log
+from ceilometer import sample
+
+LOG = log.getLogger(__name__)
+
+
+class MemoryUsagePollster(plugin.ComputePollster):
+
+    def get_samples(self, manager, cache, resources):
+        for instance in resources:
+            LOG.debug(_('Checking memory usage for instance %s'), instance.id)
+            try:
+                memory_info = manager.inspector.inspect_memory_usage(instance)
+                LOG.debug(_("MEMORY USAGE: %(instance)s %(usage)f"),
+                          ({'instance': instance.__dict__,
+                            'usage': memory_info.usage}))
+                yield util.make_sample_from_instance(
+                    instance,
+                    name='memory.usage',
+                    type=sample.TYPE_GAUGE,
+                    unit='MB',
+                    volume=memory_info.usage,
+                )
+            except virt_inspector.InstanceNotFoundException as err:
+                # Instance was deleted while getting samples. Ignore it.
+                LOG.debug(_('Exception while getting samples %s'), err)
+            except NotImplementedError:
+                # Selected inspector does not implement this pollster.
+                LOG.debug(_('Obtaining Memory Usage is not implemented for %s'
+                            ), manager.inspector.__class__.__name__)
+            except Exception as err:
+                LOG.error(_('Could not get Memory Usage for %(id)s: %(e)s'), (
+                          {'id': instance.id, 'e': err}))

ceilometer/compute/virt/inspector.py

@@ -53,6 +53,12 @@ Instance = collections.namedtuple('Instance', ['name', 'UUID'])
 #
 CPUStats = collections.namedtuple('CPUStats', ['number', 'time'])
 
+# Named tuple representing Memory usage statistics.
+#
+# usage: Amount of memory used
+#
+MemoryUsageStats = collections.namedtuple('MemoryUsageStats', ['usage'])
+
 
 # Named tuple representing vNICs.
 #
@@ -143,6 +149,14 @@ class Inspector(object):
         """
         raise NotImplementedError()
 
+    def inspect_memory_usage(self, instance):
+        """Inspect the memory usage statistics for an instance.
+
+        :param instance: the target instance
+        :return: the amount of memory used
+        """
+        raise NotImplementedError()
+
 
 def get_hypervisor_inspector():
     try:

ceilometer/compute/virt/vmware/inspector.py

@@ -0,0 +1,97 @@
+# Copyright (c) 2014 VMware, Inc.
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+"""Implementation of Inspector abstraction for VMware vSphere"""
+
+from oslo.config import cfg
+from oslo.vmware import api
+from oslo.vmware import vim
+
+from ceilometer.compute.virt import inspector as virt_inspector
+from ceilometer.compute.virt.vmware import vsphere_operations
+from ceilometer.openstack.common import units
+
+
+opt_group = cfg.OptGroup(name='vmware',
+                         title='Options for VMware')
+
+OPTS = [
+    cfg.StrOpt('host_ip',
+               default='',
+               help='IP address of the VMware Vsphere host'),
+    cfg.StrOpt('host_username',
+               default='',
+               help='Username of VMware Vsphere'),
+    cfg.StrOpt('host_password',
+               default='',
+               help='Password of VMware Vsphere'),
+    cfg.IntOpt('api_retry_count',
+               default=10,
+               help='Number of times a VMware Vsphere API must be retried'),
+    cfg.FloatOpt('task_poll_interval',
+                 default=0.5,
+                 help='Sleep time in seconds for polling an ongoing async '
+                 'task'),
+]
+
+cfg.CONF.register_group(opt_group)
+cfg.CONF.register_opts(OPTS, group=opt_group)
+
+VC_AVERAGE_MEMORY_CONSUMED_CNTR = 'mem:consumed:average'
+
+
+def get_api_session():
+    hostIp = cfg.CONF.vmware.host_ip
+    wsdl_loc = vim.Vim._get_wsdl_loc("https", hostIp)
+    api_session = api.VMwareAPISession(
+        hostIp,
+        cfg.CONF.vmware.host_username,
+        cfg.CONF.vmware.host_password,
+        cfg.CONF.vmware.api_retry_count,
+        cfg.CONF.vmware.task_poll_interval,
+        wsdl_loc=wsdl_loc)
+    return api_session
+
+
+class VsphereInspector(virt_inspector.Inspector):
+
+    def __init__(self):
+        super(VsphereInspector, self).__init__()
+        self._ops = vsphere_operations.VsphereOperations(
+            get_api_session(), 1000)
+
+    def inspect_instances(self):
+        raise NotImplementedError()
+
+    def inspect_cpus(self, instance_name):
+        raise NotImplementedError()
+
+    def inspect_vnics(self, instance_name):
+        raise NotImplementedError()
+
+    def inspect_disks(self, instance_name):
+        raise NotImplementedError()
+
+    def inspect_memory_usage(self, instance):
+        vm_moid = self._ops.get_vm_moid(instance.id)
+        if vm_moid is None:
+            raise virt_inspector.InstanceNotFoundException(
+                _('VM %s not found in VMware Vsphere') % instance.id)
+        mem_counter_id = self._ops.get_perf_counter_id(
+            VC_AVERAGE_MEMORY_CONSUMED_CNTR)
+        memory = self._ops.query_vm_aggregate_stats(vm_moid, mem_counter_id)
+        #Stat provided from VMware Vsphere is in Bytes, converting it to MB.
+        memory = memory / (units.Mi)
+        return virt_inspector.MemoryUsageStats(usage=memory)

ceilometer/tests/compute/pollsters/test_memory.py

@@ -0,0 +1,55 @@
+# Copyright (c) 2014 VMware, Inc.
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+import mock
+import six
+
+from ceilometer.compute import manager
+from ceilometer.compute.pollsters import memory
+from ceilometer.compute.virt import inspector as virt_inspector
+from ceilometer.tests.compute.pollsters import base
+
+
+class TestMemoryPollster(base.TestPollsterBase):
+
+    def setUp(self):
+        super(TestMemoryPollster, self).setUp()
+
+    @mock.patch('ceilometer.pipeline.setup_pipeline', mock.MagicMock())
+    def test_get_samples(self):
+        next_value = iter((
+            virt_inspector.MemoryUsageStats(usage=1.0),
+            virt_inspector.MemoryUsageStats(usage=2.0),
+        ))
+
+        def inspect_memory_usage(name):
+            return six.next(next_value)
+
+        self.inspector.inspect_memory_usage = \
+            mock.Mock(side_effect=inspect_memory_usage)
+
+        mgr = manager.AgentManager()
+        pollster = memory.MemoryUsagePollster()
+
+        def _verify_memory_metering(expected_memory_mb):
+            cache = {}
+            samples = list(pollster.get_samples(mgr, cache, [self.instance]))
+            self.assertEqual(1, len(samples))
+            self.assertEqual(set(['memory.usage']),
+                             set([s.name for s in samples]))
+            self.assertEqual(expected_memory_mb, samples[0].volume)
+
+        _verify_memory_metering(1.0)
+        _verify_memory_metering(2.0)

ceilometer/tests/compute/virt/vmware/test_inspector.py

@@ -0,0 +1,61 @@
+# Copyright (c) 2014 VMware, Inc.
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+"""
+Tests for VMware Vsphere inspector.
+"""
+
+import mock
+
+from oslo.vmware import api
+
+from ceilometer.compute.virt import inspector as virt_inspector
+from ceilometer.compute.virt.vmware import inspector as vsphere_inspector
+from ceilometer.openstack.common import test
+
+
+class TestVsphereInspection(test.BaseTestCase):
+
+    def setUp(self):
+        api_session = api.VMwareAPISession("test_server", "test_user",
+                                           "test_password", 0, None,
+                                           create_session=False)
+        api_session._vim = mock.MagicMock()
+        vsphere_inspector.get_api_session = mock.Mock(
+            return_value=api_session)
+        self._inspector = vsphere_inspector.VsphereInspector()
+        self._inspector._ops = mock.MagicMock()
+
+        super(TestVsphereInspection, self).setUp()
+
+    def test_inspect_memory_usage(self):
+        fake_instance_moid = 'fake_instance_moid'
+        fake_instance_id = 'fake_instance_id'
+        fake_perf_counter_id = 'fake_perf_counter_id'
+        fake_memory_value = 1048576.0
+        fake_stat = virt_inspector.MemoryUsageStats(usage=1.0)
+
+        def construct_mock_instance_object(fake_instance_id):
+            instance_object = mock.MagicMock()
+            instance_object.id = fake_instance_id
+            return instance_object
+
+        fake_instance = construct_mock_instance_object(fake_instance_id)
+        self._inspector._ops.get_vm_moid.return_value = fake_instance_moid
+        self._inspector._ops.get_perf_counter_id.return_value = \
+            fake_perf_counter_id
+        self._inspector._ops.query_vm_aggregate_stats.return_value = \
+            fake_memory_value
+        memory_stat = self._inspector.inspect_memory_usage(fake_instance)
+        self.assertEqual(fake_stat, memory_stat)

doc/source/measurements.rst

@@ -70,7 +70,8 @@ Name                           Type        Unit       Resource  Origin        No
 =============================  ==========  =========  ========  ============  ==================================================================
 instance                       Gauge       instance   inst ID   both          Existence of instance
 instance:<type>                Gauge       instance   inst ID   both          Existence of instance <type> (openstack types)
-memory                         Gauge       MB         inst ID   notification  Volume of RAM in MB
+memory                         Gauge       MB         inst ID   notification  Volume of RAM allocated in MB
+memory.usage                   Gauge       MB         inst ID   pollster      Volume of RAM used in MB
 cpu                            Cumulative  ns         inst ID   pollster      CPU time used
 cpu_util                       Gauge       %          inst ID   pollster      Average CPU utilisation
 vcpus                          Gauge       vcpu       inst ID   notification  Number of VCPUs

etc/ceilometer/ceilometer.conf.sample

@@ -982,3 +982,27 @@
 #key_file=<None>
 
 
+[vmware]
+
+#
+# Options defined in ceilometer.compute.virt.vmware.inspector
+#
+
+# IP address of the VMware Vsphere host (string value)
+#host_ip=
+
+# Username of VMware Vsphere (string value)
+#host_username=
+
+# Password of VMware Vsphere (string value)
+#host_password=
+
+# Number of times a VMware Vsphere API must be retried
+# (integer value)
+#api_retry_count=10
+
+# Sleep time in seconds for polling an ongoing async task
+# (floating point value)
+#task_poll_interval=0.5
+
+

setup.cfg

@@ -79,6 +79,7 @@ ceilometer.poll.compute =
     network.outgoing.packets = ceilometer.compute.pollsters.net:OutgoingPacketsPollster
     instance = ceilometer.compute.pollsters.instance:InstancePollster
     instance_flavor = ceilometer.compute.pollsters.instance:InstanceFlavorPollster
+    memory.usage = ceilometer.compute.pollsters.memory:MemoryUsagePollster
 
 ceilometer.poll.central =
     ip.floating = ceilometer.network.floatingip:FloatingIPPollster
@@ -139,6 +140,7 @@ ceilometer.storage =
 ceilometer.compute.virt =
     libvirt = ceilometer.compute.virt.libvirt.inspector:LibvirtInspector
     hyperv = ceilometer.compute.virt.hyperv.inspector:HyperVInspector
+    vsphere = ceilometer.compute.virt.vmware.inspector:VsphereInspector
 
 ceilometer.hardware.inspectors =
     snmp = ceilometer.hardware.inspector.snmp:SNMPInspector