Download image API to bypass vCenter

This patch introduces a new API to download images. This new API
provides a new signature which gives the ability to bypass vCenter
when downloading images.

Change-Id: Id3cf3f03e1d0622a99573bd090b8ea6b397765e0

oslo/vmware/constants.py

@@ -21,3 +21,12 @@ Shared constants across the VMware ecosystem.
 # Datacenter path for HTTP access to datastores if the target server is an ESX/
 # ESXi system: http://goo.gl/B5Htr8 for more information.
 ESX_DATACENTER_PATH = 'ha-datacenter'
+
+# User Agent for HTTP requests between OpenStack and vCenter.
+USER_AGENT = 'OpenStack-ESX-Adapter'
+
+# Key of the cookie header when using a SOAP session.
+SOAP_COOKIE_KEY = 'vmware_soap_session'
+
+# Key of the cookie header when using a CGI session.
+CGI_COOKIE_KEY = 'vmware_cgi_ticket'

oslo/vmware/image_transfer.py

@@ -26,8 +26,11 @@ from eventlet import queue
 from eventlet import timeout
 
 from oslo.vmware._i18n import _
+from oslo.vmware import constants
 from oslo.vmware import exceptions
+from oslo.vmware.objects import datastore as ds_obj
 from oslo.vmware import rw_handles
+from oslo.vmware import vim_util
 
 
 LOG = logging.getLogger(__name__)
@@ -391,6 +394,41 @@ def _start_transfer(context, timeout_secs, read_file_handle, max_data_size,
             write_file_handle.close()
 
 
+def download_image(image, image_meta, session, datastore, rel_path,
+                   bypass=True, timeout_secs=7200):
+    """Transfer an image to a datastore.
+
+    :param image: file-like iterator
+    :param image_meta: image metadata
+    :param session: VMwareAPISession object
+    :param datastore: Datastore object
+    :param rel_path: path where the file will be stored in the datastore
+    :param bypass: if set to True, bypass vCenter to download the image
+    :param timeout_secs: time in seconds to wait for the xfer to complete
+    """
+    image_size = int(image_meta['size'])
+    method = 'PUT'
+    if bypass:
+        hosts = datastore.get_connected_hosts(session)
+        host = ds_obj.Datastore.choose_host(hosts)
+        host_name = session.invoke_api(vim_util, 'get_object_property',
+                                       session.vim, host, 'name')
+        ds_url = datastore.build_url(session._scheme, host_name, rel_path,
+                                     constants.ESX_DATACENTER_PATH)
+        cookie = ds_url.get_transfer_ticket(session, method)
+        conn = ds_url.connect(method, image_size, cookie)
+    else:
+        ds_url = datastore.build_url(session._scheme, session._host, rel_path)
+        cookie = '%s=%s' % (constants.SOAP_COOKIE_KEY,
+                            session.vim.get_http_cookie().strip("\""))
+        conn = ds_url.connect(method, image_size, cookie)
+        conn.write = conn.send
+
+    read_handle = rw_handles.ImageReadHandle(image)
+    _start_transfer(None, timeout_secs, read_handle, image_size,
+                    write_file_handle=conn)
+
+
 def download_flat_image(context, timeout_secs, image_service, image_id,
                         **kwargs):
     """Download flat image from the image service to VMware server.

oslo/vmware/objects/datacenter.py

@@ -0,0 +1,27 @@
+# Copyright (c) 2014 VMware, Inc.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+from oslo.vmware._i18n import _
+
+
+class Datacenter(object):
+
+    def __init__(self, ref, name):
+        """Datacenter object holds ref and name together for convenience."""
+        if name is None:
+            raise ValueError(_("Datacenter name cannot be None"))
+        if ref is None:
+            raise ValueError(_("Datacenter reference cannot be None"))
+        self.ref = ref
+        self.name = name

oslo/vmware/objects/datastore.py

@@ -12,23 +12,33 @@
 #    License for the specific language governing permissions and limitations
 #    under the License.
 
+import httplib
+import logging
 import posixpath
+import random
 
 import six.moves.urllib.parse as urlparse
 
 from oslo.vmware._i18n import _
+from oslo.vmware import constants
+from oslo.vmware import exceptions
 from oslo.vmware import vim_util
 
+LOG = logging.getLogger(__name__)
+
 
 class Datastore(object):
 
-    def __init__(self, ref, name, capacity=None, freespace=None):
+    def __init__(self, ref, name, capacity=None, freespace=None,
+                 type=None, datacenter=None):
         """Datastore object holds ref and name together for convenience.
 
         :param ref: a vSphere reference to a datastore
         :param name: vSphere unique name for this datastore
         :param capacity: (optional) capacity in bytes of this datastore
         :param freespace: (optional) free space in bytes of datastore
+        :param type: (optional) datastore type
+        :param datacenter: (optional) oslo.vmware Datacenter object
         """
         if name is None:
             raise ValueError(_("Datastore name cannot be None"))
@@ -40,26 +50,12 @@ class Datastore(object):
             if capacity < freespace:
                 raise ValueError(_("Capacity is smaller than free space"))
 
-        self._ref = ref
+        self.ref = ref
-        self._name = name
+        self.name = name
-        self._capacity = capacity
+        self.capacity = capacity
-        self._freespace = freespace
+        self.freespace = freespace
-
+        self.type = type
-    @property
+        self.datacenter = datacenter
-    def ref(self):
-        return self._ref
-
-    @property
-    def name(self):
-        return self._name
-
-    @property
-    def capacity(self):
-        return self._capacity
-
-    @property
-    def freespace(self):
-        return self._freespace
 
     def build_path(self, *paths):
         """Constructs and returns a DatastorePath.
@@ -68,7 +64,23 @@ class Datastore(object):
                       to the root directory of the datastore
         :return: a DatastorePath object
         """
-        return DatastorePath(self._name, *paths)
+        return DatastorePath(self.name, *paths)
+
+    def build_url(self, scheme, server, rel_path, datacenter_name=None):
+        """Constructs and returns a DatastoreURL.
+
+        :param scheme: scheme of the URL (http, https).
+        :param server: hostname or ip
+        :param rel_path: relative path of the file on the datastore
+        :param datacenter_name: (optional) datacenter name
+        :return: a DatastoreURL object
+        """
+        if self.datacenter is None and datacenter_name is None:
+            raise ValueError(_("datacenter must be set to build url"))
+        if datacenter_name is None:
+            datacenter_name = self.datacenter.name
+        return DatastoreURL(scheme, server, rel_path, datacenter_name,
+                            self.name)
 
     def __str__(self):
         return '[%s]' % self._name
@@ -118,6 +130,11 @@ class Datastore(object):
 
         return writable and mounted and accessible
 
+    @staticmethod
+    def choose_host(hosts):
+        i = random.randrange(0, len(hosts))
+        return hosts[i]
+
 
 class DatastorePath(object):
 
@@ -227,6 +244,9 @@ class DatastoreURL(object):
         self._path = path
         self._datacenter_path = datacenter_path
         self._datastore_name = datastore_name
+        params = {'dcPath': self._datacenter_path,
+                  'dsName': self._datastore_name}
+        self._query = urlparse.urlencode(params)
 
     @classmethod
     def urlparse(cls, url):
@@ -258,8 +278,41 @@ class DatastoreURL(object):
         return self._datastore_name
 
     def __str__(self):
-        params = {'dcPath': self._datacenter_path,
-                  'dsName': self._datastore_name}
-        query = urlparse.urlencode(params)
         return '%s://%s/folder/%s?%s' % (self._scheme, self._server,
-                                         self.path, query)
+                                         self.path, self._query)
+
+    def connect(self, method, content_length, cookie):
+        try:
+            if self._scheme == 'http':
+                conn = httplib.HTTPConnection(self._server)
+            elif self._scheme == 'https':
+                conn = httplib.HTTPSConnection(self._server)
+            else:
+                excep_msg = _("Invalid scheme: %s.") % self._scheme
+                LOG.error(excep_msg)
+                raise ValueError(excep_msg)
+            conn.putrequest(method, '/folder/%s?%s' % (self.path, self._query))
+            conn.putheader('User-Agent', constants.USER_AGENT)
+            conn.putheader('Content-Length', content_length)
+            conn.putheader('Cookie', cookie)
+            conn.endheaders()
+            LOG.debug("Created HTTP connection to transfer the file with "
+                      "URL = %s.", str(self))
+            return conn
+        except (httplib.InvalidURL, httplib.CannotSendRequest,
+                httplib.CannotSendHeader) as excep:
+            excep_msg = _("Error occurred while creating HTTP connection "
+                          "to write to file with URL = %s.") % str(self)
+        LOG.exception(excep_msg)
+        raise exceptions.VimConnectionException(excep_msg, excep)
+
+    def get_transfer_ticket(self, session, method):
+        client_factory = session.vim.client.factory
+        spec = vim_util.get_http_service_request_spec(client_factory, method,
+                                                      str(self))
+        ticket = session.invoke_api(
+            session.vim,
+            'AcquireGenericServiceTicket',
+            session.vim.service_content.sessionManager,
+            spec=spec)
+        return '%s="%s"' % (constants.CGI_COOKIE_KEY, ticket.id)

oslo/vmware/vim_util.py

@@ -470,3 +470,17 @@ def get_inventory_path(vim, entity_ref, max_objects=100):
     if entity_name is None:
         entity_name = ""
     return '%s%s' % (path, entity_name)
+
+
+def get_http_service_request_spec(client_factory, method, uri):
+    """Build a HTTP service request spec.
+
+    :param client_factory: factory to get API input specs
+    :param method: HTTP method (GET, POST, PUT)
+    :param uri: target URL
+    """
+    http_service_request_spec = client_factory.create(
+        'ns0:SessionManagerHttpServiceRequestSpec')
+    http_service_request_spec.method = method
+    http_service_request_spec.url = uri
+    return http_service_request_spec

tests/objects/test_datacenter.py

@@ -0,0 +1,30 @@
+# Copyright (c) 2014 VMware, Inc.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+import mock
+
+from oslo.vmware.objects import datacenter
+from tests import base
+
+
+class DatacenterTestCase(base.TestCase):
+
+    """Test the Datacenter object."""
+
+    def test_dc(self):
+        self.assertRaises(ValueError, datacenter.Datacenter, None, 'dc-1')
+        self.assertRaises(ValueError, datacenter.Datacenter, mock.Mock(), None)
+        dc = datacenter.Datacenter('ref', 'name')
+        self.assertEqual('ref', dc.ref)
+        self.assertEqual('name', dc.name)

tests/objects/test_datastore.py

@@ -16,6 +16,7 @@ import mock
 import six.moves.urllib.parse as urlparse
 
 from oslo.utils import units
+from oslo.vmware import constants
 from oslo.vmware.objects import datastore
 from oslo.vmware import vim_util
 from tests import base
@@ -68,6 +69,17 @@ class DatastoreTestCase(base.TestCase):
         ds_path = ds.build_path("some_dir", "foo.vmdk")
         self.assertEqual('[ds_name] some_dir/foo.vmdk', str(ds_path))
 
+    def test_build_url(self):
+        ds = datastore.Datastore("fake_ref", "ds_name")
+        path = 'images/ubuntu.vmdk'
+        self.assertRaises(ValueError, ds.build_url, 'https', '10.0.0.2', path)
+        ds.datacenter = mock.Mock()
+        ds.datacenter.name = "dc_path"
+        ds_url = ds.build_url('https', '10.0.0.2', path)
+        self.assertEqual(ds_url.datastore_name, "ds_name")
+        self.assertEqual(ds_url.datacenter_path, "dc_path")
+        self.assertEqual(ds_url.path, path)
+
     def test_get_summary(self):
         ds_ref = vim_util.get_moref('ds-0', 'Datastore')
         ds = datastore.Datastore(ds_ref, 'ds-name')
@@ -341,3 +353,32 @@ class DatastoreURLTestCase(base.TestCase):
         url = 'https://13.37.73.31/folder/%s?%s' % (path, query)
         ds_url = datastore.DatastoreURL.urlparse(url)
         self.assertEqual(path, ds_url.path)
+
+    @mock.patch('httplib.HTTPSConnection')
+    def test_connect(self, mock_conn):
+        dc_path = 'datacenter-1'
+        ds_name = 'datastore-1'
+        params = {'dcPath': dc_path, 'dsName': ds_name}
+        query = urlparse.urlencode(params)
+        url = 'https://13.37.73.31/folder/images/aa.vmdk?%s' % query
+        ds_url = datastore.DatastoreURL.urlparse(url)
+        cookie = mock.Mock()
+        ds_url.connect('PUT', 128, cookie)
+        mock_conn.assert_called_once_with('13.37.73.31')
+
+    def test_get_transfer_ticket(self):
+        dc_path = 'datacenter-1'
+        ds_name = 'datastore-1'
+        params = {'dcPath': dc_path, 'dsName': ds_name}
+        query = urlparse.urlencode(params)
+        url = 'https://13.37.73.31/folder/images/aa.vmdk?%s' % query
+        session = mock.Mock()
+        session.invoke_api = mock.Mock()
+
+        class Ticket(object):
+            id = 'fake_id'
+        session.invoke_api.return_value = Ticket()
+        ds_url = datastore.DatastoreURL.urlparse(url)
+        ticket = ds_url.get_transfer_ticket(session, 'PUT')
+        self.assertEqual('%s="%s"' % (constants.CGI_COOKIE_KEY, 'fake_id'),
+                         ticket)