From 0fdb00522a1a6e0d9c645ac68c6dc092d6de7088 Mon Sep 17 00:00:00 2001 From: ZHU ZHU Date: Tue, 28 Jul 2015 21:47:50 -0500 Subject: [PATCH] Allow http connection pool size to be configured With multiple concurrent VM spawning, the service time from compute service flow to vcenter to trigger soap requests is getting slower. As VIM adopt requests module to access the vcenter which have default http pool size set to 10. This need to be configured to be larger in scale environment. Change-Id: Ic9e9a87de378151a66187222231eaafd7a402ac5 Closes-Bug: #1479183 --- oslo_vmware/api.py | 11 ++++++++--- oslo_vmware/pbm.py | 7 +++++-- oslo_vmware/service.py | 12 ++++++++---- oslo_vmware/tests/test_api.py | 8 ++++++-- oslo_vmware/tests/test_service.py | 6 ++++++ oslo_vmware/vim.py | 7 +++++-- 6 files changed, 38 insertions(+), 13 deletions(-) diff --git a/oslo_vmware/api.py b/oslo_vmware/api.py index 32d031a..cada825 100644 --- a/oslo_vmware/api.py +++ b/oslo_vmware/api.py @@ -138,7 +138,7 @@ class VMwareAPISession(object): def __init__(self, host, server_username, server_password, api_retry_count, task_poll_interval, scheme='https', create_session=True, wsdl_loc=None, pbm_wsdl_loc=None, - port=443, cacert=None, insecure=True): + port=443, cacert=None, insecure=True, pool_size=10): """Initializes the API session with given parameters. :param host: ESX/VC server IP address or host name @@ -158,6 +158,8 @@ class VMwareAPISession(object): TLS (https) server certificate. :param insecure: Verify HTTPS connections using system certificates, used only if cacert is not specified + :param pool_size: Maximum number of connections in http + connection pool :raises: VimException, VimFaultException, VimAttributeException, VimSessionOverLoadException """ @@ -176,6 +178,7 @@ class VMwareAPISession(object): self._pbm = None self._cacert = cacert self._insecure = insecure + self._pool_size = pool_size if create_session: self._create_session() @@ -192,7 +195,8 @@ class VMwareAPISession(object): port=self._port, wsdl_url=self._vim_wsdl_loc, cacert=self._cacert, - insecure=self._insecure) + insecure=self._insecure, + pool_maxsize=self._pool_size) return self._vim @property @@ -203,7 +207,8 @@ class VMwareAPISession(object): port=self._port, wsdl_url=self._pbm_wsdl_loc, cacert=self._cacert, - insecure=self._insecure) + insecure=self._insecure, + pool_maxsize=self._pool_size) if self._session_id: # To handle the case where pbm property is accessed after # session creation. If pbm property is accessed before session diff --git a/oslo_vmware/pbm.py b/oslo_vmware/pbm.py index 75d3e4d..e4d3643 100644 --- a/oslo_vmware/pbm.py +++ b/oslo_vmware/pbm.py @@ -41,7 +41,7 @@ class Pbm(service.Service): """Service class that provides access to the Storage Policy API.""" def __init__(self, protocol='https', host='localhost', port=443, - wsdl_url=None, cacert=None, insecure=True): + wsdl_url=None, cacert=None, insecure=True, pool_maxsize=10): """Constructs a PBM service client object. :param protocol: http or https @@ -52,10 +52,13 @@ class Pbm(service.Service): TLS (https) server certificate. :param insecure: Verify HTTPS connections using system certificates, used only if cacert is not specified + :param pool_maxsize: Maximum number of connections in http + connection pool """ base_url = service.Service.build_base_url(protocol, host, port) soap_url = base_url + '/pbm' - super(Pbm, self).__init__(wsdl_url, soap_url, cacert, insecure) + super(Pbm, self).__init__(wsdl_url, soap_url, cacert, insecure, + pool_maxsize) def set_soap_cookie(self, cookie): """Set the specified vCenter session cookie in the SOAP header diff --git a/oslo_vmware/service.py b/oslo_vmware/service.py index 554d4f6..51ca504 100644 --- a/oslo_vmware/service.py +++ b/oslo_vmware/service.py @@ -116,6 +116,9 @@ class LocalFileAdapter(requests.adapters.HTTPAdapter): See http://stackoverflow.com/a/22989322 """ + def __init__(self, pool_maxsize=10): + super(LocalFileAdapter, self).__init__(pool_connections=pool_maxsize, + pool_maxsize=pool_maxsize) def _build_response_from_file(self, request): file_path = request.url[7:] @@ -131,13 +134,14 @@ class LocalFileAdapter(requests.adapters.HTTPAdapter): class RequestsTransport(transport.Transport): - def __init__(self, cacert=None, insecure=True): + def __init__(self, cacert=None, insecure=True, pool_maxsize=10): transport.Transport.__init__(self) # insecure flag is used only if cacert is not # specified. self.verify = cacert if cacert else not insecure self.session = requests.Session() - self.session.mount('file:///', LocalFileAdapter()) + self.session.mount('file:///', + LocalFileAdapter(pool_maxsize=pool_maxsize)) self.cookiejar = self.session.cookies def open(self, request): @@ -184,12 +188,12 @@ class Service(object): """ def __init__(self, wsdl_url=None, soap_url=None, - cacert=None, insecure=True): + cacert=None, insecure=True, pool_maxsize=10): self.wsdl_url = wsdl_url self.soap_url = soap_url LOG.debug("Creating suds client with soap_url='%s' and wsdl_url='%s'", self.soap_url, self.wsdl_url) - transport = RequestsTransport(cacert, insecure) + transport = RequestsTransport(cacert, insecure, pool_maxsize) self.client = client.Client(self.wsdl_url, transport=transport, location=self.soap_url, diff --git a/oslo_vmware/tests/test_api.py b/oslo_vmware/tests/test_api.py index 31120bf..c402d47 100644 --- a/oslo_vmware/tests/test_api.py +++ b/oslo_vmware/tests/test_api.py @@ -102,6 +102,7 @@ class VMwareAPISessionTest(base.TestCase): PORT = 443 USERNAME = 'admin' PASSWORD = 'password' + POOL_SIZE = 15 def setUp(self): super(VMwareAPISessionTest, self).setUp() @@ -122,7 +123,8 @@ class VMwareAPISessionTest(base.TestCase): _create_session, port=VMwareAPISessionTest.PORT, cacert=self.cert_mock, - insecure=False) + insecure=False, + pool_size=VMwareAPISessionTest.POOL_SIZE) def test_vim(self): api_session = self._create_api_session(False) @@ -132,7 +134,9 @@ class VMwareAPISessionTest(base.TestCase): port=VMwareAPISessionTest.PORT, wsdl_url=api_session._vim_wsdl_loc, cacert=self.cert_mock, - insecure=False) + insecure=False, + pool_maxsize=VMwareAPISessionTest. + POOL_SIZE) @mock.patch.object(pbm, 'Pbm') def test_pbm(self, pbm_mock): diff --git a/oslo_vmware/tests/test_service.py b/oslo_vmware/tests/test_service.py index 4ce01ee..619a59a 100644 --- a/oslo_vmware/tests/test_service.py +++ b/oslo_vmware/tests/test_service.py @@ -440,6 +440,12 @@ class RequestsTransportTest(base.TestCase): self.assertEqual(mock.sentinel.headers, reply.headers) self.assertEqual(mock.sentinel.content, reply.message) + def test_set_conn_pool_size(self): + transport = service.RequestsTransport(pool_maxsize=100) + local_file_adapter = transport.session.adapters['file:///'] + self.assertEqual(100, local_file_adapter._pool_connections) + self.assertEqual(100, local_file_adapter._pool_maxsize) + @mock.patch('os.path.getsize') def test_send_with_local_file_url(self, get_size_mock): transport = service.RequestsTransport() diff --git a/oslo_vmware/vim.py b/oslo_vmware/vim.py index 8343de6..f984236 100644 --- a/oslo_vmware/vim.py +++ b/oslo_vmware/vim.py @@ -20,7 +20,7 @@ class Vim(service.Service): """Service class that provides access to the VIM API.""" def __init__(self, protocol='https', host='localhost', port=None, - wsdl_url=None, cacert=None, insecure=True): + wsdl_url=None, cacert=None, insecure=True, pool_maxsize=10): """Constructs a VIM service client object. :param protocol: http or https @@ -31,6 +31,8 @@ class Vim(service.Service): TLS (https) server certificate. :param insecure: Verify HTTPS connections using system certificates, used only if cacert is not specified + :param pool_maxsize: Maximum number of connections in http + connection pool :raises: VimException, VimFaultException, VimAttributeException, VimSessionOverLoadException, VimConnectionException """ @@ -38,7 +40,8 @@ class Vim(service.Service): soap_url = base_url + '/sdk' if wsdl_url is None: wsdl_url = soap_url + '/vimService.wsdl' - super(Vim, self).__init__(wsdl_url, soap_url, cacert, insecure) + super(Vim, self).__init__(wsdl_url, soap_url, cacert, insecure, + pool_maxsize) def retrieve_service_content(self): return self.RetrieveServiceContent(service.SERVICE_INSTANCE)