Drydock request factory tuning

This patchset tunes the Drydock MAAS request factory to:
a) Implement retries for requests toward MAAS_URL/api/2.0/
b) Bumps the request timeout slightly

In addition, restricts threaded actions towards nodes to
using the same MAAS client, effectively rate limiting calls
to the MAAS api.

Change-Id: I2e66105ae332adaed62c9c3bc8cddc63e1f7bf23

python/drydock_provisioner/drivers/node/maasdriver/api_client.py

@@ -17,6 +17,8 @@ import logging
 
 from oauthlib import oauth1
 import requests
+from requests.adapters import HTTPAdapter
+from requests.packages.urllib3.util.retry import Retry
 import requests.auth as req_auth
 import base64
 
@@ -58,8 +60,18 @@ class MaasRequestFactory(object):
         self.base_url = base_url + "/api/2.0/"
         self.apikey = apikey
 
+        # Adapter for maas for request retries
+        retry_strategy = Retry(
+            total=3,
+            status_forcelist=[429, 500, 502, 503, 504],
+            method_whitelist=["HEAD", "GET", "POST", "PUT", "DELETE",
+                              "OPTIONS", "TRACE"]
+        )
+        self.maas_adapter = HTTPAdapter(max_retries=retry_strategy)
+
         self.signer = MaasOauth(apikey)
         self.http_session = requests.Session()
+        self.http_session.mount(self.base_url, self.maas_adapter)
 
         # TODO(sh8121att) Get logger name from config
         self.logger = logging.getLogger('drydock')
@@ -160,7 +172,7 @@ class MaasRequestFactory(object):
         # TODO(sh8121att) timeouts should be configurable
         timeout = kwargs.pop('timeout', None)
         if timeout is None:
-            timeout = (2, 30)
+            timeout = (5, 60)
 
         request = requests.Request(
             method=method,

python/drydock_provisioner/drivers/node/maasdriver/driver.py

@@ -151,10 +151,10 @@ class MaasNodeDriver(NodeDriver):
 
             with concurrent.futures.ThreadPoolExecutor(max_workers=16) as e:
                 subtask_futures = dict()
+                maas_client = MaasRequestFactory(
+                    config.config_mgr.conf.maasdriver.maas_api_url,
+                    config.config_mgr.conf.maasdriver.maas_api_key)
                 for n in target_nodes:
-                    maas_client = MaasRequestFactory(
-                        config.config_mgr.conf.maasdriver.maas_api_url,
-                        config.config_mgr.conf.maasdriver.maas_api_key)
                     nf = self.orchestrator.create_nodefilter_from_nodelist([n])
                     subtask = self.orchestrator.create_task(
                         design_ref=task.design_ref,