stackforge/libra
Code Issues Proposed changes

LBAAS-583 - VIP assignment can fail without us noticing

Browse Source 
LBAAS-621 - Correct libra to send gearman unique messages

Change-Id: I522493e1691de876a4500ca57ddbdb21ca9ad398
This commit is contained in:
Michael Johnson 2014-07-30 20:08:50 +00:00
parent ca0fd483d5
commit a0e8b91e16
3 changed files with 53 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
libra/admin_api/device_pool/manage_pool.py
View File

@ -14,6 +14,7 @@
import ipaddress
import threading
import uuid
from datetime import datetime
from gearman.constants import JOB_UNKNOWN
@ -76,7 +77,10 @@ class Pool(object):
'action': 'DELETE_DEVICE',
'name': device.name
}
message.append(dict(task='libra_pool_mgm', data=job_data))
unique_uuid = str(uuid.uuid4())
message.append(dict(task='libra_pool_mgm',
data=job_data,
unique=unique_uuid))
counter = session.query(Counters).\
filter(Counters.name == 'devices_deleted').first()
@ -177,7 +181,10 @@ class Pool(object):
it = 0
job_data = {'action': 'BUILD_DEVICE'}
while it < count:
message.append(dict(task='libra_pool_mgm', data=job_data))
unique_uuid = str(uuid.uuid4())
message.append(dict(task='libra_pool_mgm',
data=job_data,
unique=unique_uuid))
it += 1
gear = GearmanWork()
gear.send_create_message(message)
@ -187,7 +194,10 @@ class Pool(object):
it = 0
job_data = {'action': 'BUILD_IP'}
while it < count:
message.append(dict(task='libra_pool_mgm', data=job_data))
unique_uuid = str(uuid.uuid4())
message.append(dict(task='libra_pool_mgm',
data=job_data,
unique=unique_uuid))
it += 1
gear = GearmanWork()
gear.send_vips_message(message)

27
libra/mgm/controllers/vip.py
View File

@ -85,6 +85,9 @@ class AssignIpController(object):
.format(self.msg['name'], node_id)
)
nova.vip_assign(node_id, self.msg['ip'])
self._wait_until_ip_assigned(nova, node_id, self.msg['ip'])
if cfg.CONF['mgm']['tcp_check_port']:
self.check_ip(self.msg['ip'],
cfg.CONF['mgm']['tcp_check_port'])
@ -123,6 +126,30 @@ class AssignIpController(object):
raise
time.sleep(2)
def _wait_until_ip_assigned(self, nova, node_id, vip):
current_instance_id = None
# We can check the status for up to 24 seconds since the assign
# attempts five times. All attempts must be before the Gearman
# message times out at two minutes, so let's aim for
# trying five times in ~20 secs each of the five attempts
for x in xrange(1, 6):
try:
current_instance_id = nova.vip_get_instance_id(vip)
LOG.debug("Confirmed VIP {0} is assigned to instance ID {1}"
.format(vip, current_instance_id)
)
if current_instance_id == node_id:
return
except:
pass
LOG.debug("VIP has instance ID {0} but was assigned to " \
"instance {1}, sleeping"
.format(current_instance_id, node_id)
)
if x < 5:
time.sleep(5)
raise Exception('VIP instance ID did not match assigned ' \
'instance ID after 20 secs. Failing assignment')
class RemoveIpController(object):

13
libra/mgm/nova.py
View File

@ -135,6 +135,19 @@ class Node(object):
except exceptions.ClientException:
resp, body = self.nova.delete(url)
def vip_get_instance_id(self, vip):
""" get the instance id owning the vip """
vip_id = self._find_vip_id(vip)
url = '/os-floating-ips/{0}'.format(vip_id)
resp, body = self.nova.get(url)
if resp.status_code != 200:
raise Exception(
'Response code {0}, message {1} when getting ' \
'floating IP {2} details'
.format(resp.status_code, body, vip)
)
return body['floating_ip']['instance_id']
def _find_vip_id(self, vip):
url = '/os-floating-ips'
resp, body = self.nova.get(url)