# Copyright 2015 OpenStack Foundation
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import collections
import os
import re
import shlex
import subprocess
import time
import traceback
from tempest.common.utils.linux import remote_client
from tempest.common import waiters
from tempest import config
from tempest.lib.common.utils import data_utils
from tempest.lib.common.utils import test_utils
from tempest.lib import exceptions
from tempest import test
from vmware_nsx_tempest.tests.nsxv.scenario import (
network_addon_methods as HELO)
from vmware_nsx_tempest.tests.scenario import manager
CONF = config.CONF
LOG = manager.log.getLogger(__name__)
Floating_IP_tuple = collections.namedtuple(
'Floating_IP_tuple', ['floating_ip', 'server'])
Z_VM2_DEST = "VM[%(h_ipaddr)s] %(msg)s [%(helper)s %(d_ipaddr)s]"
# Before checking for floatingIP and server connectivity, we need to wait
# x seconds for the control-plane to push configuration to data-plane
# prior to process add/update/delete requests.
WAITTIME_AFTER_DISASSOC_FLOATINGIP = CONF.scenario.waitfor_disassoc
WAITTIME_AFTER_ASSOC_FLOATINGIP = CONF.scenario.waitfor_assoc
WAITTIME_FOR_CONNECTIVITY = CONF.scenario.waitfor_connectivity
DNS_SERVERS_IPV4 = CONF.network.dns_servers
OUTSIDE_WORLD_SERVERS = CONF.scenario.outside_world_servers
# iptype
IPTYPE_FLOATING = 'floating-ip'
IPTYPE_FIXED = 'fixed-ip'
IPTYPE_OUTSIDE_SERVER = 'outside-server'
class TopoDeployScenarioManager(manager.NetworkScenarioTest):
"""Purposes for TopoDeployScenarionManager:
1. Each deployment scenarion create its network resources, so
call set_network_resource at setup_credentials() to overwrite it.
2. setUp() is for test framework. Test case topology is part of
test and is configured during test() cycle.
3. net_resources.py overwrite resourses.py so the method to add
interfaces to routers are inline with CLI, and support router
owned by admin, but subnets are primary/alt clients.
-- mechanism removed with patch 320495
-- we are relaying on the test framework to delete resources
in the reverse order of creating.
4. Ping is used for Data-plane testing. OUTSIDE_WORLD_SERVERS ping
test make sense when tenant's DNS is pirvate to provider.
5. Teardown is high cost, each test should perform its un-config to
complete the whole tenant life-cycle.
WARNING: you need to increase your quota to run in parallel as
you might run out of quota when things went wrong.
"""
# defined at test.py; used to create client managers
credentials = ['admin', 'primary', 'alt']
# router attributes used to create the tenant's router
tenant_router_attrs = {}
@classmethod
def skip_checks(cls):
super(TopoDeployScenarioManager, cls).skip_checks()
for ext in ['router', 'security-group']:
if not test.is_extension_enabled(ext, 'network'):
msg = "%s extension not enabled." % ext
raise cls.skipException(msg)
@classmethod
def check_preconditions(cls):
super(TopoDeployScenarioManager, cls).check_preconditions()
if not (CONF.network.project_networks_reachable or
CONF.network.public_network_id):
msg = ('Either project_networks_reachable must be "true", or '
'public_network_id must be defined.')
cls.enabled = False
raise cls.skipException(msg)
@classmethod
def setup_credentials(cls):
# Each client's network is created when client manager is created,
# and client manager is created at setup_credentials.
# topo-deploy scenarion manager asks not to create network resources.
cls.set_network_resources(False, False, False, False)
super(TopoDeployScenarioManager, cls).setup_credentials()
@classmethod
def resource_setup(cls):
super(TopoDeployScenarioManager, cls).resource_setup()
cls.namestart = 'topo-deploy-tenant'
cls.public_network_id = CONF.network.public_network_id
# The creation of the 2nd tenant is defined by class.credentials
# cls.alt_manager = clients.Manager(credentials=cls.alt_credentials())
cls.alt_tenant_id = cls.alt_manager.identity_client.tenant_id
@classmethod
def resource_cleanup(cls):
super(TopoDeployScenarioManager, cls).resource_cleanup()
def setUp(self):
super(TopoDeployScenarioManager, self).setUp()
self.cleanup_waits = []
self.addCleanup(self._wait_for_cleanups)
self.servers_on_net = {}
def tearDown(self):
super(TopoDeployScenarioManager, self).tearDown()
def addCleanup_with_wait(self, waiter_callable, thing_id, thing_id_param,
cleanup_callable, cleanup_args=None,
cleanup_kwargs=None, waiter_client=None):
"""Adds wait for async resource deletion at the end of cleanups
@param waiter_callable: callable to wait for the resource to delete
with the following waiter_client if specified.
@param thing_id: the id of the resource to be cleaned-up
@param thing_id_param: the name of the id param in the waiter
@param cleanup_callable: method to load pass to self.addCleanup with
the following *cleanup_args, **cleanup_kwargs.
usually a delete method.
"""
if cleanup_args is None:
cleanup_args = []
if cleanup_kwargs is None:
cleanup_kwargs = {}
self.addCleanup(cleanup_callable, *cleanup_args, **cleanup_kwargs)
wait_dict = {
'waiter_callable': waiter_callable,
thing_id_param: thing_id
}
if waiter_client:
wait_dict['client'] = waiter_client
self.cleanup_waits.append(wait_dict)
def _wait_for_cleanups(self):
# To handle async delete actions, a list of waits is added
# which will be iterated over as the last step of clearing the
# cleanup queue. That way all the delete calls are made up front
# and the tests won't succeed unless the deletes are eventually
# successful. This is the same basic approach used in the api tests to
# limit cleanup execution time except here it is multi-resource,
# because of the nature of the scenario tests.
for wait in self.cleanup_waits:
waiter_callable = wait.pop('waiter_callable')
waiter_callable(**wait)
# overwrite parent class which does not accept NSX-v extension
def _create_router(self, client_mgr=None, tenant_id=None,
namestart='topo-deploy', **kwargs):
client_mgr = client_mgr or self.manager
routers_client = getattr(client_mgr, "routers_client")
router = HELO.router_create(self, client=routers_client,
tenant_id=tenant_id,
namestart=namestart,
**kwargs)
return router
def _router_set_gateway(self, router_id, network_id, client=None):
routers_client = client or self.routers_client
return HELO.router_gateway_set(self, router_id, network_id,
client=routers_client)
def _router_clear_gateway(self, router_id, client=None):
routers_client = client or self.routers_client
return HELO.router_gateway_clear(self, router_id,
client=routers_client)
def _router_update_extra_routes(self, router_id, routes, client=None):
routers_client = client or self.routers_client
router = routers_client.update_route(self, router_id,
routes=routes)
return router['router']
def _router_delete_extra_routes(self, router_id, client=None):
routers_client = client or self.routers_client
return HELO.router_delete_extra_routes(self, router_id,
routers_client)
def _router_add_interface(self, net_router, net_subnet, client_mgr):
routers_client = client_mgr.routers_client
return HELO.router_interface_add(self, net_router['id'],
net_subnet['id'], routers_client)
def router_interface_add(self, router_id, subnet_id, client=None):
routers_client = client or self.routers_client
return HELO.router_interface_add(self, router_id, subnet_id,
routers_client)
def router_interface_delete(self, router_id, subnet_id, client=None):
routers_client = client or self.routers_client
return HELO.router_interface_delete(self, router_id, subnet_id,
routers_client)
def create_server_on_network(self, networks, security_groups=None,
name=None, image=None, wait_on_boot=True,
flavor=None, servers_client=None,
key_name=None, tenant_id=None):
name = name or data_utils.rand_name('topo-deploy-vm')
if security_groups is None:
security_groups = [{'name': 'default'}]
if type(networks) in (list, tuple):
network_ifs = [{'uuid': nw['id']} for nw in networks]
else:
network_ifs = [{'uuid': networks['id']}]
create_kwargs = {
'networks': network_ifs,
'security_groups': security_groups,
}
if key_name:
create_kwargs['key_name'] = key_name
if tenant_id:
if not (servers_client and servers_client.tenant_id == tenant_id):
create_kwargs['tenant_id'] = tenant_id
LOG.debug("TopoDeploy Create server name=%(name)s"
", create_kwargs=%(create_kwargs)s",
{'name': name, 'create_kwargs': str(create_kwargs)})
server = self.create_server(
name=name, image=image, wait_on_boot=wait_on_boot,
servers_client=servers_client, flavor=flavor,
tenant_id=tenant_id, create_kwargs=create_kwargs)
return server
# overwrite parent classes; add servers_client
# BUG https://bugs.launchpad.net/tempest/+bug/1416175
def create_server(self, name=None, image=None, flavor=None,
wait_on_boot=True, wait_on_delete=True,
servers_client=None, tenant_id=None,
create_kwargs=None):
"""Creates VM instance.
@param image: image from which to create the instance
@param wait_on_boot: wait for status ACTIVE before continue
@param wait_on_delete: force synchronous delete on cleanup
@param servers_client: the servers_client to create VM
@param create_kwargs: additional details for instance creation
@return: server dict
"""
name = name or data_utils.rand_name('topo-deploy-vm')
image = image or CONF.compute.image_ref
flavor = flavor or CONF.compute.flavor_ref
servers_client = servers_client or self.servers_client
create_kwargs = create_kwargs or {}
if type(tenant_id) in (str, unicode): # noqa
if servers_client.tenant_id != tenant_id:
create_kwargs['tenant_id'] = tenant_id
xmsg = ("Creating a server name=%(name)s, image=%(image)s"
", flavor=%(flavor)s, create_kwargs=%(create_kwargs)s" %
{'name': name, 'image': image, 'flavor': flavor,
'create_kwargs': str(create_kwargs)})
LOG.debug(xmsg)
server_resp = servers_client.create_server(
name=name, imageRef=image, flavorRef=flavor, **create_kwargs)
server = server_resp['server']
if wait_on_delete:
self.addCleanup(
waiters.wait_for_server_termination,
servers_client, server['id'])
self.addCleanup_with_wait(
waiter_callable=waiters.wait_for_server_termination,
thing_id=server['id'], thing_id_param='server_id',
waiter_client=servers_client,
cleanup_callable=test_utils.call_and_ignore_notfound_exc,
cleanup_args=[servers_client.delete_server, server['id']])
if wait_on_boot:
waiters.wait_for_server_status(
client=servers_client,
server_id=server['id'], status='ACTIVE')
# The instance retrieved on creation is missing network
# details, necessitating retrieval after it becomes active to
# ensure correct details.
server_resp = servers_client.show_server(server['id'])
server = server_resp['server']
self.assertEqual(server['name'], name)
self.servers_on_net[server['id']] = server
return server
def create_provider_network(self, client_mgr=None, create_body=None):
name = create_body.get('name', None) or data_utils.rand_name('P-net')
create_body['name'] = name
client_mgr = client_mgr or self.admin_manager
net_network = HELO.create_network(
self, client=client_mgr.networks_client, **create_body)
return net_network
def create_provider_subnet(self, client_mgr=None, create_body=None):
client_mgr = client_mgr or self.admin_manager
subnets_client = client_mgr.subnets_client
body = subnets_client.create_subnet(**create_body)
net_subnet = body['subnet']
self.addCleanup(test_utils.call_and_ignore_notfound_exc,
subnets_client.delete_subnet,
net_subnet['id'])
return net_subnet
def setup_project_network(self, external_network_id,
client_mgr=None,
namestart=None, client=None,
tenant_id=None, cidr_offset=0,
**kwargs):
"""NOTE:
Refer to create_networks@scenario/manager.py which might refer
to public_router_id which we dont' want to use.
The test class can define class variable tenant_router_attrs
to create different type of routers, or overwrite with kwargs.
"""
name = namestart or data_utils.rand_name('topo-deploy-tenant')
client_mgr = client_mgr or self.manager
# _create_router() edits distributed and router_type
# Child classes use class var tenant_router_attrs to define
# tenant's router type, however, caller can overwrite it with kwargs.
if CONF.network.backend == "nsxv3":
net_router = self._create_router(client_mgr=client_mgr,
tenant_id=tenant_id,
namestart=name)
else:
distributed = \
kwargs.get('distributed',
self.tenant_router_attrs.get('distributed'))
router_type = \
kwargs.get('router_type',
self.tenant_router_attrs.get('router_type'))
net_router = self._create_router(
client_mgr=client_mgr, tenant_id=tenant_id,
namestart=name,
distributed=distributed, router_type=router_type)
self._router_set_gateway(net_router['id'], external_network_id,
client=client_mgr.routers_client)
net_network, net_subnet = self.create_network_subnet(
client_mgr=client_mgr, name=net_router['name'],
tenant_id=tenant_id, cidr_offset=cidr_offset)
self._router_add_interface(net_router, net_subnet, client_mgr)
return net_network, net_subnet, net_router
def create_network_subnet(self, client_mgr=None,
tenant_id=None, name=None, cidr_offset=0):
client_mgr = client_mgr or self.manager
tenant_id = tenant_id or _g_tenant_id(client_mgr.networks_client)
name = name or data_utils.rand_name('topo-deploy-network')
net_network = self.create_network(
client=client_mgr.networks_client,
tenant_id=tenant_id, name=name)
net_subnet = self.create_subnet(
client=client_mgr.subnets_client,
network=net_network,
cidr_offset=cidr_offset, name=net_network['name'])
return net_network, net_subnet
# cloned from _create_network@manager.py. Allow name parameter
def create_network(self, client=None, tenant_id=None, name=None,
**kwargs):
networks_client = client or self.networks_client
tenant_id = tenant_id or _g_tenant_id(networks_client)
name = name or data_utils.rand_name('topo-deploy-network')
return HELO.create_network(self, client=networks_client,
tenant_id=tenant_id, name=name,
**kwargs)
def create_subnet(self, network, client=None,
gateway='', cidr=None, mask_bits=None,
ip_version=None, cidr_offset=0,
allocation_pools=None, dns_nameservers=None,
**kwargs):
subnets_client = client or self.subnets_client
kwargs.update(client=subnets_client, gateway=gateway,
cidr=cidr, cidr_offset=cidr_offset,
mask_bits=mask_bits, ip_version=ip_version,
allocation_pools=allocation_pools,
dns_nameservers=dns_nameservers)
return HELO.create_subnet(self, network, **kwargs)
def create_floatingip_for_server(self, server, external_network_id=None,
port_id=None, client_mgr=None,
and_check_assigned=True):
client_mgr = client_mgr or self.manager
net_floatingip = self.create_floating_ip(
server,
external_network_id=external_network_id,
port_id=port_id,
client=client_mgr.floating_ips_client)
if port_id:
# attached to port, will not check ip assignement & reachability
return net_floatingip
serv_fip = net_floatingip['floating_ip_address']
# in some condiction, remove the serv_fip from your local known_hosts
# can solve the ssh "Connection refused" problem.
rm_sshkey(serv_fip)
if not and_check_assigned:
# caller will do the floatingip assigned to server and ping tests
return net_floatingip
self._waitfor_floatingip_assigned_to_server(client_mgr.servers_client,
server.get('id'))
server_pingable = self._waitfor_associated_floatingip(net_floatingip)
STEPINTO_DEBUG_IF_TRUE(not server_pingable)
self.assertTrue(
server_pingable,
msg=("Expect server to be reachable after"
" floating-ip[%s] assigned." % serv_fip))
return net_floatingip
def _waitfor_floatingip_assigned_to_server(self, server_client, server_id,
on_network=None,
extra_timeout=60):
timeout = server_client.build_timeout + extra_timeout
interval = server_client.build_interval
start_time = time.time()
while time.time() - start_time < timeout:
sv = server_client.show_server(server_id)
sv = sv.get('server', sv)
fip = self.get_server_ip_address(sv, 'floating')
if fip:
elapse_time = time.time() - start_time
xmsg = ("%s Take %d seconds to assign floatingip to server[%s]"
% ("OS-STATS:", int(elapse_time), sv.get('name')))
LOG.debug(xmsg)
return fip
time.sleep(interval)
raise Exception(
"Server[%s] did not get its floatingip in %s seconds" %
(server_id, timeout))
def get_server_ip_address(self, server, ip_type='fixed',
network_name=None):
if network_name and server['addresses'].get(network_name):
s_if = network_name
else:
s_if = server['addresses'].keys()[0]
for s_address in server['addresses'][s_if]:
if s_address['OS-EXT-IPS:type'] == ip_type:
return s_address.get('addr')
return None
def _waitfor_associated_floatingip(self, net_floatingip):
host_ip = net_floatingip['floating_ip_address']
return self.waitfor_host_connected(host_ip)
def waitfor_host_connected(self, host_ip, ping_timeout=5, msg=None):
PING_START = 'ping-progress-start'
PING_INSESSION = 'ping-progress-in-session'
PING_DONE = 'ping-progress-completed'
PING_TIMEOUT = 'ping-progress-timeout'
if msg and type(msg) in (str, unicode): # noqa
xmsg = ("waitfor_host_connected ip=%(ip)s! %(msg)s" %
{'ip': host_ip, 'msg': msg})
LOG.debug(xmsg)
t0 = time.time()
t1 = time.time() + WAITTIME_FOR_CONNECTIVITY
LOG.debug("VM-IP[%(ip)s] %(msg)s: %(t1)s.",
{'ip': host_ip, 'msg': PING_START, 't1': t0})
while (time.time() < t1):
# waitfor backend to create floatingip & linkages
time.sleep(WAITTIME_AFTER_ASSOC_FLOATINGIP)
server_pingable = self.ping_ip_address(
host_ip, ping_timeout=ping_timeout)
if server_pingable:
xmsg = ("VM-IP[%(ip)s] %(msg)s: %(t1)s (%(t2)s)." %
{'ip': host_ip, 'msg': PING_DONE,
't1': time.time(), 't2': (time.time() - t0)})
LOG.debug(xmsg)
break
xmsg = ("VM-IP[%(ip)s] %(msg)s, redo after %(t1)s seconds." %
{'ip': host_ip, 'msg': PING_INSESSION,
't1': WAITTIME_AFTER_ASSOC_FLOATINGIP})
LOG.debug(xmsg)
if not server_pingable:
xmsg = ("VM-IP[%(ip)s] %(msg)s: %(t1)s (%(t2)s)." %
{'ip': host_ip, 'msg': PING_TIMEOUT,
't1': time.time(), 't2': (time.time() - t0)})
LOG.debug(xmsg)
return server_pingable
def disassociate_floatingip(self, net_floatingip, client=None,
and_delete=False):
floating_ips_client = client or self.floating_ips_client
kwargs = dict(port_id=None)
floating_ip = floating_ips_client.update_floatingip(
net_floatingip['id'], **kwargs)
floating_ip = floating_ip.get('floatingip', floating_ip)
self.assertIsNone(floating_ip['port_id'])
if and_delete:
floating_ips_client.delete_floatingip(floating_ip['id'])
return floating_ip
def associate_floatingip(self, net_floatingip, to_server, client=None):
floating_ips_client = client or self.floating_ips_client
port_id, _ = self._get_server_port_id_and_ip4(to_server)
kwargs = dict(port_id=port_id)
floating_ip = floating_ips_client.update_floatingip(
net_floatingip['id'], **kwargs)['floatingip']
self.assertEqual(port_id, floating_ip['port_id'])
return floating_ip
def check_networks(self, net_network, net_subnet=None, net_router=None):
return HELO.check_networks(self, net_network, net_subnet, net_router)
# use this carefully, as it expect existence of floating_ip_tuple
def check_public_network_connectivity(self, should_connect=True,
msg=None, ping_timeout=30):
"""Verifies connectivty
To a VM via public network and floating IP, and verifies
floating IP has resource status is correct.
@param should_connect: bool. determines if connectivity check is
negative or positive.
@param msg: Failure message to add to Error message. Should describe
the place in the test scenario where the method was called,
to indicate the context of the failure
"""
floating_ip, server = self.floating_ip_tuple
return self._check_floatingip_connectivity(
floating_ip, server, should_connect, msg, ping_timeout)
def _check_floatingip_connectivity(self, floating_ip, server,
should_connect=True,
msg=None, ping_timeout=30,
floating_ips_client=None):
ip_address = floating_ip['floating_ip_address']
floatingip_status = 'ACTIVE' if should_connect else 'DOWN'
is_pingable = self.ping_ip_address(ip_address,
ping_timeout=ping_timeout)
msg = msg if msg else (
"Timeout out waiting for %s to become reachable" % ip_address)
if should_connect:
self.assertTrue(is_pingable, msg=msg)
else:
self.assertFalse(is_pingable, msg=msg)
self.check_floating_ip_status(floating_ip, floatingip_status,
floating_ips_client)
def check_floating_ip_status(self, floating_ip, status,
floating_ips_client=None):
"""Verifies floatingip reaches the given status
:param dict floating_ip: floating IP dict to check status
:param status: target status
:raises: AssertionError if status doesn't match
"""
floating_ips_client = floating_ips_client or self.floating_ips_client
floatingip_id = floating_ip['id']
def refresh():
result = (floating_ips_client.
show_floatingip(floatingip_id)['floatingip'])
return status == result['status']
test_utils.call_until_true(refresh,
CONF.network.build_timeout,
CONF.network.build_interval)
floating_ip = floating_ips_client.show_floatingip(
floatingip_id)['floatingip']
self.assertEqual(status, floating_ip['status'],
message="FloatingIP: {fp} is at status: {cst}. "
"failed to reach status: {st}"
.format(fp=floating_ip, cst=floating_ip['status'],
st=status))
LOG.info("FloatingIP: {fp} is at status: {st}"
.format(fp=floating_ip, st=status))
def get_image_userpass(self):
return (CONF.validation.image_ssh_user,
CONF.validation.image_ssh_password)
def get_server_image(self):
return CONF.compute.image_ref
def get_server_flavor(self):
return CONF.compute.flavor_ref
# common utilities
def make_node_info(net_floatingip, username, password,
include_outside_servers=False):
floating_ip_address = net_floatingip['floating_ip_address']
fixed_ip_address = net_floatingip['fixed_ip_address']
node = dict(ipaddr=floating_ip_address,
username=username, password=password)
node['dest'] = [dict(ipaddr=floating_ip_address,
reachable=None, helper=IPTYPE_FLOATING),
dict(ipaddr=fixed_ip_address,
reachable=None, helper=IPTYPE_FIXED)]
if include_outside_servers:
outside_servers = dict(ipaddr=OUTSIDE_WORLD_SERVERS[0],
reachable=None, helper=IPTYPE_OUTSIDE_SERVER)
node['dest'].append(outside_servers)
return node
# we want to check the dest[iptype] is not reachable for
# at least (x_contd=2+=1 to make it is not really reachable.
def check_host_not_reachable(host, dest_list, iptype_list,
time_out=10, repeat_cnt=12,
x_contd=2):
not_connected = 0
for x in range(0, 12):
not_reachable = check_host_is_reachable(
host, dest_list, iptype_list, time_out=time_out)
if not_reachable:
not_connected += 1
else:
not_connected = 0
if not_connected > x_contd:
return True
return False
# check_hosts_connectivity
def check_host_is_reachable(host, dest_list, iptype_list, time_out=120):
rm_sshkey(host['ipaddr'])
ssh_client = get_remote_client_by_password(host['ipaddr'],
host['username'],
host['password'])
n_not_reachable = 0
for dest in dest_list:
for iptype in iptype_list:
if not dest_has_iptype(dest, iptype):
dest['reachable'] = None
continue
dest['reachable'] = is_reachable(
ssh_client, dest['ipaddr'], time_out=time_out)
if not dest['reachable']:
n_not_reachable += 1
xmsg = {'h_ipaddr': host['ipaddr'],
'msg': "can-not-reach-dest",
'helper': dest['helper'],
'd_ipaddr': dest['ipaddr']}
LOG.debug(Z_VM2_DEST, xmsg)
else:
xmsg = {'h_ipaddr': host['ipaddr'],
'msg': "can-not-dest",
'helper': dest['helper'],
'd_ipaddr': dest['ipaddr']}
LOG.debug(Z_VM2_DEST, xmsg)
return (False if n_not_reachable else True)
def dest_has_iptype(dest, iptype):
if ('helper' in dest and
re.search(iptype, dest['helper'], re.I)):
return True
return False
def check_hosts_connectivity(host, dest_list, ignore_helper=None,
time_out=120):
rm_sshkey(host['ipaddr'])
ssh_client = get_remote_client_by_password(host['ipaddr'],
host['username'],
host['password'])
n_not_reachable = 0
for dest in dest_list:
# caller can say to ignore dest ipaddr
if ('helper' in dest and
type(ignore_helper) in (str, unicode) and # noqa
re.search(ignore_helper, dest['helper'], re.I)): # noqa
dest['reachable'] = None
continue
dest['reachable'] = is_reachable(ssh_client, dest['ipaddr'],
time_out=time_out)
if not dest['reachable']:
n_not_reachable += 1
xmsg = {'h_ipaddr': host['ipaddr'],
'msg': "can-not-reach-dest",
'helper': dest['helper'],
'd_ipaddr': dest['ipaddr']}
LOG.debug(Z_VM2_DEST, xmsg)
else:
xmsg = {'h_ipaddr': host['ipaddr'],
'msg': "can-reach-dest",
'helper': dest['helper'],
'd_ipaddr': dest['ipaddr']}
LOG.debug(Z_VM2_DEST, xmsg)
return n_not_reachable
def rm_sshkey(ip_addr):
# ssh-keygen -f "/home/stack/.ssh/known_hosts" -R 10.34.57.3
kh_file = os.path.join(os.environ.get('HOME', '/home/stack'),
'.ssh/known_hosts')
cmd = ['ssh-keygen', '-f', kh_file, '-R', ip_addr]
proc = subprocess.Popen(cmd, stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
proc.communicate()
return proc.returncode
def is_reachable(ssh_client, dest_ip, time_out=60.0, ping_timeout=5.0):
for now in run_till_timeout(time_out, ping_timeout):
reachable = dest_is_reachable(ssh_client, dest_ip)
if reachable:
return True
LOG.debug("DEST[%(ip)s] NOT-REACHABLE, retry in %(t1)s seconds.",
{'ip': dest_ip, 't1': time_out})
return False
def isnot_reachable(ssh_client, dest_ip, time_out=60.0, ping_timeout=5.0,
idle_time=2.0):
if idle_time > 0.0:
time.sleep(idle_time)
for now in run_till_timeout(time_out, ping_timeout):
reachable = dest_is_reachable(ssh_client, dest_ip)
if not reachable:
return True
LOG.debug("DEST[%(ip)s] IS-REACHABLE, retry in %(t1)s seconds.",
{'ip': dest_ip, 't1': time_out})
return False
def dest_is_reachable(ssh_client, dest_ip):
XPTN = r"(\d+).*transmit.*(\d+).*receive.*(\d+).*loss"
try:
result = ssh_client.ping_host(dest_ip)
m = re.search(XPTN, result, (re.I | re.M))
if m and int(m.group(1)) > 0 and int(m.group(3)) == 0:
return True
else:
return False
except Exception:
tb_str = traceback.format_exc()
mesg = ("ERROR on testing dest_ip[%s] is reachable:\n%s" %
(dest_ip, tb_str))
LOG.debug(mesg)
return False
def run_till_timeout(seconds_to_try, interval=5.0):
now, end_time = time.time(), time.time() + seconds_to_try
while now < end_time:
yield now
time.sleep(interval)
now = time.time()
def _g_tenant_id(os_client):
try:
return os_client.tenant_id
except Exception:
return os_client.rest_client.tenant_id
def get_remote_client_by_password(client_ip, username, password):
ssh_client = remote_client.RemoteClient(client_ip, username, password)
return ssh_client
def delete_all_servers(tenant_servers_client, trys=5):
# try at least trys+1 time to delete servers, otherwise
# network resources can not be deleted
for s in tenant_servers_client.list_servers()['servers']:
tenant_servers_client.delete_server(s['id'])
for x in range(0, trys):
try:
waitfor_servers_terminated(tenant_servers_client)
return
except Exception:
pass
# last try
waitfor_servers_terminated(tenant_servers_client)
def waitfor_servers_terminated(tenant_servers_client, pause=2.0):
while (True):
s_list = tenant_servers_client.list_servers()['servers']
if len(s_list) < 1:
return
time.sleep(pause)
def copy_file_to_host(file_from, dest, host, username, pkey):
dest = "%s@%s:%s" % (username, host, dest)
cmd = "scp -v -o UserKnownHostsFile=/dev/null " \
"-o StrictHostKeyChecking=no " \
"-i %(pkey)s %(file1)s %(dest)s" % {'pkey': pkey,
'file1': file_from,
'dest': dest}
args = shlex.split(cmd.encode('utf-8'))
subprocess_args = {'stdout': subprocess.PIPE,
'stderr': subprocess.STDOUT}
proc = subprocess.Popen(args, **subprocess_args)
stdout, stderr = proc.communicate()
if proc.returncode != 0:
raise exceptions.SSHExecCommandFailed(cmd,
proc.returncode,
stdout,
stderr)
return stdout
def STEPINTO_DEBUG_IF_TRUE(want2debug=False):
"""Betting you are not set OS_TEST_TIMEOUT=24-hours running tempest"""
t_timeout = int(os.environ.get('OS_TEST_TIMEOUT', 0))
if want2debug and t_timeout > 86400:
# uncomment following statements to turn on debuggging
# import pdb
# pdb.set_trace()
pass