x/whitebox-neutron-tempest-plugin
Code Issues Proposed changes

Use run_on_master_controller to run command when possible

Browse Source 
This method logs the command executed and its output, so it could help
to troubleshoot some issues.

Change-Id: If1de40baad8d7d956724b4d666e66169b23708d3
This commit is contained in:
Eduardo Olivares 2024-12-19 12:25:08 +01:00
parent 3429e220a5
commit a575b79451
1 changed files with 30 additions and 30 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

60
whitebox_neutron_tempest_plugin/tests/scenario/base.py
View File

@ -112,6 +112,7 @@ class BaseTempestWhiteboxTestCase(base.BaseTempestTestCase):
@classmethod @classmethod
def run_on_master_controller(cls, cmd): def run_on_master_controller(cls, cmd):
LOG.debug("Command: %s", cmd)
if WB_CONF.openstack_type == 'podified': if WB_CONF.openstack_type == 'podified':
output = cls.proxy_host_client.exec_command(cmd) output = cls.proxy_host_client.exec_command(cmd)
if WB_CONF.openstack_type == 'devstack': if WB_CONF.openstack_type == 'devstack':
@ -268,7 +269,7 @@ class BaseTempestWhiteboxTestCase(base.BaseTempestTestCase):
nodes = [] nodes = []
inventory_data = yaml.safe_load( inventory_data = yaml.safe_load(
cls.proxy_host_client.exec_command( cls.run_on_master_controller(
'cat ' + WB_CONF.proxy_host_inventory_path)) 'cat ' + WB_CONF.proxy_host_inventory_path))
is_crc = False is_crc = False
ocps = inventory_data['all']['children']['ocps'] ocps = inventory_data['all']['children']['ocps']
@ -277,11 +278,11 @@ class BaseTempestWhiteboxTestCase(base.BaseTempestTestCase):
is_crc = True is_crc = True
else: else:
# create ocp_node_yaml_list # create ocp_node_yaml_list
ocp_node_list = cls.proxy_host_client.exec_command( ocp_node_list = cls.run_on_master_controller(
"{} get nodes -o name".format(cls.OC)).splitlines() "{} get nodes -o name".format(cls.OC)).splitlines()
ocp_node_yaml_list = [] ocp_node_yaml_list = []
for ocp_node in ocp_node_list: for ocp_node in ocp_node_list:
output = cls.proxy_host_client.exec_command( output = cls.run_on_master_controller(
"{} get {} -o yaml".format(cls.OC, ocp_node)) "{} get {} -o yaml".format(cls.OC, ocp_node))
ocp_node_yaml_list.append(yaml.safe_load(output)) ocp_node_yaml_list.append(yaml.safe_load(output))
@ -324,7 +325,7 @@ class BaseTempestWhiteboxTestCase(base.BaseTempestTestCase):
cls.keys_data = { cls.keys_data = {
'id_cifw_key': id_cifw_key} 'id_cifw_key': id_cifw_key}
if hasattr(cls, 'ocp_nodes_key_path'): if hasattr(cls, 'ocp_nodes_key_path'):
devscripts_key = cls.proxy_host_client.exec_command( devscripts_key = cls.run_on_master_controller(
'cat ' + cls.ocp_nodes_key_path) 'cat ' + cls.ocp_nodes_key_path)
cls.keys_data['devscripts_key'] = devscripts_key cls.keys_data['devscripts_key'] = devscripts_key
for host in cls.nodes_data: for host in cls.nodes_data:
@ -381,7 +382,7 @@ class BaseTempestWhiteboxTestCase(base.BaseTempestTestCase):
else: else:
filters = "cut -d'/' -f 2 | grep {}".format(service) filters = "cut -d'/' -f 2 | grep {}".format(service)
pods_output = cls.proxy_host_client.exec_command( pods_output = cls.run_on_master_controller(
"{} | {}; true".format(pods_list, filters)) "{} | {}; true".format(pods_list, filters))
return [pod.strip() for pod in pods_output.splitlines()] return [pod.strip() for pod in pods_output.splitlines()]
@ -391,10 +392,10 @@ class BaseTempestWhiteboxTestCase(base.BaseTempestTestCase):
# since it's the only service that existing tests are using # since it's the only service that existing tests are using
if service == 'neutron': if service == 'neutron':
pod = cls.get_pods_of_service(service)[0] pod = cls.get_pods_of_service(service)[0]
return cls.proxy_host_client.exec_command( return cls.run_on_master_controller(
'{} rsh {} find {} -type f'.format( '{} rsh {} find {} -type f'.format(
cls.OC, pod, os.path.split( cls.OC, pod, os.path.split(
cls.neutron_conf)[0])).strip().split('\n') cls.neutron_conf)[0])).split('\n')
# TODO(mblue): next gen computes configuration set should be done too, # TODO(mblue): next gen computes configuration set should be done too,
# 'oc patch' for data plane would need more steps and triggers deployment # 'oc patch' for data plane would need more steps and triggers deployment
@ -435,17 +436,17 @@ class BaseTempestWhiteboxTestCase(base.BaseTempestTestCase):
"'cp /etc/neutron/neutron.conf.d/{2} /tmp/ && " "'cp /etc/neutron/neutron.conf.d/{2} /tmp/ && "
"chmod g+w /tmp/{2}'").format( "chmod g+w /tmp/{2}'").format(
cls.OC, service_pod, custom_file) cls.OC, service_pod, custom_file)
cls.proxy_host_client.exec_command(copy_config) cls.run_on_master_controller(copy_config)
for config_option in config_list: for config_option in config_list:
combine_conf_cmd = ( combine_conf_cmd = (
"{0} rsh {1} bash -c '" "{0} rsh {1} bash -c '"
"crudini --set /tmp/{2} {3} {4} {5}'").format( "crudini --set /tmp/{2} {3} {4} {5}'").format(
cls.OC, service_pod, custom_file, config_option.section, cls.OC, service_pod, custom_file, config_option.section,
config_option.parameter, config_option.value) config_option.parameter, config_option.value)
cls.proxy_host_client.exec_command(combine_conf_cmd) cls.run_on_master_controller(combine_conf_cmd)
read_conf_cmd = "{0} rsh {1} bash -c 'cat /tmp/{2}'".format( read_conf_cmd = "{0} rsh {1} bash -c 'cat /tmp/{2}'".format(
cls.OC, service_pod, custom_file) cls.OC, service_pod, custom_file)
combined_conf = cls.proxy_host_client.exec_command(read_conf_cmd) combined_conf = cls.run_on_master_controller(read_conf_cmd)
combined_conf_ind = combined_conf.replace('\n', '\n' + 8 * ' ') combined_conf_ind = combined_conf.replace('\n', '\n' + 8 * ' ')
patch_buffer = ( patch_buffer = (
'spec:\n' 'spec:\n'
@ -459,7 +460,7 @@ class BaseTempestWhiteboxTestCase(base.BaseTempestTestCase):
cmd = ("{0} patch $({0} get oscp -o name) --type=merge " cmd = ("{0} patch $({0} get oscp -o name) --type=merge "
"--patch '".format(cls.OC) + patch_buffer + "'") "--patch '".format(cls.OC) + patch_buffer + "'")
LOG.debug("Set configuration command:\n%s", cmd) LOG.debug("Set configuration command:\n%s", cmd)
output = cls.proxy_host_client.exec_command(cmd) output = cls.run_on_master_controller(cmd)
LOG.debug("Output:\n%s", output) LOG.debug("Output:\n%s", output)
if not cfg_change and '(no change)' in output: if not cfg_change and '(no change)' in output:
# No config change done, no pod replacement. # No config change done, no pod replacement.
@ -1077,8 +1078,7 @@ class TrafficFlowTest(BaseTempestWhiteboxTestCase):
cmd = ("{} get pods --field-selector=status.phase=Running " cmd = ("{} get pods --field-selector=status.phase=Running "
"-o custom-columns=NODE:.spec.nodeName,NAME:.metadata.name " "-o custom-columns=NODE:.spec.nodeName,NAME:.metadata.name "
"-l service=ovn-controller-ovs".format(cls.OC)) "-l service=ovn-controller-ovs".format(cls.OC))
output = cls.proxy_host_client.exec_command( output = cls.run_on_master_controller(cmd).splitlines()
cmd).strip().splitlines()
for line in output: for line in output:
for node in cls.nodes: for node in cls.nodes:
# split('.')[0] ensures that we always compare short names. # split('.')[0] ensures that we always compare short names.
@ -1281,7 +1281,7 @@ class BaseTempestTestCaseOvn(BaseTempestWhiteboxTestCase):
@classmethod @classmethod
def get_podified_ovn_db_cmd(cls, db): def get_podified_ovn_db_cmd(cls, db):
# use the first pod from the list, in case of multiple replicas # use the first pod from the list, in case of multiple replicas
db_pod = cls.proxy_host_client.exec_command( db_pod = cls.run_on_master_controller(
'{} get pods -l service=ovsdbserver-{} -o name'.format( '{} get pods -l service=ovsdbserver-{} -o name'.format(
cls.OC, db)).splitlines()[0].strip() cls.OC, db)).splitlines()[0].strip()
command_items = [cls.OC, 'rsh', db_pod, 'ovn-{}ctl'.format(db)] command_items = [cls.OC, 'rsh', db_pod, 'ovn-{}ctl'.format(db)]
@ -1290,7 +1290,7 @@ class BaseTempestTestCaseOvn(BaseTempestWhiteboxTestCase):
# leader DB instance # leader DB instance
cmd = '{} get ovndbcluster ovndbcluster-{} '.format(cls.OC, db) cmd = '{} get ovndbcluster ovndbcluster-{} '.format(cls.OC, db)
cmd += '-o jsonpath="{.status.internalDbAddress}"' cmd += '-o jsonpath="{.status.internalDbAddress}"'
all_db_addresses = cls.proxy_host_client.exec_command(cmd).strip() all_db_addresses = cls.run_on_master_controller(cmd)
command_items.append('--db="{}"'.format(all_db_addresses)) command_items.append('--db="{}"'.format(all_db_addresses))
# obtain the ovsdb-server command running on this pod # obtain the ovsdb-server command running on this pod
@ -1303,7 +1303,7 @@ class BaseTempestTestCaseOvn(BaseTempestWhiteboxTestCase):
'-- setsid', '-- setsid',
'ps -o command -C ovsdb-server --no-headers -ww', 'ps -o command -C ovsdb-server --no-headers -ww',
'|| true')) '|| true'))
ovsdb_server_cmd = cls.proxy_host_client.exec_command(cmd) ovsdb_server_cmd = cls.run_on_master_controller(cmd)
assert 'ovsdb-server' in ovsdb_server_cmd, \ assert 'ovsdb-server' in ovsdb_server_cmd, \
"command '{}' returned unexpected output: {}".format( "command '{}' returned unexpected output: {}".format(
cmd, ovsdb_server_cmd) cmd, ovsdb_server_cmd)
@ -1355,14 +1355,14 @@ class BaseTempestTestCaseOvn(BaseTempestWhiteboxTestCase):
def get_router_gateway_chassis_by_id(self, chassis_id): def get_router_gateway_chassis_by_id(self, chassis_id):
res = self.run_on_master_controller( res = self.run_on_master_controller(
self.sbctl + " get chassis " + chassis_id + " hostname").rstrip() self.sbctl + " get chassis " + chassis_id + " hostname")
return res.replace('"', '').split('.')[0] return res.replace('"', '').split('.')[0]
def get_router_port_gateway_mtu(self, router_port_id): def get_router_port_gateway_mtu(self, router_port_id):
cmd = (self.nbctl + " get logical_router_port lrp-" + router_port_id + cmd = (self.nbctl + " get logical_router_port lrp-" + router_port_id +
" options:gateway_mtu") " options:gateway_mtu")
return int( return int(
self.run_on_master_controller(cmd).rstrip().strip('"')) self.run_on_master_controller(cmd).strip('"'))
def get_item_uuid(self, db, item, search_string): def get_item_uuid(self, db, item, search_string):
ovn_db = self.sbctl if db == 'sb' else self.nbctl ovn_db = self.sbctl if db == 'sb' else self.nbctl
@ -1428,7 +1428,7 @@ class BaseDisruptiveTempestTestCase(BaseTempestWhiteboxTestCase):
def resource_setup(cls): def resource_setup(cls):
super(BaseDisruptiveTempestTestCase, cls).resource_setup() super(BaseDisruptiveTempestTestCase, cls).resource_setup()
try: try:
cls.proxy_host_client.exec_command( cls.run_on_master_controller(
"timeout 10 ssh {} virsh list".format(WB_CONF.hypervisor_host)) "timeout 10 ssh {} virsh list".format(WB_CONF.hypervisor_host))
cls.hypervisor_host = WB_CONF.hypervisor_host cls.hypervisor_host = WB_CONF.hypervisor_host
return return
@ -1438,12 +1438,12 @@ class BaseDisruptiveTempestTestCase(BaseTempestWhiteboxTestCase):
".ssh/config file.", WB_CONF.hypervisor_host) ".ssh/config file.", WB_CONF.hypervisor_host)
# Depending on ci-fmw version and/or setup, .ssh/config file could # Depending on ci-fmw version and/or setup, .ssh/config file could
# include an entry for either hypervisor or hypervisor-1 # include an entry for either hypervisor or hypervisor-1
host = cls.proxy_host_client.exec_command( host = cls.run_on_master_controller(
r"grep 'Host.*\ \(hypervisor\|hypervisor-1\)$' ~/.ssh/config " r"grep 'Host.*\ \(hypervisor\|hypervisor-1\)$' ~/.ssh/config "
"| cut -d' ' -f 2").strip() "| cut -d' ' -f 2")
try: try:
cls.proxy_host_client.exec_command( cls.run_on_master_controller(
"timeout 10 ssh {} virsh list".format(host)) "timeout 10 ssh {} virsh list".format(host))
except lib_exceptions.SSHExecCommandFailed: except lib_exceptions.SSHExecCommandFailed:
raise cls.skipException( raise cls.skipException(
@ -1457,20 +1457,20 @@ class BaseDisruptiveTempestTestCase(BaseTempestWhiteboxTestCase):
cmd = ("timeout 10 ssh {} sudo virsh list --all --name " cmd = ("timeout 10 ssh {} sudo virsh list --all --name "
"| grep -w {}").format( "| grep -w {}").format(
cls.hypervisor_host, host) cls.hypervisor_host, host)
return cls.proxy_host_client.exec_command(cmd).strip() return cls.run_on_master_controller(cmd)
@classmethod @classmethod
def is_host_state_is_shut_off(cls, host): def is_host_state_is_shut_off(cls, host):
cmd = ("timeout 10 ssh {} virsh list --state-shutoff | grep -w {} " cmd = ("timeout 10 ssh {} virsh list --state-shutoff | grep -w {} "
"|| true".format(cls.hypervisor_host, host)) "|| true".format(cls.hypervisor_host, host))
output = cls.proxy_host_client.exec_command(cmd) output = cls.run_on_master_controller(cmd)
return host in output return host in output
@classmethod @classmethod
def is_host_loginable(cls, host): def is_host_loginable(cls, host):
cmd = "timeout 10 ssh {} ssh {} hostname || true".format( cmd = "timeout 10 ssh {} ssh {} hostname || true".format(
cls.hypervisor_host, host) cls.hypervisor_host, host)
output = cls.proxy_host_client.exec_command(cmd) output = cls.run_on_master_controller(cmd)
return host in output return host in output
@classmethod @classmethod
@ -1479,7 +1479,7 @@ class BaseDisruptiveTempestTestCase(BaseTempestWhiteboxTestCase):
raise cls.skipException("Power operations are not allowed") raise cls.skipException("Power operations are not allowed")
cmd = "timeout 10 ssh {} sudo virsh destroy {}".format( cmd = "timeout 10 ssh {} sudo virsh destroy {}".format(
cls.hypervisor_host, cls.find_host_virsh_name(host)) cls.hypervisor_host, cls.find_host_virsh_name(host))
cls.proxy_host_client.exec_command(cmd) cls.run_on_master_controller(cmd)
common_utils.wait_until_true( common_utils.wait_until_true(
lambda: cls.is_host_state_is_shut_off(host), lambda: cls.is_host_state_is_shut_off(host),
timeout=30, sleep=5) timeout=30, sleep=5)
@ -1490,7 +1490,7 @@ class BaseDisruptiveTempestTestCase(BaseTempestWhiteboxTestCase):
raise cls.skipException("Power operations are not allowed") raise cls.skipException("Power operations are not allowed")
cmd = "timeout 10 ssh {} sudo virsh start {}".format( cmd = "timeout 10 ssh {} sudo virsh start {}".format(
cls.hypervisor_host, cls.find_host_virsh_name(host)) cls.hypervisor_host, cls.find_host_virsh_name(host))
cls.proxy_host_client.exec_command(cmd) cls.run_on_master_controller(cmd)
# TODO(rsafrono): implement and apply additional health checks # TODO(rsafrono): implement and apply additional health checks
common_utils.wait_until_true( common_utils.wait_until_true(
lambda: cls.is_host_loginable(host), lambda: cls.is_host_loginable(host),
@ -1502,7 +1502,7 @@ class BaseDisruptiveTempestTestCase(BaseTempestWhiteboxTestCase):
raise cls.skipException("Power operations are not allowed") raise cls.skipException("Power operations are not allowed")
cmd = "timeout 10 ssh {} sudo virsh reboot {}".format( cmd = "timeout 10 ssh {} sudo virsh reboot {}".format(
cls.hypervisor_host, cls.find_host_virsh_name(host)) cls.hypervisor_host, cls.find_host_virsh_name(host))
cls.proxy_host_client.exec_command(cmd) cls.run_on_master_controller(cmd)
common_utils.wait_until_true( common_utils.wait_until_true(
lambda: cls.is_host_loginable(host), lambda: cls.is_host_loginable(host),
timeout=120, sleep=5) timeout=120, sleep=5)
@ -1510,9 +1510,9 @@ class BaseDisruptiveTempestTestCase(BaseTempestWhiteboxTestCase):
def ensure_overcloud_nodes_active(self): def ensure_overcloud_nodes_active(self):
"""Checks all openstack nodes are up, otherwise activates them. """Checks all openstack nodes are up, otherwise activates them.
""" """
hosts = self.proxy_host_client.exec_command( hosts = self.run_on_master_controller(
"timeout 10 ssh {} sudo virsh list --all --name".format( "timeout 10 ssh {} sudo virsh list --all --name".format(
self.hypervisor_host)).strip().split() self.hypervisor_host)).split()
for host in hosts: for host in hosts:
if self.is_host_state_is_shut_off(host): if self.is_host_state_is_shut_off(host):
self.power_on_host(host) self.power_on_host(host)