x/tobiko
Code Issues Proposed changes

Add SNAT background ping test

Browse Source 
Added test that checks connectivity to external IP from a VM without FIP
that uses SNAT.

Previously, background ping tests could only be executed from the test
machine. With this patch, new background ping tests can be implemented,
sending pings from an external machine (e.g. an Openstack VM instance).

TODO (will be done in follow up patches):
Refactor legacy background ping test, so that it uses the new code.
Hence, when background pings functions are called with ssh_client, they
are run from an external machine; if they are called without ssh_client,
the pings are sent from the local test machine.

Change-Id: Id862e6568623464bf87ee567c1386d56843f6919
This commit is contained in:
Eduardo Olivares 2024-12-30 16:07:49 +01:00
parent 05523dbc45
commit 5957654fdc
14 changed files with 322 additions and 97 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

21
tobiko/podified/_topology.py
View File

@ -26,6 +26,7 @@ from tobiko.podified import _openshift
from tobiko.podified import containers
from tobiko import rhosp
from tobiko.shell import iperf3
from tobiko.shell import ping
from tobiko.shell import sh
from tobiko.shell import ssh
@ -176,10 +177,22 @@ class PodifiedTopology(rhosp.RhospTopology):
node_type=EDPM_NODE)
assert isinstance(node, EdpmNode)
def check_or_start_background_vm_ping(self, server_ip):
_openshift.check_or_start_tobiko_ping_command(
server_ip=server_ip
)
def check_or_start_background_vm_ping(
self,
server_ip: typing.Union[str, netaddr.IPAddress],
ssh_client: ssh.SSHClientType = None):
if not ssh_client:
_openshift.check_or_start_tobiko_ping_command(
server_ip=server_ip
)
else:
sh.check_or_start_external_process(
start_function=ping.execute_ping_in_background,
check_function=ping.check_ping_results,
liveness_function=ping.ping_alive,
stop_function=ping.stop_ping,
address=server_ip,
ssh_client=ssh_client)
def check_or_start_background_iperf_connection(
self,

4
tobiko/rhosp/config.py
View File

@ -87,6 +87,10 @@ RHOSP_OPTIONS = [
"then working for 60 seconds and then again not working "
"for another 10 seconds. In such case this total break "
"time would be 13 seconds."),
cfg.IntOpt('max_ping_loss_allowed',
default=10,
help="maximum number of unreplied pings during the "
"background ping tests."),
]
TRIPLEO_OPTIONS = [

5
tobiko/shell/files/__init__.py
View File

@ -15,9 +15,14 @@
# under the License.
from __future__ import absolute_import
from tobiko.shell.files import _files
from tobiko.shell.files import _logs
get_home_absolute_filepath = _files.get_home_absolute_filepath
truncate_client_logfile = _files.truncate_client_logfile
remove_old_logfile = _files.remove_old_logfile
LogFileDigger = _logs.LogFileDigger
JournalLogDigger = _logs.JournalLogDigger
MultihostLogFileDigger = _logs.MultihostLogFileDigger

74
tobiko/shell/files/_files.py Normal file
View File

@ -0,0 +1,74 @@
# Copyright (c) 2025 Red Hat, Inc.
#
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
from __future__ import absolute_import
import os
import tobiko
from tobiko.shell import sh
from tobiko.shell import ssh
def get_home_absolute_filepath(path: str,
ssh_client: ssh.SSHClientType = None) -> str:
if ssh_client is None:
return _get_local_filepath(path)
else:
return _get_remote_filepath(path, ssh_client)
def _get_local_filepath(path: str) -> str:
final_dir_path = f'{sh.get_user_home_dir()}/{path}'
if not os.path.exists(final_dir_path):
os.makedirs(final_dir_path)
return final_dir_path
def _get_remote_filepath(path: str,
ssh_client: ssh.SSHClientType) -> str:
homedir = sh.execute('echo ~', ssh_client=ssh_client).stdout.rstrip()
final_dir_path = f'{homedir}/{path}'
sh.make_remote_dirs(final_dir_path, ssh_client=ssh_client)
return final_dir_path
def truncate_client_logfile(
logfile: str,
ssh_client: ssh.SSHClientType = None) -> None:
if ssh_client:
_truncate_remote_logfile(logfile, ssh_client)
else:
tobiko.truncate_logfile(logfile)
def _truncate_remote_logfile(logfile: str,
ssh_client: ssh.SSHClientType) -> None:
truncated_logfile = tobiko.get_truncated_filename(logfile)
sh.execute(f'/usr/bin/mv {logfile} {truncated_logfile}',
ssh_client=ssh_client)
def remove_old_logfile(logfile: str,
ssh_client: ssh.SSHClientType = None):
if ssh_client:
sh.execute(f'/usr/bin/rm -f {logfile}',
ssh_client=ssh_client)
else:
try:
os.remove(logfile)
except FileNotFoundError:
pass

5
tobiko/shell/find.py
View File

@ -21,6 +21,7 @@ import typing # noqa
import tobiko
from tobiko.shell import sh
from tobiko.shell import ssh
from tobiko.shell.sh import _command
class FilesNotFound(tobiko.TobikoException):
@ -32,9 +33,9 @@ NameType = typing.Union[None, str, typing.List[str]]
PathType = typing.Union[str, typing.Iterable[str]]
def find_files(path: sh.ShellCommandType,
def find_files(path: _command.ShellCommandType,
name: NameType = None,
command: sh.ShellCommandType = 'find',
command: _command.ShellCommandType = 'find',
max_depth: int = None,
modified_since: tobiko.Seconds = None,
ssh_client: ssh.SSHClientType = None,

7
tobiko/shell/grep.py
View File

@ -20,6 +20,7 @@ import typing # noqa
import tobiko
from tobiko.shell import sh
from tobiko.shell import ssh
from tobiko.shell.sh import _command
class NoMatchingLinesFound(tobiko.TobikoException):
@ -28,8 +29,8 @@ class NoMatchingLinesFound(tobiko.TobikoException):
def grep(pattern: str,
command: typing.Optional[sh.ShellCommandType] = None,
grep_command: sh.ShellCommandType = 'zgrep -Eh',
command: typing.Optional[_command.ShellCommandType] = None,
grep_command: _command.ShellCommandType = 'zgrep -Eh',
files: typing.Optional[typing.List[str]] = None,
ssh_client: ssh.SSHClientFixture = None,
blank_lines: bool = True,
@ -77,6 +78,6 @@ def grep_files(pattern: str,
def grep_lines(pattern: str,
command: sh.ShellCommandType,
command: _command.ShellCommandType,
**grep_params) -> typing.List[str]:
return grep(pattern=pattern, command=command, **grep_params)

90
tobiko/shell/iperf3/_execute.py
View File

@ -25,6 +25,7 @@ from oslo_log import log
import tobiko
from tobiko import config
from tobiko.shell import files
from tobiko.shell.iperf3 import _interface
from tobiko.shell.iperf3 import _parameters
from tobiko.shell import sh
@ -38,58 +39,11 @@ LOG = log.getLogger(__name__)
def get_iperf3_logs_filepath(address: typing.Union[str, netaddr.IPAddress],
path: str,
ssh_client: ssh.SSHClientType = None) -> str:
if ssh_client:
final_dir = _get_remote_filepath(path, ssh_client)
else:
final_dir = _get_local_filepath(path)
final_dir = files.get_home_absolute_filepath(path, ssh_client)
filename = f'iperf_{address}.log'
return os.path.join(final_dir, filename)
def _get_local_filepath(path: str) -> str:
final_dir_path = f'{sh.get_user_home_dir()}/{path}'
if not os.path.exists(final_dir_path):
os.makedirs(final_dir_path)
return final_dir_path
def _get_remote_filepath(path: str,
ssh_client: ssh.SSHClientType) -> str:
homedir = sh.execute('echo ~', ssh_client=ssh_client).stdout.rstrip()
final_dir_path = f'{homedir}/{path}'
sh.execute(f'/usr/bin/mkdir -p {final_dir_path}',
ssh_client=ssh_client)
return final_dir_path
def _truncate_iperf3_client_logfile(
logfile: str,
ssh_client: ssh.SSHClientType = None) -> None:
if ssh_client:
_truncate_remote_logfile(logfile, ssh_client)
else:
tobiko.truncate_logfile(logfile)
def _truncate_remote_logfile(logfile: str,
ssh_client: ssh.SSHClientType) -> None:
truncated_logfile = tobiko.get_truncated_filename(logfile)
sh.execute(f'/usr/bin/mv {logfile} {truncated_logfile}',
ssh_client=ssh_client)
def _remove_old_logfile(logfile: str,
ssh_client: ssh.SSHClientType = None):
if ssh_client:
sh.execute(f'/usr/bin/rm -f {logfile}',
ssh_client=ssh_client)
else:
try:
os.remove(logfile)
except FileNotFoundError:
pass
def get_bandwidth(address: typing.Union[str, netaddr.IPAddress],
bitrate: int = None,
download: bool = None,
@ -169,7 +123,7 @@ def execute_iperf3_client_in_background(
# it needs to be removed, otherwise iperf will append new log
# to the end of the existing file and this will make json output
# file to be malformed
_remove_old_logfile(output_path, ssh_client=ssh_client)
files.remove_old_logfile(output_path, ssh_client=ssh_client)
# If there is ssh client for the server where iperf3 server is going
# to run, lets make sure it is started fresh as e.g. in case of
# failure in the previous run, it may report that is still "busy" thus
@ -206,26 +160,20 @@ def _get_iperf3_pid(
port: int = None,
protocol: str = None,
ssh_client: ssh.SSHClientType = None) -> typing.Union[int, None]:
try:
iperf_pids = sh.execute(
'pidof iperf3', ssh_client=ssh_client).stdout.rstrip().split(" ")
except sh.ShellCommandFailed:
return None
for iperf_pid in iperf_pids:
proc_cmdline = sh.get_command_line(
iperf_pid,
ssh_client=ssh_client)
if address and str(address) in proc_cmdline:
# This is looking for the iperf client instance
return int(iperf_pid)
elif port and protocol:
# By looking for port and protocol we are looking
# for the iperf3 server's PID
if "-s" in proc_cmdline and f"-p {port}" in proc_cmdline:
if ((protocol.lower() == 'udp' and "-u" in proc_cmdline) or
(protocol.lower() == 'tcp' and
'-u' not in proc_cmdline)):
return int(iperf_pid)
if address:
iperf_commands = [f'iperf3 .*{address}']
elif protocol and protocol.lower() == 'udp':
iperf_commands = [f'iperf3 .*-s .*-u .*-p {port}',
f'iperf3 .*-s .*-p {port} .*-u']
else:
iperf_commands = [f'iperf3 .*-s .*-p {port}']
for iperf_command in iperf_commands:
iperf_processes = sh.list_processes(command_line=iperf_command,
ssh_client=ssh_client)
if iperf_processes:
return iperf_processes.unique.pid
LOG.debug('no iperf3 processes were found')
return None
@ -286,7 +234,7 @@ def check_iperf3_client_results(address: typing.Union[str, netaddr.IPAddress],
else:
current_break = 0
_truncate_iperf3_client_logfile(logfile, ssh_client)
files.truncate_client_logfile(logfile, ssh_client)
testcase = tobiko.get_test_case()
testcase.assertLessEqual(longest_break,
@ -319,7 +267,7 @@ def stop_iperf3_client(address: typing.Union[str, netaddr.IPAddress],
if pid:
LOG.info(f'iperf3 client process to > {address} already running '
f'with PID: {pid}')
sh.execute(f'sudo kill {pid}', ssh_client=ssh_client)
sh.execute(f'kill {pid}', ssh_client=ssh_client, sudo=True)
def start_iperf3_server(

4
tobiko/shell/ping/__init__.py
View File

@ -65,3 +65,7 @@ PingStatistics = _statistics.PingStatistics
write_ping_to_file = _ping.write_ping_to_file
check_ping_statistics = _ping.check_ping_statistics
skip_check_ping_statistics = _ping.skip_check_ping_statistics
ping_alive = _ping.ping_alive
stop_ping = _ping.stop_ping
check_ping_results = _ping.check_ping_results
execute_ping_in_background = _ping.execute_ping_in_background

4
tobiko/shell/ping/_parameters.py
View File

@ -172,8 +172,8 @@ def get_positive_integer(name, value, default=None):
return get_positive_integer(name, getattr(default, name))
if value is not None:
value = int(value)
if value <= 0:
message = "{!r} value must be positive: {!r}".format(
if value < 0:
message = "{!r} value must be zero or greater: {!r}".format(
name, value)
raise ValueError(message)
return value

142
tobiko/shell/ping/_ping.py
View File

@ -27,13 +27,17 @@ import netaddr
from oslo_log import log
import tobiko
from tobiko import config
from tobiko.shell import files
from tobiko.shell import sh
from tobiko.shell import ssh
from tobiko.shell.ping import _interface
from tobiko.shell.ping import _exception
from tobiko.shell.ping import _parameters
from tobiko.shell.ping import _statistics
CONF = config.CONF
LOG = log.getLogger(__name__)
@ -466,12 +470,16 @@ def get_vm_ping_log_files(glob_ping_log_pattern='tobiko_ping_results/ping_'
yield vm_ping_log_filename
def check_ping_statistics(failure_limit=10):
def check_ping_statistics(failure_limit=None):
"""Gets a list of ping_vm_log files and
iterates their lines, checks if max ping
failures have been reached per fip=file"""
if failure_limit is None:
failure_limit = CONF.tobiko.rhosp.max_ping_loss_allowed
ping_files_found = 0
# iterate over ping_vm_log files:
for filename in list(get_vm_ping_log_files()):
ping_files_found += 1
with io.open(filename, 'rt') as fd:
LOG.info(f'checking ping log file: {filename}, '
f'failure_limit is :{failure_limit}')
@ -496,8 +504,140 @@ def check_ping_statistics(failure_limit=10):
f'to vm fip destination: '
f'{ping_failures_list[-1]["destination"]}')
if ping_files_found == 0:
tobiko.fail('No ping log files found')
def skip_check_ping_statistics():
for filename in list(get_vm_ping_log_files()):
tobiko.truncate_logfile(filename)
LOG.info(f'skipping ping failures in ping log file: {filename}')
def _get_ping_pid(
address: typing.Union[str, netaddr.IPAddress, None] = None,
ssh_client: ssh.SSHClientType = None) -> typing.Union[int, None]:
ping_command = 'ping'
if address is not None:
ping_command += f' .*{address}'
ping_processes = sh.list_processes(command_line=ping_command,
ssh_client=ssh_client)
if not ping_processes:
LOG.debug('no ping processes were found')
return None
else:
return ping_processes.unique.pid
def ping_alive(address: typing.Union[str, netaddr.IPAddress], # noqa; pylint: disable=W0613
ssh_client: ssh.SSHClientType = None,
**kwargs) -> bool:
return bool(_get_ping_pid(address=address, ssh_client=ssh_client))
def stop_ping(address: typing.Union[str, netaddr.IPAddress],
ssh_client: ssh.SSHClientType = None,
**kwargs): # noqa; pylint: disable=W0613
pid = _get_ping_pid(address=address, ssh_client=ssh_client)
if pid:
LOG.info(f'ping process to > {address} already running '
f'with PID: {pid}')
# the SIGINT signal makes ping write the "ping statistics" block
# before exiting
sh.execute(f'kill -s SIGINT {pid}', ssh_client=ssh_client, sudo=True)
def _get_ping_logs_filepath(address: typing.Union[str, netaddr.IPAddress],
path: str,
ssh_client: ssh.SSHClientType = None) -> str:
final_dir = files.get_home_absolute_filepath(path, ssh_client)
filename = f'ping_{address}.log'
return os.path.join(final_dir, filename)
# TODO(eolivare): replace check_ping_statistics with check_ping_results
def check_ping_results(address: typing.Union[str, netaddr.IPAddress],
output_dir: str = 'tobiko_ping_results',
ssh_client: ssh.SSHClientType = None,
**kwargs): # noqa; pylint: disable=W0613
testcase = tobiko.get_test_case()
testcase.assertFalse(ping_alive(address, ssh_client))
# This function expects that the result file is available locally already
logfile = _get_ping_logs_filepath(address, output_dir, ssh_client)
try:
ping_log_raw = sh.execute(
f"cat {logfile}", ssh_client=ssh_client).stdout
except sh.ShellCommandFailed as err:
if config.is_prevent_create():
# Tobiko is not expected to create resources in this run
# so ping should be already running and log file should
# be already there, if it is not, it should fail
tobiko.fail('Failed to read ping log from the file. '
f'Ping Destination IP address: {address}; '
f'Logfile: {logfile}')
else:
# Tobiko is creating resources so it is normal that file was not
# there yet
LOG.debug(f'Failed to read ping log from the file. '
f'Error: {err}')
return
LOG.debug(f'ping log raw: {ping_log_raw}')
if not ping_log_raw:
if config.is_prevent_create():
# Tobiko is not expected to create resources in this run
# so ping should be already running and log file should
# be already there, if it is not, it should fail
tobiko.fail('Failed empty ping file.')
else:
LOG.debug('Failed ping log file empty')
return
files.truncate_client_logfile(logfile, ssh_client)
ping_stats = _statistics.parse_ping_statistics(ping_log_raw)
testcase.assertGreater(ping_stats.transmitted, 0)
testcase.assertGreater(ping_stats.received, 0)
testcase.assertLessEqual(ping_stats.transmitted - ping_stats.received,
CONF.tobiko.rhosp.max_ping_loss_allowed)
def start_background_ping(address: typing.Union[str, netaddr.IPAddress],
output_path: str,
ssh_client: ssh.SSHClientType = None):
parameters = _parameters.get_ping_parameters(host=address,
count=0,
deadline=0)
command = _interface.get_ping_command(parameters, ssh_client)
# both stdout and stderr need to be written to the provided log file
command += '2>&1'
command += f'> {output_path}'
process = sh.process(command, ssh_client=ssh_client)
process.execute()
# TODO(eolivare): replace write_ping_to_file with execute_ping_in_background
def execute_ping_in_background(address: typing.Union[str, netaddr.IPAddress],
output_dir: str = 'tobiko_ping_results',
ssh_client: ssh.SSHClientType = None,
**kwargs): # noqa; pylint: disable=W0613
output_path = _get_ping_logs_filepath(address, output_dir, ssh_client)
LOG.info(f'starting ping process to > {address} , '
f'output file is : {output_path}')
# just in case there is some leftover file from previous run,
# it needs to be removed, otherwise ping will append new log
# to the end of the existing file and this will make output
# file to be malformed
files.remove_old_logfile(output_path, ssh_client=ssh_client)
# Stop ping in case it is running
stop_ping(address, ssh_client)
# Start ping again
start_background_ping(address, output_path, ssh_client)
# if ping does not start properly, fail the test
if not ping_alive(address, ssh_client):
tobiko.fail('background ping process did not start')

3
tobiko/shell/sh/__init__.py
View File

@ -23,6 +23,7 @@ from tobiko.shell.sh import _execute
from tobiko.shell.sh import _hostname
from tobiko.shell.sh import _io
from tobiko.shell.sh import _local
from tobiko.shell.sh import _mkdirs
from tobiko.shell.sh import _nameservers
from tobiko.shell.sh import _nmcli
from tobiko.shell.sh import _path
@ -141,3 +142,5 @@ find_command = _which.find_command
get_nm_connection_ids = _nmcli.get_nm_connection_ids
get_nm_connection_values = _nmcli.get_nm_connection_values
make_remote_dirs = _mkdirs.make_remote_dirs

2
tobiko/shell/sh/_mkdirs.py
View File

@ -23,6 +23,6 @@ from tobiko.shell import ssh
def make_remote_dirs(file_name: str,
ssh_client: ssh.SSHClientType = None,
sudo: bool = None):
_execute.execute(f'mkdirs -p "{file_name}"',
_execute.execute(f'mkdir -p "{file_name}"',
ssh_client=ssh_client,
sudo=sudo)

35
tobiko/tests/scenario/neutron/test_network.py
View File

@ -65,6 +65,12 @@ class NetworkTest(BaseNetworkTest):
@pytest.mark.background
class BackgroundProcessTest(BaseNetworkTest):
"""Test designed to run in the background,
then collect results.
Logic: checks if process exists, if so stop the process,
then execute some check logic i.e. a check function.
if the process by name isn't running,
start a separate process i.e a background function"""
stack = tobiko.required_fixture(stacks.AdvancedPeerServerStackFixture)
@ -77,16 +83,31 @@ class BackgroundProcessTest(BaseNetworkTest):
'Background tests not supported by this topology class.')
def test_check_background_vm_ping(self):
""" Tests that are designed to run in the background ,
then collect results.
Logic: checks if process exists, if so stop the process,
then execute some check logic i.e. a check function.
if the process by name isn't running,
start a separate process i.e a background function"""
"""Ping from test machine/container/pod to VM with FIP,
validating north-south connectivity with SDNAT (source-destination
NAT)."""
self.topology.check_or_start_background_vm_ping(
self.stack.peer_stack.floating_ip_address)
def test_check_background_vm_ping_snat(self):
"""Ping from a VM without FIP to an external IP,
validating north-south connectivity with SNAT (source NAT)."""
# make sure the VM does not have any FIP
self.assertFalse(self.stack.has_floating_ip)
try:
ext_subnet = neutron.list_subnets(
network=self.stack.network_stack.gateway_network_id,
ip_version=4)[0]
except IndexError:
ext_subnet = neutron.list_subnets(
network=self.stack.network_stack.gateway_network_id,
ip_version=6)[0]
self.topology.check_or_start_background_vm_ping(
ext_subnet['gateway_ip'],
ssh_client=self.stack.ssh_client)
def test_east_west_tcp_traffic_background_iperf(self):
""" Test East-West TCP traffic in the existing flow.

23
tobiko/tripleo/nova.py
View File

@ -30,18 +30,29 @@ from tobiko.shell import ssh
# Test is inteded for D/S env
@overcloud.skip_if_missing_overcloud
def check_or_start_background_vm_ping(server_ip):
def check_or_start_background_vm_ping(
server_ip: typing.Union[str, netaddr.IPAddress],
ssh_client: ssh.SSHClientType = None):
"""Check if process exists, if so stop and check ping health
if not : start a new separate ping process.
Executes a Background ping to a vm floating_ip,
this test is intended to be run and picked up again
by the next tobiko run. Ping results are parsed
and a failure is raised if ping failure is above a certain amount"""
sh.check_or_start_background_process(
bg_function=ping.write_ping_to_file,
bg_process_name='tobiko_background_ping',
check_function=ping.check_ping_statistics,
ping_ip=server_ip)
if ssh_client is None:
sh.check_or_start_background_process(
bg_function=ping.write_ping_to_file,
bg_process_name='tobiko_background_ping',
check_function=ping.check_ping_statistics,
ping_ip=server_ip)
else:
sh.check_or_start_external_process(
start_function=ping.execute_ping_in_background,
check_function=ping.check_ping_results,
liveness_function=ping.ping_alive,
stop_function=ping.stop_ping,
address=server_ip,
ssh_client=ssh_client)
# Test is inteded for D/S env