x/tobiko
Code Issues Proposed changes

_check_ping_results waits until log file is available

Browse Source 
At the end of check_or_start_tobiko_command, the background ping pod is
started and immediately it checks whether the ping log file is available.
This may fail due to a race condition: the pod may take a few seconds to
start and the ping log file may be unavailable for a little more time.

This patch adds retries until the ping log file is available.

Change-Id: I9b882082b0506e8b0c6389eea8199f644bb65df2
This commit is contained in:
Eduardo Olivares 2025-04-10 08:55:12 +02:00
parent 88343162f6
commit ccf40e1a63
1 changed files with 19 additions and 12 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

31
tobiko/podified/_openshift.py
View File

@ -13,6 +13,7 @@
# under the License. # under the License.
from __future__ import absolute_import from __future__ import absolute_import
import glob
import json import json
import typing import typing
@ -345,6 +346,9 @@ def check_or_start_tobiko_command(cmd_args, pod_name, check_function):
pod_obj.delete(ignore_not_found=True) pod_obj.delete(ignore_not_found=True)
LOG.info('checked and stopped previous tobiko command ' LOG.info('checked and stopped previous tobiko command '
f'POD {pod_name}; starting a new POD.') f'POD {pod_name}; starting a new POD.')
elif config.is_prevent_create():
tobiko.fail(f'Expected POD {pod_name} not running. '
f'That POD should have been running: {cmd_args}')
else: else:
# First time the test is run: # First time the test is run:
# if POD by specific name is not present start one: # if POD by specific name is not present start one:
@ -420,18 +424,21 @@ def _check_ping_results(pod):
# tobiko.shell.ping._ping module so we can use those existing # tobiko.shell.ping._ping module so we can use those existing
# functions to check results # functions to check results
ping_results_dest = f'{sh.get_user_home_dir()}/{PING_RESULTS_DIR}' ping_results_dest = f'{sh.get_user_home_dir()}/{PING_RESULTS_DIR}'
cp = oc.oc_action( ping_log_file_pattern = f'{ping_results_dest}/ping_*.log'
pod.context, for attempt in tobiko.retry(timeout=30., interval=5.):
'cp', cp = oc.oc_action(
[f"{pod.name()}:{POD_PING_RESULTS_DIR}", ping_results_dest] pod.context,
) 'cp',
if cp.status == 0: [f"{pod.name()}:{POD_PING_RESULTS_DIR}", ping_results_dest]
ping.check_ping_statistics() )
# here we should probably move those files inside the pod to some other if cp.status == 0 and glob.glob(ping_log_file_pattern):
# location, or maybe simply delete them break
else: elif attempt.is_last:
tobiko.fail("Failed to copy ping log files from the POD " tobiko.fail("Failed to copy ping log files from the POD "
f"{pod.name()}. Error: {cp.err}") f"{pod.name()}. Error: {cp.err}")
# ping.check_ping_statistics() calls tobiko.truncate_logfile(filename) to
# rename log files to ping_<IP>.log_<date>
ping.check_ping_statistics()
def execute_in_pod(pod_name, command, container_name=None): def execute_in_pod(pod_name, command, container_name=None):