#!/usr/bin/python3 # # SPDX-License-Identifier: Apache-2.0 # """ Contains helper functions that will configure basic system settings. """ import subprocess import sys import time from consts.timeout import HostTimeout from utils import kpi, serial from utils.install_log import LOG from helper import host_helper def update_platform_cpus(stream, hostname, cpu_num=5): """ Update platform CPU allocation. """ LOG.info("Allocating %s CPUs for use by the %s platform.", cpu_num, hostname) cmd = "\nsource /etc/platform/openrc;" \ f" system host-cpu-modify {hostname} -f platform -p0 {cpu_num}" serial.send_bytes(stream, cmd, prompt="keystone", timeout=300) def set_dns(stream, dns_ip): """ Perform DNS configuration on the system. """ LOG.info("Configuring DNS to %s.", dns_ip) cmd = f"source /etc/platform/openrc; system dns-modify nameservers={dns_ip}" serial.send_bytes(stream, cmd, prompt="keystone") def config_controller(stream, password): """ Configure controller-0 using optional arguments """ LOG.info("Executing the bootstrap ansible playbook") cmd = "ansible-playbook /usr/share/ansible/stx-ansible/playbooks/bootstrap.yml" serial.send_bytes(stream, cmd, expect_prompt=False) host_helper.check_password(stream, password=password) serial.expect_bytes(stream, "~$", timeout=HostTimeout.LAB_CONFIG) cmd = "echo [$?]" serial.send_bytes(stream, cmd, expect_prompt=False, log=False) ret = serial.expect_bytes(stream, "[0]", timeout=HostTimeout.NORMAL_OP, log=False) if ret != 0: LOG.info("Configuration failed. Exiting installer.") raise SystemExit("Configcontroller failed") LOG.info("Successful bootstrap ansible playbook execution") def fault_tolerant(scale=1): """ Provides the scale argument to the fault-tolerant decorator. Args: - scale: re-attempt delay vector scale factor Returns: - fault-tolerant decorator. """ def fault_tolerant_decorator(func): """ Decorator to run a command in a fault-tolerant fashion. Args: - func: The function to be decorated. Returns: - fault-tolerant wrapper """ def fault_tolerant_wrapper(*args, **kwargs): """ Runs a command in a fault-tolerant fashion. The function provides a recovery mechanism with progressive re-attempt delays The first attempt is the normal command execution. If the command fails, the first re-attempt runs after 2s, and the re-attempt delay goes increasing until 10 min. The last re-attempts, with longer delays are intended to help the user to salvage the ongoing installation, if the system does not recover automatically. Intentionally, the function does not provide a return code, due to the following reason. To ensure system integrity, the function stops the program execution, if it can not achieve a successful result, after a maximum number of retries. Hence, any subsequent functions may safely rely on the system integrity. Args: - cmd: The command to be executed. Returns: None """ delay = HostTimeout.REATTEMPT_DELAY reattempt_delay = scale*delay max_attempts = len(reattempt_delay) attempt = 1 while True: cmd = kwargs['cmd'] try: return_code = func(*args, **kwargs) assert return_code == 0 break except AssertionError as exc: if attempt < max_attempts: LOG.warning( "#### Failed command:\n$ %s [attempt: %s/%s]\n", cmd, attempt, max_attempts ) LOG.info( "Trying again after %s ... ", kpi.get_formated_time(reattempt_delay[attempt]) ) time.sleep(reattempt_delay[attempt]) attempt = attempt + 1 else: LOG.error( "#### Failed command:\n$ %s [attempt: %s/%s]\n", cmd, attempt, max_attempts ) raise TimeoutError from exc except Exception as exc: # pylint: disable=broad-except LOG.error( "#### Failed command:\n$ %s\nError: %s", cmd, repr(exc) ) sys.exit(1) return fault_tolerant_wrapper return fault_tolerant_decorator def exec_cmd(cmd): """ Execute a local command on the host machine in a fault-tolerant fashion. Refer to the fault_tolerant decorator for more details. """ @fault_tolerant() def exec_cmd_ft(*args, **kwargs): # pylint: disable=unused-argument LOG.info("#### Executing command on the host machine:\n$ %s\n", cmd) with subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE) as process: for line in iter(process.stdout.readline, b''): LOG.info("%s", line.decode("utf-8").strip()) process.wait() return process.returncode exec_cmd_ft(**{'cmd': cmd})