From c93f1aa754e9774ecf81aa8a452d42599cec63d8 Mon Sep 17 00:00:00 2001 From: Lindley Werner Date: Thu, 4 May 2023 17:21:54 -0300 Subject: [PATCH] Adding pylint to /virtualbox/pybox Enabling automatic pylint with tox and zull for each new patchset. Test plan: PASS: Run "tox -e pylint" in the terminal, this will: - Run pylint in all python files - Show the report Story: 2005051 Task: 47900 Change-Id: I2f66a5f72e3f8746c00aae96287ad3e4edb88e28 Signed-off-by: Lindley Werner --- .zuul.yaml | 2 + pylint.rc | 2 +- requirements/test-requirements.txt | 2 +- tox.ini | 12 +- virtualbox/pybox/Parser.py | 61 +- virtualbox/pybox/README.md | 17 +- virtualbox/pybox/consts/env.py | 29 +- virtualbox/pybox/consts/networking.py | 190 +++- virtualbox/pybox/consts/node.py | 9 +- virtualbox/pybox/consts/timeout.py | 9 +- virtualbox/pybox/helper/host_helper.py | 46 +- virtualbox/pybox/helper/install_lab.py | 42 +- virtualbox/pybox/helper/vboxmanage.py | 737 +++++++++++---- virtualbox/pybox/install_vbox.py | 1186 ++++++++++++++++-------- virtualbox/pybox/requirements.txt | 1 + virtualbox/pybox/utils/install_log.py | 42 +- virtualbox/pybox/utils/kpi.py | 59 +- virtualbox/pybox/utils/serial.py | 107 ++- virtualbox/pybox/utils/sftp.py | 93 +- 19 files changed, 1846 insertions(+), 800 deletions(-) diff --git a/.zuul.yaml b/.zuul.yaml index 617cba7..d34f9e1 100644 --- a/.zuul.yaml +++ b/.zuul.yaml @@ -5,6 +5,8 @@ check: jobs: - openstack-tox-linters + - openstack-tox-pylint gate: jobs: - openstack-tox-linters + - openstack-tox-pylint diff --git a/pylint.rc b/pylint.rc index 10351b7..77ce541 100755 --- a/pylint.rc +++ b/pylint.rc @@ -234,4 +234,4 @@ valid-classmethod-first-arg=cls [EXCEPTIONS] # Exceptions that will emit a warning when being caught. Defaults to # "Exception" -overgeneral-exceptions=Exception +overgeneral-exceptions=builtins.Exception diff --git a/requirements/test-requirements.txt b/requirements/test-requirements.txt index 9349928..c4ea274 100644 --- a/requirements/test-requirements.txt +++ b/requirements/test-requirements.txt @@ -1,4 +1,4 @@ yamllint === 1.32.0 bashate === 2.1.1 pylint === 2.13.9 - +tox === 4.6.3 diff --git a/tox.ini b/tox.ini index 3c36a22..e74ce01 100644 --- a/tox.ini +++ b/tox.ini @@ -4,9 +4,7 @@ minversion = 2.3 skipsdist = True [testenv] -deps = - -r{toxinidir}/requirements/test-requirements.txt - -r{toxinidir}/virtualbox/pybox/requirements.txt +deps = -r{toxinidir}/requirements/test-requirements.txt allowlist_externals = reno [testenv:linters] @@ -23,7 +21,6 @@ commands = -print0 | xargs -0 yamllint" bash -c "find {toxinidir} \ -not \( -type d -name .?\* -prune \) \ - -not \( -type d -path {toxinidir}/toCOPY/mock_overlay -prune \) \ -type f \ -not -name \*~ \ -not -name \*.md \ @@ -33,6 +30,13 @@ commands = [testenv:pylint] basepython = python3 sitepackages = False +setenv = + BASEPATH = {toxinidir}/virtualbox/pybox + PYTHONPATH= {env:BASEPATH}:{env:BASEPATH}/helper:{env:BASEPATH}/consts:{env:BASEPATH}/utils +deps = + -r{env:BASEPATH}/requirements.txt + {[testenv]deps} +allowlist_externals = pylint commands = pylint {posargs} --rcfile=./pylint.rc virtualbox/pybox diff --git a/virtualbox/pybox/Parser.py b/virtualbox/pybox/Parser.py index 73ed60c..834f49c 100644 --- a/virtualbox/pybox/Parser.py +++ b/virtualbox/pybox/Parser.py @@ -1,3 +1,4 @@ +# pylint: disable=invalid-name #!/usr/bin/python3 # # SPDX-License-Identifier: Apache-2.0 @@ -11,7 +12,7 @@ Parser to handle command line arguments import argparse import getpass - +# pylint: disable=too-many-statements def handle_args(): """ Handle arguments supplied to the command line @@ -19,11 +20,10 @@ def handle_args(): parser = argparse.ArgumentParser(formatter_class=argparse.RawTextHelpFormatter) - """ - ************************************** - * Setup type & install configuration * - ************************************** - """ + #************************************** + #* Setup type & install configuration * + #************************************** + parser.add_argument("--setup-type", help= """ Type of setup: @@ -84,11 +84,10 @@ def handle_args(): type=str, default=None) - """ - ****************************************** - * Config folders and files configuration * - ****************************************** - """ + #****************************************** + #* Config folders and files configuration * + #****************************************** + parser.add_argument("--iso-location", help= """ Location of ISO including the filename: @@ -143,11 +142,11 @@ def handle_args(): Path to the config file to use """, action='append') - """ - ************************************** - * Disk number and size configuration * - ************************************** - """ + + #************************************** + #* Disk number and size configuration * + #************************************** + parser.add_argument("--controller-disks", help= """ Select the number of disks for a controller VM. default is 3 @@ -178,11 +177,11 @@ def handle_args(): Configure size in MiB of worker disks as a comma separated list. """, type=str) - """ - ************** - * Networking * - ************** - """ + + #************** + #* Networking * + #************** + parser.add_argument("--vboxnet-name", help= """ Which host only network to use for setup. @@ -277,11 +276,11 @@ def handle_args(): SX setups. """, type=str) - """ - ****************** - * Custom scripts * - ****************** - """ + + #****************** + #* Custom scripts * + #****************** + parser.add_argument("--script1", help= """ Name of an executable script file plus options. @@ -326,11 +325,11 @@ def handle_args(): """, default=None, type=str) - """ - ************************************** - * Other * - ************************************** - """ + + #************************************** + #* Other * + #************************************** + parser.add_argument("--list-stages", help= """ List stages that can be used by autoinstaller. diff --git a/virtualbox/pybox/README.md b/virtualbox/pybox/README.md index 8131cef..0725d14 100644 --- a/virtualbox/pybox/README.md +++ b/virtualbox/pybox/README.md @@ -136,7 +136,7 @@ will be configured and used. ```shell git clone https://opendev.org/starlingx/tools.git - cd tools/deployment/virtualbox/pybox + cd virtual-deployment/virtualbox/pybox python3 -m venv venv source ./venv/bin/activate pip install --upgrade pip @@ -150,7 +150,7 @@ will be configured and used. -O $HOME/Downloads/stx-8.iso ``` -5. Now you're ready to run the script. From the `/deployment/virtualbox/pybox` +5. Now you're ready to run the script. From the `/virtualbox/pybox` folder, do: ```shell @@ -168,8 +168,15 @@ folder, do: --snapshot ``` -The script takes a while to do all the things (from creating a VM and +The script takes a while to do all the things (from creating a VM and installing an OS in it to configuring StarlingX). Several restarts might -occur, and you might see a VirtualBox with a prompt. You don't need to type -anything. While the installation script is running it will take care of +occur, and you might see a VirtualBox with a prompt. You don't need to type +anything. While the installation script is running it will take care of everything for you. + +## Pybox folder structure +. +├── configs/aio-sx: Contains scripts and configs to set up a controller/worker +├── consts: This folder contains modules for managing virtual lab environments, including classes for Lab, Subnets, NICs, OAM, Serial, nodes, and HostTimeout. +├── helper: This folder contains modules for interacting with a StarlingX controller-0 server via a serial connection, configuring system settings, and managing virtual machines using VirtualBox. +└── utils: This folder contains modules for initializing logging, tracking and reporting KPIs, connecting and communicating with remote hosts via local domain socket, and sending files and directories to remote servers using rsync and paramiko libraries. \ No newline at end of file diff --git a/virtualbox/pybox/consts/env.py b/virtualbox/pybox/consts/env.py index 4c89401..2fa5fd4 100644 --- a/virtualbox/pybox/consts/env.py +++ b/virtualbox/pybox/consts/env.py @@ -3,6 +3,11 @@ # SPDX-License-Identifier: Apache-2.0 # +""" +This module contains a class named Lab and some supporting code. +The Lab class represents a virtual lab and has a dictionary attribute VBOX +containing information about the virtual machines in the lab. +""" import getpass from sys import platform @@ -10,18 +15,22 @@ import os user = getpass.getuser() -if platform == 'win32' or platform == 'win64': - LOGPATH = 'C:\\Temp\\pybox_logs' +if platform in ("win32", "win64"): + LOGPATH = "C:\\Temp\\pybox_logs" PORT = 10000 else: - homedir = os.environ['HOME'] - LOGPATH = '{}/vbox_installer_logs'.format(homedir) + homedir = os.environ["HOME"] + LOGPATH = f"{homedir}/vbox_installer_logs" + + +class Lab: #pylint: disable=too-few-public-methods + """The `Lab` class represents a virtual lab and contains a dictionary attribute + `VBOX` with information about the virtual machines in the lab.""" -class Lab: VBOX = { - 'floating_ip': '10.10.10.7', - 'controller-0_ip': '10.10.10.8', - 'controller-1_ip': '10.10.10.9', - 'username': 'sysadmin', - 'password': 'Li69nux*', + "floating_ip": "10.10.10.7", + "controller-0_ip": "10.10.10.8", + "controller-1_ip": "10.10.10.9", + "username": "sysadmin", + "password": "Li69nux*", } diff --git a/virtualbox/pybox/consts/networking.py b/virtualbox/pybox/consts/networking.py index 1c70bcc..dfd0e51 100644 --- a/virtualbox/pybox/consts/networking.py +++ b/virtualbox/pybox/consts/networking.py @@ -1,78 +1,194 @@ +# pylint: disable=too-few-public-methods + #!/usr/bin/python3 # # SPDX-License-Identifier: Apache-2.0 # +""" +This module defines several classes and dictionaries that contain information related +to virtual machines in a lab environment. + +Classes: +- `Subnets`: A class containing dictionaries for IPv4 and IPv6 subnets. +- `NICs`: A class containing dictionaries for NIC configurations of different types of +nodes in the virtual environment, such as `CONTROLLER`, `COMPUTE`, and `STORAGE`. +- `OAM`: A class containing an IP address and netmask for the out-of-band management (OAM) network. +- `Serial`: A class containing configurations for the serial ports. + +""" from sys import platform class Subnets: + """The `Subnets` class contains dictionaries for IPv4 and IPv6 subnets for the + management, infrastructure, and OAM networks.""" + IPV4 = { - 'mgmt_subnet': '192.168.204.0/24', - 'infra_subnet': '192.168.205.0/24', - 'oam_subnet': '10.10.10.0/24' + "mgmt_subnet": "192.168.204.0/24", + "infra_subnet": "192.168.205.0/24", + "oam_subnet": "10.10.10.0/24", } IPV6 = { - 'mgmt_subnet': 'aefd::/64', - 'infra_subnet': 'aced::/64', - 'oam_subnet': 'abcd::/64' + "mgmt_subnet": "aefd::/64", + "infra_subnet": "aced::/64", + "oam_subnet": "abcd::/64", } class NICs: - if platform == 'win32' or platform == 'win64': + """The `NICs` class contains dictionaries for NIC configurations of different types + of nodes in the virtual environment, such as `CONTROLLER`, `COMPUTE`, and `STORAGE`.""" + if platform in ("win32", "win64"): CONTROLLER = { - 'node_type': 'controller', - '1': {'nic': 'hostonly', 'intnet': '', 'nictype': '82540EM', 'nicpromisc': 'deny', 'hostonlyadapter': 'VirtualBox Host-Only Ethernet Adapter'}, - '2': {'nic': 'intnet', 'intnet': 'intnet-management', 'nictype': '82540EM', 'nicpromisc': 'allow-all', 'hostonlyadapter': 'none'}, - '3': {'nic': 'intnet', 'intnet': 'intnet-data1', 'nictype': 'virtio', 'nicpromisc': 'allow-all', 'hostonlyadapter': 'none'}, - '4': {'nic': 'intnet', 'intnet': 'intnet-data2', 'nictype': 'virtio', 'nicpromisc': 'allow-all', 'hostonlyadapter': 'none'}, + "node_type": "controller", + "1": { + "nic": "hostonly", + "intnet": "", + "nictype": "82540EM", + "nicpromisc": "deny", + "hostonlyadapter": "VirtualBox Host-Only Ethernet Adapter", + }, + "2": { + "nic": "intnet", + "intnet": "intnet-management", + "nictype": "82540EM", + "nicpromisc": "allow-all", + "hostonlyadapter": "none", + }, + "3": { + "nic": "intnet", + "intnet": "intnet-data1", + "nictype": "virtio", + "nicpromisc": "allow-all", + "hostonlyadapter": "none", + }, + "4": { + "nic": "intnet", + "intnet": "intnet-data2", + "nictype": "virtio", + "nicpromisc": "allow-all", + "hostonlyadapter": "none", + }, } else: CONTROLLER = { - 'node_type': 'controller', - '1': {'nic': 'hostonly', 'intnet': '', 'nictype': '82540EM', 'nicpromisc': 'deny', 'hostonlyadapter': 'vboxnet0'}, - '2': {'nic': 'intnet', 'intnet': 'intnet-management', 'nictype': '82540EM', 'nicpromisc': 'allow-all', 'hostonlyadapter': 'none'}, - '3': {'nic': 'intnet', 'intnet': 'intnet-data1', 'nictype': 'virtio', 'nicpromisc': 'allow-all', 'hostonlyadapter': 'none'}, - '4': {'nic': 'intnet', 'intnet': 'intnet-data2', 'nictype': 'virtio', 'nicpromisc': 'allow-all', 'hostonlyadapter': 'none'}, + "node_type": "controller", + "1": { + "nic": "hostonly", + "intnet": "", + "nictype": "82540EM", + "nicpromisc": "deny", + "hostonlyadapter": "vboxnet0", + }, + "2": { + "nic": "intnet", + "intnet": "intnet-management", + "nictype": "82540EM", + "nicpromisc": "allow-all", + "hostonlyadapter": "none", + }, + "3": { + "nic": "intnet", + "intnet": "intnet-data1", + "nictype": "virtio", + "nicpromisc": "allow-all", + "hostonlyadapter": "none", + }, + "4": { + "nic": "intnet", + "intnet": "intnet-data2", + "nictype": "virtio", + "nicpromisc": "allow-all", + "hostonlyadapter": "none", + }, } COMPUTE = { - 'node_type': 'compute', - '1': {'nic': 'intnet', 'intnet': 'intnet-unused1', 'nictype': '82540EM', 'nicpromisc': 'allow-all', 'hostonlyadapter': 'none'}, - '2': {'nic': 'intnet', 'intnet': 'intnet-management', 'nictype': '82540EM', 'nicpromisc': 'allow-all', 'hostonlyadapter': 'none'}, - '3': {'nic': 'intnet', 'intnet': 'intnet-data1', 'nictype': 'virtio', 'nicpromisc': 'allow-all', 'hostonlyadapter': 'none'}, - '4': {'nic': 'intnet', 'intnet': 'intnet-data2', 'nictype': 'virtio', 'nicpromisc': 'allow-all', 'hostonlyadapter': 'none'}, + "node_type": "compute", + "1": { + "nic": "intnet", + "intnet": "intnet-unused1", + "nictype": "82540EM", + "nicpromisc": "allow-all", + "hostonlyadapter": "none", + }, + "2": { + "nic": "intnet", + "intnet": "intnet-management", + "nictype": "82540EM", + "nicpromisc": "allow-all", + "hostonlyadapter": "none", + }, + "3": { + "nic": "intnet", + "intnet": "intnet-data1", + "nictype": "virtio", + "nicpromisc": "allow-all", + "hostonlyadapter": "none", + }, + "4": { + "nic": "intnet", + "intnet": "intnet-data2", + "nictype": "virtio", + "nicpromisc": "allow-all", + "hostonlyadapter": "none", + }, } STORAGE = { - 'node_type': 'storage', - '1': {'nic': 'intnet', 'intnet': 'intnet-unused', 'nictype': '82540EM', 'nicpromisc': 'allow-all', 'hostonlyadapter': 'none'}, - '2': {'nic': 'intnet', 'intnet': 'intnet-management', 'nictype': '82540EM', 'nicpromisc': 'allow-all', 'hostonlyadapter': 'none'}, - '3': {'nic': 'intnet', 'intnet': 'intnet-infra', 'nictype': '82540EM', 'nicpromisc': 'allow-all', 'hostonlyadapter': 'none'}, + "node_type": "storage", + "1": { + "nic": "intnet", + "intnet": "intnet-unused", + "nictype": "82540EM", + "nicpromisc": "allow-all", + "hostonlyadapter": "none", + }, + "2": { + "nic": "intnet", + "intnet": "intnet-management", + "nictype": "82540EM", + "nicpromisc": "allow-all", + "hostonlyadapter": "none", + }, + "3": { + "nic": "intnet", + "intnet": "intnet-infra", + "nictype": "82540EM", + "nicpromisc": "allow-all", + "hostonlyadapter": "none", + }, } + class OAM: + """The `OAM` class contains an IP address and netmask for the out-of-band + management (OAM) network.""" + OAM = { - 'ip': '10.10.10.254', - 'netmask': '255.255.255.0', + "ip": "10.10.10.254", + "netmask": "255.255.255.0", } + class Serial: - if platform == 'win32' or platform == 'win64': + """The `Serial` class contains configurations for the serial ports.""" + + if platform in ("win32", "win64"): SERIAL = { - 'uartbase': '0x3F8', - 'uartport': '4', - 'uartmode': 'tcpserver', - 'uartpath': '10000' + "uartbase": "0x3F8", + "uartport": "4", + "uartmode": "tcpserver", + "uartpath": "10000", } else: SERIAL = { - 'uartbase': '0x3F8', - 'uartport': '4', - 'uartmode': 'server', - 'uartpath': '/tmp/' + "uartbase": "0x3F8", + "uartport": "4", + "uartmode": "server", + "uartpath": "/tmp/", } diff --git a/virtualbox/pybox/consts/node.py b/virtualbox/pybox/consts/node.py index a53f97c..7147642 100644 --- a/virtualbox/pybox/consts/node.py +++ b/virtualbox/pybox/consts/node.py @@ -3,8 +3,15 @@ # SPDX-License-Identifier: Apache-2.0 # +""" +This module contains dictionaries for different types of nodes in a virtual environment, +such as CONTROLLER_CEPH, CONTROLLER_LVM, CONTROLLER_AIO, COMPUTE, and STORAGE. +""" + +class Nodes: #pylint: disable=too-few-public-methods + """The `Nodes` class contains dictionaries for different types of nodes in a + virtual environment.""" -class Nodes: CONTROLLER_CEPH = { 'node_type': 'controller-STORAGE', 'memory': 12288, diff --git a/virtualbox/pybox/consts/timeout.py b/virtualbox/pybox/consts/timeout.py index 61e4dcc..420ac3c 100644 --- a/virtualbox/pybox/consts/timeout.py +++ b/virtualbox/pybox/consts/timeout.py @@ -3,8 +3,15 @@ # SPDX-License-Identifier: Apache-2.0 # +""" +This module contains the HostTimeout class, which provides timeout values (in seconds) +for various operations on a host. +""" + +class HostTimeout: #pylint: disable=too-few-public-methods + """The `HostTimeout` class provides timeout values (in seconds) for various + operations on a host.""" -class HostTimeout: CONTROLLER_UNLOCK = 3600+1800 REBOOT = 900 INSTALL = 3600 diff --git a/virtualbox/pybox/helper/host_helper.py b/virtualbox/pybox/helper/host_helper.py index 6718455..213ce1e 100644 --- a/virtualbox/pybox/helper/host_helper.py +++ b/virtualbox/pybox/helper/host_helper.py @@ -3,6 +3,12 @@ # SPDX-License-Identifier: Apache-2.0 # +""" +This module provides functions to interact with a StarlingX controller-0 server via a +serial connection. The functions can be used to perform operations such as unlocking, +locking, rebooting, and installing a host. The module uses streamexpect library to +facilitate stream parsing. +""" import time import streamexpect @@ -21,15 +27,17 @@ def unlock_host(stream, hostname): - Check that host is locked - Unlock host """ + LOG.info("#### Unlock %s", hostname) - serial.send_bytes(stream, "system host-list | grep {}".format(hostname), expect_prompt=False) + serial.send_bytes(stream, f"system host-list | grep {hostname}", expect_prompt=False) try: serial.expect_bytes(stream, "locked") except streamexpect.ExpectTimeout: LOG.info("Host %s not locked", hostname) return 1 - serial.send_bytes(stream, "system host-unlock {}".format(hostname), expect_prompt=False) + serial.send_bytes(stream, f"system host-unlock {hostname}", expect_prompt=False) LOG.info("Unlocking %s", hostname) + return None def lock_host(stream, hostname): @@ -42,15 +50,17 @@ def lock_host(stream, hostname): - Check that host is unlocked - Lock host """ + LOG.info("Lock %s", hostname) - serial.send_bytes(stream, "system host-list |grep {}".format(hostname), expect_prompt=False) + serial.send_bytes(stream, f"system host-list |grep {hostname}", expect_prompt=False) try: serial.expect_bytes(stream, "unlocked") except streamexpect.ExpectTimeout: LOG.info("Host %s not unlocked", hostname) return 1 - serial.send_bytes(stream, "system host-lock {}".format(hostname), expect_prompt="keystone") + serial.send_bytes(stream, f"system host-lock {hostname}", expect_prompt="keystone") LOG.info("Locking %s", hostname) + return None def reboot_host(stream, hostname): @@ -60,8 +70,9 @@ def reboot_host(stream, hostname): stream(): hostname(str): Host to reboot """ + LOG.info("Rebooting %s", hostname) - serial.send_bytes(stream, "system host-reboot {}".format(hostname), expect_prompt=False) + serial.send_bytes(stream, f"system host-reboot {hostname}", expect_prompt=False) serial.expect_bytes(stream, "rebooting", HostTimeout.REBOOT) @@ -77,18 +88,17 @@ def install_host(stream, hostname, host_type, host_id): time.sleep(10) LOG.info("Installing %s with id %s", hostname, host_id) - if host_type is 'controller': + if host_type == 'controller': serial.send_bytes(stream, - "system host-update {} personality=controller".format(host_id), + f"system host-update {host_id} personality=controller", expect_prompt=False) - elif host_type is 'storage': + elif host_type == 'storage': serial.send_bytes(stream, - "system host-update {} personality=storage".format(host_id), + f"system host-update {host_id} personality=storage", expect_prompt=False) else: serial.send_bytes(stream, - "system host-update {} personality=compute hostname={}".format(host_id, - hostname), + f"system host-update {host_id} personality=compute hostname={hostname}", expect_prompt=False) time.sleep(30) @@ -99,6 +109,7 @@ def disable_logout(stream): Args: stream(stream): stream to cont0 """ + LOG.info('Disabling automatic logout') serial.send_bytes(stream, "export TMOUT=0") @@ -108,8 +119,8 @@ def change_password(stream, username="sysadmin", password="Li69nux*"): changes the default password on initial login. Args: stream(stream): stream to cont0 - """ + LOG.info('Changing password to Li69nux*') serial.send_bytes(stream, username, expect_prompt=False) serial.expect_bytes(stream, "Password:") @@ -131,8 +142,8 @@ def login(stream, timeout=600, username="sysadmin", password="Li69nux*"): """ serial.send_bytes(stream, "\n", expect_prompt=False) - rc = serial.expect_bytes(stream, "ogin:", fail_ok=True, timeout=timeout) - if rc != 0: + login_result = serial.expect_bytes(stream, "ogin:", fail_ok=True, timeout=timeout) + if login_result != 0: serial.send_bytes(stream, "\n", expect_prompt=False) if serial.expect_bytes(stream, "~$", timeout=10, fail_ok=True) == -1: serial.send_bytes(stream, '\n', expect_prompt=False) @@ -150,11 +161,18 @@ def logout(stream): Args: stream(stream): stream to cont0 """ + serial.send_bytes(stream, "exit", expect_prompt=False) time.sleep(5) def check_password(stream, password="Li69nux*"): + """ + Checks the password. + Args: + stream(stream): Stream to cont0 + password(str): password to check. + """ ret = serial.expect_bytes(stream, 'assword', fail_ok=True, timeout=5) if ret == 0: serial.send_bytes(stream, password, expect_prompt=False) diff --git a/virtualbox/pybox/helper/install_lab.py b/virtualbox/pybox/helper/install_lab.py index f1c9d74..a674b2e 100644 --- a/virtualbox/pybox/helper/install_lab.py +++ b/virtualbox/pybox/helper/install_lab.py @@ -8,11 +8,10 @@ Contains helper functions that will configure basic system settings. """ from consts.timeout import HostTimeout - -from helper import host_helper - from utils import serial from utils.install_log import LOG +from helper import host_helper + def update_platform_cpus(stream, hostname, cpu_num=5): """ @@ -20,9 +19,14 @@ def update_platform_cpus(stream, hostname, cpu_num=5): """ LOG.info("Allocating %s CPUs for use by the %s platform.", cpu_num, hostname) - serial.send_bytes(stream, "\nsource /etc/platform/openrc; system host-cpu-modify " - "{} -f platform -p0 {}".format(hostname, cpu_num, - prompt='keystone', timeout=300)) + serial.send_bytes( + stream, + "\nsource /etc/platform/openrc; system host-cpu-modify " + f"{hostname} -f platform -p0 {cpu_num}", + prompt="keystone", + timeout=300, + ) + def set_dns(stream, dns_ip): """ @@ -30,25 +34,31 @@ def set_dns(stream, dns_ip): """ LOG.info("Configuring DNS to %s.", dns_ip) - serial.send_bytes(stream, "source /etc/platform/openrc; system dns-modify " - "nameservers={}".format(dns_ip), prompt='keystone') + serial.send_bytes( + stream, + "source /etc/platform/openrc; system dns-modify " + f"nameservers={dns_ip}", + prompt="keystone", + ) -def config_controller(stream, config_file=None, password='Li69nux*'): +def config_controller(stream, config_file=None, password="Li69nux*"): """ Configure controller-0 using optional arguments """ - args = '' + args = "" if config_file: - args += '--config-file ' + config_file + ' ' + args += "--config-file " + config_file + " " # serial.send_bytes(stream, f'sudo config_controller {args}', expect_prompt=False) - serial.send_bytes(stream, 'ansible-playbook /usr/share/ansible/stx-ansible/playbooks/bootstrap.yml', expect_prompt=False) + serial.send_bytes( + stream, + "ansible-playbook /usr/share/ansible/stx-ansible/playbooks/bootstrap.yml", + expect_prompt=False, + ) host_helper.check_password(stream, password=password) - ret = serial.expect_bytes(stream, "~$", - timeout=HostTimeout.LAB_CONFIG) + ret = serial.expect_bytes(stream, "~$", timeout=HostTimeout.LAB_CONFIG) if ret != 0: LOG.info("Configuration failed. Exiting installer.") - raise Exception("Configcontroller failed") - + raise Exception("Configcontroller failed") # pylint: disable=E0012, W0719 diff --git a/virtualbox/pybox/helper/vboxmanage.py b/virtualbox/pybox/helper/vboxmanage.py index 7f5e060..3389cc8 100644 --- a/virtualbox/pybox/helper/vboxmanage.py +++ b/virtualbox/pybox/helper/vboxmanage.py @@ -3,6 +3,9 @@ # SPDX-License-Identifier: Apache-2.0 # +""" +This module provides functions for managing virtual machines using VirtualBox. +""" import os import subprocess @@ -12,7 +15,6 @@ import time from sys import platform from consts import env - from utils.install_log import LOG @@ -21,59 +23,90 @@ def vboxmanage_version(): Return version of vbox. """ - version = subprocess.check_output(['vboxmanage', '--version'], stderr=subprocess.STDOUT) + version = subprocess.check_output( + ["vboxmanage", "--version"], stderr=subprocess.STDOUT + ) return version -def vboxmanage_extpack(action="install"): +def vboxmanage_extpack(): """ This allows you to install, uninstall the vbox extensions" """ + output = vboxmanage_version() - version = re.match(b'(.*)r', output) - version_path = version.group(1).decode('utf-8') + version = re.match(b"(.*)r", output) + version_path = version.group(1).decode("utf-8") LOG.info("Downloading extension pack") - filename = 'Oracle_VM_VirtualBox_Extension_Pack-{}.vbox-extpack'.format(version_path) - cmd = 'http://download.virtualbox.org/virtualbox/{}/{}'.format(version_path, filename) - result = subprocess.check_output(['wget', cmd, '-P', '/tmp'], stderr=subprocess.STDOUT) + filename = f"Oracle_VM_VirtualBox_Extension_Pack-{version_path}.vbox-extpack" + cmd = f"http://download.virtualbox.org/virtualbox/{version_path}/{filename}" + result = subprocess.check_output( + ["wget", cmd, "-P", "/tmp"], stderr=subprocess.STDOUT + ) LOG.info(result) LOG.info("Installing extension pack") - result = subprocess.check_output(['vboxmanage', 'extpack', 'install', '/tmp/' + filename, - '--replace'], stderr=subprocess.STDOUT) + result = subprocess.check_output( + ["vboxmanage", "extpack", "install", "/tmp/" + filename, "--replace"], + stderr=subprocess.STDOUT, + ) LOG.info(result) def get_all_vms(labname, option="vms"): + """ + Return a list of virtual machines (VMs) belonging to a specified lab. + + Args: + labname (str): The name of the lab to which the VMs belong. + option (str, optional): The VBoxManage command option to use when listing VMs. + Defaults to "vms". + + Returns: + list: A list of strings representing the names of the VMs that belong to the specified lab. + """ + initial_node_list = [] vm_list = vboxmanage_list(option) - labname.encode('utf-8') + labname.encode("utf-8") # Reduce the number of VMs we query for item in vm_list: - if labname.encode('utf-8') in item and (b'controller-' in item or \ - b'compute-' in item or b'storage-' in item): - initial_node_list.append(item.decode('utf-8')) + if labname.encode("utf-8") in item and ( + b"controller-" in item or b"compute-" in item or b"storage-" in item + ): + initial_node_list.append(item.decode("utf-8")) # Filter by group node_list = [] - group = bytearray('"/{}"'.format(labname), 'utf-8') + group = bytearray(f'"/{labname}"', "utf-8") for item in initial_node_list: info = vboxmanage_showinfo(item).splitlines() for line in info: try: - k, v = line.split(b'=') + k_value, v_value = line.split(b"=") except ValueError: continue - if k == b'groups' and v == group: + if k_value == b"groups" and v_value == group: node_list.append(item) return node_list -def take_snapshot(labname, snapshot_name, socks=None): +def take_snapshot(labname, snapshot_name): + """ + Take a snapshot of all VMs belonging to a specified lab. + + Args: + labname (str): The name of the lab whose VMs will be snapshotted. + snapshot_name (str): The name of the snapshot to be taken. + + Returns: + None + """ + vms = get_all_vms(labname, option="vms") runningvms = get_all_vms(labname, option="runningvms") @@ -81,50 +114,103 @@ def take_snapshot(labname, snapshot_name, socks=None): LOG.info("VMs in lab %s: %s", labname, vms) LOG.info("VMs running in lab %s: %s", labname, runningvms) - hosts = len(vms) + _pause_running_vms(runningvms, vms) + + if len(vms) != 0: + vboxmanage_takesnapshot(vms, snapshot_name) + + _resume_running_vms(runningvms) + + time.sleep(10) + + if runningvms: + _wait_for_vms_to_run(labname, runningvms, vms) + + +def _pause_running_vms(runningvms, vms): + """Pause running virtual machines. + + Args: + runningvms (list): A list of strings representing the names of running virtual machines. + vms (list): A list of strings representing the names of all virtual machines. + + Returns: + None + """ - # Pause running VMs to take snapshot if len(runningvms) > 1: for node in runningvms: newpid = os.fork() if newpid == 0: vboxmanage_controlvms([node], "pause") - os._exit(0) + os._exit(0) # pylint: disable=protected-access for node in vms: os.waitpid(0, 0) time.sleep(2) - if hosts != 0: - vboxmanage_takesnapshot(vms, snapshot_name) - # Resume VMs after snapshot was taken +def _resume_running_vms(runningvms): + """Resume paused virtual machines. + + Args: + runningvms (list): A list of strings representing the names of running virtual machines. + + Returns: + None + """ + if len(runningvms) > 1: for node in runningvms: newpid = os.fork() if newpid == 0: vboxmanage_controlvms([node], "resume") - os._exit(0) + os._exit(0) # pylint: disable=protected-access for node in runningvms: os.waitpid(0, 0) - time.sleep(10) # Wait for VM serial port to stabilize, otherwise it may refuse to connect - if runningvms: - new_vms = get_all_vms(labname, option="runningvms") - retry = 0 - while retry < 20: - LOG.info("Waiting for VMs to come up running after taking snapshot..." - "Up VMs are %s ", new_vms) - if len(runningvms) < len(new_vms): - time.sleep(1) - new_vms = get_all_vms(labname, option="runningvms") - retry += 1 - else: - LOG.info("All VMs %s are up running after taking snapshot...", vms) - break +def _wait_for_vms_to_run(labname, runningvms, vms): + """Wait for virtual machines to finish running. + + Args: + labname (str): The name of the lab whose virtual machines are being waited for. + runningvms (list): A list of strings representing the names of running virtual machines. + vms (list): A list of strings representing the names of all virtual machines. + + Returns: + None + """ + + new_vms = get_all_vms(labname, option="runningvms") + retry = 0 + while retry < 20: + LOG.info( + "Waiting for VMs to come up running after taking snapshot..." + "Up VMs are %s ", + new_vms, + ) + if len(runningvms) < len(new_vms): + time.sleep(1) + new_vms = get_all_vms(labname, option="runningvms") + retry += 1 + else: + LOG.info("All VMs %s are up running after taking snapshot...", vms) + break def restore_snapshot(node_list, name): + """ + Restore a snapshot of a list of virtual machines. + + Args: + node_list (list): A list of strings representing the names of the virtual machines + whose snapshot will be restored. + name (str): The name of the snapshot to restore. + + Returns: + None + """ + LOG.info("Restore snapshot of %s for hosts %s", name, node_list) if len(node_list) != 0: vboxmanage_controlvms(node_list, "poweroff") @@ -147,7 +233,10 @@ def vboxmanage_list(option="vms"): """ This returns a list of vm names. """ - result = subprocess.check_output(['vboxmanage', 'list', option], stderr=subprocess.STDOUT) + + result = subprocess.check_output( + ["vboxmanage", "list", option], stderr=subprocess.STDOUT + ) vms_list = [] for item in result.splitlines(): vm_name = re.match(b'"(.*?)"', item) @@ -160,10 +249,13 @@ def vboxmanage_showinfo(host): """ This returns info about the host """ + if not isinstance(host, str): - host.decode('utf-8') - result = subprocess.check_output(['vboxmanage', 'showvminfo', host, '--machinereadable'], - stderr=subprocess.STDOUT) + host.decode("utf-8") + result = subprocess.check_output( + ["vboxmanage", "showvminfo", host, "--machinereadable"], + stderr=subprocess.STDOUT, + ) return result @@ -176,9 +268,21 @@ def vboxmanage_createvm(hostname, labname): assert labname, "Labname is required" group = "/" + labname LOG.info("Creating VM %s", hostname) - result = subprocess.check_output(['vboxmanage', 'createvm', '--name', hostname, '--register', - '--ostype', 'Linux_64', '--groups', group], - stderr=subprocess.STDOUT) + subprocess.check_output( + [ + "vboxmanage", + "createvm", + "--name", + hostname, + "--register", + "--ostype", + "Linux_64", + "--groups", + group, + ], + stderr=subprocess.STDOUT, + ) + def vboxmanage_deletevms(hosts=None): """ @@ -190,124 +294,295 @@ def vboxmanage_deletevms(hosts=None): if len(hosts) != 0: for hostname in hosts: LOG.info("Deleting VM %s", hostname) - result = subprocess.check_output(['vboxmanage', 'unregistervm', hostname, '--delete'], - stderr=subprocess.STDOUT) + subprocess.check_output( + ["vboxmanage", "unregistervm", hostname, "--delete"], + stderr=subprocess.STDOUT, + ) time.sleep(10) # in case medium is still present after delete vboxmanage_deletemedium(hostname) vms_list = vboxmanage_list("vms") for items in hosts: - assert items not in vms_list, "The following vms are unexpectedly" \ - "present {}".format(vms_list) + assert ( + items not in vms_list + ), f"The following vms are unexpectedly present {vms_list}" -def vboxmanage_hostonlyifcreate(name="vboxnet0", ip=None, netmask=None): +def vboxmanage_hostonlyifcreate(name="vboxnet0", oam_ip=None, netmask=None): """ This creates a hostonly network for systems to communicate. """ assert name, "Must provide network name" - assert ip, "Must provide an OAM IP" + assert oam_ip, "Must provide an OAM IP" assert netmask, "Must provide an OAM Netmask" LOG.info("Creating Host-only Network") - result = subprocess.check_output(['vboxmanage', 'hostonlyif', 'create'], - stderr=subprocess.STDOUT) + subprocess.check_output( + ["vboxmanage", "hostonlyif", "create"], stderr=subprocess.STDOUT + ) - LOG.info("Provisioning %s with IP %s and Netmask %s", name, ip, netmask) - result = subprocess.check_output(['vboxmanage', 'hostonlyif', 'ipconfig', name, '--ip', - ip, '--netmask', netmask], stderr=subprocess.STDOUT) + LOG.info("Provisioning %s with IP %s and Netmask %s", name, oam_ip, netmask) + subprocess.check_output( + [ + "vboxmanage", + "hostonlyif", + "ipconfig", + name, + "--ip", + oam_ip, + "--netmask", + netmask, + ], + stderr=subprocess.STDOUT, + ) def vboxmanage_hostonlyifdelete(name="vboxnet0"): """ Deletes hostonly network. This is used as a work around for creating too many hostonlyifs. - """ + assert name, "Must provide network name" LOG.info("Removing Host-only Network") - result = subprocess.check_output(['vboxmanage', 'hostonlyif', 'remove', name], - stderr=subprocess.STDOUT) + subprocess.check_output( + ["vboxmanage", "hostonlyif", "remove", name], stderr=subprocess.STDOUT + ) -def vboxmanage_modifyvm(hostname=None, cpus=None, memory=None, nic=None, - nictype=None, nicpromisc=None, nicnum=None, - intnet=None, hostonlyadapter=None, - natnetwork=None, uartbase=None, uartport=None, - uartmode=None, uartpath=None, nicbootprio2=1, prefix=""): +def vboxmanage_modifyvm(hostname, vm_config=None): """ - This modifies a VM with a specified name. + Modify a virtual machine according to a specified configuration. + + Args: + hostname(str): Name of host to modify + vm_config (dict): A dictionary representing the configuration options + for the virtual machine. Possible key values: cpus, memory, nic, nictype, + nicpromisc, nicnum, intnet, hostonlyadapter, natnetwork, uartbase, + uartport, uartmode, uartpath, nicbootprio2=1, prefix="" + + Returns: + None """ - assert hostname, "Hostname is required" - # Add more semantic checks - cmd = ['vboxmanage', 'modifyvm', hostname] - if cpus: - cmd.extend(['--cpus', cpus]) - if memory: - cmd.extend(['--memory', memory]) - if nic and nictype and nicpromisc and nicnum: - cmd.extend(['--nic{}'.format(nicnum), nic]) - cmd.extend(['--nictype{}'.format(nicnum), nictype]) - cmd.extend(['--nicpromisc{}'.format(nicnum), nicpromisc]) - if intnet: - if prefix: - intnet = "{}-{}".format(prefix, intnet) - else: - intnet = "{}".format(intnet) - cmd.extend(['--intnet{}'.format(nicnum), intnet]) - if hostonlyadapter: - cmd.extend(['--hostonlyadapter{}'.format(nicnum), hostonlyadapter]) - if natnetwork: - cmd.extend(['--nat-network{}'.format(nicnum), natnetwork]) - elif nicnum and nictype == 'nat': - cmd.extend(['--nic{}'.format(nicnum), 'nat']) - if uartbase and uartport and uartmode and uartpath: - cmd.extend(['--uart1']) - cmd.extend(['{}'.format(uartbase)]) - cmd.extend(['{}'.format(uartport)]) - cmd.extend(['--uartmode1']) - cmd.extend(['{}'.format(uartmode)]) - if platform == 'win32' or platform == 'win64': - cmd.extend(['{}'.format(env.PORT)]) - env.PORT += 1 - else: - if prefix: - prefix = "{}_".format(prefix) - if 'controller-0' in hostname: - cmd.extend(['{}{}{}_serial'.format(uartpath, prefix, hostname)]) - else: - cmd.extend(['{}{}{}'.format(uartpath, prefix, hostname)]) - if nicbootprio2: - cmd.extend(['--nicbootprio2']) - cmd.extend(['{}'.format(nicbootprio2)]) - cmd.extend(['--boot4']) - cmd.extend(['net']) + #put default values in nicbootprio2 and prefix if they not exist + vm_config["nicbootprio2"] = vm_config.get("nicbootprio2", 1) + vm_config["prefix"] = vm_config.get("prefix", "") + + cmd = ["vboxmanage", "modifyvm", hostname] + nic_cmd = [] + + if _contains_value("cpus", vm_config): + cmd.extend(["--cpus", vm_config["cpus"]]) + + if _contains_value("memory", vm_config): + cmd.extend(["--memory", vm_config["memory"]]) + + if _is_network_configured(vm_config): + nic_cmd = _get_network_configuration(vm_config) + cmd.extend(nic_cmd) + + elif _is_nat_network_configured(vm_config): + cmd.extend([f'--nic{vm_config["nicnum"]}', "nat"]) + + if _is_uart_configured(vm_config): + uart_config = _add_uart(hostname, vm_config) + cmd.extend(uart_config) + + if _contains_value("nicbootprio2", vm_config): + cmd.extend(["--nicbootprio2"]) + cmd.extend([f'{vm_config["nicbootprio2"]}']) + + cmd.extend(["--boot4"]) + cmd.extend(["net"]) + LOG.info(cmd) LOG.info("Updating VM %s configuration", hostname) - result = subprocess.check_output(cmd, stderr=subprocess.STDOUT) + subprocess.check_output(cmd, stderr=subprocess.STDOUT) + + +def _is_network_configured(vm_config): + """ + Checks whether a network interface is configured in the given VM configuration. + + Args: + vm_config (dict): A dictionary representing the configuration options for the VM. + + Returns: + bool: True if a network interface is configured, False otherwise. + """ + + return (_contains_value("nic", vm_config) + and _contains_value("nictype", vm_config) + and _contains_value("nicpromisc", vm_config) + and _contains_value("nicnum", vm_config) + ) + + +def _get_network_configuration(vm_config): + """ + Constructs a list of options for the network interface based on the values in vm_config. + + Args: + vm_config (dict): A dictionary representing the configuration options for the VM. + + Returns: + list: A list of command-line options for the network interface. + """ + + nic_cmd = [f'--nic{vm_config["nicnum"]}', vm_config["nic"]] + nic_cmd.extend([f'--nictype{vm_config["nicnum"]}', vm_config["nictype"]]) + nic_cmd.extend([f'--nicpromisc{vm_config["nicnum"]}', vm_config["nicpromisc"]]) + + if _contains_value("intnet", vm_config): + intnet = vm_config["intnet"] + if _contains_value("prefix", vm_config): + intnet = f"{vm_config['prefix']}-{intnet}" + else: + intnet = f"{intnet}" + nic_cmd.extend([f'--intnet{vm_config["nicnum"]}', intnet]) + + if _contains_value("hostonlyadapter", vm_config): + nic_cmd.extend( + [ + f'--hostonlyadapter{vm_config["nicnum"]}', + vm_config["hostonlyadapter"], + ] + ) + + if _contains_value("natnetwork", vm_config): + nic_cmd.extend( + [f'--nat-network{vm_config["nicnum"]}', vm_config["natnetwork"]] + ) + + return nic_cmd + + +def _is_nat_network_configured(vm_config): + """ + Checks whether the NAT network is configured in the given VM configuration. + + Args: + vm_config (dict): A dictionary representing the configuration options for the VM. + + Returns: + bool: True if the NAT network is configured, False otherwise. + """ + + return _contains_value("nicnum", vm_config) and vm_config.get("nictype") == "nat" + + +def _is_uart_configured(vm_config): + """ + Checks whether the UART device is configured in the given VM configuration. + + Args: + vm_config (dict): A dictionary representing the configuration options for the VM. + + Returns: + bool: True if the UART device is configured, False otherwise. + """ + + return ( + _contains_value("uartbase", vm_config) + and _contains_value("uartport", vm_config) + and _contains_value("uartmode", vm_config) + and _contains_value("uartpath", vm_config) + ) + + +def _add_uart(hostname, vm_config): + """ + Constructs a list of options for the UART device based on the values in vm_config. + + Args: + hostname (str): Name of the virtual machine. + vm_config (dict): A dictionary representing the configuration options for the VM. + + Returns: + list: A list of command-line options for the UART device. + """ + + uart_config = ["--uart1"] + uart_config.extend([f'{vm_config["uartbase"]}']) + uart_config.extend([f'{vm_config["uartport"]}']) + uart_config.extend(["--uartmode1"]) + uart_config.extend([f'{vm_config["uartmode"]}']) + + if platform in ("win32", "win64"): + uart_config.extend([f"{env.PORT}"]) + env.PORT += 1 + else: + if _contains_value("prefix", vm_config): + prefix = f'{vm_config["prefix"]}_' + if "controller-0" in hostname: + uart_config.extend([f'{vm_config["uartpath"]}{prefix}{hostname}_serial']) + else: + uart_config.extend([f'{vm_config["uartpath"]}{prefix}{hostname}']) + + return uart_config + + +def _contains_value(key, dictionary): + return key in dictionary and dictionary[key] + def vboxmanage_port_forward(hostname, network, local_port, guest_port, guest_ip): + """ + Configures port forwarding for a NAT network in VirtualBox. + + Args: + hostname (str): Name of the virtual machine. + network (str): Name of the NAT network. + local_port (int): The local port number to forward. + guest_port (int): The port number on the guest to forward to. + guest_ip (str): The IP address of the guest to forward to. + + Returns: + None + """ + # VBoxManage natnetwork modify --netname natnet1 --port-forward-4 # "ssh:tcp:[]:1022:[192.168.15.5]:22" - rule_name = "{}-{}".format(hostname, guest_port) + rule_name = f"{hostname}-{guest_port}" # Delete previous entry, if any - LOG.info("Removing previous forwarding rule '%s' from NAT network '%s'", rule_name, network) - cmd = ['vboxmanage', 'natnetwork', 'modify', '--netname', network, - '--port-forward-4', 'delete', rule_name] + LOG.info( + "Removing previous forwarding rule '%s' from NAT network '%s'", + rule_name, + network, + ) + cmd = [ + "vboxmanage", + "natnetwork", + "modify", + "--netname", + network, + "--port-forward-4", + "delete", + rule_name, + ] try: - result = subprocess.check_output(cmd, stderr=subprocess.STDOUT) + subprocess.check_output(cmd, stderr=subprocess.STDOUT) except subprocess.CalledProcessError: pass # Add new rule - rule = "{}:tcp:[]:{}:[{}]:{}".format(rule_name, local_port, guest_ip, guest_port) + rule = f"{rule_name}:tcp:[]:{local_port}:[{guest_ip}]:{guest_port}" LOG.info("Updating port-forwarding rule to: %s", rule) - cmd = ['vboxmanage', 'natnetwork', 'modify', '--netname', network, '--port-forward-4', rule] - result = subprocess.check_output(cmd, stderr=subprocess.STDOUT) + cmd = [ + "vboxmanage", + "natnetwork", + "modify", + "--netname", + network, + "--port-forward-4", + rule, + ] + subprocess.check_output(cmd, stderr=subprocess.STDOUT) + def vboxmanage_storagectl(hostname=None, storectl="sata", hostiocache="off"): """ @@ -317,62 +592,132 @@ def vboxmanage_storagectl(hostname=None, storectl="sata", hostiocache="off"): assert hostname, "Hostname is required" assert storectl, "Type of storage controller is required" LOG.info("Creating %s storage controller on VM %s", storectl, hostname) - result = subprocess.check_output(['vboxmanage', 'storagectl', - hostname, '--name', storectl, - '--add', storectl, '--hostiocache', - hostiocache], stderr=subprocess.STDOUT) + subprocess.check_output( + [ + "vboxmanage", + "storagectl", + hostname, + "--name", + storectl, + "--add", + storectl, + "--hostiocache", + hostiocache, + ], + stderr=subprocess.STDOUT, + ) -def vboxmanage_storageattach(hostname=None, storectl="sata", - storetype="hdd", disk=None, port_num="0", device_num="0"): +def vboxmanage_storageattach(hostname, storage_config): """ - This attaches a disk to a controller. + Attaches a disk to a storage controller. + + Args: + hostname (str): Name of the virtual machine. + storage_config (dict): A dictionary containing the config options for the storage device. + Possible key values: storectl, storetype, disk, port_num, device_num. + + Returns: + str: The output of the VBoxManage command. """ assert hostname, "Hostname is required" + assert storage_config and isinstance(storage_config, dict), "Storage configuration is required" + + storectl = storage_config.get("storectl", "sata") + storetype = storage_config.get("storetype", "hdd") + disk = storage_config.get("disk") + port_num = storage_config.get("port_num", "0") + device_num = storage_config.get("device_num", "0") + assert disk, "Disk name is required" assert storectl, "Name of storage controller is required" assert storetype, "Type of storage controller is required" - LOG.info("Attaching %s storage to storage controller %s on VM %s", - storetype, storectl, hostname) - result = subprocess.check_output(['vboxmanage', 'storageattach', - hostname, '--storagectl', storectl, - '--medium', disk, '--type', - storetype, '--port', port_num, - '--device', device_num], stderr=subprocess.STDOUT) - return result -def vboxmanage_deletemedium(hostname, vbox_home_dir='/home'): + LOG.info( + "Attaching %s storage to storage controller %s on VM %s", + storetype, + storectl, + hostname, + ) + + return subprocess.check_output( + [ + "vboxmanage", + "storageattach", + hostname, + "--storagectl", + storectl, + "--medium", + disk, + "--type", + storetype, + "--port", + port_num, + "--device", + device_num, + ], + stderr=subprocess.STDOUT, + ) + + +def vboxmanage_deletemedium(hostname, vbox_home_dir="/home"): + """ + Deletes the disk medium associated with a virtual machine. + + Args: + hostname (str): The name of the virtual machine to which the disk medium is attached. + vbox_home_dir (str): The directory in which the disk medium files are stored. + Defaults to "/home". + + Returns: + None + """ + assert hostname, "Hostname is required" - if platform == 'win32' or platform == 'win64': + if platform in ("win32", "win64"): return username = getpass.getuser() - vbox_home_dir = "{}/{}/vbox_disks/".format(vbox_home_dir, username) + vbox_home_dir = f"{vbox_home_dir}/{username}/vbox_disks/" - disk_list = [f for f in os.listdir(vbox_home_dir) if - os.path.isfile(os.path.join(vbox_home_dir, f)) and hostname in f] + disk_list = [ + f + for f in os.listdir(vbox_home_dir) + if os.path.isfile(os.path.join(vbox_home_dir, f)) and hostname in f + ] LOG.info("Disk mediums to delete: %s", disk_list) for disk in disk_list: LOG.info("Disconnecting disk %s from vbox.", disk) try: - result = subprocess.check_output(['vboxmanage', 'closemedium', 'disk', - "{}{}".format(vbox_home_dir, disk), '--delete'], - stderr=subprocess.STDOUT) + result = subprocess.check_output( + [ + "vboxmanage", + "closemedium", + "disk", + "{vbox_home_dir}{disk}", + "--delete", + ], + stderr=subprocess.STDOUT, + ) LOG.info(result) - except subprocess.CalledProcessError as e: + except subprocess.CalledProcessError as exception: # Continue if failures, disk may not be present - LOG.info("Error disconnecting disk, continuing. " - "Details: stdout: %s stderr: %s", e.stdout, e.stderr) + LOG.info( + "Error disconnecting disk, continuing. " + "Details: stdout: %s stderr: %s", + exception.stdout, + exception.stderr, + ) LOG.info("Removing backing file %s", disk) try: - os.remove("{}{}".format(vbox_home_dir, disk)) - except: + os.remove("{vbox_home_dir}{disk}") + except: # pylint: disable=bare-except pass -def vboxmanage_createmedium(hostname=None, disk_list=None, vbox_home_dir='/home'): +def vboxmanage_createmedium(hostname=None, disk_list=None, vbox_home_dir="/home"): """ This creates the required disks. """ @@ -385,28 +730,63 @@ def vboxmanage_createmedium(hostname=None, disk_list=None, vbox_home_dir='/home' port_num = 0 disk_count = 1 for disk in disk_list: - if platform == 'win32' or platform == 'win64': - file_name = "C:\\Users\\" + username + "\\vbox_disks\\" + \ - hostname + "_disk_{}".format(disk_count) + if platform in ("win32", "win64"): + file_name = ( + "C:\\Users\\" + + username + + "\\vbox_disks\\" + + hostname + + f"_disk_{disk_count}" + ) else: - file_name = vbox_home_dir + '/' + username + "/vbox_disks/" \ - + hostname + "_disk_{}".format(disk_count) - LOG.info("Creating disk %s of size %s on VM %s on device %s port %s", - file_name, disk, hostname, device_num, port_num) + file_name = ( + vbox_home_dir + + "/" + + username + + "/vbox_disks/" + + hostname + + f"_disk_{disk_count}" + ) + LOG.info( + "Creating disk %s of size %s on VM %s on device %s port %s", + file_name, + disk, + hostname, + device_num, + port_num, + ) try: - result = subprocess.check_output(['vboxmanage', 'createmedium', - 'disk', '--size', str(disk), - '--filename', file_name, - '--format', 'vdi', - '--variant', 'standard'], - stderr=subprocess.STDOUT) + result = subprocess.check_output( + [ + "vboxmanage", + "createmedium", + "disk", + "--size", + str(disk), + "--filename", + file_name, + "--format", + "vdi", + "--variant", + "standard", + ], + stderr=subprocess.STDOUT, + ) LOG.info(result) - except subprocess.CalledProcessError as e: - LOG.info("Error stdout: %s stderr: %s", e.stdout, e.stderr) + except subprocess.CalledProcessError as exception: + LOG.info("Error stdout: %s stderr: %s", exception.stdout, exception.stderr) raise - vboxmanage_storageattach(hostname, "sata", "hdd", file_name + \ - ".vdi", str(port_num), str(device_num)) + vboxmanage_storageattach( + hostname, + { + "storectl": "sata", + "storetype": "hdd", + "disk": file_name + ".vdi", + "port_num": str(port_num), + "device_num": str(device_num), + }, + ) disk_count += 1 port_num += 1 time.sleep(5) @@ -425,12 +805,13 @@ def vboxmanage_startvm(hostname=None, force=False): else: running_vms = [] - if hostname.encode('utf-8') in running_vms: + if hostname.encode("utf-8") in running_vms: LOG.info("Host %s is already started", hostname) else: LOG.info("Powering on VM %s", hostname) - result = subprocess.check_output(['vboxmanage', 'startvm', - hostname], stderr=subprocess.STDOUT) + result = subprocess.check_output( + ["vboxmanage", "startvm", hostname], stderr=subprocess.STDOUT + ) LOG.info(result) # Wait for VM to start @@ -438,11 +819,11 @@ def vboxmanage_startvm(hostname=None, force=False): while tmout: tmout -= 1 running_vms = vboxmanage_list(option="runningvms") - if hostname.encode('utf-8') in running_vms: + if hostname.encode("utf-8") in running_vms: break time.sleep(1) else: - raise "Failed to start VM: {}".format(hostname) + raise f"Failed to start VM: {hostname}" LOG.info("VM '%s' started.", hostname) @@ -456,8 +837,9 @@ def vboxmanage_controlvms(hosts=None, action=None): for host in hosts: LOG.info("Executing %s action on VM %s", action, host) - result = subprocess.call(["vboxmanage", "controlvm", host, - action], stderr=subprocess.STDOUT) + subprocess.call( + ["vboxmanage", "controlvm", host, action], stderr=subprocess.STDOUT + ) time.sleep(1) @@ -471,8 +853,9 @@ def vboxmanage_takesnapshot(hosts=None, name=None): for host in hosts: LOG.info("Taking snapshot %s on VM %s", name, host) - result = subprocess.call(["vboxmanage", "snapshot", host, "take", - name], stderr=subprocess.STDOUT) + subprocess.call( + ["vboxmanage", "snapshot", host, "take", name], stderr=subprocess.STDOUT + ) def vboxmanage_restoresnapshot(host=None, name=None): @@ -484,7 +867,7 @@ def vboxmanage_restoresnapshot(host=None, name=None): assert name, "Need to provide the snapshot to restore" LOG.info("Restoring snapshot %s on VM %s", name, host) - result = subprocess.call(["vboxmanage", "snapshot", host, "restore", - name], stderr=subprocess.STDOUT) + subprocess.call( + ["vboxmanage", "snapshot", host, "restore", name], stderr=subprocess.STDOUT + ) time.sleep(10) - diff --git a/virtualbox/pybox/install_vbox.py b/virtualbox/pybox/install_vbox.py index 7cec686..c483566 100755 --- a/virtualbox/pybox/install_vbox.py +++ b/virtualbox/pybox/install_vbox.py @@ -1,4 +1,5 @@ -#!/usr/bin/python3 +# pylint: disable=too-many-lines +# !/usr/bin/python3 # # SPDX-License-Identifier: Apache-2.0 # @@ -35,9 +36,8 @@ from consts.timeout import HostTimeout from Parser import handle_args - # Global vars -vboxoptions = None +V_BOX_OPTIONS = None def menu_selector(stream, setup_type, @@ -83,7 +83,8 @@ def setup_networking(stream, ctrlr0_ip, gateway_ip, password='Li69nux*'): """ Setup initial networking so we can transfer files. """ - ip = ctrlr0_ip + + ip_addr = ctrlr0_ip interface = "enp0s3" ret = serial.send_bytes( stream, @@ -96,34 +97,34 @@ def setup_networking(stream, ctrlr0_ip, gateway_ip, password='Li69nux*'): else: LOG.info("Skipping networking setup") return - LOG.info("%s being set up with ip %s", interface, ip) + LOG.info("%s being set up with ip %s", interface, ip_addr) serial.send_bytes(stream, - "sudo /sbin/ip addr add {}/24 dev {}".format(ip, interface), + f"sudo /sbin/ip addr add {ip_addr}/24 dev {interface}", expect_prompt=False) host_helper.check_password(stream, password=password) time.sleep(2) serial.send_bytes(stream, - "sudo /sbin/ip link set {} up".format(interface), + f"sudo /sbin/ip link set {interface} up", expect_prompt=False) host_helper.check_password(stream, password=password) time.sleep(2) serial.send_bytes(stream, - "sudo route add default gw {}".format(gateway_ip), + f"sudo route add default gw {gateway_ip}", expect_prompt=False) host_helper.check_password(stream, password=password) - if vboxoptions.vboxnet_type == 'hostonly': - LOG.info("Pinging controller-0 at: %s...", ip) + if V_BOX_OPTIONS.vboxnet_type == 'hostonly': + LOG.info("Pinging controller-0 at: %s...", ip_addr) tmout = HostTimeout.NETWORKING_OPERATIONAL while tmout: # Ping from machine hosting virtual box to virtual machine - rc = subprocess.call(['ping', '-c', '1', ip]) - if rc == 0: + return_code = subprocess.call(['ping', '-c', '1', ip_addr]) + if return_code == 0: break tmout -= 1 else: - raise "Failed to establish connection in {}s " \ - "to controller-0 at: {}!" + raise ConnectionError(f"Failed to establish connection in {tmout}s " \ + "to controller-0 at: {ip_addr}!") LOG.info("Ping succeeded!") @@ -132,46 +133,74 @@ def fix_networking(stream, release, password='Li69nux*'): Vbox/linux bug: Sometimes after resuming a VM networking fails to comes up. Setting VM interface down then up again fixes it. """ + if release == "R2": interface = "eth0" else: interface = "enp0s3" LOG.info("Fixing networking ...") serial.send_bytes(stream, - "sudo /sbin/ip link set {} down".format(interface), + f"sudo /sbin/ip link set {interface} down", expect_prompt=False) host_helper.check_password(stream, password=password) time.sleep(1) serial.send_bytes( stream, - "sudo /sbin/ip link set {} up".format(interface), + f"sudo /sbin/ip link set {interface} up", expect_prompt=False) host_helper.check_password(stream, password=password) time.sleep(2) -def install_controller_0(cont0_stream, setup_type, securityprofile, lowlatency, - install_mode, ctrlr0_ip, gateway_ip, - username='wrsroot', password='Li69nux*'): +def install_controller_0(cont0_stream, menu_select_dict, network_dict): """ - Installation of controller-0. + Installs controller-0 node by performing the following steps: + 1. Selects setup type, security profile, low latency, and install mode using menu_selector. + 2. Expects "login:" prompt in the installation console. + 3. Changes the password on initial login. + 4. Disables user logout. + 5. Sets up basic networking. + + Args: + cont0_stream (stream): The installation console stream for controller-0. + menu_select_dict (dict): A dictionary containing the following keys: + - setup_type (str): The type of setup (Simplex, Duplex, etc.). + - securityprofile (str): The security profile (Standard, FIPS, etc.). + - lowlatency (bool): Whether or not to enable low latency. + - install_mode (str): The install mode (standard, patch, etc.). + network_dict (dict): A dictionary containing the following keys: + - ctrlr0_ip (str): The IP address for controller-0. + - gateway_ip (str): The IP address for the gateway. + - username (str, optional): The username for the SSH connection. Defaults to "wrsroot". + - password (str, optional): The password for the SSH connection. Defaults to "Li69nux*". + + Raises: + Exception: If there is a failure in the installation process. + + Note: + The function waits for certain durations between each step. """ + + username = network_dict.get("username", "wrsroot") + password = network_dict.get("password", "Li69nux*") + LOG.info("Starting installation of controller-0") start_time = time.time() menu_selector( cont0_stream, - setup_type, - securityprofile, - lowlatency, - install_mode) + menu_select_dict["setup_type"], + menu_select_dict["securityprofile"], + menu_select_dict["lowlatency"], + menu_select_dict["install_mode"] + ) try: serial.expect_bytes( cont0_stream, "login:", timeout=HostTimeout.INSTALL) - except Exception as e: - LOG.info("Connection failed for controller-0 with %s", e) + except Exception as exception: # pylint: disable=E0012, W0703 + LOG.info("Connection failed for controller-0 with %s", exception) # Sometimes we get UnicodeDecodeError exception due to the output # of installation. So try one more time maybe LOG.info("So ignore the exception and wait for controller-0 to be installed again.") @@ -193,7 +222,12 @@ def install_controller_0(cont0_stream, setup_type, securityprofile, lowlatency, host_helper.disable_logout(cont0_stream) # Setup basic networking time.sleep(1) - setup_networking(cont0_stream, ctrlr0_ip, gateway_ip, password=password) + setup_networking( + cont0_stream, + network_dict["ctrlr0_ip"], + network_dict["gateway_ip"], + password=password + ) def delete_lab(labname, force=False): @@ -211,9 +245,8 @@ def delete_lab(labname, force=False): choice = input().lower() if choice == 'y': break - else: - LOG.info("Aborting!") - exit(1) + LOG.info("Aborting!") + sys.exit(1) LOG.info("#### Deleting lab %s.", labname) LOG.info("VMs in lab: %s.", node_list) vboxmanage.vboxmanage_controlvms(node_list, "poweroff") @@ -225,19 +258,19 @@ def get_disk_sizes(comma_list): """ Return the disk sizes as taken from the command line. """ + sizes = comma_list.split(',') for size in sizes: - try: - val = int(size) - if val < 0: - raise Exception() - except: + val = int(size) + if val < 0: LOG.info("Disk sizes must be a comma separated list of positive integers.") + # pylint: disable=E0012, W0719 raise Exception("Disk sizes must be a comma separated list of positive integers.") return sizes -def create_lab(vboxoptions): +# pylint: disable=too-many-locals, too-many-branches, too-many-statements +def create_lab(m_vboxoptions): """ Creates vms using the arguments in vboxoptions. """ @@ -247,88 +280,95 @@ def create_lab(vboxoptions): for attr in dir(Nodes) if not attr.startswith('__')] nic_config = [getattr(NICs, attr) for attr in dir(NICs) if not attr.startswith('__')] - oam_config = [getattr(OAM, attr) - for attr in dir(OAM) if not attr.startswith('__')][0] + # oam_config = [getattr(OAM, attr) + # for attr in dir(OAM) if not attr.startswith('__')][0] serial_config = [getattr(Serial, attr) for attr in dir(Serial) if not attr.startswith('__')] # Create nodes list nodes_list = [] - if vboxoptions.controllers: - for node_id in range(0, vboxoptions.controllers): - node_name = vboxoptions.labname + "-controller-{}".format(node_id) + if m_vboxoptions.controllers: + for node_id in range(0, m_vboxoptions.controllers): + node_name = m_vboxoptions.labname + f"-controller-{node_id}" nodes_list.append(node_name) - if vboxoptions.workers: - for node_id in range(0, vboxoptions.workers): - node_name = vboxoptions.labname + "-worker-{}".format(node_id) + if m_vboxoptions.workers: + for node_id in range(0, m_vboxoptions.workers): + node_name = m_vboxoptions.labname + f"-worker-{node_id}" nodes_list.append(node_name) - if vboxoptions.storages: - for node_id in range(0, vboxoptions.storages): - node_name = vboxoptions.labname + "-storage-{}".format(node_id) + if m_vboxoptions.storages: + for node_id in range(0, m_vboxoptions.storages): + node_name = m_vboxoptions.labname + f"-storage-{node_id}" nodes_list.append(node_name) LOG.info("#### We will create the following nodes: %s", nodes_list) port = 10000 + # pylint: disable=too-many-nested-blocks for node in nodes_list: LOG.info("#### Creating node: %s", node) - vboxmanage.vboxmanage_createvm(node, vboxoptions.labname) + vboxmanage.vboxmanage_createvm(node, m_vboxoptions.labname) vboxmanage.vboxmanage_storagectl( node, storectl="sata", - hostiocache=vboxoptions.hostiocache) + hostiocache=m_vboxoptions.hostiocache) disk_sizes = None no_disks = 0 if "controller" in node: - if vboxoptions.setup_type in [AIO_DX, AIO_SX]: + if m_vboxoptions.setup_type in [AIO_DX, AIO_SX]: node_type = "controller-AIO" else: - node_type = "controller-{}".format(vboxoptions.setup_type) - if vboxoptions.controller_disk_sizes: - disk_sizes = get_disk_sizes(vboxoptions.controller_disk_sizes) + node_type = f"controller-{m_vboxoptions.setup_type}" + if m_vboxoptions.controller_disk_sizes: + disk_sizes = get_disk_sizes(m_vboxoptions.controller_disk_sizes) else: - no_disks = vboxoptions.controller_disks + no_disks = m_vboxoptions.controller_disks elif "worker" in node: node_type = "worker" - if vboxoptions.worker_disk_sizes: - disk_sizes = get_disk_sizes(vboxoptions.worker_disk_sizes) + if m_vboxoptions.worker_disk_sizes: + disk_sizes = get_disk_sizes(m_vboxoptions.worker_disk_sizes) else: - no_disks = vboxoptions.worker_disks + no_disks = m_vboxoptions.worker_disks elif "storage" in node: node_type = "storage" - if vboxoptions.storage_disk_sizes: - disk_sizes = get_disk_sizes(vboxoptions.storage_disk_sizes) + if m_vboxoptions.storage_disk_sizes: + disk_sizes = get_disk_sizes(m_vboxoptions.storage_disk_sizes) else: - no_disks = vboxoptions.storage_disks + no_disks = m_vboxoptions.storage_disks for item in node_config: if item['node_type'] == node_type: vboxmanage.vboxmanage_modifyvm( node, - cpus=str(item['cpus']), - memory=str(item['memory'])) + { + "cpus": str(item['cpus']), + "memory": str(item['memory']), + }, + ) if not disk_sizes: disk_sizes = item['disks'][no_disks] vboxmanage.vboxmanage_createmedium(node, disk_sizes, - vbox_home_dir=vboxoptions.vbox_home_dir) - if platform == 'win32' or platform == 'win64': + vbox_home_dir=m_vboxoptions.vbox_home_dir) + if platform in ("win32", "win64"): vboxmanage.vboxmanage_modifyvm( - node, uartbase=serial_config[0]['uartbase'], - uartport=serial_config[ - 0]['uartport'], - uartmode=serial_config[ - 0]['uartmode'], - uartpath=port) + node, + { + "uartbase": serial_config[0]['uartbase'], + "uartport": serial_config[0]['uartport'], + "uartmode": serial_config[0]['uartmode'], + "uartpath": port, + }, + ) port += 1 else: vboxmanage.vboxmanage_modifyvm( - node, uartbase=serial_config[0]['uartbase'], - uartport=serial_config[ - 0]['uartport'], - uartmode=serial_config[ - 0]['uartmode'], - uartpath=serial_config[ - 0]['uartpath'], - prefix=vboxoptions.userid) + node, + { + "uartbase": serial_config[0]['uartbase'], + "uartport": serial_config[0]['uartport'], + "uartmode": serial_config[0]['uartmode'], + "uartpath": serial_config[0]['uartpath'], + "prefix": m_vboxoptions.userid, + }, + ) if "controller" in node: node_type = "controller" @@ -340,86 +380,109 @@ def create_lab(vboxoptions): if adapter.isdigit(): last_adapter += 1 data = item[adapter] - if vboxoptions.vboxnet_name is not 'none' and data['nic'] is 'hostonly': - if vboxoptions.vboxnet_type == 'nat': + if m_vboxoptions.vboxnet_name != 'none' and data['nic'] == 'hostonly': + if m_vboxoptions.vboxnet_type == 'nat': data['nic'] = 'natnetwork' - data['natnetwork'] = vboxoptions.vboxnet_name + data['natnetwork'] = m_vboxoptions.vboxnet_name data['hostonlyadapter'] = None data['intnet'] = None # data['nicpromisc1'] = None else: data[ - 'hostonlyadapter'] = vboxoptions.vboxnet_name + 'hostonlyadapter'] = m_vboxoptions.vboxnet_name data['natnetwork'] = None else: data['natnetwork'] = None - vboxmanage.vboxmanage_modifyvm(node, - nic=data['nic'], nictype=data['nictype'], - nicpromisc=data['nicpromisc'], - nicnum=int(adapter), intnet=data['intnet'], - hostonlyadapter=data['hostonlyadapter'], - natnetwork=data['natnetwork'], - prefix="{}-{}".format(vboxoptions.userid, - vboxoptions.labname)) + vboxmanage.vboxmanage_modifyvm( + node, + { + "nic": data['nic'], + "nictype": data['nictype'], + "nicpromisc": data['nicpromisc'], + "nicnum": int(adapter), + "intnet": data['intnet'], + "hostonlyadapter": data['hostonlyadapter'], + "natnetwork": data['natnetwork'], + "prefix": f"{m_vboxoptions.userid}-{m_vboxoptions.labname}", + }, + ) - if vboxoptions.add_nat_interface: + if m_vboxoptions.add_nat_interface: last_adapter += 1 - vboxmanage.vboxmanage_modifyvm(node, nicnum=adapter, nictype='nat') + vboxmanage.vboxmanage_modifyvm( + node, + { + # "nicnum": adapter, #TODO where this adapter come from? #pylint: disable=fixme + "nictype": 'nat', + }, + ) # Add port forwarding rules for controllers nat interfaces - if vboxoptions.vboxnet_type == 'nat' and 'controller' in node: + if m_vboxoptions.vboxnet_type == 'nat' and 'controller' in node: if 'controller-0' in node: - local_port = vboxoptions.nat_controller0_local_ssh_port - ip = vboxoptions.controller0_ip + local_port = m_vboxoptions.nat_controller0_local_ssh_port + ip_addr = m_vboxoptions.controller0_ip elif 'controller-1' in node: - local_port = vboxoptions.nat_controller1_local_ssh_port - ip = vboxoptions.controller1_ip - vboxmanage.vboxmanage_port_forward(node, vboxoptions.vboxnet_name, - local_port=local_port, guest_port='22', guest_ip=ip) + local_port = m_vboxoptions.nat_controller1_local_ssh_port + ip_addr = m_vboxoptions.controller1_ip + vboxmanage.vboxmanage_port_forward( + node, + m_vboxoptions.vboxnet_name, + local_port=local_port, + guest_port='22', + guest_ip=ip_addr + ) # Floating ip port forwarding - if vboxoptions.vboxnet_type == 'nat' and vboxoptions.setup_type != 'AIO-SX': - local_port = vboxoptions.nat_controller_floating_local_ssh_port - ip = vboxoptions.controller_floating_ip - name = vboxoptions.labname + 'controller-float' - vboxmanage.vboxmanage_port_forward(name, vboxoptions.vboxnet_name, - local_port=local_port, guest_port='22', guest_ip=ip) + if m_vboxoptions.vboxnet_type == 'nat' and m_vboxoptions.setup_type != 'AIO-SX': + local_port = m_vboxoptions.nat_controller_floating_local_ssh_port + ip_addr = m_vboxoptions.controller_floating_ip + name = m_vboxoptions.labname + 'controller-float' + vboxmanage.vboxmanage_port_forward(name, m_vboxoptions.vboxnet_name, + local_port=local_port, guest_port='22', guest_ip=ip_addr) - ctrlr0 = vboxoptions.labname + '-controller-0' + ctrlr0 = m_vboxoptions.labname + '-controller-0' vboxmanage.vboxmanage_storagectl( ctrlr0, storectl="ide", - hostiocache=vboxoptions.hostiocache) + hostiocache=m_vboxoptions.hostiocache) vboxmanage.vboxmanage_storageattach( - ctrlr0, storectl="ide", storetype="dvddrive", - disk=vboxoptions.iso_location, device_num="0", port_num="1") + ctrlr0, + { + "storectl": "ide", + "storetype": "dvddrive", + "disk": m_vboxoptions.iso_location, + "port_num": "1", + "device_num": "0", + }, + ) -def get_hostnames(ignore=None, personalities=['controller', 'storage', 'worker']): +def get_hostnames(ignore=None, personalities=('controller', 'storage', 'worker')): """ Based on the number of nodes defined on the command line, construct the hostnames of each node. """ hostnames = {} - if vboxoptions.controllers and 'controller' in personalities: - for node_id in range(0, vboxoptions.controllers): - node_name = vboxoptions.labname + "-controller-{}".format(node_id) + if V_BOX_OPTIONS.controllers and 'controller' in personalities: + for node_id in range(0, V_BOX_OPTIONS.controllers): + node_name = V_BOX_OPTIONS.labname + f"-controller-{node_id}" if ignore and node_name in ignore: continue - hostnames[node_name] = 'controller-{}'.format(id) - if vboxoptions.workers and 'worker' in personalities: - for node_id in range(0, vboxoptions.workers): - node_name = vboxoptions.labname + "-worker-{}".format(node_id) + hostnames[node_name] = f"controller-{id}" + if V_BOX_OPTIONS.workers and 'worker' in personalities: + for node_id in range(0, V_BOX_OPTIONS.workers): + node_name = V_BOX_OPTIONS.labname + f"-worker-{node_id}" if ignore and node_name in ignore: continue - hostnames[node_name] = 'worker-{}'.format(id) - if vboxoptions.storages and 'storage' in personalities: - for node_id in range(0, vboxoptions.storages): - node_name = vboxoptions.labname + "-storage-{}".format(node_id) + hostnames[node_name] = f"worker-{id}" + if V_BOX_OPTIONS.storages and 'storage' in personalities: + for node_id in range(0, V_BOX_OPTIONS.storages): + node_name = V_BOX_OPTIONS.labname + f"-storage-{node_id}" if ignore and node_name in ignore: continue - hostnames[node_name] = 'storage-{}'.format(node_id) + hostnames[node_name] = f'storage-{node_id}' return hostnames @@ -430,21 +493,21 @@ def get_personalities(ignore=None): """ personalities = {} - if vboxoptions.controllers: - for node_id in range(0, vboxoptions.controllers): - node_name = vboxoptions.labname + "-controller-{}".format(node_id) + if V_BOX_OPTIONS.controllers: + for node_id in range(0, V_BOX_OPTIONS.controllers): + node_name = V_BOX_OPTIONS.labname + f"-controller-{node_id}" if ignore and node_name in ignore: continue personalities[node_name] = 'controller' - if vboxoptions.workers: - for node_id in range(0, vboxoptions.workers): - node_name = vboxoptions.labname + "-worker-{}".format(node_id) + if V_BOX_OPTIONS.workers: + for node_id in range(0, V_BOX_OPTIONS.workers): + node_name = V_BOX_OPTIONS.labname + f"-worker-{node_id}" if ignore and node_name in ignore: continue personalities[node_name] = 'worker' - if vboxoptions.storages: - for node_id in range(0, vboxoptions.storages): - node_name = vboxoptions.labname + "-storage-{}".format(node_id) + if V_BOX_OPTIONS.storages: + for node_id in range(0, V_BOX_OPTIONS.storages): + node_name = V_BOX_OPTIONS.labname + f"-storage-{node_id}" if ignore and node_name in ignore: continue personalities[node_name] = 'storage' @@ -481,24 +544,25 @@ def create_host_bulk_add(): """ + LOG.info("Creating content for 'system host-bulk-add'") - vms = vboxmanage.get_all_vms(vboxoptions.labname, option="vms") - ctrl0 = vboxoptions.labname + "-controller-0" + vms = vboxmanage.get_all_vms(V_BOX_OPTIONS.labname, option="vms") + ctrl0 = V_BOX_OPTIONS.labname + "-controller-0" vms.remove(ctrl0) # Get management macs macs = {} - for vm in vms: - info = vboxmanage.vboxmanage_showinfo(vm).splitlines() + for virtual_machine in vms: + info = vboxmanage.vboxmanage_showinfo(virtual_machine).splitlines() for line in info: try: - k, v = line.split(b'=') + key, value = line.split(b'=') except ValueError: continue - if k == b'macaddress2': - orig_mac = v.decode('utf-8').replace("\"", "") + if key == b'macaddress2': + orig_mac = value.decode('utf-8').replace("\"", "") # Do for e.g.: 080027C95571 -> 08:00:27:C9:55:71 - macs[vm] = ":".join(re.findall(r"..", orig_mac)) + macs[virtual_machine] = ":".join(re.findall(r"..", orig_mac)) # Get personalities personalities = get_personalities(ignore=[ctrl0]) @@ -507,18 +571,18 @@ def create_host_bulk_add(): # Create file host_xml = ('\n' '\n') - for vm in vms: + for virtual_machine in vms: host_xml += ' \n' - host_xml += ' {}\n'.format(hostnames[vm]) - host_xml += ' {}\n'.format( - personalities[vm]) - host_xml += ' {}\n'.format(macs[vm]) + host_xml += f' {hostnames[virtual_machine]}\n' + host_xml += f' {personalities[virtual_machine]}\n' + host_xml += f' {macs[virtual_machine]}\n' host_xml += ' \n' host_xml += '\n' return host_xml -serial_prompt_configured = False + +# serial_prompt_configured = False def wait_for_hosts(ssh_client, hostnames, status, @@ -528,12 +592,12 @@ def wait_for_hosts(ssh_client, hostnames, status, status. """ - start = time.time() + start_time = time.time() while hostnames: LOG.info("Hosts not %s: %s", status, hostnames) - if (time.time() - start) > HostTimeout.HOST_INSTALL: + if (time.time() - start_time) > HostTimeout.HOST_INSTALL: LOG.info("VMs not booted in %s, aborting: %s", timeout, hostnames) - raise Exception("VMs failed to go %s!", status) + raise Exception(f"VMs failed to go {status}!") # pylint: disable=E0012, W0719 # Get host list host_statuses, _, _ = run_ssh_cmd( ssh_client, 'source /etc/platform/openrc; system host-list', timeout=30) @@ -546,10 +610,11 @@ def wait_for_hosts(ssh_client, hostnames, status, LOG.info("Waiting %s sec before re-checking host status.", interval) time.sleep(interval) + CONSOLE_UNKNOWN_MODE = 'disconnected' CONSOLE_USER_MODE = 'user' CONSOLE_ROOT_MODE = 'root' -serial_console_mode = CONSOLE_UNKNOWN_MODE +SERIAL_CONSOLE_MODE = CONSOLE_UNKNOWN_MODE def run_ssh_cmd(ssh_client, cmd, timeout=5, @@ -560,11 +625,11 @@ def run_ssh_cmd(ssh_client, cmd, timeout=5, if mode == CONSOLE_ROOT_MODE: LOG.info(">>>>>") - cmd = "sudo {}".format(cmd) + cmd = f"sudo {cmd}" LOG.info("#### Running command over ssh: '%s'", cmd) stdin, stdout, stderr = ssh_client.exec_command(cmd, timeout, get_pty=True) if mode == CONSOLE_ROOT_MODE: - stdin.write('{}\n'.format(vboxoptions.password)) + stdin.write(f'{V_BOX_OPTIONS.password}\n') stdin.flush() stdout_lines = [] while True: @@ -593,12 +658,12 @@ def set_serial_prompt_mode(stream, mode): password). """ - global serial_console_mode + global SERIAL_CONSOLE_MODE # pylint: disable=global-statement - if serial_console_mode == mode: + if SERIAL_CONSOLE_MODE == mode: LOG.info("Serial console prompt already set to '%s' mode.", mode) return - if serial_console_mode != CONSOLE_USER_MODE: + if SERIAL_CONSOLE_MODE != CONSOLE_USER_MODE: # Set mode to user first, even if we later go to root serial.send_bytes(stream, "exit\n", expect_prompt=False) if serial.expect_bytes(stream, "ogin:", fail_ok=True, timeout=4): @@ -606,21 +671,21 @@ def set_serial_prompt_mode(stream, mode): if serial.expect_bytes(stream, "ogin:", fail_ok=True, timeout=4): LOG.info("Expected login prompt, connect to console" \ "stop any running processes and log out.") - raise Exception("Failure getting login prompt on serial console!") + raise Exception("Failure getting login prompt on serial console!") # pylint: disable=E0012, W0719 serial.send_bytes( stream, - vboxoptions.username, + V_BOX_OPTIONS.username, prompt="assword:", timeout=30) - if serial.send_bytes(stream, vboxoptions.password, prompt="~$", fail_ok=True, timeout=30): - raise Exception("Login failure, invalid password?") + if serial.send_bytes(stream, V_BOX_OPTIONS.password, prompt="~$", fail_ok=True, timeout=30): + raise Exception("Login failure, invalid password?") # pylint: disable=E0012, W0719 if mode == CONSOLE_USER_MODE: serial.send_bytes(stream, "source /etc/platform/openrc\n", timeout=30, prompt='keystone') - serial_console_mode = CONSOLE_USER_MODE - if mode == 'root' and serial_console_mode != 'root': + SERIAL_CONSOLE_MODE = CONSOLE_USER_MODE + if mode == 'root' and SERIAL_CONSOLE_MODE != 'root': serial.send_bytes(stream, 'sudo su -', expect_prompt=False) - host_helper.check_password(stream, password=vboxoptions.password) + host_helper.check_password(stream, password=V_BOX_OPTIONS.password) serial.send_bytes( stream, "cd /home/wrsroot", @@ -628,33 +693,59 @@ def set_serial_prompt_mode(stream, mode): timeout=30) serial.send_bytes(stream, "source /etc/platform/openrc\n", timeout=30, prompt='keystone') - serial_console_mode = CONSOLE_ROOT_MODE + SERIAL_CONSOLE_MODE = CONSOLE_ROOT_MODE serial.send_bytes(stream, "export TMOUT=0", timeout=10, prompt='keystone') # also reset OAM networking? def serial_prompt_mode(mode): + """ + A decorator function that sets the serial console login prompt to the specified + mode before calling the decorated function. + + Args: + mode (str): The login prompt mode to set. Valid values are "admin" and "root". + + Returns: + function: A decorator function that sets the serial console login prompt to the specified mode. + """ + def real_decorator(func): def func_wrapper(*args, **kwargs): try: set_serial_prompt_mode(kwargs['stream'], mode) - except: + except: # pylint: disable=bare-except LOG.info("Serial console login as '%s' failed. Retrying once.", mode) set_serial_prompt_mode(kwargs['stream'], mode) return func(*args, **kwargs) + return func_wrapper + return real_decorator -def _connect_to_serial(vm=None): - if not vm: - vm = vboxoptions.labname + "-controller-0" - sock = serial.connect(vm, 10000, getpass.getuser()) +def _connect_to_serial(virtual_machine=None): + if not virtual_machine: + virtual_machine = V_BOX_OPTIONS.labname + "-controller-0" + sock = serial.connect(virtual_machine, 10000, getpass.getuser()) return sock, streamexpect.wrap(sock, echo=True, close_stream=False) def connect_to_serial(func): + """ + A decorator function that establishes a connection to the serial console before + calling the decorated function. + + Args: + func (function): The function to be decorated. + + Returns: + function: A wrapper function that establishes a connection to the serial console, + calls the decorated function, and then disconnects from the serial console. + """ + def func_wrapper(*args, **kwargs): + sock = None try: sock, kwargs['stream'] = _connect_to_serial() return func(*args, **kwargs) @@ -665,37 +756,47 @@ def connect_to_serial(func): def _connect_to_ssh(): - # Get ip and port for ssh on floating ip - ip, port = get_ssh_ip_and_port() + ip_addr, port = get_ssh_ip_and_port() # Remove ssh key # For hostonly adapter we remove port 22 of controller ip # for nat interfaces we remove the specific port on 127.0.0.1 as # we have port forwarding enabled. - if vboxoptions.vboxnet_type == 'nat': - keygen_arg = "[127.0.0.1]:{}".format(port) + # pylint: disable=R0801 + if V_BOX_OPTIONS.vboxnet_type == 'nat': + keygen_arg = f"[127.0.0.1]:{port}" else: - keygen_arg = ip - cmd = 'ssh-keygen -f "/home/{}/.ssh/known_hosts" -R {}'.format( - getpass.getuser(), keygen_arg) + keygen_arg = ip_addr + cmd = f'ssh-keygen -f "/home/{getpass.getuser()}/.ssh/known_hosts" -R {keygen_arg}' LOG.info("CMD: %s", cmd) - process = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE) - for line in iter(process.stdout.readline, b''): - LOG.info("%s", line.decode("utf-8").strip()) - process.wait() + with subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE) as process: + for line in iter(process.stdout.readline, b''): + LOG.info("%s", line.decode("utf-8").strip()) + process.wait() # Connect to ssh ssh = paramiko.SSHClient() ssh.load_system_host_keys() ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) - ssh.connect(ip, port=port, username=vboxoptions.username, - password=vboxoptions.password, look_for_keys=False, allow_agent=False) + ssh.connect(ip_addr, port=port, username=V_BOX_OPTIONS.username, + password=V_BOX_OPTIONS.password, look_for_keys=False, allow_agent=False) return ssh def connect_to_ssh(func): + """ + Decorator function to establish a SSH connection before executing the function + and close the connection afterwards. + + Args: + - func: The function to be decorated. + + Returns: + - The decorated function that has a SSH connection established before executing the function. + """ + def func_wrapper(*args, **kwargs): try: ssh = _connect_to_ssh() @@ -703,29 +804,56 @@ def connect_to_ssh(func): return func(*args, **kwargs) finally: ssh.close() + return func_wrapper def stage_test_success(): + """Prints a log message indicating the execution of a test stage.""" + LOG.info("Executing stage_test_success") def stage_test_fail(): + """ + Prints a log message indicating the execution of a test stage and raises an exception. + + Raises: + - Exception: Always raises an exception. + """ + LOG.info("Executing stage_test_success") - raise Exception("exception as of stage_test_fail") + raise Exception("exception as of stage_test_fail") # pylint: disable=E0012, W0719 def stage_create_lab(): - delete_lab(vboxoptions.labname, vboxoptions.force_delete_lab) - create_lab(vboxoptions) + """ + Wrapper function for deleting an existing virtual lab and creating a new one + using `vboxoptions`. + """ + + delete_lab(V_BOX_OPTIONS.labname, V_BOX_OPTIONS.force_delete_lab) + create_lab(V_BOX_OPTIONS) # time.sleep(2) def stage_install_controller0(): - node_list = vboxmanage.get_all_vms(vboxoptions.labname, option="vms") + """ + Starts the `controller-0` VM, establishes a serial connection to it, and installs + the OS on it using the `install_controller_0` function with the parameters specified + in `vboxoptions`. + + Args: + - None + + Raises: + - AssertionError: If `controller-0` is not in the list of available VMs. + """ + + node_list = vboxmanage.get_all_vms(V_BOX_OPTIONS.labname, option="vms") LOG.info("Found nodes: %s", node_list) - ctrlr0 = vboxoptions.labname + "-controller-0" + ctrlr0 = V_BOX_OPTIONS.labname + "-controller-0" assert ctrlr0 in node_list, "controller-0 not in vm list. Stopping installation." vboxmanage.vboxmanage_startvm(ctrlr0) @@ -734,150 +862,235 @@ def stage_install_controller0(): cont0_stream = streamexpect.wrap(sock, echo=True, close_stream=False) install_controller_0( - cont0_stream, vboxoptions.setup_type, vboxoptions.securityprofile, - vboxoptions.lowlatency, - install_mode=vboxoptions.install_mode, ctrlr0_ip=vboxoptions.controller0_ip, - gateway_ip=vboxoptions.vboxnet_ip, - username=vboxoptions.username, password=vboxoptions.password) + cont0_stream, + menu_select_dict={ + "setup_type": V_BOX_OPTIONS.setup_type, + "securityprofile": V_BOX_OPTIONS.securityprofile, + "lowlatency": V_BOX_OPTIONS.lowlatency, + "install_mode": V_BOX_OPTIONS.install_mode, + }, + network_dict={ + "ctrlr0_ip": V_BOX_OPTIONS.controller0_ip, + "gateway_ip": V_BOX_OPTIONS.vboxnet_ip, + "username": V_BOX_OPTIONS.username, + "password": V_BOX_OPTIONS.password + } + ) serial.disconnect(sock) time.sleep(5) @connect_to_serial -def stage_config_controller(stream): - ip, port = get_ssh_ip_and_port( - 'controller-0') # Floating ip is not yet configured +def stage_config_controller(stream): # pylint: disable=too-many-locals + """ + Stage to configure controller-0 networking settings and upload the configuration + file to the controller. + Args: + stream (obj): Serial console stream. + + Raises: + Exception: If there is an error in the configuration or upload process, + raises an exception with the error message. + + Note: + This method assumes that the controller-0 virtual machine has been previously + installed and that its serial console stream is open. + """ + + ip_addr, port = get_ssh_ip_and_port( + 'controller-0') # Floating ip is not yet configured + # if True: # Updated config file LOG.info("#### Updating config_controller ini file networking" \ "settings and uploading it to controller.") destination = "/home/" + \ - vboxoptions.username + "/stx_config.ini_centos" + V_BOX_OPTIONS.username + "/stx_config.ini_centos" configini = configparser.ConfigParser() configini.optionxform = str - configini.read(vboxoptions.config_controller_ini) + configini.read(V_BOX_OPTIONS.config_controller_ini) old_cidr = configini['OAM_NETWORK']['CIDR'] - new_cidr = vboxoptions.ini_oam_cidr + new_cidr = V_BOX_OPTIONS.ini_oam_cidr LOG.info("Replacing OAM_NETWORK/CIDR from %s to %s", old_cidr, new_cidr) configini['OAM_NETWORK']['CIDR'] = new_cidr old_gateway = configini['OAM_NETWORK']['GATEWAY'] - new_gateway = vboxoptions.vboxnet_ip + new_gateway = V_BOX_OPTIONS.vboxnet_ip LOG.info("Replacing OAM_NETWORK/GATEWAY from %s to %s", old_gateway, new_gateway) configini['OAM_NETWORK']['GATEWAY'] = new_gateway - if vboxoptions.setup_type == AIO_SX: + if V_BOX_OPTIONS.setup_type == AIO_SX: old_ip_address = configini['OAM_NETWORK']['IP_ADDRESS'] - new_ip_address = vboxoptions.controller0_ip + new_ip_address = V_BOX_OPTIONS.controller0_ip LOG.info("Replacing OAM_NETWORK/IP_ADDRESS from %s to %s", old_ip_address, new_ip_address) configini['OAM_NETWORK']['IP_ADDRESS'] = new_ip_address else: old_start_addr = configini['OAM_NETWORK']['IP_START_ADDRESS'] - new_start_addr = vboxoptions.ini_oam_ip_start_address + new_start_addr = V_BOX_OPTIONS.ini_oam_ip_start_address LOG.info("Replacing OAM_NETWORK/IP_START_ADDRESS from %s to %s", old_start_addr, new_start_addr) configini['OAM_NETWORK']['IP_START_ADDRESS'] = new_start_addr old_end_addr = configini['OAM_NETWORK']['IP_END_ADDRESS'] - new_end_addr = vboxoptions.ini_oam_ip_end_address + new_end_addr = V_BOX_OPTIONS.ini_oam_ip_end_address LOG.info("Replacing OAM_NETWORK/IP_END_ADDRESS from %s to %s", old_end_addr, new_end_addr) configini['OAM_NETWORK']['IP_END_ADDRESS'] = new_end_addr # Take updated config file and copy it to controller - with tempfile.NamedTemporaryFile(mode='w') as fp: - configini.write(fp, space_around_delimiters=False) - fp.flush() + with tempfile.NamedTemporaryFile(mode='w') as file: + configini.write(file, space_around_delimiters=False) + file.flush() sftp_send( - fp.name, remote_host=ip, remote_port=port, destination=destination, - username=vboxoptions.username, password=vboxoptions.password) + file.name, + destination, + { + "remote_host": ip_addr, + "remote_port": port, + "username": V_BOX_OPTIONS.username, + "password": V_BOX_OPTIONS.password + } + ) LOG.info("Copying Ansible configuration file") - destination_ansible = f'/home/{vboxoptions.username}/localhost.yml' + destination_ansible = f'/home/{V_BOX_OPTIONS.username}/localhost.yml' sftp_send( - vboxoptions.ansible_controller_config, remote_host=ip, remote_port=port, destination=destination_ansible, - username=vboxoptions.username, password=vboxoptions.password) + V_BOX_OPTIONS.ansible_controller_config, + destination_ansible, + { + "remote_host": ip_addr, + "remote_port": port, + "username": V_BOX_OPTIONS.username, + "password": V_BOX_OPTIONS.password + } + ) # Run config_controller LOG.info("#### Running config_controller") install_lab.config_controller(stream, config_file=destination, - password=vboxoptions.password) + password=V_BOX_OPTIONS.password) # Wait for services to stabilize time.sleep(120) - if vboxoptions.setup_type == AIO_SX: + if V_BOX_OPTIONS.setup_type == AIO_SX: # Increase AIO responsiveness by allocating more cores to platform install_lab.update_platform_cpus(stream, 'controller-0') def get_ssh_ip_and_port(node='floating'): - if vboxoptions.vboxnet_type == 'nat': - ip = '127.0.0.1' + """ + This function returns the IP address and port of the specified node to use for + an SSH connection. + + Args: + node (str, optional): The node to get the IP address and port for. + Valid values are "floating" (default), "controller-0", and "controller-1". + + Returns: + tuple: A tuple containing the IP address and port of the specified node. + + Raises: + Exception: If an undefined node is specified. + """ + + if V_BOX_OPTIONS.vboxnet_type == 'nat': + ip_addr = '127.0.0.1' if node == 'floating': - if vboxoptions.setup_type != 'AIO-SX': - port = vboxoptions.nat_controller_floating_local_ssh_port + if V_BOX_OPTIONS.setup_type != 'AIO-SX': + port = V_BOX_OPTIONS.nat_controller_floating_local_ssh_port else: - port = vboxoptions.nat_controller0_local_ssh_port + port = V_BOX_OPTIONS.nat_controller0_local_ssh_port elif node == 'controller-0': - port = vboxoptions.nat_controller0_local_ssh_port + port = V_BOX_OPTIONS.nat_controller0_local_ssh_port elif node == 'controller-1': - port = vboxoptions.nat_controller_1_local_ssh_port + port = V_BOX_OPTIONS.nat_controller_1_local_ssh_port else: - raise Exception("Undefined node '{}'".format(node)) + raise Exception(f"Undefined node '{node}'") # pylint: disable=E0012, W0719 else: if node == 'floating': - if vboxoptions.setup_type != 'AIO-SX': - ip = vboxoptions.controller_floating_ip + if V_BOX_OPTIONS.setup_type != 'AIO-SX': + ip_addr = V_BOX_OPTIONS.controller_floating_ip else: - ip = vboxoptions.controller0_ip + ip_addr = V_BOX_OPTIONS.controller0_ip elif node == 'controller-0': - ip = vboxoptions.controller0_ip + ip_addr = V_BOX_OPTIONS.controller0_ip elif node == 'controller-1': - ip = vboxoptions.controller1_ip + ip_addr = V_BOX_OPTIONS.controller1_ip else: - raise Exception("Undefined node '{}'".format(node)) + raise Exception(f"Undefined node '{node}'") # pylint: disable=E0012, W0719 port = 22 - return ip, port + return ip_addr, port -#@connect_to_serial -#@serial_prompt_mode(CONSOLE_USER_MODE) + +# @connect_to_serial +# @serial_prompt_mode(CONSOLE_USER_MODE) def stage_rsync_config(): - if not vboxoptions.config_files_dir and not vboxoptions.config_files_dir_dont_follow_links: + """ + Rsync the local configuration files with the remote host's configuration files. + + This method copies the configuration files to the controller. It uses rsync to + synchronize the local configuration files with the remote host's configuration files. + + If the `config_files_dir` or `config_files_dir_dont_follow_links` option is set, this + method copies the files to the remote host. If both are not set, then this method does + nothing. + + Args: + None. + + Returns: + None. + """ + + if not V_BOX_OPTIONS.config_files_dir and not V_BOX_OPTIONS.config_files_dir_dont_follow_links: LOG.info("No rsync done! Please set config-files-dir " "and/or config-files-dir-dont-follow-links") return # Get ip and port for ssh on floating ip - ip, port = get_ssh_ip_and_port() - + ip_addr, port = get_ssh_ip_and_port() # Copy config files to controller - if vboxoptions.config_files_dir: - local_path = vboxoptions.config_files_dir + if V_BOX_OPTIONS.config_files_dir: + local_path = V_BOX_OPTIONS.config_files_dir follow_links = True - send_dir(source=local_path, remote_host=ip, remote_port=port, - destination='/home/' + vboxoptions.username + '/', - username=vboxoptions.username, password=vboxoptions.password, - follow_links=follow_links) + send_dir( + { + "source": local_path, + "remote_host": ip_addr, + "remote_port": port, + "destination": '/home/' + V_BOX_OPTIONS.username + '/', + "username": V_BOX_OPTIONS.username, + "password": V_BOX_OPTIONS.password, + "follow_links": follow_links + } + ) - if vboxoptions.config_files_dir_dont_follow_links: - local_path = vboxoptions.config_files_dir_dont_follow_links + if V_BOX_OPTIONS.config_files_dir_dont_follow_links: + local_path = V_BOX_OPTIONS.config_files_dir_dont_follow_links follow_links = False - send_dir(source=local_path, remote_host=ip, remote_port=port, - destination='/home/' + vboxoptions.username + '/', - username=vboxoptions.username, password=vboxoptions.password, - follow_links=follow_links) + send_dir( + { + "source": local_path, + "remote_host": ip_addr, + "remote_port": port, + "destination": '/home/' + V_BOX_OPTIONS.username + '/', + "username": V_BOX_OPTIONS.username, + "password": V_BOX_OPTIONS.password, + "follow_links": follow_links + } + ) @connect_to_serial @serial_prompt_mode(CONSOLE_USER_MODE) def _run_lab_setup_serial(stream): conf_str = "" - for cfg_file in vboxoptions.lab_setup_conf: - conf_str = conf_str + " -f {}".format(cfg_file) + for cfg_file in V_BOX_OPTIONS.lab_setup_conf: + conf_str = conf_str + f" -f {cfg_file}" - serial.send_bytes(stream, "sh lab_setup.sh {}".format(conf_str), + serial.send_bytes(stream, f"sh lab_setup.sh {conf_str}", timeout=HostTimeout.LAB_INSTALL, prompt='keystone') LOG.info("Lab setup execution completed. Checking if return code is 0.") serial.send_bytes(stream, "echo \"Return code: [$?]\"", @@ -885,48 +1098,68 @@ def _run_lab_setup_serial(stream): @connect_to_ssh -def _run_lab_setup(stage, ssh_client): +def _run_lab_setup(m_stage, ssh_client): conf_str = "" - for cfg_file in vboxoptions.lab_setup_conf: - conf_str = conf_str + " -f {}".format(cfg_file) + for cfg_file in V_BOX_OPTIONS.lab_setup_conf: + conf_str = conf_str + f" -f {cfg_file}" command = f'source /etc/platform/openrc; export ' \ f'PATH="$PATH:/usr/local/bin"; export PATH="$PATH:/usr/bin"; ' \ f'export PATH="$PATH:/usr/local/sbin"; export ' \ - f'PATH="$PATH:/usr/sbin"; sh lab_setup{stage}.sh' + f'PATH="$PATH:/usr/sbin"; sh lab_setup{m_stage}.sh' _, _, exitcode = run_ssh_cmd(ssh_client, command, timeout=HostTimeout.LAB_INSTALL) if exitcode != 0: - msg = "Lab setup failed, expecting exit code of 0 but got {}.".format( - exitcode) + msg = f"Lab setup failed, expecting exit code of 0 but got {exitcode}." LOG.info(msg) - raise Exception(msg) + raise Exception(msg) # pylint: disable=E0012, W0719 def stage_lab_setup1(): - _run_lab_setup(1) + """Calls _run_lab_setup with ssh_client 1""" + + _run_lab_setup(1) # pylint: disable=no-value-for-parameter def stage_lab_setup2(): - _run_lab_setup(2) + """Calls _run_lab_setup with ssh_client 2""" + + _run_lab_setup(2) # pylint: disable=no-value-for-parameter def stage_lab_setup3(): - _run_lab_setup(3) + """Calls _run_lab_setup with ssh_client 3""" + + _run_lab_setup(3) # pylint: disable=no-value-for-parameter def stage_lab_setup4(): - _run_lab_setup(4) + """Calls _run_lab_setup with ssh_client 4""" + + _run_lab_setup(4) # pylint: disable=no-value-for-parameter def stage_lab_setup5(): - _run_lab_setup(5) + """Calls _run_lab_setup with ssh_client 5""" + + _run_lab_setup(5) # pylint: disable=no-value-for-parameter @connect_to_ssh @connect_to_serial def stage_unlock_controller0(stream, ssh_client): + """ + Unlocks the controller-0 node and waits for it to reboot. + + Args: + - stream (obj): Serial stream to send and receive data + - ssh_client (obj): SSH client connection to execute remote commands + + Returns: + None. + """ + LOG.info("#### Unlocking controller-0") _, _, _ = run_ssh_cmd(ssh_client, 'source /etc/platform/openrc; system host-unlock controller-0', @@ -942,13 +1175,23 @@ def stage_unlock_controller0(stream, ssh_client): time.sleep(120) # Make sure we login again, after reboot we are not logged in. - serial_console_mode = CONSOLE_UNKNOWN_MODE + SERIAL_CONSOLE_MODE = CONSOLE_UNKNOWN_MODE # pylint: disable=redefined-outer-name, invalid-name, unused-variable @connect_to_serial @serial_prompt_mode(CONSOLE_USER_MODE) def stage_unlock_controller0_serial(stream): - global serial_console_mode + """ + Unlock the controller-0 host via serial console and wait for services to activate. + + Args: + - stream (stream object): The serial console stream. + + Returns: + None. + """ + + global SERIAL_CONSOLE_MODE # pylint: disable=global-statement if host_helper.unlock_host(stream, 'controller-0'): LOG.info("Host is unlocked, nothing to do. Exiting stage.") return @@ -962,55 +1205,72 @@ def stage_unlock_controller0_serial(stream): time.sleep(120) # Make sure we login again - serial_console_mode = CONSOLE_UNKNOWN_MODE # After reboot we are not logged in. + SERIAL_CONSOLE_MODE = CONSOLE_UNKNOWN_MODE # After reboot we are not logged in. @connect_to_ssh def stage_install_nodes(ssh_client): + """ + Install nodes in the environment using SSH. + + Args: + - ssh_client (paramiko SSH client object): The SSH client to use for connecting + to the environment. + + Returns: + None. + """ + # Create and transfer host_bulk_add.xml to ctrl-0 host_xml = create_host_bulk_add() LOG.info("host_bulk_add.xml content:\n%s", host_xml) # Send file to controller - destination = "/home/" + vboxoptions.username + "/host_bulk_add.xml" - with tempfile.NamedTemporaryFile() as fp: - fp.write(host_xml.encode('utf-8')) - fp.flush() + destination = "/home/" + V_BOX_OPTIONS.username + "/host_bulk_add.xml" + with tempfile.NamedTemporaryFile() as file: + file.write(host_xml.encode('utf-8')) + file.flush() # Connection to NAT interfaces is local - if vboxoptions.vboxnet_type == 'nat': - ip = '127.0.0.1' - port = vboxoptions.nat_controller0_local_ssh_port + if V_BOX_OPTIONS.vboxnet_type == 'nat': + ip_addr = '127.0.0.1' + port = V_BOX_OPTIONS.nat_controller0_local_ssh_port else: - ip = vboxoptions.controller0_ip + ip_addr = V_BOX_OPTIONS.controller0_ip port = 22 - sftp_send(source=fp.name, remote_host=ip, remote_port=port, - destination=destination, - username=vboxoptions.username, password=vboxoptions.password) + sftp_send( + file.name, + destination, + { + "remote_host": ip_addr, + "remote_port": port, + "username": V_BOX_OPTIONS.username, + "password": V_BOX_OPTIONS.password + } + ) # Apply host-bulk-add _, _, exitcode = run_ssh_cmd(ssh_client, - 'source /etc/platform/openrc; ', - 'system host-bulk-add {}'.format(destination), + f'source /etc/platform/openrc; system host-bulk-add {destination}', timeout=60) if exitcode != 0: msg = "Host bulk add failed, expecting exit code of 0 but got %s", exitcode LOG.info(msg) - raise Exception(msg) + raise Exception(msg) # pylint: disable=E0012, W0719 # Start hosts one by one, wait 10s between each start - vms = vboxmanage.get_all_vms(vboxoptions.labname, option="vms") + vms = vboxmanage.get_all_vms(V_BOX_OPTIONS.labname, option="vms") runningvms = vboxmanage.get_all_vms( - vboxoptions.labname, + V_BOX_OPTIONS.labname, option="runningvms") powered_off = list(set(vms) - set(runningvms)) LOG.info("#### Powered off VMs: %s", powered_off) - for vm in powered_off: - LOG.info("#### Powering on VM: %s", vm) - vboxmanage.vboxmanage_startvm(vm, force=True) + for virtual_machine in powered_off: + LOG.info("#### Powering on VM: %s", virtual_machine) + vboxmanage.vboxmanage_startvm(virtual_machine, force=True) LOG.info("Give VM 20s to boot.") time.sleep(20) - ctrl0 = vboxoptions.labname + "-controller-0" + ctrl0 = V_BOX_OPTIONS.labname + "-controller-0" hostnames = list(get_hostnames(ignore=[ctrl0]).values()) wait_for_hosts(ssh_client, hostnames, 'online') @@ -1018,6 +1278,17 @@ def stage_install_nodes(ssh_client): @connect_to_ssh def stage_unlock_controller1(ssh_client): + """ + Unlock controller-1 host via SSH. + + Args: + - ssh_client (paramiko SSH client object): The SSH client to use for + connecting to the environment. + + Returns: + None. + """ + # Fast for standard, wait for storage hostnames = list(get_hostnames().values()) if 'controller-1' not in hostnames: @@ -1035,12 +1306,23 @@ def stage_unlock_controller1(ssh_client): @connect_to_ssh def stage_unlock_storages(ssh_client): + """ + Unlock storage nodes via SSH. + + Args: + - ssh_client (paramiko SSH client object): The SSH client to use for + connecting to the environment. + + Returns: + None. + """ + # Unlock storage nodes, wait for them to be 'available' storages = list(get_hostnames(personalities=['storage']).values()) for storage in storages: run_ssh_cmd(ssh_client, - 'source /etc/platform/openrc; system host-unlock {}'.format(storage), + f'source /etc/platform/openrc; system host-unlock {storage}', timeout=60) LOG.info("Waiting 15s before next unlock") time.sleep(15) @@ -1051,14 +1333,25 @@ def stage_unlock_storages(ssh_client): @connect_to_ssh def stage_unlock_workers(ssh_client): + """ + Unlock worker nodes via SSH. + + Args: + - ssh_client (paramiko SSH client object): The SSH client to use for + connecting to the environment. + + Returns: + None. + """ + # Unlock all, wait for all hosts, except ctrl0 to be 'available' workers = list(get_hostnames(personalities=['worker']).values()) - ctrl0 = vboxoptions.labname + '-controller-0' + ctrl0 = V_BOX_OPTIONS.labname + '-controller-0' for worker in workers: run_ssh_cmd( ssh_client, - 'source /etc/platform/openrc; system host-unlock {}'.format(worker), + f'source /etc/platform/openrc; system host-unlock {worker}', timeout=60) LOG.info("Waiting 15s before next unlock") time.sleep(15) @@ -1070,17 +1363,30 @@ def stage_unlock_workers(ssh_client): def run_custom_script(script, timeout, console, mode): + """ + Run a custom script on the environment. + + Args: + - script (str): The name of the script to run. + - timeout (int): The timeout for the script. + - console (str): The console to use for running the script. + - mode (str): The mode to use for running the script. + + Returns: + None. + """ + LOG.info("#### Running custom script %s with options:", script) LOG.info(" timeout: %s", timeout) LOG.info(" console mode: %s", console) LOG.info(" user mode: %s", mode) if console == 'ssh': ssh_client = _connect_to_ssh() - _, __, return_code = run_ssh_cmd(ssh_client, "./{}".format(script), - timeout=timeout, mode=mode) + # pylint: disable=W0703, C0103 + _, __, return_code = run_ssh_cmd(ssh_client, f"./{script}", timeout=timeout, mode=mode) if return_code != 0: LOG.info("Custom script '%s' return code is not 0. Aborting.", script) - raise Exception("Script execution failed with return code: {}".format(return_code)) + raise Exception(f"Script execution failed with return code: {return_code}") # pylint: disable=E0012, W0719 else: sock, stream = _connect_to_serial() try: @@ -1090,20 +1396,30 @@ def run_custom_script(script, timeout, console, mode): serial.send_bytes(stream, 'sudo su -', expect_prompt=False) host_helper.check_password( stream, - password=vboxoptions.password) + password=V_BOX_OPTIONS.password) else: set_serial_prompt_mode(stream, CONSOLE_USER_MODE) - serial.send_bytes(stream, "./{}".format(script), + serial.send_bytes(stream, f"./{script}", timeout=timeout, prompt='keystone') LOG.info("Script execution completed. Checking if return code is 0.") serial.send_bytes(stream, - "echo \"Return code: [$?]\"".format(script), + f"echo 'Return code: [{script}]'", timeout=3, prompt='Return code: [0]') finally: sock.close() def get_custom_script_options(options_list): + """ + Parse options for a custom script. + + Args: + - options_list (str): The list of options for the script. + + Returns: + A tuple containing the script name, timeout, console, and mode. + """ + LOG.info("Parsing custom script options: %s", options_list) # defaults script = "" @@ -1111,15 +1427,15 @@ def get_custom_script_options(options_list): console = 'serial' mode = 'user' # supported options - CONSOLES = ['serial', 'ssh'] - MODES = ['user', 'root'] + consoles = ['serial', 'ssh'] + modes = ['user', 'root'] # No spaces or special chars allowed not_allowed = ['\n', ' ', '*'] - for c in not_allowed: - if c in options_list: - LOG.info("Char '%s' not allowed in options list: %s.", c, options_list) - raise Exception("Char not allowed in options_list") + for char in not_allowed: + if char in options_list: + LOG.info("Char '%s' not allowed in options list: %s.", char, options_list) + raise Exception("Char not allowed in options_list") # pylint: disable=E0012, W0719 # get options options = options_list.split(',') @@ -1129,20 +1445,26 @@ def get_custom_script_options(options_list): timeout = int(options[1]) if len(options) >= 3: console = options[2] - if console not in CONSOLES: - raise "Console must be one of {}, not {}.".format( - CONSOLES, console) + if console not in consoles: + raise f"Console must be one of {consoles}, not {console}." if len(options) >= 4: mode = options[3] - if mode not in MODES: - raise "Mode must be one of {}, not {}.".format(MODES, mode) + if mode not in modes: + raise f"Mode must be one of {modes}, not {mode}." return script, timeout, console, mode def stage_custom_script1(): - if vboxoptions.script1: + """ + Run the first custom script. + + Returns: + None. + """ + + if V_BOX_OPTIONS.script1: script, timeout, console, mode = get_custom_script_options( - vboxoptions.script1) + V_BOX_OPTIONS.script1) else: script = "custom_script1.sh" timeout = 3600 @@ -1152,9 +1474,16 @@ def stage_custom_script1(): def stage_custom_script2(): - if vboxoptions.script2: + """ + Run the second custom script. + + Returns: + None. + """ + + if V_BOX_OPTIONS.script2: script, timeout, console, mode = get_custom_script_options( - vboxoptions.script2) + V_BOX_OPTIONS.script2) else: script = "custom_script2.sh" timeout = 3600 @@ -1164,9 +1493,16 @@ def stage_custom_script2(): def stage_custom_script3(): - if vboxoptions.script3: + """ + Run the third custom script. + + Returns: + None. + """ + + if V_BOX_OPTIONS.script3: script, timeout, console, mode = get_custom_script_options( - vboxoptions.script3) + V_BOX_OPTIONS.script3) else: script = "custom_script3.sh" timeout = 3600 @@ -1176,9 +1512,16 @@ def stage_custom_script3(): def stage_custom_script4(): - if vboxoptions.script4: + """ + Run the fourth custom script. + + Returns: + None. + """ + + if V_BOX_OPTIONS.script4: script, timeout, console, mode = get_custom_script_options( - vboxoptions.script4) + V_BOX_OPTIONS.script4) else: script = "custom_script4.sh" timeout = 3600 @@ -1188,9 +1531,16 @@ def stage_custom_script4(): def stage_custom_script5(): - if vboxoptions.script5: + """ + Run the fifth custom script. + + Returns: + None. + """ + + if V_BOX_OPTIONS.script5: script, timeout, console, mode = get_custom_script_options( - vboxoptions.script5) + V_BOX_OPTIONS.script5) else: script = "custom_script5.sh" timeout = 3600 @@ -1198,6 +1548,7 @@ def stage_custom_script5(): mode = 'user' run_custom_script(script, timeout, console, mode) + STG_CREATE_LAB = "create-lab" STG_INSTALL_CONTROLLER0 = "install-controller-0" STG_CONFIG_CONTROLLER = "config-controller" @@ -1386,170 +1737,201 @@ AVAILABLE_CHAINS = [AIO_SX, AIO_DX, STANDARD, STORAGE] def load_config(): - global vboxoptions - vboxoptions = handle_args().parse_args() + """ + Loads and updates the configuration options specified in the command-line arguments. + It also sets defaults for some options. + """ + + global V_BOX_OPTIONS # pylint: disable=global-statement + V_BOX_OPTIONS = handle_args().parse_args() lab_config = [getattr(env.Lab, attr) for attr in dir(env.Lab) if not attr.startswith('__')] oam_config = [getattr(OAM, attr) for attr in dir(OAM) if not attr.startswith('__')] - if vboxoptions.controller0_ip is None: - vboxoptions.controller0_ip = lab_config[0]['controller-0_ip'] + if V_BOX_OPTIONS.controller0_ip is None: + V_BOX_OPTIONS.controller0_ip = lab_config[0]['controller-0_ip'] - if vboxoptions.vboxnet_ip is None: - vboxoptions.vboxnet_ip = oam_config[0]['ip'] + if V_BOX_OPTIONS.vboxnet_ip is None: + V_BOX_OPTIONS.vboxnet_ip = oam_config[0]['ip'] - if vboxoptions.username is None: - vboxoptions.username = lab_config[0]['username'] + if V_BOX_OPTIONS.username is None: + V_BOX_OPTIONS.username = lab_config[0]['username'] - if vboxoptions.password is None: - vboxoptions.password = lab_config[0]['password'] - if vboxoptions.hostiocache: - vboxoptions.hostiocache = 'on' + if V_BOX_OPTIONS.password is None: + V_BOX_OPTIONS.password = lab_config[0]['password'] + if V_BOX_OPTIONS.hostiocache: + V_BOX_OPTIONS.hostiocache = 'on' else: - vboxoptions.hostiocache = 'off' - if vboxoptions.lab_setup_conf is None: - vboxoptions.lab_setup_conf = {"~/lab_setup.conf"} + V_BOX_OPTIONS.hostiocache = 'off' + if V_BOX_OPTIONS.lab_setup_conf is None: + V_BOX_OPTIONS.lab_setup_conf = {"~/lab_setup.conf"} else: - vboxoptions.lab_setup_conf = vboxoptions.lab_setup_conf + V_BOX_OPTIONS.lab_setup_conf = V_BOX_OPTIONS.lab_setup_conf - if vboxoptions.setup_type == AIO_SX: - vboxoptions.controllers = 1 - vboxoptions.workers = 0 - vboxoptions.storages = 0 - elif vboxoptions.setup_type == AIO_DX: - vboxoptions.controllers = 2 - vboxoptions.workers = 0 - vboxoptions.storages = 0 - elif vboxoptions.setup_type == STANDARD: - vboxoptions.storages = 0 + if V_BOX_OPTIONS.setup_type == AIO_SX: + V_BOX_OPTIONS.controllers = 1 + V_BOX_OPTIONS.workers = 0 + V_BOX_OPTIONS.storages = 0 + elif V_BOX_OPTIONS.setup_type == AIO_DX: + V_BOX_OPTIONS.controllers = 2 + V_BOX_OPTIONS.workers = 0 + V_BOX_OPTIONS.storages = 0 + elif V_BOX_OPTIONS.setup_type == STANDARD: + V_BOX_OPTIONS.storages = 0 -def pre_validate(vboxoptions): +def pre_validate(m_vboxoptions): + """ + Checks that required options have been set and prints an error message and exits + with an error code if any of them are missing. + """ + err = False - if not vboxoptions.setup_type: + if not m_vboxoptions.setup_type: print("Please set --setup-type") err = True - if not vboxoptions.labname: + if not m_vboxoptions.labname: print("Please set --labname") err = True - if not vboxoptions.config_controller_ini: + if not m_vboxoptions.config_controller_ini: print("Please set --iso-location") err = True if err: print("\nMissing arguments. Please check --help and --list-stages for usage.") - exit(5) + sys.exit(5) -def validate(vboxoptions, stages): +def validate(v_box_opt, m_stages): + """ + Validates the values of the configuration options based on the stages that are going + to be executed. Checks that required options have been set and prints an error + message and exits with an error code if any of them are missing. It also performs + additional validation depending on the stage that is going to be executed. + """ + err = False # Generic - if vboxoptions.vboxnet_type == 'nat': - if vboxoptions.setup_type != AIO_SX: - if not vboxoptions.nat_controller_floating_local_ssh_port: + if v_box_opt.vboxnet_type == 'nat': + if v_box_opt.setup_type != AIO_SX: + if not v_box_opt.nat_controller_floating_local_ssh_port: print("Please set --nat-controller-floating-local-ssh-port") err = True - if not vboxoptions.nat_controller0_local_ssh_port: + if not v_box_opt.nat_controller0_local_ssh_port: print("Please set --nat-controller0-local-ssh-port") err = True - if vboxoptions.controllers > 1 and not vboxoptions.nat_controller1_local_ssh_port: + if v_box_opt.controllers > 1 and not v_box_opt.nat_controller1_local_ssh_port: print("Second controller is configured, please set --nat-controller1-local-ssh-port") err = True else: - if vboxoptions.setup_type != AIO_SX: - if not vboxoptions.controller_floating_ip: + if v_box_opt.setup_type != AIO_SX: + if not v_box_opt.controller_floating_ip: print("Please set --controller-floating-ip") err = True - if not vboxoptions.controller0_ip: + if not v_box_opt.controller0_ip: print("Please set --controller0-ip") err = True - if vboxoptions.controllers > 1 and not vboxoptions.controller1_ip: + if v_box_opt.controllers > 1 and not v_box_opt.controller1_ip: print("Second controller is configured, please set --controller1-ip") err = True - if STG_CONFIG_CONTROLLER in stages: - if not vboxoptions.config_controller_ini: - print("Please set --config-controller-ini " - "as needed by stage {}".format(STG_CONFIG_CONTROLLER)) + if STG_CONFIG_CONTROLLER in m_stages: + if not v_box_opt.config_controller_ini: + print(f"Please set --config-controller-ini as needed by stage {STG_CONFIG_CONTROLLER}") err = True - if STG_RSYNC_CONFIG in stages: - if not vboxoptions.config_files_dir and not vboxoptions.config_files_dir_dont_follow_links: + if STG_RSYNC_CONFIG in m_stages: + if not v_box_opt.config_files_dir and not v_box_opt.config_files_dir_dont_follow_links: print("Please set --config-files-dir and/or --config-files-dir-dont-follow-links " - "as needed by stage {} and {}".format(STG_RSYNC_CONFIG, - STG_LAB_SETUP1)) + f"as needed by stage {STG_RSYNC_CONFIG} and {STG_LAB_SETUP1}") err = True - if (STG_LAB_SETUP1 in stages or STG_LAB_SETUP2 in stages - or STG_LAB_SETUP3 in stages or STG_LAB_SETUP4 in stages - or STG_LAB_SETUP5 in stages): - if not vboxoptions.lab_setup_conf: + if (STG_LAB_SETUP1 in m_stages or STG_LAB_SETUP2 in m_stages + or STG_LAB_SETUP3 in m_stages or STG_LAB_SETUP4 in m_stages + or STG_LAB_SETUP5 in m_stages): + if not v_box_opt.lab_setup_conf: print("Please set at least one --lab-setup-conf file as needed by lab-setup stages") err = True - FILE = ["lab_setup.sh"] + # file = ["lab_setup.sh"] dirs = [] - if vboxoptions.config_files_dir: - dirs.append(vboxoptions.config_files_dir) - if vboxoptions.config_files_dir_dont_follow_links: - dirs.append(vboxoptions.config_files_dir_dont_follow_links) - for directory in dirs: - pass + if v_box_opt.config_files_dir: + dirs.append(v_box_opt.config_files_dir) + if v_box_opt.config_files_dir_dont_follow_links: + dirs.append(v_box_opt.config_files_dir_dont_follow_links) + # for directory in dirs: + # pass if err: print("\nMissing arguments. Please check --help and --list-stages for usage.") - exit(5) + sys.exit(5) -def wrap_stage_help(stage, stage_callbacks, number=None): +def wrap_stage_help(m_stage, stage_callbacks, number=None): + """ + Returns a formatted string containing the name of the stage, its number (if given), + and its description, separated by "#" symbol. m_stage is a string with the name of + the stage. stage_callbacks is a string with the description of the stage. + Number is an optional integer with the number of the stage. + """ + if number: - text = " {}. {}".format(number, stage) + text = f" {number}. {m_stage}" else: - text = " {}".format(stage) - LEN = 30 - fill = LEN - len(text) + text = f" {m_stage}" + length = 30 + fill = length - len(text) text += " " * fill - text += "# {}".format(stage_callbacks) + text += f"# {stage_callbacks}" return text + # Define signal handler for ctrl+c -def signal_handler(sig, frame): +def signal_handler(): + """ + This function is called when the user presses Ctrl+C. It prints a message to the + console and exits the script. Additionally, it calls the print_kpi_metrics() + function from the kpi module to print KPI metrics. + """ + print('You pressed Ctrl+C!') kpi.print_kpi_metrics() sys.exit(1) + +# pylint: disable=invalid-name if __name__ == "__main__": kpi.init_kpi_metrics() signal.signal(signal.SIGINT, signal_handler) load_config() - if vboxoptions.list_stages: - print("Defined setups: {}".format(list(STAGES_CHAINS.keys()))) - if vboxoptions.setup_type and vboxoptions.setup_type in AVAILABLE_CHAINS: - AVAILABLE_CHAINS = [vboxoptions.setup_type] + if V_BOX_OPTIONS.list_stages: + print(f"Defined setups: {list(STAGES_CHAINS.keys())}") + if V_BOX_OPTIONS.setup_type and V_BOX_OPTIONS.setup_type in AVAILABLE_CHAINS: + AVAILABLE_CHAINS = [V_BOX_OPTIONS.setup_type] for setup in AVAILABLE_CHAINS: i = 1 - print("Stages for setup: {}".format(setup)) + print(f"Stages for setup: {setup}") for stage in STAGES_CHAINS[setup]: print(wrap_stage_help(stage, STAGE_CALLBACKS[stage][HELP], i)) i += 1 print("Available stages that can be used for --custom-stages:") for stage in AVAILABLE_STAGES: print(wrap_stage_help(stage, STAGE_CALLBACKS[stage][HELP])) - exit(0) + sys.exit(0) - pre_validate(vboxoptions) + pre_validate(V_BOX_OPTIONS) - init_logging(vboxoptions.labname, vboxoptions.logpath) + init_logging(V_BOX_OPTIONS.labname, V_BOX_OPTIONS.logpath) LOG.info("Logging to directory: %s", (get_log_dir() + "/")) LOG.info("Install manages: %s controllers, %s workers, %s storages.", - vboxoptions.controllers, vboxoptions.workers, vboxoptions.storages) + V_BOX_OPTIONS.controllers, V_BOX_OPTIONS.workers, V_BOX_OPTIONS.storages) # Setup stages to run based on config install_stages = [] - if vboxoptions.custom_stages: + if V_BOX_OPTIONS.custom_stages: # Custom stages - install_stages = vboxoptions.custom_stages.split(',') + install_stages = V_BOX_OPTIONS.custom_stages.split(',') for stage in install_stages: invalid_stages = [] if stage not in AVAILABLE_STAGES: @@ -1557,29 +1939,29 @@ if __name__ == "__main__": if invalid_stages: LOG.info("Following custom stages are not supported: %s.\n" \ "Choose from: %s", invalid_stages, AVAILABLE_STAGES) - exit(1) + sys.exit(1) else: # List all stages between 'from-stage' to 'to-stage' - stages = STAGES_CHAINS[vboxoptions.setup_type] + stages = STAGES_CHAINS[V_BOX_OPTIONS.setup_type] from_index = 0 to_index = None - if vboxoptions.from_stage: - if vboxoptions.from_stage == 'start': + if V_BOX_OPTIONS.from_stage: + if V_BOX_OPTIONS.from_stage == 'start': from_index = 0 else: - from_index = stages.index(vboxoptions.from_stage) - if vboxoptions.to_stage: - if vboxoptions.from_stage == 'end': + from_index = stages.index(V_BOX_OPTIONS.from_stage) + if V_BOX_OPTIONS.to_stage: + if V_BOX_OPTIONS.from_stage == 'end': to_index = -1 else: - to_index = stages.index(vboxoptions.to_stage) + 1 + to_index = stages.index(V_BOX_OPTIONS.to_stage) + 1 if to_index is not None: install_stages = stages[from_index:to_index] else: install_stages = stages[from_index:] LOG.info("Executing %s stage(s): %s.", len(install_stages), install_stages) - validate(vboxoptions, install_stages) + validate(V_BOX_OPTIONS, install_stages) stg_no = 0 prev_stage = None @@ -1594,10 +1976,10 @@ if __name__ == "__main__": STAGE_CALLBACKS[stage][CALLBACK]() # Take snapshot if configured - if vboxoptions.snapshot: + if V_BOX_OPTIONS.snapshot: vboxmanage.take_snapshot( - vboxoptions.labname, - "snapshot-AFTER-{}".format(stage)) + V_BOX_OPTIONS.labname, + f"snapshot-AFTER-{stage}") # Compute KPIs duration = time.time() - start diff --git a/virtualbox/pybox/requirements.txt b/virtualbox/pybox/requirements.txt index 089f779..4ffaafe 100644 --- a/virtualbox/pybox/requirements.txt +++ b/virtualbox/pybox/requirements.txt @@ -3,4 +3,5 @@ paramiko pytest git+https://github.com/digidotcom/python-streamexpect#egg=streamexpect pexpect +ruamel.yaml diff --git a/virtualbox/pybox/utils/install_log.py b/virtualbox/pybox/utils/install_log.py index 2aa6ac1..37ee8ec 100644 --- a/virtualbox/pybox/utils/install_log.py +++ b/virtualbox/pybox/utils/install_log.py @@ -3,36 +3,45 @@ # SPDX-License-Identifier: Apache-2.0 # +""" +This module provides functionality to initialize logging for a lab, create a sub-directory +for the current run, set the logging level, and create a symbolic link to the latest logs +for the lab. +""" import os import datetime import logging from consts.env import LOGPATH -log_dir = "" +LOG_DIR = "" LOG = logging.getLogger() + def init_logging(lab_name, log_path=None): - global LOG, log_dir + """ + This method initializes the logging for a lab. It creates a sub-directory for the + current run and sets the logging level to INFO. It also creates a symbolic link to + the latest logs for the lab. The method takes in the lab name and an optional log path + parameter. If no log path is specified, it uses the default path provided by the + LOGPATH constant in the env module. + """ + + global LOG, LOG_DIR # pylint: disable=global-statement, global-variable-not-assigned if not log_path: log_path = LOGPATH lab_log_path = log_path + "/" + lab_name # Setup log sub-directory for current run current_time = datetime.datetime.now() - log_dir = "{}/{}_{}_{}_{}_{}_{}".format(lab_log_path, - current_time.year, - current_time.month, - current_time.day, - current_time.hour, - current_time.minute, - current_time.second) - if not os.path.exists(log_dir): - os.makedirs(log_dir) + LOG_DIR = f"{lab_log_path}/{current_time.year}_{current_time.month}_\ + {current_time.day}_{current_time.hour}_{current_time.minute}_{current_time.second}" + if not os.path.exists(LOG_DIR): + os.makedirs(LOG_DIR) LOG.setLevel(logging.INFO) formatter = logging.Formatter("%(asctime)s: %(message)s") - log_file = "{}/install.log".format(log_dir) + log_file = f"{LOG_DIR}/install.log" handler = logging.FileHandler(log_file) handler.setFormatter(formatter) handler.setLevel(logging.INFO) @@ -44,9 +53,12 @@ def init_logging(lab_name, log_path=None): # Create symbolic link to latest logs of this lab try: os.unlink(lab_log_path + "/latest") - except: + except: # pylint: disable=bare-except pass - os.symlink(log_dir, lab_log_path + "/latest") + os.symlink(LOG_DIR, lab_log_path + "/latest") + def get_log_dir(): - return log_dir + """This method returns the directory path of the current logging run.""" + + return LOG_DIR diff --git a/virtualbox/pybox/utils/kpi.py b/virtualbox/pybox/utils/kpi.py index 503c841..efa291e 100644 --- a/virtualbox/pybox/utils/kpi.py +++ b/virtualbox/pybox/utils/kpi.py @@ -3,55 +3,80 @@ # SPDX-License-Identifier: Apache-2.0 # +""" +This module provides functions to track and report key performance indicators (KPIs) for a program. +""" import time from utils.install_log import LOG STAGES = [] METRICS = {} -start = 0 +START = 0 def init_kpi_metrics(): - global start - start = time.time() + """ + Initializes the global variable START with the current time to start tracking the + duration of a program. + """ + + global START # pylint: disable=global-statement + START = time.time() + def get_formated_time(sec): + """ + Takes the duration in seconds and formats it in hours, minutes and seconds. + Returns the formatted string. + """ + hours = sec // 3600 sec %= 3600 minutes = sec // 60 sec %= 60 seconds = sec if hours: - return "{:.0f}h {:.0f}m {:.2f}s".format(hours, minutes, seconds) - elif minutes: - return "{:.0f}m {:.2f}s".format(minutes, seconds) - elif seconds: - return "{:.2f}s".format(seconds) + return f"{hours:.0f}h {minutes:.0f}m {seconds:.2f}s" + if minutes: + return f"{minutes:.0f}m {seconds:.2f}s" + return f"{seconds:.2f}s" + def set_kpi_metric(metric, duration): - global METRICS, STAGES + """Sets the duration of a metric and adds the metric to the global list of STAGES.""" + + global METRICS, STAGES # pylint: disable=global-statement, global-variable-not-assigned METRICS[metric] = duration STAGES.append(metric) + def print_kpi(metric): + """Takes a metric as input and prints the duration of that metric using the LOG module.""" + if metric in STAGES: sec = METRICS[metric] LOG.info(" Time in stage '%s': %s ", metric, get_formated_time(sec)) - elif metric == 'total' and start: - duration = time.time() - start + elif metric == 'total' and START: + duration = time.time() - START LOG.info(" Total time: %s", get_formated_time(duration)) + def get_kpi_str(metric): + """Takes a metric as input and returns the duration of that metric as a formatted string.""" + msg = "" if metric in STAGES: sec = METRICS[metric] - msg += (" Time in stage '{}': {} \n".format(metric, get_formated_time(sec))) - elif metric == 'total' and start: - duration = time.time() - start - msg += (" Total time: {}\n".format(get_formated_time(duration))) + msg += (f" Time in stage '{metric}': {get_formated_time(sec)} \n") + elif metric == 'total' and START: + duration = time.time() - START + msg += (f" Total time: {get_formated_time(duration)}\n") return msg + def get_kpi_metrics_str(): + """Returns a formatted string with all the metrics and their durations.""" + msg = "===================== Metrics ====================\n" for stage in STAGES: msg += get_kpi_str(stage) @@ -59,10 +84,12 @@ def get_kpi_metrics_str(): msg += "===============================================\n" return msg + def print_kpi_metrics(): + """Prints all the metrics and their durations using the LOG module.""" + LOG.info("===================== Metrics ====================") for stage in STAGES: print_kpi(stage) print_kpi('total') LOG.info("==================================================") - diff --git a/virtualbox/pybox/utils/serial.py b/virtualbox/pybox/utils/serial.py index b22352d..271990e 100644 --- a/virtualbox/pybox/utils/serial.py +++ b/virtualbox/pybox/utils/serial.py @@ -3,6 +3,10 @@ # SPDX-License-Identifier: Apache-2.0 # +""" +This module provides functionality to connect and communicate with a remote host +using local domain socket. +""" import re import socket @@ -22,27 +26,27 @@ def connect(hostname, port=10000, prefix=""): """ if prefix: - prefix = "{}_".format(prefix) - socketname = "/tmp/{}{}".format(prefix, hostname) - if 'controller-0'in hostname: + prefix = f"{prefix}_" + socketname = f"/tmp/{prefix}{hostname}" + if 'controller-0' in hostname: socketname += '_serial' LOG.info("Connecting to %s at %s", hostname, socketname) - if platform == 'win32' or platform == 'win64': + if platform in ('win32', 'win64'): sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM, socket.IPPROTO_TCP) else: sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) try: - if platform == 'win32' or platform == 'win64': + if platform in ('win32', 'win64'): sock.setsockopt(socket.SOL_SOCKET, socket.SO_KEEPALIVE, 1) sock.connect(('localhost', port)) else: sock.connect(socketname) - except: + except: # pylint: disable=bare-except LOG.info("Connection failed") - pass + pass # pylint: disable=unnecessary-pass # disconnect(sock) sock = None - # TODO (WEI): double check this + # TODO (WEI): double check this # pylint: disable=fixme sock.setblocking(0) return sock @@ -61,14 +65,18 @@ def disconnect(sock): sock.shutdown(socket.SHUT_RDWR) sock.close() -def get_output(stream, cmd, prompts=None, timeout=5, log=True, as_lines=True, flush=True): - #TODO: Not testested, will not work if kernel or other processes throw data on stdout or stderr + +# pylint: disable=too-many-arguments, too-many-locals, too-many-branches +def get_output(stream, prompts=None, timeout=5, log=True, as_lines=True, flush=True): + # pylint: disable=fixme + # TODO: Not tested, will not work if kernel or other processes throw data on stdout or stderr """ Execute a command and get its output. Make sure no other command is executing. And 'dmesg -D' was executed. """ - POLL_PERIOD = 0.1 - MAX_READ_BUFFER = 1024 + + poll_period = 0.1 + max_read_buffer = 1024 data = "" line_buf = "" lines = [] @@ -82,13 +90,13 @@ def get_output(stream, cmd, prompts=None, timeout=5, log=True, as_lines=True, fl try: LOG.info("Buffer has bytes before cmd execution: %s", trash.decode('utf-8')) - except Exception: + except Exception: # pylint: disable=W0703 pass except streamexpect.ExpectTimeout: pass # Send command - stream.sendall("{}\n".format(cmd).encode('utf-8')) + stream.sendall("{cmd}\n".encode('utf-8')) # Get response patterns = [] @@ -98,20 +106,21 @@ def get_output(stream, cmd, prompts=None, timeout=5, log=True, as_lines=True, fl now = time.time() end_time = now + float(timeout) prev_timeout = stream.gettimeout() - stream.settimeout(POLL_PERIOD) + stream.settimeout(poll_period) incoming = None + # pylint: disable=too-many-nested-blocks try: while (end_time - now) >= 0: try: - incoming = stream.recv(MAX_READ_BUFFER) + incoming = stream.recv(max_read_buffer) except socket.timeout: pass if incoming: data += incoming if log: - for c in incoming: - if c != '\n': - line_buf += c + for char in incoming: + if char != '\n': + line_buf += char else: LOG.info(line_buf) lines.append(line_buf) @@ -120,8 +129,7 @@ def get_output(stream, cmd, prompts=None, timeout=5, log=True, as_lines=True, fl if pattern.search(data): if as_lines: return lines - else: - return data + return data now = time.time() raise streamexpect.ExpectTimeout() finally: @@ -132,23 +140,24 @@ def expect_bytes(stream, text, timeout=180, fail_ok=False, flush=True): """ Wait for user specified text from stream. """ + time.sleep(1) if timeout < 60: LOG.info("Expecting text within %s seconds: %s\n", timeout, text) else: - LOG.info("Expecting text within %s minutes: %s\n", timeout/60, text) + LOG.info("Expecting text within %s minutes: %s\n", timeout / 60, text) try: - stream.expect_bytes("{}".format(text).encode('utf-8'), timeout=timeout) + stream.expect_bytes(f"{text}".encode('utf-8'), timeout=timeout) except streamexpect.ExpectTimeout: if fail_ok: return -1 - else: - stdout.write('\n') - LOG.error("Did not find expected text") - # disconnect(stream) - raise - except Exception as e: - LOG.info("Connection failed with %s", e) + + stdout.write('\n') + LOG.error("Did not find expected text") + # disconnect(stream) + raise + except Exception as exception: + LOG.info("Connection failed with %s", exception) raise stdout.write('\n') @@ -162,8 +171,8 @@ def expect_bytes(stream, text, timeout=180, fail_ok=False, flush=True): incoming += b'\n' try: LOG.info(">>> expect_bytes: Buffer has bytes!") - stdout.write(incoming.decode('utf-8')) # streamexpect hardcodes it - except Exception: + stdout.write(incoming.decode('utf-8')) # streamexpect hardcodes it + except Exception: # pylint: disable=W0703 pass except streamexpect.ExpectTimeout: pass @@ -171,11 +180,13 @@ def expect_bytes(stream, text, timeout=180, fail_ok=False, flush=True): return 0 +# pylint: disable=inconsistent-return-statements def send_bytes(stream, text, fail_ok=False, expect_prompt=True, prompt=None, timeout=180, send=True, flush=True): """ Send user specified text to stream. """ + time.sleep(1) if flush: try: @@ -184,8 +195,8 @@ def send_bytes(stream, text, fail_ok=False, expect_prompt=True, incoming += b'\n' try: LOG.info(">>> send_bytes: Buffer has bytes!") - stdout.write(incoming.decode('utf-8')) # streamexpect hardcodes it - except Exception: + stdout.write(incoming.decode('utf-8')) # streamexpect hardcodes it + except Exception: # pylint: disable=W0703 pass except streamexpect.ExpectTimeout: pass @@ -193,28 +204,28 @@ def send_bytes(stream, text, fail_ok=False, expect_prompt=True, LOG.info("Sending text: %s", text) try: if send: - stream.sendall("{}\n".format(text).encode('utf-8')) + stream.sendall(f"{text}\n".encode('utf-8')) else: - stream.sendall("{}".format(text).encode('utf-8')) + stream.sendall(f"{text}".encode('utf-8')) if expect_prompt: time.sleep(1) if prompt: return expect_bytes(stream, prompt, timeout=timeout, fail_ok=fail_ok) - else: - rc = expect_bytes(stream, "~$", timeout=timeout, fail_ok=True) - if rc != 0: - send_bytes(stream, '\n', expect_prompt=False) - expect_bytes(stream, 'keystone', timeout=timeout) - return + + return_code = expect_bytes(stream, "~$", timeout=timeout, fail_ok=True) + if return_code != 0: + send_bytes(stream, '\n', expect_prompt=False) + expect_bytes(stream, 'keystone', timeout=timeout) + return except streamexpect.ExpectTimeout: if fail_ok: return -1 - else: - LOG.error("Failed to send text, logging out.") - stream.sendall("exit".encode('utf-8')) - raise - except Exception as e: - LOG.info("Connection failed with %s.", e) + + LOG.error("Failed to send text, logging out.") + stream.sendall("exit".encode('utf-8')) + raise + except Exception as exception: + LOG.info("Connection failed with %s.", exception) raise return 0 diff --git a/virtualbox/pybox/utils/sftp.py b/virtualbox/pybox/utils/sftp.py index 1fd0388..7be9c6f 100644 --- a/virtualbox/pybox/utils/sftp.py +++ b/virtualbox/pybox/utils/sftp.py @@ -3,6 +3,10 @@ # SPDX-License-Identifier: Apache-2.0 # +""" +This module provides functionality to send files and directories to remote servers using the +rsync and paramiko libraries. +""" import getpass import os @@ -12,10 +16,14 @@ import paramiko from utils.install_log import LOG -def sftp_send(source, remote_host, remote_port, destination, username, password): +def sftp_send(source, destination, client_dict): """ Send files to remote server """ + + remote_host = client_dict["remote_host"] + username = client_dict["username"] + LOG.info("Connecting to server %s with username %s", remote_host, username) ssh_client = paramiko.SSHClient() @@ -25,12 +33,17 @@ def sftp_send(source, remote_host, remote_port, destination, username, password) retry = 0 while retry < 8: try: - ssh_client.connect(remote_host, port=remote_port, - username=username, password=password, - look_for_keys=False, allow_agent=False) + ssh_client.connect( + remote_host, + port=client_dict["remote_port"], + username=username, + password=client_dict["password"], + look_for_keys=False, + allow_agent=False + ) sftp_client = ssh_client.open_sftp() retry = 8 - except Exception as e: + except Exception: # pylint: disable=W0703 LOG.info("******* try again") retry += 1 time.sleep(10) @@ -42,8 +55,41 @@ def sftp_send(source, remote_host, remote_port, destination, username, password) ssh_client.close() -def send_dir(source, remote_host, remote_port, destination, username, - password, follow_links=True, clear_known_hosts=True): +# pylint: disable=R0801 +def send_dir(params_dict): + """ + Send directory `source` to remote host `remote_host` at port `remote_port` and destination + `destination` using `rsync` over `ssh`. + + Args: + params_dict (dict): A dictionary containing the following keys: + - source (str): The local directory to be sent. + - remote_host (str): The IP address or domain name of the remote host. + - remote_port (int): The port number of the remote host to connect to. + - destination (str): The remote directory to copy `source` into. + - username (str): The username for the SSH connection. + - password (str): The password for the SSH connection. + - follow_links (bool, optional): Whether or not to follow symbolic links when + copying files. Default is True. + - clear_known_hosts (bool, optional): Whether or not to clear the known_hosts file + before making the SSH connection. Default is True. + + Raises: + Exception: If there is an error in `rsync`, raises an exception with the return code. + + Note: + This method only works from a Linux environment. + """ + + source = params_dict['source'] + remote_host = params_dict['remote_host'] + remote_port = params_dict['remote_port'] + destination = params_dict['destination'] + username = params_dict['username'] + password = params_dict['password'] + follow_links = params_dict.get('follow_links', True) + clear_known_hosts = params_dict.get('clear_known_hosts', True) + # Only works from linux for now if not source.endswith('/') or not source.endswith('\\'): source = source + '/' @@ -51,29 +97,28 @@ def send_dir(source, remote_host, remote_port, destination, username, follow_links = "L" if follow_links else "" if clear_known_hosts: if remote_host == '127.0.0.1': - keygen_arg = "[127.0.0.1]:{}".format(remote_port) + keygen_arg = f"[127.0.0.1]:{remote_port}" else: keygen_arg = remote_host cmd = f'ssh-keygen -f "/home/{getpass.getuser()}/.ssh/known_hosts" -R {keygen_arg}' LOG.info("CMD: %s", cmd) - process = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE) - for line in iter(process.stdout.readline, b''): - LOG.info("%s", line.decode("utf-8").strip()) - process.wait() + with subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE) as process: + for line in iter(process.stdout.readline, b''): + LOG.info("%s", line.decode("utf-8").strip()) + process.wait() - LOG.info(f'Running rsync of dir: {source} -> {username}@{remote_host}' - f':{destination}') + LOG.info('Running rsync of dir: %s -> %s@%s:%s', source, username, remote_host, destination) cmd = (f'rsync -av{follow_links} --rsh="/usr/bin/sshpass -p {password} ' f'ssh -p {remote_port} -o StrictHostKeyChecking=no -l {username}" ' f'{source}* {username}@{remote_host}:{destination}') LOG.info("CMD: %s", cmd) - process = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE) - for line in iter(process.stdout.readline, b''): - LOG.info("%s", line.decode("utf-8").strip()) - process.wait() + with subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE) as process: + for line in iter(process.stdout.readline, b''): + LOG.info("%s", line.decode("utf-8").strip()) + process.wait() if process.returncode: - raise Exception(f'Error in rsync, return code: {process.returncode}') + raise Exception(f"Error in rsync, return code:{process.returncode}") # pylint: disable=E0012, W0719 def send_dir_fallback(source, remote_host, destination, username, password): @@ -87,10 +132,17 @@ def send_dir_fallback(source, remote_host, destination, username, password): e.g. myhost.com - destination: where to store the file on host: /home/myuser/ """ + LOG.info("Connecting to server %s with username %s", remote_host, username) ssh_client = paramiko.SSHClient() ssh_client.set_missing_host_key_policy(paramiko.AutoAddPolicy()) - ssh_client.connect(remote_host, username=username, password=password, look_for_keys=False, allow_agent=False) + ssh_client.connect( + remote_host, + username=username, + password=password, + look_for_keys=False, + allow_agent=False + ) sftp_client = ssh_client.open_sftp() path = '' send_img = False @@ -113,4 +165,3 @@ def send_dir_fallback(source, remote_host, destination, username, password): ssh_client.close() if send_img: time.sleep(10) -