starlingx/virtual-deployment
Code Issues Proposed changes

Adding pylint to /virtualbox/pybox

Browse Source 
Enabling automatic pylint with tox and zull for each new patchset.

Test plan:
PASS: Run "tox -e pylint" in the terminal, this will:
  - Run pylint in all python files
  - Show the report

Story: 2005051
Task: 47900
Change-Id: I2f66a5f72e3f8746c00aae96287ad3e4edb88e28
Signed-off-by: Lindley Werner <lindley.vieira@encora.com>
This commit is contained in:
Lindley Werner 2023-05-04 17:21:54 -03:00
parent f73ed26012
commit c93f1aa754
19 changed files with 1846 additions and 800 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
.zuul.yaml
View File

@ -5,6 +5,8 @@
check:
jobs:
- openstack-tox-linters
- openstack-tox-pylint
gate:
jobs:
- openstack-tox-linters
- openstack-tox-pylint

2
pylint.rc
View File

@ -234,4 +234,4 @@ valid-classmethod-first-arg=cls
[EXCEPTIONS]
# Exceptions that will emit a warning when being caught. Defaults to
# "Exception"
overgeneral-exceptions=Exception
overgeneral-exceptions=builtins.Exception

2
requirements/test-requirements.txt
View File

@ -1,4 +1,4 @@
yamllint === 1.32.0
bashate === 2.1.1
pylint === 2.13.9
tox === 4.6.3

12
tox.ini
View File

@ -4,9 +4,7 @@ minversion = 2.3
skipsdist = True
[testenv]
deps =
-r{toxinidir}/requirements/test-requirements.txt
-r{toxinidir}/virtualbox/pybox/requirements.txt
deps = -r{toxinidir}/requirements/test-requirements.txt
allowlist_externals = reno
[testenv:linters]
@ -23,7 +21,6 @@ commands =
-print0 | xargs -0 yamllint"
bash -c "find {toxinidir} \
-not \( -type d -name .?\* -prune \) \
-not \( -type d -path {toxinidir}/toCOPY/mock_overlay -prune \) \
-type f \
-not -name \*~ \
-not -name \*.md \
@ -33,6 +30,13 @@ commands =
[testenv:pylint]
basepython = python3
sitepackages = False
setenv =
BASEPATH = {toxinidir}/virtualbox/pybox
PYTHONPATH= {env:BASEPATH}:{env:BASEPATH}/helper:{env:BASEPATH}/consts:{env:BASEPATH}/utils
deps =
-r{env:BASEPATH}/requirements.txt
{[testenv]deps}
allowlist_externals = pylint
commands =
pylint {posargs} --rcfile=./pylint.rc virtualbox/pybox

61
virtualbox/pybox/Parser.py
View File

@ -1,3 +1,4 @@
# pylint: disable=invalid-name
#!/usr/bin/python3
#
# SPDX-License-Identifier: Apache-2.0
@ -11,7 +12,7 @@ Parser to handle command line arguments
import argparse
import getpass
# pylint: disable=too-many-statements
def handle_args():
"""
Handle arguments supplied to the command line
@ -19,11 +20,10 @@ def handle_args():
parser = argparse.ArgumentParser(formatter_class=argparse.RawTextHelpFormatter)
"""
**************************************
* Setup type & install configuration *
**************************************
"""
#**************************************
#* Setup type & install configuration *
#**************************************
parser.add_argument("--setup-type", help=
"""
Type of setup:
@ -84,11 +84,10 @@ def handle_args():
type=str,
default=None)
"""
******************************************
* Config folders and files configuration *
******************************************
"""
#******************************************
#* Config folders and files configuration *
#******************************************
parser.add_argument("--iso-location", help=
"""
Location of ISO including the filename:
@ -143,11 +142,11 @@ def handle_args():
Path to the config file to use
""",
action='append')
"""
**************************************
* Disk number and size configuration *
**************************************
"""
#**************************************
#* Disk number and size configuration *
#**************************************
parser.add_argument("--controller-disks", help=
"""
Select the number of disks for a controller VM. default is 3
@ -178,11 +177,11 @@ def handle_args():
Configure size in MiB of worker disks as a comma separated list.
""",
type=str)
"""
**************
* Networking *
**************
"""
#**************
#* Networking *
#**************
parser.add_argument("--vboxnet-name", help=
"""
Which host only network to use for setup.
@ -277,11 +276,11 @@ def handle_args():
SX setups.
""",
type=str)
"""
******************
* Custom scripts *
******************
"""
#******************
#* Custom scripts *
#******************
parser.add_argument("--script1", help=
"""
Name of an executable script file plus options.
@ -326,11 +325,11 @@ def handle_args():
""",
default=None,
type=str)
"""
**************************************
* Other *
**************************************
"""
#**************************************
#* Other *
#**************************************
parser.add_argument("--list-stages", help=
"""
List stages that can be used by autoinstaller.

17
virtualbox/pybox/README.md
View File

@ -136,7 +136,7 @@ will be configured and used.
```shell
git clone https://opendev.org/starlingx/tools.git
cd tools/deployment/virtualbox/pybox
cd virtual-deployment/virtualbox/pybox
python3 -m venv venv
source ./venv/bin/activate
pip install --upgrade pip
@ -150,7 +150,7 @@ will be configured and used.
-O $HOME/Downloads/stx-8.iso
```
5. Now you're ready to run the script. From the `/deployment/virtualbox/pybox`
5. Now you're ready to run the script. From the `/virtualbox/pybox`
folder, do:
```shell
@ -168,8 +168,15 @@ folder, do:
--snapshot
```
The script takes a while to do all the things (from creating a VM and
The script takes a while to do all the things (from creating a VM and
installing an OS in it to configuring StarlingX). Several restarts might
occur, and you might see a VirtualBox with a prompt. You don't need to type
anything. While the installation script is running it will take care of
occur, and you might see a VirtualBox with a prompt. You don't need to type
anything. While the installation script is running it will take care of
everything for you.
## Pybox folder structure
.
├── configs/aio-sx: Contains scripts and configs to set up a controller/worker
├── consts: This folder contains modules for managing virtual lab environments, including classes for Lab, Subnets, NICs, OAM, Serial, nodes, and HostTimeout.
├── helper: This folder contains modules for interacting with a StarlingX controller-0 server via a serial connection, configuring system settings, and managing virtual machines using VirtualBox.
└── utils: This folder contains modules for initializing logging, tracking and reporting KPIs, connecting and communicating with remote hosts via local domain socket, and sending files and directories to remote servers using rsync and paramiko libraries.

29
virtualbox/pybox/consts/env.py
View File

@ -3,6 +3,11 @@
# SPDX-License-Identifier: Apache-2.0
#
"""
This module contains a class named Lab and some supporting code.
The Lab class represents a virtual lab and has a dictionary attribute VBOX
containing information about the virtual machines in the lab.
"""
import getpass
from sys import platform
@ -10,18 +15,22 @@ import os
user = getpass.getuser()
if platform == 'win32' or platform == 'win64':
LOGPATH = 'C:\\Temp\\pybox_logs'
if platform in ("win32", "win64"):
LOGPATH = "C:\\Temp\\pybox_logs"
PORT = 10000
else:
homedir = os.environ['HOME']
LOGPATH = '{}/vbox_installer_logs'.format(homedir)
homedir = os.environ["HOME"]
LOGPATH = f"{homedir}/vbox_installer_logs"
class Lab: #pylint: disable=too-few-public-methods
"""The `Lab` class represents a virtual lab and contains a dictionary attribute
`VBOX` with information about the virtual machines in the lab."""
class Lab:
VBOX = {
'floating_ip': '10.10.10.7',
'controller-0_ip': '10.10.10.8',
'controller-1_ip': '10.10.10.9',
'username': 'sysadmin',
'password': 'Li69nux*',
"floating_ip": "10.10.10.7",
"controller-0_ip": "10.10.10.8",
"controller-1_ip": "10.10.10.9",
"username": "sysadmin",
"password": "Li69nux*",
}

190
virtualbox/pybox/consts/networking.py
View File

@ -1,78 +1,194 @@
# pylint: disable=too-few-public-methods
#!/usr/bin/python3
#
# SPDX-License-Identifier: Apache-2.0
#
"""
This module defines several classes and dictionaries that contain information related
to virtual machines in a lab environment.
Classes:
- `Subnets`: A class containing dictionaries for IPv4 and IPv6 subnets.
- `NICs`: A class containing dictionaries for NIC configurations of different types of
nodes in the virtual environment, such as `CONTROLLER`, `COMPUTE`, and `STORAGE`.
- `OAM`: A class containing an IP address and netmask for the out-of-band management (OAM) network.
- `Serial`: A class containing configurations for the serial ports.
"""
from sys import platform
class Subnets:
"""The `Subnets` class contains dictionaries for IPv4 and IPv6 subnets for the
management, infrastructure, and OAM networks."""
IPV4 = {
'mgmt_subnet': '192.168.204.0/24',
'infra_subnet': '192.168.205.0/24',
'oam_subnet': '10.10.10.0/24'
"mgmt_subnet": "192.168.204.0/24",
"infra_subnet": "192.168.205.0/24",
"oam_subnet": "10.10.10.0/24",
}
IPV6 = {
'mgmt_subnet': 'aefd::/64',
'infra_subnet': 'aced::/64',
'oam_subnet': 'abcd::/64'
"mgmt_subnet": "aefd::/64",
"infra_subnet": "aced::/64",
"oam_subnet": "abcd::/64",
}
class NICs:
if platform == 'win32' or platform == 'win64':
"""The `NICs` class contains dictionaries for NIC configurations of different types
of nodes in the virtual environment, such as `CONTROLLER`, `COMPUTE`, and `STORAGE`."""
if platform in ("win32", "win64"):
CONTROLLER = {
'node_type': 'controller',
'1': {'nic': 'hostonly', 'intnet': '', 'nictype': '82540EM', 'nicpromisc': 'deny', 'hostonlyadapter': 'VirtualBox Host-Only Ethernet Adapter'},
'2': {'nic': 'intnet', 'intnet': 'intnet-management', 'nictype': '82540EM', 'nicpromisc': 'allow-all', 'hostonlyadapter': 'none'},
'3': {'nic': 'intnet', 'intnet': 'intnet-data1', 'nictype': 'virtio', 'nicpromisc': 'allow-all', 'hostonlyadapter': 'none'},
'4': {'nic': 'intnet', 'intnet': 'intnet-data2', 'nictype': 'virtio', 'nicpromisc': 'allow-all', 'hostonlyadapter': 'none'},
"node_type": "controller",
"1": {
"nic": "hostonly",
"intnet": "",
"nictype": "82540EM",
"nicpromisc": "deny",
"hostonlyadapter": "VirtualBox Host-Only Ethernet Adapter",
},
"2": {
"nic": "intnet",
"intnet": "intnet-management",
"nictype": "82540EM",
"nicpromisc": "allow-all",
"hostonlyadapter": "none",
},
"3": {
"nic": "intnet",
"intnet": "intnet-data1",
"nictype": "virtio",
"nicpromisc": "allow-all",
"hostonlyadapter": "none",
},
"4": {
"nic": "intnet",
"intnet": "intnet-data2",
"nictype": "virtio",
"nicpromisc": "allow-all",
"hostonlyadapter": "none",
},
}
else:
CONTROLLER = {
'node_type': 'controller',
'1': {'nic': 'hostonly', 'intnet': '', 'nictype': '82540EM', 'nicpromisc': 'deny', 'hostonlyadapter': 'vboxnet0'},
'2': {'nic': 'intnet', 'intnet': 'intnet-management', 'nictype': '82540EM', 'nicpromisc': 'allow-all', 'hostonlyadapter': 'none'},
'3': {'nic': 'intnet', 'intnet': 'intnet-data1', 'nictype': 'virtio', 'nicpromisc': 'allow-all', 'hostonlyadapter': 'none'},
'4': {'nic': 'intnet', 'intnet': 'intnet-data2', 'nictype': 'virtio', 'nicpromisc': 'allow-all', 'hostonlyadapter': 'none'},
"node_type": "controller",
"1": {
"nic": "hostonly",
"intnet": "",
"nictype": "82540EM",
"nicpromisc": "deny",
"hostonlyadapter": "vboxnet0",
},
"2": {
"nic": "intnet",
"intnet": "intnet-management",
"nictype": "82540EM",
"nicpromisc": "allow-all",
"hostonlyadapter": "none",
},
"3": {
"nic": "intnet",
"intnet": "intnet-data1",
"nictype": "virtio",
"nicpromisc": "allow-all",
"hostonlyadapter": "none",
},
"4": {
"nic": "intnet",
"intnet": "intnet-data2",
"nictype": "virtio",
"nicpromisc": "allow-all",
"hostonlyadapter": "none",
},
}
COMPUTE = {
'node_type': 'compute',
'1': {'nic': 'intnet', 'intnet': 'intnet-unused1', 'nictype': '82540EM', 'nicpromisc': 'allow-all', 'hostonlyadapter': 'none'},
'2': {'nic': 'intnet', 'intnet': 'intnet-management', 'nictype': '82540EM', 'nicpromisc': 'allow-all', 'hostonlyadapter': 'none'},
'3': {'nic': 'intnet', 'intnet': 'intnet-data1', 'nictype': 'virtio', 'nicpromisc': 'allow-all', 'hostonlyadapter': 'none'},
'4': {'nic': 'intnet', 'intnet': 'intnet-data2', 'nictype': 'virtio', 'nicpromisc': 'allow-all', 'hostonlyadapter': 'none'},
"node_type": "compute",
"1": {
"nic": "intnet",
"intnet": "intnet-unused1",
"nictype": "82540EM",
"nicpromisc": "allow-all",
"hostonlyadapter": "none",
},
"2": {
"nic": "intnet",
"intnet": "intnet-management",
"nictype": "82540EM",
"nicpromisc": "allow-all",
"hostonlyadapter": "none",
},
"3": {
"nic": "intnet",
"intnet": "intnet-data1",
"nictype": "virtio",
"nicpromisc": "allow-all",
"hostonlyadapter": "none",
},
"4": {
"nic": "intnet",
"intnet": "intnet-data2",
"nictype": "virtio",
"nicpromisc": "allow-all",
"hostonlyadapter": "none",
},
}
STORAGE = {
'node_type': 'storage',
'1': {'nic': 'intnet', 'intnet': 'intnet-unused', 'nictype': '82540EM', 'nicpromisc': 'allow-all', 'hostonlyadapter': 'none'},
'2': {'nic': 'intnet', 'intnet': 'intnet-management', 'nictype': '82540EM', 'nicpromisc': 'allow-all', 'hostonlyadapter': 'none'},
'3': {'nic': 'intnet', 'intnet': 'intnet-infra', 'nictype': '82540EM', 'nicpromisc': 'allow-all', 'hostonlyadapter': 'none'},
"node_type": "storage",
"1": {
"nic": "intnet",
"intnet": "intnet-unused",
"nictype": "82540EM",
"nicpromisc": "allow-all",
"hostonlyadapter": "none",
},
"2": {
"nic": "intnet",
"intnet": "intnet-management",
"nictype": "82540EM",
"nicpromisc": "allow-all",
"hostonlyadapter": "none",
},
"3": {
"nic": "intnet",
"intnet": "intnet-infra",
"nictype": "82540EM",
"nicpromisc": "allow-all",
"hostonlyadapter": "none",
},
}
class OAM:
"""The `OAM` class contains an IP address and netmask for the out-of-band
management (OAM) network."""
OAM = {
'ip': '10.10.10.254',
'netmask': '255.255.255.0',
"ip": "10.10.10.254",
"netmask": "255.255.255.0",
}
class Serial:
if platform == 'win32' or platform == 'win64':
"""The `Serial` class contains configurations for the serial ports."""
if platform in ("win32", "win64"):
SERIAL = {
'uartbase': '0x3F8',
'uartport': '4',
'uartmode': 'tcpserver',
'uartpath': '10000'
"uartbase": "0x3F8",
"uartport": "4",
"uartmode": "tcpserver",
"uartpath": "10000",
}
else:
SERIAL = {
'uartbase': '0x3F8',
'uartport': '4',
'uartmode': 'server',
'uartpath': '/tmp/'
"uartbase": "0x3F8",
"uartport": "4",
"uartmode": "server",
"uartpath": "/tmp/",
}

9
virtualbox/pybox/consts/node.py
View File

@ -3,8 +3,15 @@
# SPDX-License-Identifier: Apache-2.0
#
"""
This module contains dictionaries for different types of nodes in a virtual environment,
such as CONTROLLER_CEPH, CONTROLLER_LVM, CONTROLLER_AIO, COMPUTE, and STORAGE.
"""
class Nodes: #pylint: disable=too-few-public-methods
"""The `Nodes` class contains dictionaries for different types of nodes in a
virtual environment."""
class Nodes:
CONTROLLER_CEPH = {
'node_type': 'controller-STORAGE',
'memory': 12288,

9
virtualbox/pybox/consts/timeout.py
View File

@ -3,8 +3,15 @@
# SPDX-License-Identifier: Apache-2.0
#
"""
This module contains the HostTimeout class, which provides timeout values (in seconds)
for various operations on a host.
"""
class HostTimeout: #pylint: disable=too-few-public-methods
"""The `HostTimeout` class provides timeout values (in seconds) for various
operations on a host."""
class HostTimeout:
CONTROLLER_UNLOCK = 3600+1800
REBOOT = 900
INSTALL = 3600

46
virtualbox/pybox/helper/host_helper.py
View File

@ -3,6 +3,12 @@
# SPDX-License-Identifier: Apache-2.0
#
"""
This module provides functions to interact with a StarlingX controller-0 server via a
serial connection. The functions can be used to perform operations such as unlocking,
locking, rebooting, and installing a host. The module uses streamexpect library to
facilitate stream parsing.
"""
import time
import streamexpect
@ -21,15 +27,17 @@ def unlock_host(stream, hostname):
- Check that host is locked
- Unlock host
"""
LOG.info("#### Unlock %s", hostname)
serial.send_bytes(stream, "system host-list | grep {}".format(hostname), expect_prompt=False)
serial.send_bytes(stream, f"system host-list | grep {hostname}", expect_prompt=False)
try:
serial.expect_bytes(stream, "locked")
except streamexpect.ExpectTimeout:
LOG.info("Host %s not locked", hostname)
return 1
serial.send_bytes(stream, "system host-unlock {}".format(hostname), expect_prompt=False)
serial.send_bytes(stream, f"system host-unlock {hostname}", expect_prompt=False)
LOG.info("Unlocking %s", hostname)
return None
def lock_host(stream, hostname):
@ -42,15 +50,17 @@ def lock_host(stream, hostname):
- Check that host is unlocked
- Lock host
"""
LOG.info("Lock %s", hostname)
serial.send_bytes(stream, "system host-list |grep {}".format(hostname), expect_prompt=False)
serial.send_bytes(stream, f"system host-list |grep {hostname}", expect_prompt=False)
try:
serial.expect_bytes(stream, "unlocked")
except streamexpect.ExpectTimeout:
LOG.info("Host %s not unlocked", hostname)
return 1
serial.send_bytes(stream, "system host-lock {}".format(hostname), expect_prompt="keystone")
serial.send_bytes(stream, f"system host-lock {hostname}", expect_prompt="keystone")
LOG.info("Locking %s", hostname)
return None
def reboot_host(stream, hostname):
@ -60,8 +70,9 @@ def reboot_host(stream, hostname):
stream():
hostname(str): Host to reboot
"""
LOG.info("Rebooting %s", hostname)
serial.send_bytes(stream, "system host-reboot {}".format(hostname), expect_prompt=False)
serial.send_bytes(stream, f"system host-reboot {hostname}", expect_prompt=False)
serial.expect_bytes(stream, "rebooting", HostTimeout.REBOOT)
@ -77,18 +88,17 @@ def install_host(stream, hostname, host_type, host_id):
time.sleep(10)
LOG.info("Installing %s with id %s", hostname, host_id)
if host_type is 'controller':
if host_type == 'controller':
serial.send_bytes(stream,
"system host-update {} personality=controller".format(host_id),
f"system host-update {host_id} personality=controller",
expect_prompt=False)
elif host_type is 'storage':
elif host_type == 'storage':
serial.send_bytes(stream,
"system host-update {} personality=storage".format(host_id),
f"system host-update {host_id} personality=storage",
expect_prompt=False)
else:
serial.send_bytes(stream,
"system host-update {} personality=compute hostname={}".format(host_id,
hostname),
f"system host-update {host_id} personality=compute hostname={hostname}",
expect_prompt=False)
time.sleep(30)
@ -99,6 +109,7 @@ def disable_logout(stream):
Args:
stream(stream): stream to cont0
"""
LOG.info('Disabling automatic logout')
serial.send_bytes(stream, "export TMOUT=0")
@ -108,8 +119,8 @@ def change_password(stream, username="sysadmin", password="Li69nux*"):
changes the default password on initial login.
Args:
stream(stream): stream to cont0
"""
LOG.info('Changing password to Li69nux*')
serial.send_bytes(stream, username, expect_prompt=False)
serial.expect_bytes(stream, "Password:")
@ -131,8 +142,8 @@ def login(stream, timeout=600, username="sysadmin", password="Li69nux*"):
"""
serial.send_bytes(stream, "\n", expect_prompt=False)
rc = serial.expect_bytes(stream, "ogin:", fail_ok=True, timeout=timeout)
if rc != 0:
login_result = serial.expect_bytes(stream, "ogin:", fail_ok=True, timeout=timeout)
if login_result != 0:
serial.send_bytes(stream, "\n", expect_prompt=False)
if serial.expect_bytes(stream, "~$", timeout=10, fail_ok=True) == -1:
serial.send_bytes(stream, '\n', expect_prompt=False)
@ -150,11 +161,18 @@ def logout(stream):
Args:
stream(stream): stream to cont0
"""
serial.send_bytes(stream, "exit", expect_prompt=False)
time.sleep(5)
def check_password(stream, password="Li69nux*"):
"""
Checks the password.
Args:
stream(stream): Stream to cont0
password(str): password to check.
"""
ret = serial.expect_bytes(stream, 'assword', fail_ok=True, timeout=5)
if ret == 0:
serial.send_bytes(stream, password, expect_prompt=False)

42
virtualbox/pybox/helper/install_lab.py
View File

@ -8,11 +8,10 @@ Contains helper functions that will configure basic system settings.
"""
from consts.timeout import HostTimeout
from helper import host_helper
from utils import serial
from utils.install_log import LOG
from helper import host_helper
def update_platform_cpus(stream, hostname, cpu_num=5):
"""
@ -20,9 +19,14 @@ def update_platform_cpus(stream, hostname, cpu_num=5):
"""
LOG.info("Allocating %s CPUs for use by the %s platform.", cpu_num, hostname)
serial.send_bytes(stream, "\nsource /etc/platform/openrc; system host-cpu-modify "
"{} -f platform -p0 {}".format(hostname, cpu_num,
prompt='keystone', timeout=300))
serial.send_bytes(
stream,
"\nsource /etc/platform/openrc; system host-cpu-modify "
f"{hostname} -f platform -p0 {cpu_num}",
prompt="keystone",
timeout=300,
)
def set_dns(stream, dns_ip):
"""
@ -30,25 +34,31 @@ def set_dns(stream, dns_ip):
"""
LOG.info("Configuring DNS to %s.", dns_ip)
serial.send_bytes(stream, "source /etc/platform/openrc; system dns-modify "
"nameservers={}".format(dns_ip), prompt='keystone')
serial.send_bytes(
stream,
"source /etc/platform/openrc; system dns-modify "
f"nameservers={dns_ip}",
prompt="keystone",
)
def config_controller(stream, config_file=None, password='Li69nux*'):
def config_controller(stream, config_file=None, password="Li69nux*"):
"""
Configure controller-0 using optional arguments
"""
args = ''
args = ""
if config_file:
args += '--config-file ' + config_file + ' '
args += "--config-file " + config_file + " "
# serial.send_bytes(stream, f'sudo config_controller {args}', expect_prompt=False)
serial.send_bytes(stream, 'ansible-playbook /usr/share/ansible/stx-ansible/playbooks/bootstrap.yml', expect_prompt=False)
serial.send_bytes(
stream,
"ansible-playbook /usr/share/ansible/stx-ansible/playbooks/bootstrap.yml",
expect_prompt=False,
)
host_helper.check_password(stream, password=password)
ret = serial.expect_bytes(stream, "~$",
timeout=HostTimeout.LAB_CONFIG)
ret = serial.expect_bytes(stream, "~$", timeout=HostTimeout.LAB_CONFIG)
if ret != 0:
LOG.info("Configuration failed. Exiting installer.")
raise Exception("Configcontroller failed")
raise Exception("Configcontroller failed") # pylint: disable=E0012, W0719

737
virtualbox/pybox/helper/vboxmanage.py
View File

@ -3,6 +3,9 @@
# SPDX-License-Identifier: Apache-2.0
#
"""
This module provides functions for managing virtual machines using VirtualBox.
"""
import os
import subprocess
@ -12,7 +15,6 @@ import time
from sys import platform
from consts import env
from utils.install_log import LOG
@ -21,59 +23,90 @@ def vboxmanage_version():
Return version of vbox.
"""
version = subprocess.check_output(['vboxmanage', '--version'], stderr=subprocess.STDOUT)
version = subprocess.check_output(
["vboxmanage", "--version"], stderr=subprocess.STDOUT
)
return version
def vboxmanage_extpack(action="install"):
def vboxmanage_extpack():
"""
This allows you to install, uninstall the vbox extensions"
"""
output = vboxmanage_version()
version = re.match(b'(.*)r', output)
version_path = version.group(1).decode('utf-8')
version = re.match(b"(.*)r", output)
version_path = version.group(1).decode("utf-8")
LOG.info("Downloading extension pack")
filename = 'Oracle_VM_VirtualBox_Extension_Pack-{}.vbox-extpack'.format(version_path)
cmd = 'http://download.virtualbox.org/virtualbox/{}/{}'.format(version_path, filename)
result = subprocess.check_output(['wget', cmd, '-P', '/tmp'], stderr=subprocess.STDOUT)
filename = f"Oracle_VM_VirtualBox_Extension_Pack-{version_path}.vbox-extpack"
cmd = f"http://download.virtualbox.org/virtualbox/{version_path}/{filename}"
result = subprocess.check_output(
["wget", cmd, "-P", "/tmp"], stderr=subprocess.STDOUT
)
LOG.info(result)
LOG.info("Installing extension pack")
result = subprocess.check_output(['vboxmanage', 'extpack', 'install', '/tmp/' + filename,
'--replace'], stderr=subprocess.STDOUT)
result = subprocess.check_output(
["vboxmanage", "extpack", "install", "/tmp/" + filename, "--replace"],
stderr=subprocess.STDOUT,
)
LOG.info(result)
def get_all_vms(labname, option="vms"):
"""
Return a list of virtual machines (VMs) belonging to a specified lab.
Args:
labname (str): The name of the lab to which the VMs belong.
option (str, optional): The VBoxManage command option to use when listing VMs.
Defaults to "vms".
Returns:
list: A list of strings representing the names of the VMs that belong to the specified lab.
"""
initial_node_list = []
vm_list = vboxmanage_list(option)
labname.encode('utf-8')
labname.encode("utf-8")
# Reduce the number of VMs we query
for item in vm_list:
if labname.encode('utf-8') in item and (b'controller-' in item or \
b'compute-' in item or b'storage-' in item):
initial_node_list.append(item.decode('utf-8'))
if labname.encode("utf-8") in item and (
b"controller-" in item or b"compute-" in item or b"storage-" in item
):
initial_node_list.append(item.decode("utf-8"))
# Filter by group
node_list = []
group = bytearray('"/{}"'.format(labname), 'utf-8')
group = bytearray(f'"/{labname}"', "utf-8")
for item in initial_node_list:
info = vboxmanage_showinfo(item).splitlines()
for line in info:
try:
k, v = line.split(b'=')
k_value, v_value = line.split(b"=")
except ValueError:
continue
if k == b'groups' and v == group:
if k_value == b"groups" and v_value == group:
node_list.append(item)
return node_list
def take_snapshot(labname, snapshot_name, socks=None):
def take_snapshot(labname, snapshot_name):
"""
Take a snapshot of all VMs belonging to a specified lab.
Args:
labname (str): The name of the lab whose VMs will be snapshotted.
snapshot_name (str): The name of the snapshot to be taken.
Returns:
None
"""
vms = get_all_vms(labname, option="vms")
runningvms = get_all_vms(labname, option="runningvms")
@ -81,50 +114,103 @@ def take_snapshot(labname, snapshot_name, socks=None):
LOG.info("VMs in lab %s: %s", labname, vms)
LOG.info("VMs running in lab %s: %s", labname, runningvms)
hosts = len(vms)
_pause_running_vms(runningvms, vms)
if len(vms) != 0:
vboxmanage_takesnapshot(vms, snapshot_name)
_resume_running_vms(runningvms)
time.sleep(10)
if runningvms:
_wait_for_vms_to_run(labname, runningvms, vms)
def _pause_running_vms(runningvms, vms):
"""Pause running virtual machines.
Args:
runningvms (list): A list of strings representing the names of running virtual machines.
vms (list): A list of strings representing the names of all virtual machines.
Returns:
None
"""
# Pause running VMs to take snapshot
if len(runningvms) > 1:
for node in runningvms:
newpid = os.fork()
if newpid == 0:
vboxmanage_controlvms([node], "pause")
os._exit(0)
os._exit(0) # pylint: disable=protected-access
for node in vms:
os.waitpid(0, 0)
time.sleep(2)
if hosts != 0:
vboxmanage_takesnapshot(vms, snapshot_name)
# Resume VMs after snapshot was taken
def _resume_running_vms(runningvms):
"""Resume paused virtual machines.
Args:
runningvms (list): A list of strings representing the names of running virtual machines.
Returns:
None
"""
if len(runningvms) > 1:
for node in runningvms:
newpid = os.fork()
if newpid == 0:
vboxmanage_controlvms([node], "resume")
os._exit(0)
os._exit(0) # pylint: disable=protected-access
for node in runningvms:
os.waitpid(0, 0)
time.sleep(10) # Wait for VM serial port to stabilize, otherwise it may refuse to connect
if runningvms:
new_vms = get_all_vms(labname, option="runningvms")
retry = 0
while retry < 20:
LOG.info("Waiting for VMs to come up running after taking snapshot..."
"Up VMs are %s ", new_vms)
if len(runningvms) < len(new_vms):
time.sleep(1)
new_vms = get_all_vms(labname, option="runningvms")
retry += 1
else:
LOG.info("All VMs %s are up running after taking snapshot...", vms)
break
def _wait_for_vms_to_run(labname, runningvms, vms):
"""Wait for virtual machines to finish running.
Args:
labname (str): The name of the lab whose virtual machines are being waited for.
runningvms (list): A list of strings representing the names of running virtual machines.
vms (list): A list of strings representing the names of all virtual machines.
Returns:
None
"""
new_vms = get_all_vms(labname, option="runningvms")
retry = 0
while retry < 20:
LOG.info(
"Waiting for VMs to come up running after taking snapshot..."
"Up VMs are %s ",
new_vms,
)
if len(runningvms) < len(new_vms):
time.sleep(1)
new_vms = get_all_vms(labname, option="runningvms")
retry += 1
else:
LOG.info("All VMs %s are up running after taking snapshot...", vms)
break
def restore_snapshot(node_list, name):
"""
Restore a snapshot of a list of virtual machines.
Args:
node_list (list): A list of strings representing the names of the virtual machines
whose snapshot will be restored.
name (str): The name of the snapshot to restore.
Returns:
None
"""
LOG.info("Restore snapshot of %s for hosts %s", name, node_list)
if len(node_list) != 0:
vboxmanage_controlvms(node_list, "poweroff")
@ -147,7 +233,10 @@ def vboxmanage_list(option="vms"):
"""
This returns a list of vm names.
"""
result = subprocess.check_output(['vboxmanage', 'list', option], stderr=subprocess.STDOUT)
result = subprocess.check_output(
["vboxmanage", "list", option], stderr=subprocess.STDOUT
)
vms_list = []
for item in result.splitlines():
vm_name = re.match(b'"(.*?)"', item)
@ -160,10 +249,13 @@ def vboxmanage_showinfo(host):
"""
This returns info about the host
"""
if not isinstance(host, str):
host.decode('utf-8')
result = subprocess.check_output(['vboxmanage', 'showvminfo', host, '--machinereadable'],
stderr=subprocess.STDOUT)
host.decode("utf-8")
result = subprocess.check_output(
["vboxmanage", "showvminfo", host, "--machinereadable"],
stderr=subprocess.STDOUT,
)
return result
@ -176,9 +268,21 @@ def vboxmanage_createvm(hostname, labname):
assert labname, "Labname is required"
group = "/" + labname
LOG.info("Creating VM %s", hostname)
result = subprocess.check_output(['vboxmanage', 'createvm', '--name', hostname, '--register',
'--ostype', 'Linux_64', '--groups', group],
stderr=subprocess.STDOUT)
subprocess.check_output(
[
"vboxmanage",
"createvm",
"--name",
hostname,
"--register",
"--ostype",
"Linux_64",
"--groups",
group,
],
stderr=subprocess.STDOUT,
)
def vboxmanage_deletevms(hosts=None):
"""
@ -190,124 +294,295 @@ def vboxmanage_deletevms(hosts=None):
if len(hosts) != 0:
for hostname in hosts:
LOG.info("Deleting VM %s", hostname)
result = subprocess.check_output(['vboxmanage', 'unregistervm', hostname, '--delete'],
stderr=subprocess.STDOUT)
subprocess.check_output(
["vboxmanage", "unregistervm", hostname, "--delete"],
stderr=subprocess.STDOUT,
)
time.sleep(10)
# in case medium is still present after delete
vboxmanage_deletemedium(hostname)
vms_list = vboxmanage_list("vms")
for items in hosts:
assert items not in vms_list, "The following vms are unexpectedly" \
"present {}".format(vms_list)
assert (
items not in vms_list
), f"The following vms are unexpectedly present {vms_list}"
def vboxmanage_hostonlyifcreate(name="vboxnet0", ip=None, netmask=None):
def vboxmanage_hostonlyifcreate(name="vboxnet0", oam_ip=None, netmask=None):
"""
This creates a hostonly network for systems to communicate.
"""
assert name, "Must provide network name"
assert ip, "Must provide an OAM IP"
assert oam_ip, "Must provide an OAM IP"
assert netmask, "Must provide an OAM Netmask"
LOG.info("Creating Host-only Network")
result = subprocess.check_output(['vboxmanage', 'hostonlyif', 'create'],
stderr=subprocess.STDOUT)
subprocess.check_output(
["vboxmanage", "hostonlyif", "create"], stderr=subprocess.STDOUT
)
LOG.info("Provisioning %s with IP %s and Netmask %s", name, ip, netmask)
result = subprocess.check_output(['vboxmanage', 'hostonlyif', 'ipconfig', name, '--ip',
ip, '--netmask', netmask], stderr=subprocess.STDOUT)
LOG.info("Provisioning %s with IP %s and Netmask %s", name, oam_ip, netmask)
subprocess.check_output(
[
"vboxmanage",
"hostonlyif",
"ipconfig",
name,
"--ip",
oam_ip,
"--netmask",
netmask,
],
stderr=subprocess.STDOUT,
)
def vboxmanage_hostonlyifdelete(name="vboxnet0"):
"""
Deletes hostonly network. This is used as a work around for creating too many hostonlyifs.
"""
assert name, "Must provide network name"
LOG.info("Removing Host-only Network")
result = subprocess.check_output(['vboxmanage', 'hostonlyif', 'remove', name],
stderr=subprocess.STDOUT)
subprocess.check_output(
["vboxmanage", "hostonlyif", "remove", name], stderr=subprocess.STDOUT
)
def vboxmanage_modifyvm(hostname=None, cpus=None, memory=None, nic=None,
nictype=None, nicpromisc=None, nicnum=None,
intnet=None, hostonlyadapter=None,
natnetwork=None, uartbase=None, uartport=None,
uartmode=None, uartpath=None, nicbootprio2=1, prefix=""):
def vboxmanage_modifyvm(hostname, vm_config=None):
"""
This modifies a VM with a specified name.
Modify a virtual machine according to a specified configuration.
Args:
hostname(str): Name of host to modify
vm_config (dict): A dictionary representing the configuration options
for the virtual machine. Possible key values: cpus, memory, nic, nictype,
nicpromisc, nicnum, intnet, hostonlyadapter, natnetwork, uartbase,
uartport, uartmode, uartpath, nicbootprio2=1, prefix=""
Returns:
None
"""
assert hostname, "Hostname is required"
# Add more semantic checks
cmd = ['vboxmanage', 'modifyvm', hostname]
if cpus:
cmd.extend(['--cpus', cpus])
if memory:
cmd.extend(['--memory', memory])
if nic and nictype and nicpromisc and nicnum:
cmd.extend(['--nic{}'.format(nicnum), nic])
cmd.extend(['--nictype{}'.format(nicnum), nictype])
cmd.extend(['--nicpromisc{}'.format(nicnum), nicpromisc])
if intnet:
if prefix:
intnet = "{}-{}".format(prefix, intnet)
else:
intnet = "{}".format(intnet)
cmd.extend(['--intnet{}'.format(nicnum), intnet])
if hostonlyadapter:
cmd.extend(['--hostonlyadapter{}'.format(nicnum), hostonlyadapter])
if natnetwork:
cmd.extend(['--nat-network{}'.format(nicnum), natnetwork])
elif nicnum and nictype == 'nat':
cmd.extend(['--nic{}'.format(nicnum), 'nat'])
if uartbase and uartport and uartmode and uartpath:
cmd.extend(['--uart1'])
cmd.extend(['{}'.format(uartbase)])
cmd.extend(['{}'.format(uartport)])
cmd.extend(['--uartmode1'])
cmd.extend(['{}'.format(uartmode)])
if platform == 'win32' or platform == 'win64':
cmd.extend(['{}'.format(env.PORT)])
env.PORT += 1
else:
if prefix:
prefix = "{}_".format(prefix)
if 'controller-0' in hostname:
cmd.extend(['{}{}{}_serial'.format(uartpath, prefix, hostname)])
else:
cmd.extend(['{}{}{}'.format(uartpath, prefix, hostname)])
if nicbootprio2:
cmd.extend(['--nicbootprio2'])
cmd.extend(['{}'.format(nicbootprio2)])
cmd.extend(['--boot4'])
cmd.extend(['net'])
#put default values in nicbootprio2 and prefix if they not exist
vm_config["nicbootprio2"] = vm_config.get("nicbootprio2", 1)
vm_config["prefix"] = vm_config.get("prefix", "")
cmd = ["vboxmanage", "modifyvm", hostname]
nic_cmd = []
if _contains_value("cpus", vm_config):
cmd.extend(["--cpus", vm_config["cpus"]])
if _contains_value("memory", vm_config):
cmd.extend(["--memory", vm_config["memory"]])
if _is_network_configured(vm_config):
nic_cmd = _get_network_configuration(vm_config)
cmd.extend(nic_cmd)
elif _is_nat_network_configured(vm_config):
cmd.extend([f'--nic{vm_config["nicnum"]}', "nat"])
if _is_uart_configured(vm_config):
uart_config = _add_uart(hostname, vm_config)
cmd.extend(uart_config)
if _contains_value("nicbootprio2", vm_config):
cmd.extend(["--nicbootprio2"])
cmd.extend([f'{vm_config["nicbootprio2"]}'])
cmd.extend(["--boot4"])
cmd.extend(["net"])
LOG.info(cmd)
LOG.info("Updating VM %s configuration", hostname)
result = subprocess.check_output(cmd, stderr=subprocess.STDOUT)
subprocess.check_output(cmd, stderr=subprocess.STDOUT)
def _is_network_configured(vm_config):
"""
Checks whether a network interface is configured in the given VM configuration.
Args:
vm_config (dict): A dictionary representing the configuration options for the VM.
Returns:
bool: True if a network interface is configured, False otherwise.
"""
return (_contains_value("nic", vm_config)
and _contains_value("nictype", vm_config)
and _contains_value("nicpromisc", vm_config)
and _contains_value("nicnum", vm_config)
)
def _get_network_configuration(vm_config):
"""
Constructs a list of options for the network interface based on the values in vm_config.
Args:
vm_config (dict): A dictionary representing the configuration options for the VM.
Returns:
list: A list of command-line options for the network interface.
"""
nic_cmd = [f'--nic{vm_config["nicnum"]}', vm_config["nic"]]
nic_cmd.extend([f'--nictype{vm_config["nicnum"]}', vm_config["nictype"]])
nic_cmd.extend([f'--nicpromisc{vm_config["nicnum"]}', vm_config["nicpromisc"]])
if _contains_value("intnet", vm_config):
intnet = vm_config["intnet"]
if _contains_value("prefix", vm_config):
intnet = f"{vm_config['prefix']}-{intnet}"
else:
intnet = f"{intnet}"
nic_cmd.extend([f'--intnet{vm_config["nicnum"]}', intnet])
if _contains_value("hostonlyadapter", vm_config):
nic_cmd.extend(
[
f'--hostonlyadapter{vm_config["nicnum"]}',
vm_config["hostonlyadapter"],
]
)
if _contains_value("natnetwork", vm_config):
nic_cmd.extend(
[f'--nat-network{vm_config["nicnum"]}', vm_config["natnetwork"]]
)
return nic_cmd
def _is_nat_network_configured(vm_config):
"""
Checks whether the NAT network is configured in the given VM configuration.
Args:
vm_config (dict): A dictionary representing the configuration options for the VM.
Returns:
bool: True if the NAT network is configured, False otherwise.
"""
return _contains_value("nicnum", vm_config) and vm_config.get("nictype") == "nat"
def _is_uart_configured(vm_config):
"""
Checks whether the UART device is configured in the given VM configuration.
Args:
vm_config (dict): A dictionary representing the configuration options for the VM.
Returns:
bool: True if the UART device is configured, False otherwise.
"""
return (
_contains_value("uartbase", vm_config)
and _contains_value("uartport", vm_config)
and _contains_value("uartmode", vm_config)
and _contains_value("uartpath", vm_config)
)
def _add_uart(hostname, vm_config):
"""
Constructs a list of options for the UART device based on the values in vm_config.
Args:
hostname (str): Name of the virtual machine.
vm_config (dict): A dictionary representing the configuration options for the VM.
Returns:
list: A list of command-line options for the UART device.
"""
uart_config = ["--uart1"]
uart_config.extend([f'{vm_config["uartbase"]}'])
uart_config.extend([f'{vm_config["uartport"]}'])
uart_config.extend(["--uartmode1"])
uart_config.extend([f'{vm_config["uartmode"]}'])
if platform in ("win32", "win64"):
uart_config.extend([f"{env.PORT}"])
env.PORT += 1
else:
if _contains_value("prefix", vm_config):
prefix = f'{vm_config["prefix"]}_'
if "controller-0" in hostname:
uart_config.extend([f'{vm_config["uartpath"]}{prefix}{hostname}_serial'])
else:
uart_config.extend([f'{vm_config["uartpath"]}{prefix}{hostname}'])
return uart_config
def _contains_value(key, dictionary):
return key in dictionary and dictionary[key]
def vboxmanage_port_forward(hostname, network, local_port, guest_port, guest_ip):
"""
Configures port forwarding for a NAT network in VirtualBox.
Args:
hostname (str): Name of the virtual machine.
network (str): Name of the NAT network.
local_port (int): The local port number to forward.
guest_port (int): The port number on the guest to forward to.
guest_ip (str): The IP address of the guest to forward to.
Returns:
None
"""
# VBoxManage natnetwork modify --netname natnet1 --port-forward-4
# "ssh:tcp:[]:1022:[192.168.15.5]:22"
rule_name = "{}-{}".format(hostname, guest_port)
rule_name = f"{hostname}-{guest_port}"
# Delete previous entry, if any
LOG.info("Removing previous forwarding rule '%s' from NAT network '%s'", rule_name, network)
cmd = ['vboxmanage', 'natnetwork', 'modify', '--netname', network,
'--port-forward-4', 'delete', rule_name]
LOG.info(
"Removing previous forwarding rule '%s' from NAT network '%s'",
rule_name,
network,
)
cmd = [
"vboxmanage",
"natnetwork",
"modify",
"--netname",
network,
"--port-forward-4",
"delete",
rule_name,
]
try:
result = subprocess.check_output(cmd, stderr=subprocess.STDOUT)
subprocess.check_output(cmd, stderr=subprocess.STDOUT)
except subprocess.CalledProcessError:
pass
# Add new rule
rule = "{}:tcp:[]:{}:[{}]:{}".format(rule_name, local_port, guest_ip, guest_port)
rule = f"{rule_name}:tcp:[]:{local_port}:[{guest_ip}]:{guest_port}"
LOG.info("Updating port-forwarding rule to: %s", rule)
cmd = ['vboxmanage', 'natnetwork', 'modify', '--netname', network, '--port-forward-4', rule]
result = subprocess.check_output(cmd, stderr=subprocess.STDOUT)
cmd = [
"vboxmanage",
"natnetwork",
"modify",
"--netname",
network,
"--port-forward-4",
rule,
]
subprocess.check_output(cmd, stderr=subprocess.STDOUT)
def vboxmanage_storagectl(hostname=None, storectl="sata", hostiocache="off"):
"""
@ -317,62 +592,132 @@ def vboxmanage_storagectl(hostname=None, storectl="sata", hostiocache="off"):
assert hostname, "Hostname is required"
assert storectl, "Type of storage controller is required"
LOG.info("Creating %s storage controller on VM %s", storectl, hostname)
result = subprocess.check_output(['vboxmanage', 'storagectl',
hostname, '--name', storectl,
'--add', storectl, '--hostiocache',
hostiocache], stderr=subprocess.STDOUT)
subprocess.check_output(
[
"vboxmanage",
"storagectl",
hostname,
"--name",
storectl,
"--add",
storectl,
"--hostiocache",
hostiocache,
],
stderr=subprocess.STDOUT,
)
def vboxmanage_storageattach(hostname=None, storectl="sata",
storetype="hdd", disk=None, port_num="0", device_num="0"):
def vboxmanage_storageattach(hostname, storage_config):
"""
This attaches a disk to a controller.
Attaches a disk to a storage controller.
Args:
hostname (str): Name of the virtual machine.
storage_config (dict): A dictionary containing the config options for the storage device.
Possible key values: storectl, storetype, disk, port_num, device_num.
Returns:
str: The output of the VBoxManage command.
"""
assert hostname, "Hostname is required"
assert storage_config and isinstance(storage_config, dict), "Storage configuration is required"
storectl = storage_config.get("storectl", "sata")
storetype = storage_config.get("storetype", "hdd")
disk = storage_config.get("disk")
port_num = storage_config.get("port_num", "0")
device_num = storage_config.get("device_num", "0")
assert disk, "Disk name is required"
assert storectl, "Name of storage controller is required"
assert storetype, "Type of storage controller is required"
LOG.info("Attaching %s storage to storage controller %s on VM %s",
storetype, storectl, hostname)
result = subprocess.check_output(['vboxmanage', 'storageattach',
hostname, '--storagectl', storectl,
'--medium', disk, '--type',
storetype, '--port', port_num,
'--device', device_num], stderr=subprocess.STDOUT)
return result
def vboxmanage_deletemedium(hostname, vbox_home_dir='/home'):
LOG.info(
"Attaching %s storage to storage controller %s on VM %s",
storetype,
storectl,
hostname,
)
return subprocess.check_output(
[
"vboxmanage",
"storageattach",
hostname,
"--storagectl",
storectl,
"--medium",
disk,
"--type",
storetype,
"--port",
port_num,
"--device",
device_num,
],
stderr=subprocess.STDOUT,
)
def vboxmanage_deletemedium(hostname, vbox_home_dir="/home"):
"""
Deletes the disk medium associated with a virtual machine.
Args:
hostname (str): The name of the virtual machine to which the disk medium is attached.
vbox_home_dir (str): The directory in which the disk medium files are stored.
Defaults to "/home".
Returns:
None
"""
assert hostname, "Hostname is required"
if platform == 'win32' or platform == 'win64':
if platform in ("win32", "win64"):
return
username = getpass.getuser()
vbox_home_dir = "{}/{}/vbox_disks/".format(vbox_home_dir, username)
vbox_home_dir = f"{vbox_home_dir}/{username}/vbox_disks/"
disk_list = [f for f in os.listdir(vbox_home_dir) if
os.path.isfile(os.path.join(vbox_home_dir, f)) and hostname in f]
disk_list = [
f
for f in os.listdir(vbox_home_dir)
if os.path.isfile(os.path.join(vbox_home_dir, f)) and hostname in f
]
LOG.info("Disk mediums to delete: %s", disk_list)
for disk in disk_list:
LOG.info("Disconnecting disk %s from vbox.", disk)
try:
result = subprocess.check_output(['vboxmanage', 'closemedium', 'disk',
"{}{}".format(vbox_home_dir, disk), '--delete'],
stderr=subprocess.STDOUT)
result = subprocess.check_output(
[
"vboxmanage",
"closemedium",
"disk",
"{vbox_home_dir}{disk}",
"--delete",
],
stderr=subprocess.STDOUT,
)
LOG.info(result)
except subprocess.CalledProcessError as e:
except subprocess.CalledProcessError as exception:
# Continue if failures, disk may not be present
LOG.info("Error disconnecting disk, continuing. "
"Details: stdout: %s stderr: %s", e.stdout, e.stderr)
LOG.info(
"Error disconnecting disk, continuing. "
"Details: stdout: %s stderr: %s",
exception.stdout,
exception.stderr,
)
LOG.info("Removing backing file %s", disk)
try:
os.remove("{}{}".format(vbox_home_dir, disk))
except:
os.remove("{vbox_home_dir}{disk}")
except: # pylint: disable=bare-except
pass
def vboxmanage_createmedium(hostname=None, disk_list=None, vbox_home_dir='/home'):
def vboxmanage_createmedium(hostname=None, disk_list=None, vbox_home_dir="/home"):
"""
This creates the required disks.
"""
@ -385,28 +730,63 @@ def vboxmanage_createmedium(hostname=None, disk_list=None, vbox_home_dir='/home'
port_num = 0
disk_count = 1
for disk in disk_list:
if platform == 'win32' or platform == 'win64':
file_name = "C:\\Users\\" + username + "\\vbox_disks\\" + \
hostname + "_disk_{}".format(disk_count)
if platform in ("win32", "win64"):
file_name = (
"C:\\Users\\"
+ username
+ "\\vbox_disks\\"
+ hostname
+ f"_disk_{disk_count}"
)
else:
file_name = vbox_home_dir + '/' + username + "/vbox_disks/" \
+ hostname + "_disk_{}".format(disk_count)
LOG.info("Creating disk %s of size %s on VM %s on device %s port %s",
file_name, disk, hostname, device_num, port_num)
file_name = (
vbox_home_dir
+ "/"
+ username
+ "/vbox_disks/"
+ hostname
+ f"_disk_{disk_count}"
)
LOG.info(
"Creating disk %s of size %s on VM %s on device %s port %s",
file_name,
disk,
hostname,
device_num,
port_num,
)
try:
result = subprocess.check_output(['vboxmanage', 'createmedium',
'disk', '--size', str(disk),
'--filename', file_name,
'--format', 'vdi',
'--variant', 'standard'],
stderr=subprocess.STDOUT)
result = subprocess.check_output(
[
"vboxmanage",
"createmedium",
"disk",
"--size",
str(disk),
"--filename",
file_name,
"--format",
"vdi",
"--variant",
"standard",
],
stderr=subprocess.STDOUT,
)
LOG.info(result)
except subprocess.CalledProcessError as e:
LOG.info("Error stdout: %s stderr: %s", e.stdout, e.stderr)
except subprocess.CalledProcessError as exception:
LOG.info("Error stdout: %s stderr: %s", exception.stdout, exception.stderr)
raise
vboxmanage_storageattach(hostname, "sata", "hdd", file_name + \
".vdi", str(port_num), str(device_num))
vboxmanage_storageattach(
hostname,
{
"storectl": "sata",
"storetype": "hdd",
"disk": file_name + ".vdi",
"port_num": str(port_num),
"device_num": str(device_num),
},
)
disk_count += 1
port_num += 1
time.sleep(5)
@ -425,12 +805,13 @@ def vboxmanage_startvm(hostname=None, force=False):
else:
running_vms = []
if hostname.encode('utf-8') in running_vms:
if hostname.encode("utf-8") in running_vms:
LOG.info("Host %s is already started", hostname)
else:
LOG.info("Powering on VM %s", hostname)
result = subprocess.check_output(['vboxmanage', 'startvm',
hostname], stderr=subprocess.STDOUT)
result = subprocess.check_output(
["vboxmanage", "startvm", hostname], stderr=subprocess.STDOUT
)
LOG.info(result)
# Wait for VM to start
@ -438,11 +819,11 @@ def vboxmanage_startvm(hostname=None, force=False):
while tmout:
tmout -= 1
running_vms = vboxmanage_list(option="runningvms")
if hostname.encode('utf-8') in running_vms:
if hostname.encode("utf-8") in running_vms:
break
time.sleep(1)
else:
raise "Failed to start VM: {}".format(hostname)
raise f"Failed to start VM: {hostname}"
LOG.info("VM '%s' started.", hostname)
@ -456,8 +837,9 @@ def vboxmanage_controlvms(hosts=None, action=None):
for host in hosts:
LOG.info("Executing %s action on VM %s", action, host)
result = subprocess.call(["vboxmanage", "controlvm", host,
action], stderr=subprocess.STDOUT)
subprocess.call(
["vboxmanage", "controlvm", host, action], stderr=subprocess.STDOUT
)
time.sleep(1)
@ -471,8 +853,9 @@ def vboxmanage_takesnapshot(hosts=None, name=None):
for host in hosts:
LOG.info("Taking snapshot %s on VM %s", name, host)
result = subprocess.call(["vboxmanage", "snapshot", host, "take",
name], stderr=subprocess.STDOUT)
subprocess.call(
["vboxmanage", "snapshot", host, "take", name], stderr=subprocess.STDOUT
)
def vboxmanage_restoresnapshot(host=None, name=None):
@ -484,7 +867,7 @@ def vboxmanage_restoresnapshot(host=None, name=None):
assert name, "Need to provide the snapshot to restore"
LOG.info("Restoring snapshot %s on VM %s", name, host)
result = subprocess.call(["vboxmanage", "snapshot", host, "restore",
name], stderr=subprocess.STDOUT)
subprocess.call(
["vboxmanage", "snapshot", host, "restore", name], stderr=subprocess.STDOUT
)
time.sleep(10)

1186
virtualbox/pybox/install_vbox.py
View File

File diff suppressed because it is too large Load Diff

1
virtualbox/pybox/requirements.txt
View File

@ -3,4 +3,5 @@ paramiko
pytest
git+https://github.com/digidotcom/python-streamexpect#egg=streamexpect
pexpect
ruamel.yaml

42
virtualbox/pybox/utils/install_log.py
View File

@ -3,36 +3,45 @@
# SPDX-License-Identifier: Apache-2.0
#
"""
This module provides functionality to initialize logging for a lab, create a sub-directory
for the current run, set the logging level, and create a symbolic link to the latest logs
for the lab.
"""
import os
import datetime
import logging
from consts.env import LOGPATH
log_dir = ""
LOG_DIR = ""
LOG = logging.getLogger()
def init_logging(lab_name, log_path=None):
global LOG, log_dir
"""
This method initializes the logging for a lab. It creates a sub-directory for the
current run and sets the logging level to INFO. It also creates a symbolic link to
the latest logs for the lab. The method takes in the lab name and an optional log path
parameter. If no log path is specified, it uses the default path provided by the
LOGPATH constant in the env module.
"""
global LOG, LOG_DIR # pylint: disable=global-statement, global-variable-not-assigned
if not log_path:
log_path = LOGPATH
lab_log_path = log_path + "/" + lab_name
# Setup log sub-directory for current run
current_time = datetime.datetime.now()
log_dir = "{}/{}_{}_{}_{}_{}_{}".format(lab_log_path,
current_time.year,
current_time.month,
current_time.day,
current_time.hour,
current_time.minute,
current_time.second)
if not os.path.exists(log_dir):
os.makedirs(log_dir)
LOG_DIR = f"{lab_log_path}/{current_time.year}_{current_time.month}_\
{current_time.day}_{current_time.hour}_{current_time.minute}_{current_time.second}"
if not os.path.exists(LOG_DIR):
os.makedirs(LOG_DIR)
LOG.setLevel(logging.INFO)
formatter = logging.Formatter("%(asctime)s: %(message)s")
log_file = "{}/install.log".format(log_dir)
log_file = f"{LOG_DIR}/install.log"
handler = logging.FileHandler(log_file)
handler.setFormatter(formatter)
handler.setLevel(logging.INFO)
@ -44,9 +53,12 @@ def init_logging(lab_name, log_path=None):
# Create symbolic link to latest logs of this lab
try:
os.unlink(lab_log_path + "/latest")
except:
except: # pylint: disable=bare-except
pass
os.symlink(log_dir, lab_log_path + "/latest")
os.symlink(LOG_DIR, lab_log_path + "/latest")
def get_log_dir():
return log_dir
"""This method returns the directory path of the current logging run."""
return LOG_DIR

59
virtualbox/pybox/utils/kpi.py
View File

@ -3,55 +3,80 @@
# SPDX-License-Identifier: Apache-2.0
#
"""
This module provides functions to track and report key performance indicators (KPIs) for a program.
"""
import time
from utils.install_log import LOG
STAGES = []
METRICS = {}
start = 0
START = 0
def init_kpi_metrics():
global start
start = time.time()
"""
Initializes the global variable START with the current time to start tracking the
duration of a program.
"""
global START # pylint: disable=global-statement
START = time.time()
def get_formated_time(sec):
"""
Takes the duration in seconds and formats it in hours, minutes and seconds.
Returns the formatted string.
"""
hours = sec // 3600
sec %= 3600
minutes = sec // 60
sec %= 60
seconds = sec
if hours:
return "{:.0f}h {:.0f}m {:.2f}s".format(hours, minutes, seconds)
elif minutes:
return "{:.0f}m {:.2f}s".format(minutes, seconds)
elif seconds:
return "{:.2f}s".format(seconds)
return f"{hours:.0f}h {minutes:.0f}m {seconds:.2f}s"
if minutes:
return f"{minutes:.0f}m {seconds:.2f}s"
return f"{seconds:.2f}s"
def set_kpi_metric(metric, duration):
global METRICS, STAGES
"""Sets the duration of a metric and adds the metric to the global list of STAGES."""
global METRICS, STAGES # pylint: disable=global-statement, global-variable-not-assigned
METRICS[metric] = duration
STAGES.append(metric)
def print_kpi(metric):
"""Takes a metric as input and prints the duration of that metric using the LOG module."""
if metric in STAGES:
sec = METRICS[metric]
LOG.info(" Time in stage '%s': %s ", metric, get_formated_time(sec))
elif metric == 'total' and start:
duration = time.time() - start
elif metric == 'total' and START:
duration = time.time() - START
LOG.info(" Total time: %s", get_formated_time(duration))
def get_kpi_str(metric):
"""Takes a metric as input and returns the duration of that metric as a formatted string."""
msg = ""
if metric in STAGES:
sec = METRICS[metric]
msg += (" Time in stage '{}': {} \n".format(metric, get_formated_time(sec)))
elif metric == 'total' and start:
duration = time.time() - start
msg += (" Total time: {}\n".format(get_formated_time(duration)))
msg += (f" Time in stage '{metric}': {get_formated_time(sec)} \n")
elif metric == 'total' and START:
duration = time.time() - START
msg += (f" Total time: {get_formated_time(duration)}\n")
return msg
def get_kpi_metrics_str():
"""Returns a formatted string with all the metrics and their durations."""
msg = "===================== Metrics ====================\n"
for stage in STAGES:
msg += get_kpi_str(stage)
@ -59,10 +84,12 @@ def get_kpi_metrics_str():
msg += "===============================================\n"
return msg
def print_kpi_metrics():
"""Prints all the metrics and their durations using the LOG module."""
LOG.info("===================== Metrics ====================")
for stage in STAGES:
print_kpi(stage)
print_kpi('total')
LOG.info("==================================================")

107
virtualbox/pybox/utils/serial.py
View File

@ -3,6 +3,10 @@
# SPDX-License-Identifier: Apache-2.0
#
"""
This module provides functionality to connect and communicate with a remote host
using local domain socket.
"""
import re
import socket
@ -22,27 +26,27 @@ def connect(hostname, port=10000, prefix=""):
"""
if prefix:
prefix = "{}_".format(prefix)
socketname = "/tmp/{}{}".format(prefix, hostname)
if 'controller-0'in hostname:
prefix = f"{prefix}_"
socketname = f"/tmp/{prefix}{hostname}"
if 'controller-0' in hostname:
socketname += '_serial'
LOG.info("Connecting to %s at %s", hostname, socketname)
if platform == 'win32' or platform == 'win64':
if platform in ('win32', 'win64'):
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM, socket.IPPROTO_TCP)
else:
sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
try:
if platform == 'win32' or platform == 'win64':
if platform in ('win32', 'win64'):
sock.setsockopt(socket.SOL_SOCKET, socket.SO_KEEPALIVE, 1)
sock.connect(('localhost', port))
else:
sock.connect(socketname)
except:
except: # pylint: disable=bare-except
LOG.info("Connection failed")
pass
pass # pylint: disable=unnecessary-pass
# disconnect(sock)
sock = None
# TODO (WEI): double check this
# TODO (WEI): double check this # pylint: disable=fixme
sock.setblocking(0)
return sock
@ -61,14 +65,18 @@ def disconnect(sock):
sock.shutdown(socket.SHUT_RDWR)
sock.close()
def get_output(stream, cmd, prompts=None, timeout=5, log=True, as_lines=True, flush=True):
#TODO: Not testested, will not work if kernel or other processes throw data on stdout or stderr
# pylint: disable=too-many-arguments, too-many-locals, too-many-branches
def get_output(stream, prompts=None, timeout=5, log=True, as_lines=True, flush=True):
# pylint: disable=fixme
# TODO: Not tested, will not work if kernel or other processes throw data on stdout or stderr
"""
Execute a command and get its output. Make sure no other command is executing.
And 'dmesg -D' was executed.
"""
POLL_PERIOD = 0.1
MAX_READ_BUFFER = 1024
poll_period = 0.1
max_read_buffer = 1024
data = ""
line_buf = ""
lines = []
@ -82,13 +90,13 @@ def get_output(stream, cmd, prompts=None, timeout=5, log=True, as_lines=True, fl
try:
LOG.info("Buffer has bytes before cmd execution: %s",
trash.decode('utf-8'))
except Exception:
except Exception: # pylint: disable=W0703
pass
except streamexpect.ExpectTimeout:
pass
# Send command
stream.sendall("{}\n".format(cmd).encode('utf-8'))
stream.sendall("{cmd}\n".encode('utf-8'))
# Get response
patterns = []
@ -98,20 +106,21 @@ def get_output(stream, cmd, prompts=None, timeout=5, log=True, as_lines=True, fl
now = time.time()
end_time = now + float(timeout)
prev_timeout = stream.gettimeout()
stream.settimeout(POLL_PERIOD)
stream.settimeout(poll_period)
incoming = None
# pylint: disable=too-many-nested-blocks
try:
while (end_time - now) >= 0:
try:
incoming = stream.recv(MAX_READ_BUFFER)
incoming = stream.recv(max_read_buffer)
except socket.timeout:
pass
if incoming:
data += incoming
if log:
for c in incoming:
if c != '\n':
line_buf += c
for char in incoming:
if char != '\n':
line_buf += char
else:
LOG.info(line_buf)
lines.append(line_buf)
@ -120,8 +129,7 @@ def get_output(stream, cmd, prompts=None, timeout=5, log=True, as_lines=True, fl
if pattern.search(data):
if as_lines:
return lines
else:
return data
return data
now = time.time()
raise streamexpect.ExpectTimeout()
finally:
@ -132,23 +140,24 @@ def expect_bytes(stream, text, timeout=180, fail_ok=False, flush=True):
"""
Wait for user specified text from stream.
"""
time.sleep(1)
if timeout < 60:
LOG.info("Expecting text within %s seconds: %s\n", timeout, text)
else:
LOG.info("Expecting text within %s minutes: %s\n", timeout/60, text)
LOG.info("Expecting text within %s minutes: %s\n", timeout / 60, text)
try:
stream.expect_bytes("{}".format(text).encode('utf-8'), timeout=timeout)
stream.expect_bytes(f"{text}".encode('utf-8'), timeout=timeout)
except streamexpect.ExpectTimeout:
if fail_ok:
return -1
else:
stdout.write('\n')
LOG.error("Did not find expected text")
# disconnect(stream)
raise
except Exception as e:
LOG.info("Connection failed with %s", e)
stdout.write('\n')
LOG.error("Did not find expected text")
# disconnect(stream)
raise
except Exception as exception:
LOG.info("Connection failed with %s", exception)
raise
stdout.write('\n')
@ -162,8 +171,8 @@ def expect_bytes(stream, text, timeout=180, fail_ok=False, flush=True):
incoming += b'\n'
try:
LOG.info(">>> expect_bytes: Buffer has bytes!")
stdout.write(incoming.decode('utf-8')) # streamexpect hardcodes it
except Exception:
stdout.write(incoming.decode('utf-8')) # streamexpect hardcodes it
except Exception: # pylint: disable=W0703
pass
except streamexpect.ExpectTimeout:
pass
@ -171,11 +180,13 @@ def expect_bytes(stream, text, timeout=180, fail_ok=False, flush=True):
return 0
# pylint: disable=inconsistent-return-statements
def send_bytes(stream, text, fail_ok=False, expect_prompt=True,
prompt=None, timeout=180, send=True, flush=True):
"""
Send user specified text to stream.
"""
time.sleep(1)
if flush:
try:
@ -184,8 +195,8 @@ def send_bytes(stream, text, fail_ok=False, expect_prompt=True,
incoming += b'\n'
try:
LOG.info(">>> send_bytes: Buffer has bytes!")
stdout.write(incoming.decode('utf-8')) # streamexpect hardcodes it
except Exception:
stdout.write(incoming.decode('utf-8')) # streamexpect hardcodes it
except Exception: # pylint: disable=W0703
pass
except streamexpect.ExpectTimeout:
pass
@ -193,28 +204,28 @@ def send_bytes(stream, text, fail_ok=False, expect_prompt=True,
LOG.info("Sending text: %s", text)
try:
if send:
stream.sendall("{}\n".format(text).encode('utf-8'))
stream.sendall(f"{text}\n".encode('utf-8'))
else:
stream.sendall("{}".format(text).encode('utf-8'))
stream.sendall(f"{text}".encode('utf-8'))
if expect_prompt:
time.sleep(1)
if prompt:
return expect_bytes(stream, prompt, timeout=timeout, fail_ok=fail_ok)
else:
rc = expect_bytes(stream, "~$", timeout=timeout, fail_ok=True)
if rc != 0:
send_bytes(stream, '\n', expect_prompt=False)
expect_bytes(stream, 'keystone', timeout=timeout)
return
return_code = expect_bytes(stream, "~$", timeout=timeout, fail_ok=True)
if return_code != 0:
send_bytes(stream, '\n', expect_prompt=False)
expect_bytes(stream, 'keystone', timeout=timeout)
return
except streamexpect.ExpectTimeout:
if fail_ok:
return -1
else:
LOG.error("Failed to send text, logging out.")
stream.sendall("exit".encode('utf-8'))
raise
except Exception as e:
LOG.info("Connection failed with %s.", e)
LOG.error("Failed to send text, logging out.")
stream.sendall("exit".encode('utf-8'))
raise
except Exception as exception:
LOG.info("Connection failed with %s.", exception)
raise
return 0

93
virtualbox/pybox/utils/sftp.py
View File

@ -3,6 +3,10 @@
# SPDX-License-Identifier: Apache-2.0
#
"""
This module provides functionality to send files and directories to remote servers using the
rsync and paramiko libraries.
"""
import getpass
import os
@ -12,10 +16,14 @@ import paramiko
from utils.install_log import LOG
def sftp_send(source, remote_host, remote_port, destination, username, password):
def sftp_send(source, destination, client_dict):
"""
Send files to remote server
"""
remote_host = client_dict["remote_host"]
username = client_dict["username"]
LOG.info("Connecting to server %s with username %s", remote_host, username)
ssh_client = paramiko.SSHClient()
@ -25,12 +33,17 @@ def sftp_send(source, remote_host, remote_port, destination, username, password)
retry = 0
while retry < 8:
try:
ssh_client.connect(remote_host, port=remote_port,
username=username, password=password,
look_for_keys=False, allow_agent=False)
ssh_client.connect(
remote_host,
port=client_dict["remote_port"],
username=username,
password=client_dict["password"],
look_for_keys=False,
allow_agent=False
)
sftp_client = ssh_client.open_sftp()
retry = 8
except Exception as e:
except Exception: # pylint: disable=W0703
LOG.info("******* try again")
retry += 1
time.sleep(10)
@ -42,8 +55,41 @@ def sftp_send(source, remote_host, remote_port, destination, username, password)
ssh_client.close()
def send_dir(source, remote_host, remote_port, destination, username,
password, follow_links=True, clear_known_hosts=True):
# pylint: disable=R0801
def send_dir(params_dict):
"""
Send directory `source` to remote host `remote_host` at port `remote_port` and destination
`destination` using `rsync` over `ssh`.
Args:
params_dict (dict): A dictionary containing the following keys:
- source (str): The local directory to be sent.
- remote_host (str): The IP address or domain name of the remote host.
- remote_port (int): The port number of the remote host to connect to.
- destination (str): The remote directory to copy `source` into.
- username (str): The username for the SSH connection.
- password (str): The password for the SSH connection.
- follow_links (bool, optional): Whether or not to follow symbolic links when
copying files. Default is True.
- clear_known_hosts (bool, optional): Whether or not to clear the known_hosts file
before making the SSH connection. Default is True.
Raises:
Exception: If there is an error in `rsync`, raises an exception with the return code.
Note:
This method only works from a Linux environment.
"""
source = params_dict['source']
remote_host = params_dict['remote_host']
remote_port = params_dict['remote_port']
destination = params_dict['destination']
username = params_dict['username']
password = params_dict['password']
follow_links = params_dict.get('follow_links', True)
clear_known_hosts = params_dict.get('clear_known_hosts', True)
# Only works from linux for now
if not source.endswith('/') or not source.endswith('\\'):
source = source + '/'
@ -51,29 +97,28 @@ def send_dir(source, remote_host, remote_port, destination, username,
follow_links = "L" if follow_links else ""
if clear_known_hosts:
if remote_host == '127.0.0.1':
keygen_arg = "[127.0.0.1]:{}".format(remote_port)
keygen_arg = f"[127.0.0.1]:{remote_port}"
else:
keygen_arg = remote_host
cmd = f'ssh-keygen -f "/home/{getpass.getuser()}/.ssh/known_hosts" -R {keygen_arg}'
LOG.info("CMD: %s", cmd)
process = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE)
for line in iter(process.stdout.readline, b''):
LOG.info("%s", line.decode("utf-8").strip())
process.wait()
with subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE) as process:
for line in iter(process.stdout.readline, b''):
LOG.info("%s", line.decode("utf-8").strip())
process.wait()
LOG.info(f'Running rsync of dir: {source} -> {username}@{remote_host}'
f':{destination}')
LOG.info('Running rsync of dir: %s -> %s@%s:%s', source, username, remote_host, destination)
cmd = (f'rsync -av{follow_links} --rsh="/usr/bin/sshpass -p {password} '
f'ssh -p {remote_port} -o StrictHostKeyChecking=no -l {username}" '
f'{source}* {username}@{remote_host}:{destination}')
LOG.info("CMD: %s", cmd)
process = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE)
for line in iter(process.stdout.readline, b''):
LOG.info("%s", line.decode("utf-8").strip())
process.wait()
with subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE) as process:
for line in iter(process.stdout.readline, b''):
LOG.info("%s", line.decode("utf-8").strip())
process.wait()
if process.returncode:
raise Exception(f'Error in rsync, return code: {process.returncode}')
raise Exception(f"Error in rsync, return code:{process.returncode}") # pylint: disable=E0012, W0719
def send_dir_fallback(source, remote_host, destination, username, password):
@ -87,10 +132,17 @@ def send_dir_fallback(source, remote_host, destination, username, password):
e.g. myhost.com
- destination: where to store the file on host: /home/myuser/
"""
LOG.info("Connecting to server %s with username %s", remote_host, username)
ssh_client = paramiko.SSHClient()
ssh_client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
ssh_client.connect(remote_host, username=username, password=password, look_for_keys=False, allow_agent=False)
ssh_client.connect(
remote_host,
username=username,
password=password,
look_for_keys=False,
allow_agent=False
)
sftp_client = ssh_client.open_sftp()
path = ''
send_img = False
@ -113,4 +165,3 @@ def send_dir_fallback(source, remote_host, destination, username, password):
ssh_client.close()
if send_img:
time.sleep(10)