# pylint: disable=too-many-lines
# !/usr/bin/python3
#
# SPDX-License-Identifier: Apache-2.0
#
"""
This tool is an automated installer to allow users to easily install
StarlingX on VirtualBox.
"""
import subprocess
import getpass
import time
import re
import tempfile
import signal
import sys
from sys import platform
import paramiko
import streamexpect
import ruamel.yaml
from utils import kpi, serial
from utils.install_log import init_logging, get_log_dir, LOG
from utils.sftp import sftp_send, send_dir
from helper import vboxmanage
from helper import install_lab
from helper import host_helper
from consts.node import Nodes
from consts.networking import NICs, OAM, Serial
from consts.timeout import HostTimeout
from Parser import handle_args
# Global vars
V_BOX_OPTIONS = None
def menu_selector(stream, setup_type,
securityprofile, lowlatency, install_mode='serial'):
"""
Select the correct install option.
"""
# Wait for menu to load (add sleep so we can see what is picked)
serial.expect_bytes(stream, "Press")
# Pick install type
if setup_type in [AIO_SX, AIO_DX]:
LOG.info("Selecting All-in-one Install")
serial.send_bytes(stream, "\033[B", expect_prompt=False, send=False)
if lowlatency is True:
LOG.info("Selecting All-in-one (lowlatency) Install")
serial.send_bytes(stream, "\033[B", expect_prompt=False, send=False)
else:
LOG.info("Selecting Controller Install")
serial.send_bytes(stream, "\n", expect_prompt=False, send=False)
time.sleep(4)
# Serial or Graphical menu (picking Serial by default)
if install_mode == "graphical":
LOG.info("Selecting Graphical menu")
serial.send_bytes(stream, "\033[B", expect_prompt=False, send=False)
else:
LOG.info("Selecting Serial menu")
serial.send_bytes(stream, "\n", expect_prompt=False, send=False)
time.sleep(6)
# Security profile menu
if securityprofile == "extended":
LOG.info("Selecting extended security profile")
serial.send_bytes(stream, "\033[B", expect_prompt=False, send=False)
time.sleep(2)
serial.send_bytes(stream, "\n", expect_prompt=False, send=False)
time.sleep(4)
def setup_networking(stream, ctrlr0_ip, gateway_ip, password):
"""
Setup initial networking so we can transfer files.
"""
ip_addr = ctrlr0_ip
interface = "enp0s3"
ret = serial.send_bytes(
stream,
"/sbin/ip address list",
prompt=ctrlr0_ip,
fail_ok=True,
timeout=10)
if ret != 0:
LOG.info("Setting networking up.")
else:
LOG.info("Skipping networking setup")
return
LOG.info("%s being set up with ip %s", interface, ip_addr)
serial.send_bytes(stream,
f"sudo /sbin/ip addr add {ip_addr}/24 dev {interface}",
expect_prompt=False)
host_helper.check_password(stream, password=password)
time.sleep(2)
serial.send_bytes(stream,
f"sudo /sbin/ip link set {interface} up",
expect_prompt=False)
host_helper.check_password(stream, password=password)
time.sleep(2)
serial.send_bytes(stream,
f"sudo route add default gw {gateway_ip}",
expect_prompt=False)
host_helper.check_password(stream, password=password)
if V_BOX_OPTIONS.vboxnet_type == 'hostonly':
LOG.info("Pinging controller-0 at: %s...", ip_addr)
tmout = HostTimeout.NETWORKING_OPERATIONAL
while tmout:
# Ping from machine hosting virtual box to virtual machine
return_code = subprocess.call(['ping', '-c', '1', ip_addr])
if return_code == 0:
break
tmout -= 1
else:
raise ConnectionError(f"Failed to establish connection in {tmout}s " \
"to controller-0 at: {ip_addr}!")
LOG.info("Ping succeeded!")
def fix_networking(stream, release, password):
"""
Vbox/linux bug: Sometimes after resuming a VM networking fails to comes up.
Setting VM interface down then up again fixes it.
"""
if release == "R2":
interface = "eth0"
else:
interface = "enp0s3"
LOG.info("Fixing networking ...")
serial.send_bytes(stream,
f"sudo /sbin/ip link set {interface} down",
expect_prompt=False)
host_helper.check_password(stream, password=password)
time.sleep(1)
serial.send_bytes(
stream,
f"sudo /sbin/ip link set {interface} up",
expect_prompt=False)
host_helper.check_password(stream, password=password)
time.sleep(2)
def install_controller_0(cont0_stream, menu_select_dict, network_dict):
"""
Installs controller-0 node by performing the following steps:
1. Selects setup type, security profile, low latency, and install mode using menu_selector.
2. Expects "login:" prompt in the installation console.
3. Changes the password on initial login.
4. Disables user logout.
5. Sets up basic networking.
Args:
cont0_stream (stream): The installation console stream for controller-0.
menu_select_dict (dict): A dictionary containing the following keys:
- setup_type (str): The type of setup (Simplex, Duplex, etc.).
- securityprofile (str): The security profile (Standard, FIPS, etc.).
- lowlatency (bool): Whether or not to enable low latency.
- install_mode (str): The install mode (standard, patch, etc.).
network_dict (dict): A dictionary containing the following keys:
- ctrlr0_ip (str): The IP address for controller-0.
- gateway_ip (str): The IP address for the gateway.
- username (str, optional): The username for the SSH connection.
- password (str, optional): The password for the SSH connection.
Raises:
Exception: If there is a failure in the installation process.
Note:
The function waits for certain durations between each step.
"""
username = network_dict.get("username")
password = network_dict.get("password")
LOG.info("Starting installation of controller-0")
start_time = time.time()
menu_selector(
cont0_stream,
menu_select_dict["setup_type"],
menu_select_dict["securityprofile"],
menu_select_dict["lowlatency"],
menu_select_dict["install_mode"]
)
try:
serial.expect_bytes(
cont0_stream,
"login:",
timeout=HostTimeout.INSTALL)
except Exception as exception: # pylint: disable=E0012, W0703
LOG.info("Connection failed for controller-0 with %s", exception)
# Sometimes we get UnicodeDecodeError exception due to the output
# of installation. So try one more time maybe
LOG.info("So ignore the exception and wait for controller-0 to be installed again.")
if HostTimeout.INSTALL > (time.time() - start_time):
serial.expect_bytes(
cont0_stream,
"login:",
timeout=HostTimeout.INSTALL - (time.time() - start_time))
LOG.info("Completed installation of controller-0.")
# Change password on initial login
time.sleep(20)
host_helper.change_password(
cont0_stream,
username=username,
password=password)
# Disable user logout
time.sleep(10)
host_helper.disable_logout(cont0_stream)
# Setup basic networking
time.sleep(1)
setup_networking(
cont0_stream,
network_dict["ctrlr0_ip"],
network_dict["gateway_ip"],
password=password
)
def delete_lab(labname, force=False):
"""
This allows for the deletion of an existing lab.
"""
node_list = vboxmanage.get_all_vms(labname, option="vms")
if len(node_list) != 0:
if not force:
LOG.info("This will delete lab %s with vms: %s", labname, node_list)
LOG.info("Continue? (y/N)")
while True:
choice = input().lower()
if choice == 'y':
break
LOG.info("Aborting!")
sys.exit(1)
LOG.info("#### Deleting lab %s.", labname)
LOG.info("VMs in lab: %s.", node_list)
vboxmanage.vboxmanage_controlvms(node_list, "poweroff")
time.sleep(2)
vboxmanage.vboxmanage_deletevms(node_list)
def get_disk_sizes(comma_list):
"""
Return the disk sizes as taken from the command line.
"""
sizes = comma_list.split(',')
for size in sizes:
val = int(size)
if val < 0:
LOG.info("Disk sizes must be a comma separated list of positive integers.")
# pylint: disable=E0012, W0719
raise Exception("Disk sizes must be a comma separated list of positive integers.")
return sizes
def create_port_forward(hostname, network, local_port, guest_port, guest_ip):
"""
Create a port forwarding rule for a NAT network in VirtualBox.
Args:
hostname (str): Name of the virtual machine.
network (str): Name of the NAT network.
local_port (str): The local port number to forward.
guest_port (str): The port number on the guest to forward to.
guest_ip (str): The IP address of the guest to forward to.
Returns:
None
"""
if not vboxmanage.vboxmanage_addportforward(hostname, local_port, guest_ip, guest_port, network):
rule_name = vboxmanage.vboxmanage_getrulename(network, local_port)
if not rule_name:
LOG.info(
"Could not add a port-forwarding rule using port %s, and could not find any rule already using it. Check the Nat Network and/or local port.", local_port)
LOG.info("Aborting!")
sys.exit(1)
LOG.info(
"Trying to create a port-forwarding rule with port: %s, but it is already in use with rule name: %s",
local_port,
rule_name)
LOG.info("Rewrite rule? (y/n)")
choice = input().lower()
if choice == 'y':
vboxmanage.vboxmanage_deleteportforward(rule_name, network)
vboxmanage.vboxmanage_addportforward(hostname, local_port, guest_ip, guest_port, network)
else:
LOG.info("Ignoring the creation of the port-forward rule and continuing installation!")
# pylint: disable=too-many-locals, too-many-branches, too-many-statements
def create_lab(m_vboxoptions):
"""
Creates vms using the arguments in vboxoptions.
"""
# Pull in node configuration
node_config = [getattr(Nodes, attr)
for attr in dir(Nodes) if not attr.startswith('__')]
nic_config = [getattr(NICs, attr)
for attr in dir(NICs) if not attr.startswith('__')]
# oam_config = [getattr(OAM, attr)
# for attr in dir(OAM) if not attr.startswith('__')][0]
serial_config = [getattr(Serial, attr)
for attr in dir(Serial) if not attr.startswith('__')]
# Create nodes list
nodes_list = []
if m_vboxoptions.controllers:
for node_id in range(0, m_vboxoptions.controllers):
node_name = m_vboxoptions.labname + f"-controller-{node_id}"
nodes_list.append(node_name)
if m_vboxoptions.workers:
for node_id in range(0, m_vboxoptions.workers):
node_name = m_vboxoptions.labname + f"-worker-{node_id}"
nodes_list.append(node_name)
if m_vboxoptions.storages:
for node_id in range(0, m_vboxoptions.storages):
node_name = m_vboxoptions.labname + f"-storage-{node_id}"
nodes_list.append(node_name)
LOG.info("#### We will create the following nodes: %s", nodes_list)
port = 10000
# pylint: disable=too-many-nested-blocks
for node in nodes_list:
LOG.info("#### Creating node: %s", node)
vboxmanage.vboxmanage_createvm(node, m_vboxoptions.labname)
vboxmanage.vboxmanage_storagectl(
node,
storectl="sata",
hostiocache=m_vboxoptions.hostiocache)
disk_sizes = None
no_disks = 0
if "controller" in node:
if m_vboxoptions.setup_type in [AIO_DX, AIO_SX]:
node_type = "controller-AIO"
else:
node_type = f"controller-{m_vboxoptions.setup_type}"
if m_vboxoptions.controller_disk_sizes:
disk_sizes = get_disk_sizes(m_vboxoptions.controller_disk_sizes)
else:
no_disks = m_vboxoptions.controller_disks
elif "worker" in node:
node_type = "worker"
if m_vboxoptions.worker_disk_sizes:
disk_sizes = get_disk_sizes(m_vboxoptions.worker_disk_sizes)
else:
no_disks = m_vboxoptions.worker_disks
elif "storage" in node:
node_type = "storage"
if m_vboxoptions.storage_disk_sizes:
disk_sizes = get_disk_sizes(m_vboxoptions.storage_disk_sizes)
else:
no_disks = m_vboxoptions.storage_disks
for item in node_config:
if item['node_type'] == node_type:
vboxmanage.vboxmanage_modifyvm(
node,
{
"cpus": str(item['cpus']),
"memory": str(item['memory']),
},
)
if not disk_sizes:
disk_sizes = item['disks'][no_disks]
vboxmanage.vboxmanage_createmedium(node, disk_sizes,
vbox_home_dir=m_vboxoptions.vbox_home_dir)
if platform in ("win32", "win64"):
vboxmanage.vboxmanage_modifyvm(
node,
{
"uartbase": serial_config[0]['uartbase'],
"uartport": serial_config[0]['uartport'],
"uartmode": serial_config[0]['uartmode'],
"uartpath": port,
},
)
port += 1
else:
vboxmanage.vboxmanage_modifyvm(
node,
{
"uartbase": serial_config[0]['uartbase'],
"uartport": serial_config[0]['uartport'],
"uartmode": serial_config[0]['uartmode'],
"uartpath": serial_config[0]['uartpath'],
"prefix": m_vboxoptions.userid,
},
)
if "controller" in node:
node_type = "controller"
last_adapter = 1
for item in nic_config:
if item['node_type'] == node_type:
for adapter in item.keys():
if adapter.isdigit():
last_adapter += 1
data = item[adapter]
if m_vboxoptions.vboxnet_name != 'none' and data['nic'] == 'hostonly':
if m_vboxoptions.vboxnet_type == 'nat':
data['nic'] = 'natnetwork'
data['natnetwork'] = m_vboxoptions.vboxnet_name
data['hostonlyadapter'] = None
data['intnet'] = None
# data['nicpromisc1'] = None
else:
data[
'hostonlyadapter'] = m_vboxoptions.vboxnet_name
data['natnetwork'] = None
else:
data['natnetwork'] = None
vboxmanage.vboxmanage_modifyvm(
node,
{
"nic": data['nic'],
"nictype": data['nictype'],
"nicpromisc": data['nicpromisc'],
"nicnum": int(adapter),
"intnet": data['intnet'],
"hostonlyadapter": data['hostonlyadapter'],
"natnetwork": data['natnetwork'],
"prefix": f"{m_vboxoptions.userid}-{m_vboxoptions.labname}",
},
)
if m_vboxoptions.add_nat_interface:
last_adapter += 1
vboxmanage.vboxmanage_modifyvm(
node,
{
# "nicnum": adapter, #TODO where this adapter come from? #pylint: disable=fixme
"nictype": 'nat',
},
)
# Add port forwarding rules for controllers nat interfaces
if m_vboxoptions.vboxnet_type == 'nat' and 'controller' in node:
if 'controller-0' in node:
local_port = m_vboxoptions.nat_controller0_local_ssh_port
ip_addr = m_vboxoptions.controller0_ip
# Add port forward rule for StarlingX dashboard visualization at 8080
rule_name = m_vboxoptions.labname + "-horizon-dashbord"
create_port_forward(rule_name,
m_vboxoptions.vboxnet_name,
local_port=m_vboxoptions.horizon_dashboard_port,
guest_port='8080',
guest_ip=ip_addr)
elif 'controller-1' in node:
local_port = m_vboxoptions.nat_controller1_local_ssh_port
ip_addr = m_vboxoptions.controller1_ip
create_port_forward(
node,
m_vboxoptions.vboxnet_name,
local_port=local_port,
guest_port='22',
guest_ip=ip_addr
)
# Floating ip port forwarding
if m_vboxoptions.vboxnet_type == 'nat' and m_vboxoptions.setup_type != 'AIO-SX':
local_port = m_vboxoptions.nat_controller_floating_local_ssh_port
ip_addr = m_vboxoptions.controller_floating_ip
name = m_vboxoptions.labname + 'controller-float'
create_port_forward(name, m_vboxoptions.vboxnet_name,
local_port=local_port, guest_port='22', guest_ip=ip_addr)
ctrlr0 = m_vboxoptions.labname + '-controller-0'
vboxmanage.vboxmanage_storagectl(
ctrlr0,
storectl="ide",
hostiocache=m_vboxoptions.hostiocache)
vboxmanage.vboxmanage_storageattach(
ctrlr0,
{
"storectl": "ide",
"storetype": "dvddrive",
"disk": m_vboxoptions.iso_location,
"port_num": "1",
"device_num": "0",
},
)
def override_ansible_become_pass():
"""
Override the ansible_become_pass value in the localhost.yml
with the password passed via terminal in the python call
"""
file = V_BOX_OPTIONS.ansible_controller_config
new_file = "/tmp/localhost.yml"
#Load Ansible config file
try:
with open(file, encoding="utf-8") as stream:
yaml = ruamel.yaml.YAML()
yaml.preserve_quotes = True
yaml.explicit_start = True
loaded = yaml.load(stream)
except FileNotFoundError:
print(f'\n Ansible configuration file not found in {file} \n')
sys.exit(1)
except ruamel.yaml.YAMLError:
print("\n Error while parsing YAML file \n")
sys.exit(1)
# modify the password with the one passed on the python call
loaded['ansible_become_pass'] = V_BOX_OPTIONS.password
#Save it again
try:
with open(new_file, mode='w', encoding="utf-8") as stream:
yaml.dump(loaded, stream)
except ruamel.yaml.YAMLError as exc:
print(exc)
return new_file
# pylint: disable=W0102
def get_hostnames(ignore=None, personalities=['controller', 'storage', 'worker']):
"""
Based on the number of nodes defined on the command line, construct
the hostnames of each node.
"""
hostnames = {}
if V_BOX_OPTIONS.controllers and 'controller' in personalities:
for node_id in range(0, V_BOX_OPTIONS.controllers):
node_name = V_BOX_OPTIONS.labname + f"-controller-{node_id}"
if ignore and node_name in ignore:
continue
hostnames[node_name] = f"controller-{id}"
if V_BOX_OPTIONS.workers and 'worker' in personalities:
for node_id in range(0, V_BOX_OPTIONS.workers):
node_name = V_BOX_OPTIONS.labname + f"-worker-{node_id}"
if ignore and node_name in ignore:
continue
hostnames[node_name] = f"worker-{id}"
if V_BOX_OPTIONS.storages and 'storage' in personalities:
for node_id in range(0, V_BOX_OPTIONS.storages):
node_name = V_BOX_OPTIONS.labname + f"-storage-{node_id}"
if ignore and node_name in ignore:
continue
hostnames[node_name] = f'storage-{node_id}'
return hostnames
def get_personalities(ignore=None):
"""
Map the target to the node type.
"""
personalities = {}
if V_BOX_OPTIONS.controllers:
for node_id in range(0, V_BOX_OPTIONS.controllers):
node_name = V_BOX_OPTIONS.labname + f"-controller-{node_id}"
if ignore and node_name in ignore:
continue
personalities[node_name] = 'controller'
if V_BOX_OPTIONS.workers:
for node_id in range(0, V_BOX_OPTIONS.workers):
node_name = V_BOX_OPTIONS.labname + f"-worker-{node_id}"
if ignore and node_name in ignore:
continue
personalities[node_name] = 'worker'
if V_BOX_OPTIONS.storages:
for node_id in range(0, V_BOX_OPTIONS.storages):
node_name = V_BOX_OPTIONS.labname + f"-storage-{node_id}"
if ignore and node_name in ignore:
continue
personalities[node_name] = 'storage'
return personalities
def create_host_bulk_add():
"""
Sample xml:
<?xml version="1.0" encoding="UTF-8" ?>
<hosts>
<host>
<personality>controller</personality>
<mgmt_mac>08:00:27:4B:6A:6A</mgmt_mac>
</host>
<host>
<personality>storage</personality>
<mgmt_mac>08:00:27:36:14:3D</mgmt_mac>
</host>
<host>
<personality>storage</personality>
<mgmt_mac>08:00:27:B3:D0:69</mgmt_mac>
</host>
<host>
<hostname>worker-0</hostname>
<personality>worker</personality>
<mgmt_mac>08:00:27:47:68:52</mgmt_mac>
</host>
<host>
<hostname>worker-1</hostname>
<personality>worker</personality>
<mgmt_mac>08:00:27:31:15:48</mgmt_mac>
</host>
</hosts>
"""
LOG.info("Creating content for 'system host-bulk-add'")
vms = vboxmanage.get_all_vms(V_BOX_OPTIONS.labname, option="vms")
ctrl0 = V_BOX_OPTIONS.labname + "-controller-0"
vms.remove(ctrl0)
# Get management macs
macs = {}
for virtual_machine in vms:
info = vboxmanage.vboxmanage_showinfo(virtual_machine).splitlines()
for line in info:
try:
key, value = line.split(b'=')
except ValueError:
continue
if key == b'macaddress2':
orig_mac = value.decode('utf-8').replace("\"", "")
# Do for e.g.: 080027C95571 -> 08:00:27:C9:55:71
macs[virtual_machine] = ":".join(re.findall(r"..", orig_mac))
# Get personalities
personalities = get_personalities(ignore=[ctrl0])
hostnames = get_hostnames(ignore=[ctrl0])
# Create file
host_xml = ('<?xml version="1.0" encoding="UTF-8" ?>\n'
'<hosts>\n')
for virtual_machine in vms:
host_xml += ' <host>\n'
host_xml += f' <hostname>{hostnames[virtual_machine]}</hostname>\n'
host_xml += f' <personality>{personalities[virtual_machine]}</personality>\n'
host_xml += f' <mgmt_mac>{macs[virtual_machine]}</mgmt_mac>\n'
host_xml += ' </host>\n'
host_xml += '</hosts>\n'
return host_xml
# serial_prompt_configured = False
def wait_for_hosts(ssh_client, hostnames, status,
timeout=HostTimeout.HOST_INSTALL, interval=20):
"""
Wait for a given interval for the host(s) to reach the expected
status.
"""
start_time = time.time()
while hostnames:
LOG.info("Hosts not %s: %s", status, hostnames)
if (time.time() - start_time) > HostTimeout.HOST_INSTALL:
LOG.info("VMs not booted in %s, aborting: %s", timeout, hostnames)
raise Exception(f"VMs failed to go {status}!") # pylint: disable=E0012, W0719
# Get host list
host_statuses, _, _ = run_ssh_cmd(
ssh_client, 'source /etc/platform/openrc; system host-list', timeout=30)
host_statuses = host_statuses[1:-1]
for host_status in host_statuses:
for host in hostnames:
if host in host_status and status in host_status:
hostnames.remove(host)
if hostnames:
LOG.info("Waiting %s sec before re-checking host status.", interval)
time.sleep(interval)
CONSOLE_UNKNOWN_MODE = 'disconnected'
CONSOLE_USER_MODE = 'user'
CONSOLE_ROOT_MODE = 'root'
SERIAL_CONSOLE_MODE = CONSOLE_UNKNOWN_MODE
def run_ssh_cmd(ssh_client, cmd, timeout=5,
log_output=True, mode=CONSOLE_USER_MODE):
"""
Execute an arbitrary command on a target.
"""
if mode == CONSOLE_ROOT_MODE:
LOG.info(">>>>>")
cmd = f"sudo {cmd}"
LOG.info("#### Running command over ssh: '%s'", cmd)
stdin, stdout, stderr = ssh_client.exec_command(cmd, timeout, get_pty=True)
if mode == CONSOLE_ROOT_MODE:
stdin.write(f'{V_BOX_OPTIONS.password}\n')
stdin.flush()
stdout_lines = []
while True:
if stdout.channel.exit_status_ready():
break
stdout_lines.append(stdout.readline().rstrip('\n'))
if log_output and stdout:
LOG.info("|%s", stdout_lines[-1])
stderr_lines = stderr.readlines()
if log_output and stderr_lines:
LOG.info("stderr:|\n%s", "".join(stderr_lines))
return_code = stdout.channel.recv_exit_status()
LOG.info("Return code: %s", return_code)
if mode == CONSOLE_ROOT_MODE:
# Cut sudo's password echo and "Password:" string from output
stdout_lines = stdout_lines[2:]
return stdout_lines, stderr_lines, return_code
def set_serial_prompt_mode(stream, mode):
"""
To make sure that we are at the correct prompt,
we first logout, then login back again.
Note that logging out also helps fixing some problems with passwords
not getting accepted in some cases (prompt just hangs after inserting
password).
"""
global SERIAL_CONSOLE_MODE # pylint: disable=global-statement
if SERIAL_CONSOLE_MODE == mode:
LOG.info("Serial console prompt already set to '%s' mode.", mode)
return
if SERIAL_CONSOLE_MODE != CONSOLE_USER_MODE:
# Set mode to user first, even if we later go to root
serial.send_bytes(stream, "exit\n", expect_prompt=False)
if serial.expect_bytes(stream, "ogin:", fail_ok=True, timeout=4):
serial.send_bytes(stream, "exit\n", expect_prompt=False)
if serial.expect_bytes(stream, "ogin:", fail_ok=True, timeout=4):
LOG.info("Expected login prompt, connect to console" \
"stop any running processes and log out.")
raise Exception("Failure getting login prompt on serial console!") # pylint: disable=E0012, W0719
serial.send_bytes(
stream,
V_BOX_OPTIONS.username,
prompt="assword:",
timeout=30)
if serial.send_bytes(stream, V_BOX_OPTIONS.password, prompt="~$", fail_ok=True, timeout=30):
raise Exception("Login failure, invalid password?") # pylint: disable=E0012, W0719
if mode == CONSOLE_USER_MODE:
serial.send_bytes(stream, "source /etc/platform/openrc\n",
timeout=30, prompt='keystone')
SERIAL_CONSOLE_MODE = CONSOLE_USER_MODE
if mode == 'root' and SERIAL_CONSOLE_MODE != 'root':
serial.send_bytes(stream, 'sudo su -', expect_prompt=False)
host_helper.check_password(stream, password=V_BOX_OPTIONS.password)
serial.send_bytes(
stream,
"cd /home/wrsroot",
prompt="/home/wrsroot# ",
timeout=30)
serial.send_bytes(stream, "source /etc/platform/openrc\n",
timeout=30, prompt='keystone')
SERIAL_CONSOLE_MODE = CONSOLE_ROOT_MODE
serial.send_bytes(stream, "export TMOUT=0", timeout=10, prompt='keystone')
# also reset OAM networking?
def serial_prompt_mode(mode):
"""
A decorator function that sets the serial console login prompt to the specified
mode before calling the decorated function.
Args:
mode (str): The login prompt mode to set. Valid values are "admin" and "root".
Returns:
function: A decorator function that sets the serial console login prompt to the specified mode.
"""
def real_decorator(func):
def func_wrapper(*args, **kwargs):
try:
set_serial_prompt_mode(kwargs['stream'], mode)
except: # pylint: disable=bare-except
LOG.info("Serial console login as '%s' failed. Retrying once.", mode)
set_serial_prompt_mode(kwargs['stream'], mode)
return func(*args, **kwargs)
return func_wrapper
return real_decorator
def _connect_to_serial(virtual_machine=None):
if not virtual_machine:
virtual_machine = V_BOX_OPTIONS.labname + "-controller-0"
sock = serial.connect(virtual_machine, 10000, getpass.getuser())
return sock, streamexpect.wrap(sock, echo=True, close_stream=False)
def connect_to_serial(func):
"""
A decorator function that establishes a connection to the serial console before
calling the decorated function.
Args:
func (function): The function to be decorated.
Returns:
function: A wrapper function that establishes a connection to the serial console,
calls the decorated function, and then disconnects from the serial console.
"""
def func_wrapper(*args, **kwargs):
sock = None
try:
sock, kwargs['stream'] = _connect_to_serial()
return func(*args, **kwargs)
finally:
serial.disconnect(sock)
return func_wrapper
def _connect_to_ssh():
# Get ip and port for ssh on floating ip
ip_addr, port = get_ssh_ip_and_port()
# Remove ssh key
# For hostonly adapter we remove port 22 of controller ip
# for nat interfaces we remove the specific port on 127.0.0.1 as
# we have port forwarding enabled.
# pylint: disable=R0801
if V_BOX_OPTIONS.vboxnet_type == 'nat':
keygen_arg = f"[127.0.0.1]:{port}"
else:
keygen_arg = ip_addr
cmd = f'ssh-keygen -f "/home/{getpass.getuser()}/.ssh/known_hosts" -R {keygen_arg}'
LOG.info("CMD: %s", cmd)
with subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE) as process:
for line in iter(process.stdout.readline, b''):
LOG.info("%s", line.decode("utf-8").strip())
process.wait()
# Connect to ssh
ssh = paramiko.SSHClient()
ssh.load_system_host_keys()
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
ssh.connect(ip_addr, port=port, username=V_BOX_OPTIONS.username,
password=V_BOX_OPTIONS.password, look_for_keys=False, allow_agent=False)
return ssh
def connect_to_ssh(func):
"""
Decorator function to establish a SSH connection before executing the function
and close the connection afterwards.
Args:
- func: The function to be decorated.
Returns:
- The decorated function that has a SSH connection established before executing the function.
"""
def func_wrapper(*args, **kwargs):
try:
ssh = _connect_to_ssh()
kwargs['ssh_client'] = ssh
return func(*args, **kwargs)
finally:
ssh.close()
return func_wrapper
def stage_test_success():
"""Prints a log message indicating the execution of a test stage."""
LOG.info("Executing stage_test_success")
def stage_test_fail():
"""
Prints a log message indicating the execution of a test stage and raises an exception.
Raises:
- Exception: Always raises an exception.
"""
LOG.info("Executing stage_test_success")
raise Exception("exception as of stage_test_fail") # pylint: disable=E0012, W0719
def stage_create_lab():
"""
Wrapper function for deleting an existing virtual lab and creating a new one
using `vboxoptions`.
"""
delete_lab(V_BOX_OPTIONS.labname, V_BOX_OPTIONS.force_delete_lab)
create_lab(V_BOX_OPTIONS)
# time.sleep(2)
def stage_install_controller0():
"""
Starts the `controller-0` VM, establishes a serial connection to it, and installs
the OS on it using the `install_controller_0` function with the parameters specified
in `vboxoptions`.
Args:
- None
Raises:
- AssertionError: If `controller-0` is not in the list of available VMs.
"""
node_list = vboxmanage.get_all_vms(V_BOX_OPTIONS.labname, option="vms")
LOG.info("Found nodes: %s", node_list)
ctrlr0 = V_BOX_OPTIONS.labname + "-controller-0"
assert ctrlr0 in node_list, "controller-0 not in vm list. Stopping installation."
vboxmanage.vboxmanage_startvm(ctrlr0, V_BOX_OPTIONS.headless)
sock = serial.connect(ctrlr0, 10000, getpass.getuser())
cont0_stream = streamexpect.wrap(sock, echo=True, close_stream=False)
install_controller_0(
cont0_stream,
menu_select_dict={
"setup_type": V_BOX_OPTIONS.setup_type,
"securityprofile": V_BOX_OPTIONS.securityprofile,
"lowlatency": V_BOX_OPTIONS.lowlatency,
"install_mode": V_BOX_OPTIONS.install_mode,
},
network_dict={
"ctrlr0_ip": V_BOX_OPTIONS.controller0_ip,
"gateway_ip": V_BOX_OPTIONS.vboxnet_ip,
"username": V_BOX_OPTIONS.username,
"password": V_BOX_OPTIONS.password
}
)
serial.disconnect(sock)
time.sleep(5)
@connect_to_serial
def stage_config_controller(stream): # pylint: disable=too-many-locals
"""
Stage to configure controller-0 networking settings and upload the configuration
file to the controller.
Args:
stream (obj): Serial console stream.
Raises:
Exception: If there is an error in the configuration or upload process,
raises an exception with the error message.
Note:
This method assumes that the controller-0 virtual machine has been previously
installed and that its serial console stream is open.
"""
ip_addr, port = get_ssh_ip_and_port(
'controller-0') # Floating ip is not yet configured
#Update localhost.yml with system password
new_config_ansible = override_ansible_become_pass()
#Send Ansible configuration file to VM
LOG.info("Copying Ansible configuration file")
destination_ansible = f'/home/{V_BOX_OPTIONS.username}/localhost.yml'
sftp_send(
new_config_ansible,
destination_ansible,
{
"remote_host": ip_addr,
"remote_port": port,
"username": V_BOX_OPTIONS.username,
"password": V_BOX_OPTIONS.password
}
)
# Run config_controller
LOG.info("#### Running config_controller")
install_lab.config_controller(stream, V_BOX_OPTIONS.password)
# Wait for services to stabilize
time.sleep(120)
if V_BOX_OPTIONS.setup_type == AIO_SX:
# Increase AIO responsiveness by allocating more cores to platform
install_lab.update_platform_cpus(stream, 'controller-0')
def get_ssh_ip_and_port(node='floating'):
"""
This function returns the IP address and port of the specified node to use for
an SSH connection.
Args:
node (str, optional): The node to get the IP address and port for.
Valid values are "floating" (default), "controller-0", and "controller-1".
Returns:
tuple: A tuple containing the IP address and port of the specified node.
Raises:
Exception: If an undefined node is specified.
"""
if V_BOX_OPTIONS.vboxnet_type == 'nat':
ip_addr = '127.0.0.1'
if node == 'floating':
if V_BOX_OPTIONS.setup_type != 'AIO-SX':
port = V_BOX_OPTIONS.nat_controller_floating_local_ssh_port
else:
port = V_BOX_OPTIONS.nat_controller0_local_ssh_port
elif node == 'controller-0':
port = V_BOX_OPTIONS.nat_controller0_local_ssh_port
elif node == 'controller-1':
port = V_BOX_OPTIONS.nat_controller_1_local_ssh_port
else:
raise Exception(f"Undefined node '{node}'") # pylint: disable=E0012, W0719
else:
if node == 'floating':
if V_BOX_OPTIONS.setup_type != 'AIO-SX':
ip_addr = V_BOX_OPTIONS.controller_floating_ip
else:
ip_addr = V_BOX_OPTIONS.controller0_ip
elif node == 'controller-0':
ip_addr = V_BOX_OPTIONS.controller0_ip
elif node == 'controller-1':
ip_addr = V_BOX_OPTIONS.controller1_ip
else:
raise Exception(f"Undefined node '{node}'") # pylint: disable=E0012, W0719
port = 22
return ip_addr, port
# @connect_to_serial
# @serial_prompt_mode(CONSOLE_USER_MODE)
def stage_rsync_config():
"""
Rsync the local configuration files with the remote host's configuration files.
This method copies the configuration files to the controller. It uses rsync to
synchronize the local configuration files with the remote host's configuration files.
If the `config_files_dir` or `config_files_dir_dont_follow_links` option is set, this
method copies the files to the remote host. If both are not set, then this method does
nothing.
Args:
None.
Returns:
None.
"""
if not V_BOX_OPTIONS.config_files_dir and not V_BOX_OPTIONS.config_files_dir_dont_follow_links:
LOG.info("No rsync done! Please set config-files-dir "
"and/or config-files-dir-dont-follow-links")
return
# Get ip and port for ssh on floating ip
ip_addr, port = get_ssh_ip_and_port()
# Copy config files to controller
if V_BOX_OPTIONS.config_files_dir:
local_path = V_BOX_OPTIONS.config_files_dir
follow_links = True
send_dir(
{
"source": local_path,
"remote_host": ip_addr,
"remote_port": port,
"destination": '/home/' + V_BOX_OPTIONS.username + '/',
"username": V_BOX_OPTIONS.username,
"password": V_BOX_OPTIONS.password,
"follow_links": follow_links
}
)
if V_BOX_OPTIONS.config_files_dir_dont_follow_links:
local_path = V_BOX_OPTIONS.config_files_dir_dont_follow_links
follow_links = False
send_dir(
{
"source": local_path,
"remote_host": ip_addr,
"remote_port": port,
"destination": '/home/' + V_BOX_OPTIONS.username + '/',
"username": V_BOX_OPTIONS.username,
"password": V_BOX_OPTIONS.password,
"follow_links": follow_links
}
)
@connect_to_serial
@serial_prompt_mode(CONSOLE_USER_MODE)
def _run_lab_setup_serial(stream):
conf_str = ""
for cfg_file in V_BOX_OPTIONS.lab_setup_conf:
conf_str = conf_str + f" -f {cfg_file}"
serial.send_bytes(stream, f"sh lab_setup.sh {conf_str}",
timeout=HostTimeout.LAB_INSTALL, prompt='keystone')
LOG.info("Lab setup execution completed. Checking if return code is 0.")
serial.send_bytes(stream, "echo \"Return code: [$?]\"",
timeout=3, prompt='Return code: [0]')
@connect_to_ssh
def _run_lab_setup(m_stage, ssh_client):
conf_str = ""
for cfg_file in V_BOX_OPTIONS.lab_setup_conf:
conf_str = conf_str + f" -f {cfg_file}"
command = f'source /etc/platform/openrc; export ' \
f'PATH="$PATH:/usr/local/bin"; export PATH="$PATH:/usr/bin"; ' \
f'export PATH="$PATH:/usr/local/sbin"; export ' \
f'PATH="$PATH:/usr/sbin"; sh lab_setup{m_stage}.sh'
_, _, exitcode = run_ssh_cmd(ssh_client, command, timeout=HostTimeout.LAB_INSTALL)
if exitcode != 0:
msg = f"Lab setup failed, expecting exit code of 0 but got {exitcode}."
LOG.info(msg)
raise Exception(msg) # pylint: disable=E0012, W0719
def stage_lab_setup1():
"""Calls _run_lab_setup with ssh_client 1"""
_run_lab_setup(1) # pylint: disable=no-value-for-parameter
def stage_lab_setup2():
"""Calls _run_lab_setup with ssh_client 2"""
_run_lab_setup(2) # pylint: disable=no-value-for-parameter
def stage_lab_setup3():
"""Calls _run_lab_setup with ssh_client 3"""
_run_lab_setup(3) # pylint: disable=no-value-for-parameter
def stage_lab_setup4():
"""Calls _run_lab_setup with ssh_client 4"""
_run_lab_setup(4) # pylint: disable=no-value-for-parameter
def stage_lab_setup5():
"""Calls _run_lab_setup with ssh_client 5"""
_run_lab_setup(5) # pylint: disable=no-value-for-parameter
@connect_to_ssh
@connect_to_serial
def stage_unlock_controller0(stream, ssh_client):
"""
Unlocks the controller-0 node and waits for it to reboot.
Args:
- stream (obj): Serial stream to send and receive data
- ssh_client (obj): SSH client connection to execute remote commands
Returns:
None.
"""
LOG.info("#### Unlocking controller-0")
_, _, _ = run_ssh_cmd(ssh_client,
'source /etc/platform/openrc; system host-unlock controller-0',
timeout=HostTimeout.CONTROLLER_UNLOCK)
LOG.info("#### Waiting for controller-0 to reboot")
serial.expect_bytes(
stream,
'login:',
timeout=HostTimeout.CONTROLLER_UNLOCK)
LOG.info("Waiting 120s for services to activate.")
time.sleep(120)
# Make sure we login again, after reboot we are not logged in.
SERIAL_CONSOLE_MODE = CONSOLE_UNKNOWN_MODE # pylint: disable=redefined-outer-name, invalid-name, unused-variable
@connect_to_serial
@serial_prompt_mode(CONSOLE_USER_MODE)
def stage_unlock_controller0_serial(stream):
"""
Unlock the controller-0 host via serial console and wait for services to activate.
Args:
- stream (stream object): The serial console stream.
Returns:
None.
"""
global SERIAL_CONSOLE_MODE # pylint: disable=global-statement
if host_helper.unlock_host(stream, 'controller-0'):
LOG.info("Host is unlocked, nothing to do. Exiting stage.")
return
serial.expect_bytes(
stream,
'login:',
timeout=HostTimeout.CONTROLLER_UNLOCK)
LOG.info("Waiting 120s for services to activate.")
time.sleep(120)
# Make sure we login again
SERIAL_CONSOLE_MODE = CONSOLE_UNKNOWN_MODE # After reboot we are not logged in.
@connect_to_ssh
def stage_install_nodes(ssh_client):
"""
Install nodes in the environment using SSH.
Args:
- ssh_client (paramiko SSH client object): The SSH client to use for connecting
to the environment.
Returns:
None.
"""
# Create and transfer host_bulk_add.xml to ctrl-0
host_xml = create_host_bulk_add()
LOG.info("host_bulk_add.xml content:\n%s", host_xml)
# Send file to controller
destination = "/home/" + V_BOX_OPTIONS.username + "/host_bulk_add.xml"
with tempfile.NamedTemporaryFile() as file:
file.write(host_xml.encode('utf-8'))
file.flush()
# Connection to NAT interfaces is local
if V_BOX_OPTIONS.vboxnet_type == 'nat':
ip_addr = '127.0.0.1'
port = V_BOX_OPTIONS.nat_controller0_local_ssh_port
else:
ip_addr = V_BOX_OPTIONS.controller0_ip
port = 22
sftp_send(
file.name,
destination,
{
"remote_host": ip_addr,
"remote_port": port,
"username": V_BOX_OPTIONS.username,
"password": V_BOX_OPTIONS.password
}
)
# Apply host-bulk-add
_, _, exitcode = run_ssh_cmd(ssh_client,
f'source /etc/platform/openrc; system host-bulk-add {destination}',
timeout=60)
if exitcode != 0:
msg = "Host bulk add failed, expecting exit code of 0 but got %s", exitcode
LOG.info(msg)
raise Exception(msg) # pylint: disable=E0012, W0719
# Start hosts one by one, wait 10s between each start
vms = vboxmanage.get_all_vms(V_BOX_OPTIONS.labname, option="vms")
runningvms = vboxmanage.get_all_vms(
V_BOX_OPTIONS.labname,
option="runningvms")
powered_off = list(set(vms) - set(runningvms))
LOG.info("#### Powered off VMs: %s", powered_off)
for virtual_machine in powered_off:
LOG.info("#### Powering on VM: %s", virtual_machine)
vboxmanage.vboxmanage_startvm(virtual_machine, V_BOX_OPTIONS.headless, force=True)
LOG.info("Give VM 20s to boot.")
time.sleep(20)
ctrl0 = V_BOX_OPTIONS.labname + "-controller-0"
hostnames = list(get_hostnames(ignore=[ctrl0]).values())
wait_for_hosts(ssh_client, hostnames, 'online')
@connect_to_ssh
def stage_unlock_controller1(ssh_client):
"""
Unlock controller-1 host via SSH.
Args:
- ssh_client (paramiko SSH client object): The SSH client to use for
connecting to the environment.
Returns:
None.
"""
# Fast for standard, wait for storage
hostnames = list(get_hostnames().values())
if 'controller-1' not in hostnames:
LOG.info("Controller-1 not configured, skipping unlock.")
return
LOG.info("#### Unlocking controller-1")
run_ssh_cmd(ssh_client,
'source /etc/platform/openrc; system host-unlock controller-1',
timeout=60)
LOG.info("#### waiting for controller-1 to be available.")
wait_for_hosts(ssh_client, ['controller-1'], 'available')
@connect_to_ssh
def stage_unlock_storages(ssh_client):
"""
Unlock storage nodes via SSH.
Args:
- ssh_client (paramiko SSH client object): The SSH client to use for
connecting to the environment.
Returns:
None.
"""
# Unlock storage nodes, wait for them to be 'available'
storages = list(get_hostnames(personalities=['storage']).values())
for storage in storages:
run_ssh_cmd(ssh_client,
f'source /etc/platform/openrc; system host-unlock {storage}',
timeout=60)
LOG.info("Waiting 15s before next unlock")
time.sleep(15)
LOG.info("#### Waiting for all hosts to be available.")
wait_for_hosts(ssh_client, storages, 'available')
@connect_to_ssh
def stage_unlock_workers(ssh_client):
"""
Unlock worker nodes via SSH.
Args:
- ssh_client (paramiko SSH client object): The SSH client to use for
connecting to the environment.
Returns:
None.
"""
# Unlock all, wait for all hosts, except ctrl0 to be 'available'
workers = list(get_hostnames(personalities=['worker']).values())
ctrl0 = V_BOX_OPTIONS.labname + '-controller-0'
for worker in workers:
run_ssh_cmd(
ssh_client,
f'source /etc/platform/openrc; system host-unlock {worker}',
timeout=60)
LOG.info("Waiting 15s before next unlock")
time.sleep(15)
# Wait for all hosts, except ctrl0 to be available
# At this stage we expect ctrl1 to also be available
hosts = list(get_hostnames(ignore=[ctrl0]).values())
wait_for_hosts(ssh_client, hosts, 'available')
@connect_to_ssh
def stage_enable_kubernetes(ssh_client):
ip_addr, port = get_ssh_ip_and_port()
local_path = V_BOX_OPTIONS.kubernetes_config_files
send_dir(
{
"source": local_path,
"remote_host": ip_addr,
"remote_port": port,
"destination":'/home/' + V_BOX_OPTIONS.username + '/',
"username": V_BOX_OPTIONS.username, "password": V_BOX_OPTIONS.password
}
)
LOG.info("###### Adding port-forward rule for kubernetes dashboard ######")
# Add port forward rule for Kubernetes dashboard visualization at 32000
ip_addr = V_BOX_OPTIONS.controller0_ip
rule_name = V_BOX_OPTIONS.labname + "-kubernetes-dasboard"
create_port_forward(rule_name,
V_BOX_OPTIONS.vboxnet_name,
local_port=V_BOX_OPTIONS.kubernetes_dashboard_port,
guest_port='32000',
guest_ip=ip_addr)
LOG.info("###### Installing Kubernetes dashboard ######")
_, _, exitcode = run_ssh_cmd(ssh_client,
'source /etc/platform/openrc && '
'source /etc/profile && '
'cp /etc/kubernetes/admin.conf ~/.kube/config && '
'helm repo update; helm repo add kubernetes-dashboard https://kubernetes.github.io/dashboard/ && '
'helm install kubernetes-dashboard kubernetes-dashboard/kubernetes-dashboard -f dashboard-values.yaml', timeout=60)
if exitcode == 0:
LOG.info("###### Creating an admin-user service account with cluster-admin provileges ######")
_, _, exitcode2 = run_ssh_cmd(ssh_client,
'kubectl apply -f admin-login.yaml && kubectl -n kube-system '
'describe secret $(kubectl get secret | grep admin-user-sa-token | awk "{print $1}") | tee $HOME/token.txt', timeout=60)
if exitcode2 == 0:
send_token()
LOG.info('##### TOKEN CREATED AND FILE CONTAINING TOKEN SENT TO HOST AT /home/%s #####', getpass.getuser())
if exitcode != 0 or exitcode2 != 0:
msg = f'Installation of Kubernetes dashboard failed, expecting exit code of 0 but got {exitcode}.'
LOG.info(msg)
raise Exception(msg)
def send_token():
LOG.info('###### Sending token.txt to /home/%s ######', getpass.getuser())
ip_addr, port = get_ssh_ip_and_port()
username =V_BOX_OPTIONS.username
password = V_BOX_OPTIONS.password
source = f'/home/{username}/token.txt'
destination = f'/home/{getpass.getuser()}'
# Send token file to HOME/Desktop using rsync
LOG.info("###### rsync command ######")
cmd = (f'rsync -avL --rsh="/usr/bin/sshpass -p {password} '
f'ssh -p {port} -o StrictHostKeyChecking=no -l {username}" '
f'{username}@{ip_addr}:{source}* {destination}')
LOG.info('CMD: %s', cmd)
with subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE) as process:
for line in iter(process.stdout.readline, b''):
LOG.info("%s", line.decode("utf-8").strip())
process.wait()
if process.returncode:
raise Exception(f'Error in rsync, return code: {process.returncode}')
def run_custom_script(script, timeout, console, mode):
"""
Run a custom script on the environment.
Args:
- script (str): The name of the script to run.
- timeout (int): The timeout for the script.
- console (str): The console to use for running the script.
- mode (str): The mode to use for running the script.
Returns:
None.
"""
LOG.info("#### Running custom script %s with options:", script)
LOG.info(" timeout: %s", timeout)
LOG.info(" console mode: %s", console)
LOG.info(" user mode: %s", mode)
if console == 'ssh':
ssh_client = _connect_to_ssh()
# pylint: disable=W0703, C0103
_, __, return_code = run_ssh_cmd(ssh_client, f"./{script}", timeout=timeout, mode=mode)
if return_code != 0:
LOG.info("Custom script '%s' return code is not 0. Aborting.", script)
raise Exception(f"Script execution failed with return code: {return_code}") # pylint: disable=E0012, W0719
else:
sock, stream = _connect_to_serial()
try:
if mode == 'root':
set_serial_prompt_mode(stream, CONSOLE_ROOT_MODE)
# Login as root
serial.send_bytes(stream, 'sudo su -', expect_prompt=False)
host_helper.check_password(
stream,
password=V_BOX_OPTIONS.password)
else:
set_serial_prompt_mode(stream, CONSOLE_USER_MODE)
serial.send_bytes(stream, f"./{script}",
timeout=timeout, prompt='keystone')
LOG.info("Script execution completed. Checking if return code is 0.")
serial.send_bytes(stream,
f"echo 'Return code: [{script}]'",
timeout=3, prompt='Return code: [0]')
finally:
sock.close()
def get_custom_script_options(options_list):
"""
Parse options for a custom script.
Args:
- options_list (str): The list of options for the script.
Returns:
A tuple containing the script name, timeout, console, and mode.
"""
LOG.info("Parsing custom script options: %s", options_list)
# defaults
script = ""
timeout = 5
console = 'serial'
mode = 'user'
# supported options
consoles = ['serial', 'ssh']
modes = ['user', 'root']
# No spaces or special chars allowed
not_allowed = ['\n', ' ', '*']
for char in not_allowed:
if char in options_list:
LOG.info("Char '%s' not allowed in options list: %s.", char, options_list)
raise Exception("Char not allowed in options_list") # pylint: disable=E0012, W0719
# get options
options = options_list.split(',')
if len(options) >= 1:
script = options[0]
if len(options) >= 2:
timeout = int(options[1])
if len(options) >= 3:
console = options[2]
if console not in consoles:
raise f"Console must be one of {consoles}, not {console}."
if len(options) >= 4:
mode = options[3]
if mode not in modes:
raise f"Mode must be one of {modes}, not {mode}."
return script, timeout, console, mode
def stage_custom_script1():
"""
Run the first custom script.
Returns:
None.
"""
if V_BOX_OPTIONS.script1:
script, timeout, console, mode = get_custom_script_options(
V_BOX_OPTIONS.script1)
else:
script = "custom_script1.sh"
timeout = 3600
console = 'serial'
mode = 'user'
run_custom_script(script, timeout, console, mode)
def stage_custom_script2():
"""
Run the second custom script.
Returns:
None.
"""
if V_BOX_OPTIONS.script2:
script, timeout, console, mode = get_custom_script_options(
V_BOX_OPTIONS.script2)
else:
script = "custom_script2.sh"
timeout = 3600
console = 'serial'
mode = 'user'
run_custom_script(script, timeout, console, mode)
def stage_custom_script3():
"""
Run the third custom script.
Returns:
None.
"""
if V_BOX_OPTIONS.script3:
script, timeout, console, mode = get_custom_script_options(
V_BOX_OPTIONS.script3)
else:
script = "custom_script3.sh"
timeout = 3600
console = 'serial'
mode = 'user'
run_custom_script(script, timeout, console, mode)
def stage_custom_script4():
"""
Run the fourth custom script.
Returns:
None.
"""
if V_BOX_OPTIONS.script4:
script, timeout, console, mode = get_custom_script_options(
V_BOX_OPTIONS.script4)
else:
script = "custom_script4.sh"
timeout = 3600
console = 'serial'
mode = 'user'
run_custom_script(script, timeout, console, mode)
def stage_custom_script5():
"""
Run the fifth custom script.
Returns:
None.
"""
if V_BOX_OPTIONS.script5:
script, timeout, console, mode = get_custom_script_options(
V_BOX_OPTIONS.script5)
else:
script = "custom_script5.sh"
timeout = 3600
console = 'serial'
mode = 'user'
run_custom_script(script, timeout, console, mode)
STG_CREATE_LAB = "create-lab"
STG_INSTALL_CONTROLLER0 = "install-controller-0"
STG_CONFIG_CONTROLLER = "config-controller"
STG_RSYNC_CONFIG = "rsync-config"
STG_LAB_SETUP1 = "lab-setup1"
STG_UNLOCK_CONTROLLER0 = "unlock-controller-0"
STG_LAB_SETUP2 = "lab-setup2"
STG_INSTALL_NODES = "install-nodes"
STG_UNLOCK_CONTROLLER1 = "unlock-controller-1"
STG_LAB_SETUP3 = "lab-setup3"
STG_UNLOCK_STORAGES = "unlock-storages"
STG_LAB_SETUP4 = "lab-setup4"
STG_UNLOCK_WORKERS = "unlock-workers"
STG_LAB_SETUP5 = "lab-setup5"
STG_ENABLE_KUBERNETES = "enable-kubernetes"
STG_CUSTOM_SCRIPT1 = "custom-script1"
STG_CUSTOM_SCRIPT2 = "custom-script2"
STG_CUSTOM_SCRIPT3 = "custom-script3"
STG_CUSTOM_SCRIPT4 = "custom-script4"
STG_CUSTOM_SCRIPT5 = "custom-script5"
# For internal testing only, one stage is always successful
# the other one always raises an exception.
STC_TEST_SUCCESS = "test-success"
STG_TEST_FAIL = "test-fail"
CALLBACK = 'callback'
HELP = 'help'
STAGE_CALLBACKS = {
STG_CREATE_LAB:
{CALLBACK: stage_create_lab,
HELP: "Create VMs in vbox: controller-0, controller-1..."},
STG_INSTALL_CONTROLLER0:
{CALLBACK: stage_install_controller0,
HELP: "Install controller-0 from --iso-location"},
STG_CONFIG_CONTROLLER:
{CALLBACK: stage_config_controller,
HELP: "Run config controller using the --ansible-controller-config" \
"updated based on --ini-* options."},
STG_RSYNC_CONFIG:
{CALLBACK: stage_rsync_config,
HELP: "Rsync all files from --config-files-dir and --config-files-dir* to /home/wrsroot."},
STG_LAB_SETUP1:
{CALLBACK: stage_lab_setup1,
HELP: "Run lab_setup with one or more --lab-setup-conf files from controller-0."},
STG_UNLOCK_CONTROLLER0:
{CALLBACK: stage_unlock_controller0,
HELP: "Unlock controller-0 and wait for it to reboot."},
STG_LAB_SETUP2:
{CALLBACK: stage_lab_setup2,
HELP: "Run lab_setup with one or more --lab-setup-conf files from controller-0."},
STG_INSTALL_NODES:
{CALLBACK: stage_install_nodes,
HELP: "Generate a host-bulk-add.xml, apply it and install all" \
"other nodes, wait for them to be 'online."},
STG_UNLOCK_CONTROLLER1:
{CALLBACK: stage_unlock_controller1,
HELP: "Unlock controller-1, wait for it to be 'available'"},
STG_LAB_SETUP3:
{CALLBACK: stage_lab_setup3,
HELP: "Run lab_setup with one or more --lab-setup-conf files from controller-0."},
STG_UNLOCK_STORAGES:
{CALLBACK: stage_unlock_storages,
HELP: "Unlock all storage nodes, wait for them to be 'available'"},
STG_LAB_SETUP4:
{CALLBACK: stage_lab_setup4,
HELP: "Run lab_setup with one or more --lab-setup-conf files from controller-0."},
STG_UNLOCK_WORKERS:
{CALLBACK: stage_unlock_workers,
HELP: "Unlock all workers, wait for them to be 'available"},
STG_LAB_SETUP5:
{CALLBACK: stage_lab_setup5,
HELP: "Run lab_setup with one or more --lab-setup-conf files from controller-0."},
STG_ENABLE_KUBERNETES:
{CALLBACK: stage_enable_kubernetes,
HELP: "Installation and configuration of Kubernetes dashboard"},
STG_CUSTOM_SCRIPT1:
{CALLBACK: stage_custom_script1,
HELP: "Run a custom script from /home/wrsroot, make sure you" \
"upload it in the rsync-config stage and it is +x. See help."},
STG_CUSTOM_SCRIPT2:
{CALLBACK: stage_custom_script2,
HELP: "Run a custom script from /home/wrsroot, make sure you" \
"upload it in the rsync-config stage and it is +x. See help."},
STG_CUSTOM_SCRIPT3:
{CALLBACK: stage_custom_script3,
HELP: "Run a custom script from /home/wrsroot, make sure you" \
"upload it in the rsync-config stage and it is +x. See help."},
STG_CUSTOM_SCRIPT4:
{CALLBACK: stage_custom_script4,
HELP: "Run a custom script from /home/wrsroot, make sure you" \
"upload it in the rsync-config stage and it is +x. See help."},
STG_CUSTOM_SCRIPT5:
{CALLBACK: stage_custom_script5,
HELP: "Run a custom script from /home/wrsroot, make sure you" \
"upload it in the rsync-config stage and it is +x. See help."},
# internal testing
STC_TEST_SUCCESS: {CALLBACK: stage_test_success,
HELP: "Internal only, does not do anything, used for testing."},
STG_TEST_FAIL: {CALLBACK: stage_test_fail,
HELP: "Internal only, raises exception, used for testing."},
}
AVAILABLE_STAGES = [STG_CREATE_LAB,
STG_INSTALL_CONTROLLER0,
STG_CONFIG_CONTROLLER,
STG_RSYNC_CONFIG,
STG_LAB_SETUP1,
STG_UNLOCK_CONTROLLER0,
STG_LAB_SETUP2,
STG_INSTALL_NODES,
STG_UNLOCK_CONTROLLER1,
STG_LAB_SETUP3,
STG_UNLOCK_STORAGES,
STG_LAB_SETUP4,
STG_UNLOCK_WORKERS,
STG_LAB_SETUP5,
STG_ENABLE_KUBERNETES,
STG_CUSTOM_SCRIPT1,
STG_CUSTOM_SCRIPT2,
STG_CUSTOM_SCRIPT3,
STG_CUSTOM_SCRIPT4,
STG_CUSTOM_SCRIPT5,
STC_TEST_SUCCESS,
STG_TEST_FAIL]
AIO_SX_STAGES = [
STG_CREATE_LAB,
STG_INSTALL_CONTROLLER0,
STG_CONFIG_CONTROLLER,
STG_RSYNC_CONFIG,
STG_LAB_SETUP1,
STG_UNLOCK_CONTROLLER0,
STG_ENABLE_KUBERNETES,
]
AIO_DX_STAGES = [
STG_CREATE_LAB,
STG_INSTALL_CONTROLLER0,
STG_CONFIG_CONTROLLER,
STG_RSYNC_CONFIG,
STG_LAB_SETUP1,
STG_UNLOCK_CONTROLLER0,
STG_INSTALL_NODES,
STG_LAB_SETUP2,
STG_UNLOCK_CONTROLLER1,
STG_LAB_SETUP3,
STG_ENABLE_KUBERNETES,
]
STD_STAGES = [
STG_CREATE_LAB,
STG_INSTALL_CONTROLLER0,
STG_CONFIG_CONTROLLER,
STG_RSYNC_CONFIG,
STG_LAB_SETUP1,
STG_UNLOCK_CONTROLLER0,
STG_INSTALL_NODES,
STG_LAB_SETUP2,
STG_UNLOCK_CONTROLLER1,
STG_LAB_SETUP3,
STG_UNLOCK_WORKERS,
STG_ENABLE_KUBERNETES,
]
STORAGE_STAGES = [
STG_CREATE_LAB,
STG_INSTALL_CONTROLLER0,
STG_CONFIG_CONTROLLER,
STG_RSYNC_CONFIG,
STG_LAB_SETUP1,
STG_UNLOCK_CONTROLLER0,
STG_INSTALL_NODES,
STG_LAB_SETUP2,
STG_UNLOCK_CONTROLLER1,
STG_LAB_SETUP3,
STG_UNLOCK_STORAGES,
STG_LAB_SETUP4,
STG_UNLOCK_WORKERS,
STG_LAB_SETUP5,
STG_ENABLE_KUBERNETES,
]
AIO_SX = 'AIO-SX'
AIO_DX = 'AIO-DX'
STANDARD = 'STANDARD'
STORAGE = 'STORAGE'
STAGES_CHAINS = {AIO_SX: AIO_SX_STAGES,
AIO_DX: AIO_DX_STAGES,
STANDARD: STD_STAGES,
STORAGE: STORAGE_STAGES}
AVAILABLE_CHAINS = [AIO_SX, AIO_DX, STANDARD, STORAGE]
def load_config():
"""
Loads and updates the configuration options specified in the command-line arguments.
It also sets defaults for some options.
"""
global V_BOX_OPTIONS # pylint: disable=global-statement
V_BOX_OPTIONS = handle_args().parse_args()
oam_config = [getattr(OAM, attr)
for attr in dir(OAM) if not attr.startswith('__')]
if V_BOX_OPTIONS.vboxnet_ip is None:
V_BOX_OPTIONS.vboxnet_ip = oam_config[0]['ip']
if V_BOX_OPTIONS.hostiocache:
V_BOX_OPTIONS.hostiocache = 'on'
else:
V_BOX_OPTIONS.hostiocache = 'off'
if V_BOX_OPTIONS.lab_setup_conf is None:
V_BOX_OPTIONS.lab_setup_conf = {"~/lab_setup.conf"}
else:
V_BOX_OPTIONS.lab_setup_conf = V_BOX_OPTIONS.lab_setup_conf
try:
with open(V_BOX_OPTIONS.ansible_controller_config, encoding="utf-8") as stream:
loaded = ruamel.yaml.safe_load(stream)
if V_BOX_OPTIONS.setup_type != AIO_SX:
V_BOX_OPTIONS.controller_floating_ip = loaded.get('external_oam_floating_address')
V_BOX_OPTIONS.controller0_ip = loaded.get('external_oam_node_0_address')
V_BOX_OPTIONS.controller1_ip = loaded.get('external_oam_node_1_address')
assert V_BOX_OPTIONS.controller_floating_ip, "Missing external_oam_floating_address from ansible config file"
assert V_BOX_OPTIONS.controller0_ip, "Missing external_oam_node_0_address from ansible config file"
assert V_BOX_OPTIONS.controller1_ip, "Missing external_oam_node_1_address from ansible config file"
else:
V_BOX_OPTIONS.controller_floating_ip = None
# In a AIO-SX configuration the ip of controller-0 must be the same as the floating defined in ansible config file.
V_BOX_OPTIONS.controller0_ip = loaded.get('external_oam_floating_address')
V_BOX_OPTIONS.controller1_ip = None
assert V_BOX_OPTIONS.controller0_ip, "Missing external_oam_floating_address from ansible config file"
except FileNotFoundError:
print (f' \n Ansible configuration file not found in {V_BOX_OPTIONS.ansible_controller_config} \n')
sys.exit(1)
except ruamel.yaml.YAMLError:
print("\n Error while parsing YAML file \n")
sys.exit()
if V_BOX_OPTIONS.setup_type == AIO_SX:
V_BOX_OPTIONS.controllers = 1
V_BOX_OPTIONS.workers = 0
V_BOX_OPTIONS.storages = 0
elif V_BOX_OPTIONS.setup_type == AIO_DX:
V_BOX_OPTIONS.controllers = 2
V_BOX_OPTIONS.workers = 0
V_BOX_OPTIONS.storages = 0
elif V_BOX_OPTIONS.setup_type == STANDARD:
V_BOX_OPTIONS.storages = 0
def validate(v_box_opt, m_stages):
"""
Validates the values of the configuration options based on the stages that are going
to be executed. Checks that required options have been set and prints an error
message and exits with an error code if any of them are missing. It also performs
additional validation depending on the stage that is going to be executed.
"""
err = False
# Generic
if v_box_opt.vboxnet_type == 'nat':
if v_box_opt.setup_type != AIO_SX:
if not v_box_opt.nat_controller_floating_local_ssh_port:
print("Please set --nat-controller-floating-local-ssh-port")
err = True
if not v_box_opt.nat_controller0_local_ssh_port:
print("Please set --nat-controller0-local-ssh-port")
err = True
if v_box_opt.controllers > 1 and not v_box_opt.nat_controller1_local_ssh_port:
print("Second controller is configured, please set --nat-controller1-local-ssh-port")
err = True
else:
if v_box_opt.setup_type != AIO_SX:
if not v_box_opt.controller_floating_ip:
print("Please set --controller-floating-ip")
err = True
if not v_box_opt.controller0_ip:
print("Please set --controller0-ip")
err = True
if v_box_opt.controllers > 1 and not v_box_opt.controller1_ip:
print("Second controller is configured, please set --controller1-ip")
err = True
if STG_CONFIG_CONTROLLER in m_stages:
if not v_box_opt.ansible_controller_config:
print(f"Please set --ansible-controller-config as needed by stage {STG_CONFIG_CONTROLLER}")
err = True
if STG_RSYNC_CONFIG in m_stages:
if not v_box_opt.config_files_dir and not v_box_opt.config_files_dir_dont_follow_links:
print("Please set --config-files-dir and/or --config-files-dir-dont-follow-links "
f"as needed by stage {STG_RSYNC_CONFIG} and {STG_LAB_SETUP1}")
err = True
if (STG_LAB_SETUP1 in m_stages or STG_LAB_SETUP2 in m_stages
or STG_LAB_SETUP3 in m_stages or STG_LAB_SETUP4 in m_stages
or STG_LAB_SETUP5 in m_stages):
if not v_box_opt.lab_setup_conf:
print("Please set at least one --lab-setup-conf file as needed by lab-setup stages")
err = True
# file = ["lab_setup.sh"]
dirs = []
if v_box_opt.config_files_dir:
dirs.append(v_box_opt.config_files_dir)
if v_box_opt.config_files_dir_dont_follow_links:
dirs.append(v_box_opt.config_files_dir_dont_follow_links)
# for directory in dirs:
# pass
if err:
print("\nMissing arguments. Please check --help and --list-stages for usage.")
sys.exit(5)
def wrap_stage_help(m_stage, stage_callbacks, number=None):
"""
Returns a formatted string containing the name of the stage, its number (if given),
and its description, separated by "#" symbol. m_stage is a string with the name of
the stage. stage_callbacks is a string with the description of the stage.
Number is an optional integer with the number of the stage.
"""
if number:
text = f" {number}. {m_stage}"
else:
text = f" {m_stage}"
length = 30
fill = length - len(text)
text += " " * fill
text += f"# {stage_callbacks}"
return text
# Define signal handler for ctrl+c
def signal_handler():
"""
This function is called when the user presses Ctrl+C. It prints a message to the
console and exits the script. Additionally, it calls the print_kpi_metrics()
function from the kpi module to print KPI metrics.
"""
print('You pressed Ctrl+C!')
kpi.print_kpi_metrics()
sys.exit(1)
# pylint: disable=invalid-name
if __name__ == "__main__":
kpi.init_kpi_metrics()
signal.signal(signal.SIGINT, signal_handler)
load_config()
if V_BOX_OPTIONS.list_stages:
print(f"Defined setups: {list(STAGES_CHAINS.keys())}")
if V_BOX_OPTIONS.setup_type and V_BOX_OPTIONS.setup_type in AVAILABLE_CHAINS:
AVAILABLE_CHAINS = [V_BOX_OPTIONS.setup_type]
for setup in AVAILABLE_CHAINS:
i = 1
print(f"Stages for setup: {setup}")
for stage in STAGES_CHAINS[setup]:
print(wrap_stage_help(stage, STAGE_CALLBACKS[stage][HELP], i))
i += 1
print("Available stages that can be used for --custom-stages:")
for stage in AVAILABLE_STAGES:
print(wrap_stage_help(stage, STAGE_CALLBACKS[stage][HELP]))
sys.exit(0)
init_logging(V_BOX_OPTIONS.labname, V_BOX_OPTIONS.logpath)
LOG.info("Logging to directory: %s", (get_log_dir() + "/"))
LOG.info("Install manages: %s controllers, %s workers, %s storages.",
V_BOX_OPTIONS.controllers, V_BOX_OPTIONS.workers, V_BOX_OPTIONS.storages)
# Setup stages to run based on config
install_stages = []
if V_BOX_OPTIONS.custom_stages:
# Custom stages
install_stages = V_BOX_OPTIONS.custom_stages.split(',')
for stage in install_stages:
invalid_stages = []
if stage not in AVAILABLE_STAGES:
invalid_stages.append(stage)
if invalid_stages:
LOG.info("Following custom stages are not supported: %s.\n" \
"Choose from: %s", invalid_stages, AVAILABLE_STAGES)
sys.exit(1)
else:
# List all stages between 'from-stage' to 'to-stage'
stages = STAGES_CHAINS[V_BOX_OPTIONS.setup_type]
from_index = 0
to_index = None
if V_BOX_OPTIONS.from_stage:
if V_BOX_OPTIONS.from_stage == 'start':
from_index = 0
else:
from_index = stages.index(V_BOX_OPTIONS.from_stage)
if V_BOX_OPTIONS.to_stage:
if V_BOX_OPTIONS.from_stage == 'end':
to_index = -1
else:
to_index = stages.index(V_BOX_OPTIONS.to_stage) + 1
if to_index is not None:
install_stages = stages[from_index:to_index]
else:
install_stages = stages[from_index:]
LOG.info("Executing %s stage(s): %s.", len(install_stages), install_stages)
validate(V_BOX_OPTIONS, install_stages)
stg_no = 0
prev_stage = None
for stage in install_stages:
stg_no += 1
start = time.time()
try:
LOG.info("######## (%s/%s) Entering stage %s ########",
stg_no,
len(install_stages),
stage)
STAGE_CALLBACKS[stage][CALLBACK]()
# Take snapshot if configured
if V_BOX_OPTIONS.snapshot:
vboxmanage.take_snapshot(
V_BOX_OPTIONS.labname,
f"snapshot-AFTER-{stage}")
# Compute KPIs
duration = time.time() - start
kpi.set_kpi_metric(stage, duration)
kpi.print_kpi(stage)
kpi.print_kpi('total')
except Exception as e:
duration = time.time() - start
kpi.set_kpi_metric(stage, duration)
LOG.info("INSTALL FAILED, ABORTING!")
kpi.print_kpi_metrics()
LOG.info("Exception details: %s", e)
raise
# Stage completed
prev_stage = stage
LOG.info("INSTALL SUCCEEDED!")
kpi.print_kpi_metrics()