virtual-deployment/virtualbox/pybox/helper/vboxmanage.py

#!/usr/bin/python3
#
# SPDX-License-Identifier: Apache-2.0
#

"""
This module provides functions for managing virtual machines using VirtualBox.
"""

import os
import subprocess
import re
import getpass
import time

from sys import platform
from utils.install_log import LOG


def vboxmanage_version():
    """
    Return version of vbox.
    """

    cmd = ["vboxmanage", "--version"]
    version = subprocess.check_output(cmd, stderr=subprocess.STDOUT)
    return version


def vboxmanage_extpack():
    """
    This allows you to install, uninstall the vbox extensions"
    """

    output = vboxmanage_version()
    version = re.match(b"(.*)r", output)
    version_path = version.group(1).decode("utf-8")

    LOG.info("Downloading extension pack")
    filename = f"Oracle_VM_VirtualBox_Extension_Pack-{version_path}.vbox-extpack"
    cmd = [
        "wget",
        f"http://download.virtualbox.org/virtualbox/{version_path}/{filename}",
        "-P",
        "/tmp"
    ]
    result = subprocess.check_output(cmd, stderr=subprocess.STDOUT)
    LOG.info(result)

    LOG.info("Installing extension pack")
    cmd = ["vboxmanage", "extpack", "install", "/tmp/" + filename, "--replace"]
    result = subprocess.check_output(cmd, stderr=subprocess.STDOUT)
    LOG.info(result)


def get_all_vms(labname, option="vms"):
    """
    Return a list of virtual machines (VMs) belonging to a specified lab.

    Args:
        labname (str): The name of the lab to which the VMs belong.
        option (str, optional): The vboxmanage command option to use when listing VMs.
            Defaults to "vms".

    Returns:
        list: A list of strings representing the names of the VMs that belong to the specified lab.
    """

    initial_node_list = []
    vm_list = vboxmanage_list(option)

    labname.encode("utf-8")
    # Reduce the number of VMs we query
    for item in vm_list:
        if labname.encode("utf-8") in item and (
            b"controller-" in item or b"compute-" in item or b"storage-" in item
        ):
            initial_node_list.append(item.decode("utf-8"))

    # Filter by group
    node_list = []
    group = bytearray(f'"/{labname}"', "utf-8")
    for item in initial_node_list:
        info = vboxmanage_showinfo(item).splitlines()
        for line in info:
            try:
                k_value, v_value = line.split(b"=")
            except ValueError:
                continue
            if k_value == b"groups" and v_value == group:
                node_list.append(item)

    return node_list


def take_snapshot(labname, snapshot_name):
    """
    Take a snapshot of all VMs belonging to a specified lab.

    Args:
        labname (str): The name of the lab whose VMs will be snapshotted.
        snapshot_name (str): The name of the snapshot to be taken.

    Returns:
        None
    """

    vms = get_all_vms(labname, option="vms")
    runningvms = get_all_vms(labname, option="runningvms")

    LOG.info("#### Taking snapshot %s of lab %s", snapshot_name, labname)
    LOG.info("VMs in lab %s: %s", labname, vms)
    LOG.info("VMs running in lab %s: %s", labname, runningvms)

    _pause_running_vms(runningvms, vms)

    if len(vms) != 0:
        vboxmanage_takesnapshot(vms, snapshot_name)

    _resume_running_vms(runningvms)

    LOG.info("Waiting 10s before running VMs")
    time.sleep(10)

    if runningvms:
        _wait_for_vms_to_run(labname, runningvms, vms)


def _pause_running_vms(runningvms, vms):
    """Pause running virtual machines.

    Args:
        runningvms (list): A list of strings representing the names of running virtual machines.
        vms (list): A list of strings representing the names of all virtual machines.

    Returns:
        None
    """

    if len(runningvms) > 1:
        for node in runningvms:
            newpid = os.fork()
            if newpid == 0:
                vboxmanage_controlvms([node], "pause")
                os._exit(0)  # pylint: disable=protected-access
        for node in vms:
            os.waitpid(0, 0)
        time.sleep(2)


def _resume_running_vms(runningvms):
    """Resume paused virtual machines.

    Args:
        runningvms (list): A list of strings representing the names of running virtual machines.

    Returns:
        None
    """

    if len(runningvms) > 1:
        for node in runningvms:
            newpid = os.fork()
            if newpid == 0:
                vboxmanage_controlvms([node], "resume")
                os._exit(0)  # pylint: disable=protected-access
        for node in runningvms:
            os.waitpid(0, 0)


def _wait_for_vms_to_run(labname, runningvms, vms):
    """Wait for virtual machines to finish running.

    Args:
        labname (str): The name of the lab whose virtual machines are being waited for.
        runningvms (list): A list of strings representing the names of running virtual machines.
        vms (list): A list of strings representing the names of all virtual machines.

    Returns:
        None
    """

    new_vms = get_all_vms(labname, option="runningvms")
    retry = 0
    while retry < 20:
        LOG.info(
            "Waiting for VMs to come up running after taking snapshot..."
            "Up VMs are %s ",
            new_vms,
        )
        if len(runningvms) < len(new_vms):
            time.sleep(1)
            new_vms = get_all_vms(labname, option="runningvms")
            retry += 1
        else:
            LOG.info("All VMs %s are up running after taking snapshot...", vms)
            break


def restore_snapshot(node_list, name):
    """
    Restore a snapshot of a list of virtual machines.

    Args:
        node_list (list): A list of strings representing the names of the virtual machines
            whose snapshot will be restored.
        name (str): The name of the snapshot to restore.

    Returns:
        None
    """

    LOG.info("Restore snapshot of %s for hosts %s", name, node_list)
    if len(node_list) != 0:
        vboxmanage_controlvms(node_list, "poweroff")
        LOG.info("Waiting 5s")
        time.sleep(5)
    if len(node_list) != 0:
        for host in node_list:
            vboxmanage_restoresnapshot(host, name)
            LOG.info("Waiting 5s")
            time.sleep(5)
        for host in node_list:
            if "controller-0" not in host:
                vboxmanage_startvm(host)
                LOG.info("Waiting 10s")
                time.sleep(10)
        for host in node_list:
            if "controller-0" in host:
                vboxmanage_startvm(host)
                LOG.info("Waiting 10s")
                time.sleep(10)


def vboxmanage_list(option="vms"):
    """
    This returns a list of vm names.
    """

    cmd = ["vboxmanage", "list", option]
    result = subprocess.check_output(cmd, stderr=subprocess.STDOUT)
    vms_list = []
    for item in result.splitlines():
        vm_name = re.match(b'"(.*?)"', item)
        vms_list.append(vm_name.group(1))

    return vms_list


def vboxmanage_showinfo(host):
    """
    This returns info about the host
    """

    if not isinstance(host, str):
        host.decode("utf-8")
    cmd = ["vboxmanage", "showvminfo", host, "--machinereadable"]
    result = subprocess.check_output(cmd, stderr=subprocess.STDOUT)
    return result


def vboxmanage_createvm(hostname, labname):
    """
    This creates a VM with the specified name.
    """

    assert hostname, "Hostname is required"
    assert labname, "Labname is required"
    group = "/" + labname
    LOG.info("Creating VM %s", hostname)
    cmd = [
        "vboxmanage",
        "createvm",
        "--name",
        hostname,
        "--register",
        "--ostype",
        "Linux_64",
        "--groups",
        group,
    ]
    subprocess.check_output(cmd, stderr=subprocess.STDOUT)


def vboxmanage_deletevms(hosts=None):
    """
    Deletes a list of VMs
    """

    assert hosts, "A list of hostname(s) is required"

    if len(hosts) != 0:
        for hostname in hosts:
            LOG.info("Deleting VM %s", hostname)
            cmd = ["vboxmanage", "unregistervm", hostname, "--delete"]
            subprocess.check_output(cmd, stderr=subprocess.STDOUT)
            LOG.info("Waiting 10s")
            time.sleep(10)
            # in case medium is still present after delete
            vboxmanage_deletemedium(hostname)

    vms_list = vboxmanage_list("vms")
    for items in hosts:
        assert (
            items not in vms_list
        ), f"The following vms are unexpectedly present {vms_list}"


def vboxmanage_hostonlyifcreate(name="vboxnet0", oam_ip=None, netmask=None):
    """
    This creates a hostonly network for systems to communicate.
    """

    assert name, "Must provide network name"
    assert oam_ip, "Must provide an OAM IP"
    assert netmask, "Must provide an OAM Netmask"

    LOG.info("Creating Host-only Network")
    cmd = ["vboxmanage", "hostonlyif", "create"]
    subprocess.check_output(cmd, stderr=subprocess.STDOUT)

    LOG.info("Provisioning %s with IP %s and Netmask %s", name, oam_ip, netmask)
    cmd = [
        "vboxmanage",
        "hostonlyif",
        "ipconfig",
        name,
        "--ip",
        oam_ip,
        "--netmask",
        netmask,
    ]
    subprocess.check_output(cmd, stderr=subprocess.STDOUT)


def vboxmanage_hostonlyifdelete(name="vboxnet0"):
    """
    Deletes hostonly network. This is used as a work around for creating too many hostonlyifs.
    """

    assert name, "Must provide network name"
    LOG.info("Removing Host-only Network")
    cmd = ["vboxmanage", "hostonlyif", "remove", name]
    subprocess.check_output(cmd, stderr=subprocess.STDOUT)


def vboxmanage_modifyvm(hostname, vm_config=None):
    """
    Modify a virtual machine according to a specified configuration.

    Args:
        hostname(str): Name of host to modify
        vm_config (dict): A dictionary representing the configuration options
        for the virtual machine. Possible key values: cpus, memory, nic, nictype,
        nicpromisc, nicnum, intnet, hostonlyadapter, natnetwork, uartbase,
        uartport, uartmode, uartpath, nicbootprio2=1, prefix=""

    Returns:
        None
    """

    #put default values in nicbootprio2 and prefix if they not exist
    vm_config["nicbootprio2"] = vm_config.get("nicbootprio2", 1)
    vm_config["prefix"] = vm_config.get("prefix", "")

    cmd = ["vboxmanage", "modifyvm", hostname]
    nic_cmd = []

    if _contains_value("cpus", vm_config):
        cmd.extend(["--cpus", vm_config["cpus"]])

    if _contains_value("memory", vm_config):
        cmd.extend(["--memory", vm_config["memory"]])

    if _is_network_configured(vm_config):
        nic_cmd = _get_network_configuration(vm_config)
        cmd.extend(nic_cmd)

    elif _is_nat_network_configured(vm_config):
        cmd.extend([f'--nic{vm_config["nicnum"]}', "nat"])

    if _is_uart_configured(vm_config):
        uart_config = _add_uart(vm_config)
        cmd.extend(uart_config)

    if _contains_value("nicbootprio2", vm_config):
        cmd.extend(["--nicbootprio2"])
        cmd.extend([f'{vm_config["nicbootprio2"]}'])

    cmd.extend(["--boot4"])
    cmd.extend(["net"])

    LOG.info("#### Updating VM %s configuration", hostname)
    LOG.info("#### Executing command on the host machine:\n$ %s\n", ' '.join(str(i) for i in cmd))
    subprocess.check_output(cmd, stderr=subprocess.STDOUT)


def _is_network_configured(vm_config):
    """
    Checks whether a network interface is configured in the given VM configuration.

    Args:
        vm_config (dict): A dictionary representing the configuration options for the VM.

    Returns:
        bool: True if a network interface is configured, False otherwise.
    """

    return (_contains_value("nic", vm_config)
        and _contains_value("nictype", vm_config)
        and _contains_value("nicpromisc", vm_config)
        and _contains_value("nicnum", vm_config)
    )


def _get_network_configuration(vm_config):
    """
    Constructs a list of options for the network interface based on the values in vm_config.

    Args:
        vm_config (dict): A dictionary representing the configuration options for the VM.

    Returns:
        list: A list of command-line options for the network interface.
    """

    nic_cmd = [f'--nic{vm_config["nicnum"]}', vm_config["nic"]]
    nic_cmd.extend([f'--nictype{vm_config["nicnum"]}', vm_config["nictype"]])
    nic_cmd.extend([f'--nicpromisc{vm_config["nicnum"]}', vm_config["nicpromisc"]])

    if _contains_value("intnet", vm_config):
        intnet = vm_config["intnet"]
        if _contains_value("prefix", vm_config):
            intnet = f"{vm_config['prefix']}-{intnet}"
        else:
            intnet = f"{intnet}"
        nic_cmd.extend([f'--intnet{vm_config["nicnum"]}', intnet])

    if _contains_value("hostonlyadapter", vm_config):
        nic_cmd.extend(
            [
                f'--hostonlyadapter{vm_config["nicnum"]}',
                vm_config["hostonlyadapter"],
            ]
        )

    if _contains_value("natnetwork", vm_config):
        nic_cmd.extend(
            [f'--nat-network{vm_config["nicnum"]}', vm_config["natnetwork"]]
        )

    return nic_cmd


def _is_nat_network_configured(vm_config):
    """
    Checks whether the NAT network is configured in the given VM configuration.

    Args:
        vm_config (dict): A dictionary representing the configuration options for the VM.

    Returns:
        bool: True if the NAT network is configured, False otherwise.
    """

    return _contains_value("nicnum", vm_config) and vm_config.get("nictype") == "nat"


def _is_uart_configured(vm_config):
    """
    Checks whether the UART device is configured in the given VM configuration.

    Args:
        vm_config (dict): A dictionary representing the configuration options for the VM.

    Returns:
        bool: True if the UART device is configured, False otherwise.
    """

    return (
        _contains_value("uartbase", vm_config)
        and _contains_value("uartport", vm_config)
        and _contains_value("uartmode", vm_config)
        and _contains_value("uartpath", vm_config)
    )


def _add_uart(vm_config):
    """
    Constructs a list of options for the UART device based on the values in vm_config.

    Args:
        vm_config (dict): A dictionary representing the configuration options for the VM.

    Returns:
        list: A list of command-line options for the UART device.
    """

    uart_config = ["--uart1"]
    uart_config.extend([f'{vm_config["uartbase"]}'])
    uart_config.extend([f'{vm_config["uartport"]}'])
    uart_config.extend(["--uartmode1"])
    uart_config.extend([f'{vm_config["uartmode"]}'])
    uart_config.extend([f'{vm_config["uartpath"]}'])

    return uart_config


def _contains_value(key, dictionary):
    return key in dictionary and dictionary[key]


def vboxmanage_storagectl(hostname=None, storectl="sata", hostiocache="off"):
    """
    This creates a storage controller on the host.
    """

    assert hostname, "Hostname is required"
    assert storectl, "Type of storage controller is required"

    LOG.info("Creating %s storage controller on VM %s", storectl, hostname)
    cmd = [
        "vboxmanage",
        "storagectl",
        hostname,
        "--name",
        storectl,
        "--add",
        storectl,
        "--hostiocache",
        hostiocache,
    ]
    subprocess.check_output(cmd, stderr=subprocess.STDOUT)


def vboxmanage_storageattach(hostname, storage_config):
    """
    Attaches a disk to a storage controller.

    Args:
        hostname (str): Name of the virtual machine.
        storage_config (dict): A dictionary containing the config options for the storage device.
            Possible key values: storectl, storetype, disk, port_num, device_num.

    Returns:
        str: The output of the vboxmanage command.
    """

    assert hostname, "Hostname is required"
    assert storage_config and isinstance(storage_config, dict), "Storage configuration is required"

    storectl = storage_config.get("storectl", "sata")
    storetype = storage_config.get("storetype", "hdd")
    disk = storage_config.get("disk")
    port_num = storage_config.get("port_num", "0")
    device_num = storage_config.get("device_num", "0")

    assert disk, "Disk name is required"
    assert storectl, "Name of storage controller is required"
    assert storetype, "Type of storage controller is required"

    LOG.info(
        "Attaching %s storage to storage controller %s on VM %s",
        storetype,
        storectl,
        hostname,
    )
    cmd = [
        "vboxmanage",
        "storageattach",
        hostname,
        "--storagectl",
        storectl,
        "--medium",
        disk,
        "--type",
        storetype,
        "--port",
        port_num,
        "--device",
        device_num,
    ]
    return subprocess.check_output(cmd, stderr=subprocess.STDOUT)


def vboxmanage_deletemedium(hostname, vbox_home_dir="/home"):
    """
    Deletes the disk medium associated with a virtual machine.

    Args:
        hostname (str): The name of the virtual machine to which the disk medium is attached.
        vbox_home_dir (str): The directory in which the disk medium files are stored.
        Defaults to "/home".

    Returns:
        None
    """

    assert hostname, "Hostname is required"

    if platform in ("win32", "win64"):
        return

    username = getpass.getuser()
    vbox_home_dir = f"{vbox_home_dir}/{username}/vbox_disks/"

    disk_list = [
        f
        for f in os.listdir(vbox_home_dir)
        if os.path.isfile(os.path.join(vbox_home_dir, f)) and hostname in f
    ]
    LOG.info("Disk mediums to delete: %s", disk_list)
    for disk in disk_list:
        LOG.info("Disconnecting disk %s from vbox.", disk)
        try:
            cmd = [
                "vboxmanage",
                "closemedium",
                "disk",
                f"{vbox_home_dir}{disk}",
                "--delete",
            ]
            result = subprocess.check_output(cmd, stderr=subprocess.STDOUT)
            LOG.info(result)
        except subprocess.CalledProcessError as exception:
            # Continue if failures, disk may not be present
            LOG.warning(
                "Error disconnecting disk, continuing. "
                "Details: stdout: %s stderr: %s",
                exception.stdout,
                exception.stderr,
            )
        LOG.info("Removing backing file %s", disk)
        try:
            os.remove(f"{vbox_home_dir}{disk}")
        except Exception as exc:
            LOG.debug("Failure at removing backing file\nError: %s\n", repr(exc))


def vboxmanage_createmedium(hostname=None, disk_list=None, vbox_home_dir="/home"):
    """
    This creates the required disks.
    """

    assert hostname, "Hostname is required"
    assert disk_list, "A list of disk sizes is required"

    username = getpass.getuser()
    device_num = 0
    port_num = 0
    disk_count = 1
    for disk in disk_list:
        if platform in ("win32", "win64"):
            file_name = (
                "C:\\Users\\"
                + username
                + "\\vbox_disks\\"
                + hostname
                + f"_disk_{disk_count}"
            )
        else:
            file_name = (
                vbox_home_dir
                + "/"
                + username
                + "/vbox_disks/"
                + hostname
                + f"_disk_{disk_count}"
            )
        LOG.info(
            "Creating disk %s of size %s on VM %s on device %s port %s",
            file_name,
            disk,
            hostname,
            device_num,
            port_num,
        )

        try:
            cmd = [
                "vboxmanage",
                "createmedium",
                "disk",
                "--size",
                str(disk),
                "--filename",
                file_name,
                "--format",
                "vdi",
                "--variant",
                "standard",
            ]
            result = subprocess.check_output(cmd, stderr=subprocess.STDOUT)
            LOG.info(result)
        except subprocess.CalledProcessError as exception:
            LOG.error("Error stdout: %s stderr: %s", exception.stdout, exception.stderr)
            raise
        vboxmanage_storageattach(
            hostname,
            {
                "storectl": "sata",
                "storetype": "hdd",
                "disk": file_name + ".vdi",
                "port_num": str(port_num),
                "device_num": str(device_num),
            },
        )
        disk_count += 1
        port_num += 1

    LOG.info("Waiting 5s")
    time.sleep(5)


def vboxmanage_startvm(hostname=None, headless=False, force=False):
    """
    This allows you to power on a VM.
    """

    assert hostname, "Hostname is required"

    if not force:
        LOG.info("Check if VM is running")
        running_vms = vboxmanage_list(option="runningvms")
    else:
        running_vms = []

    interface_type = "gui"
    if headless:
        interface_type = "headless"

    if hostname.encode("utf-8") in running_vms:
        LOG.info("Host %s is already started", hostname)
    else:
        LOG.info("Powering on VM %s", hostname)
        cmd = ["vboxmanage", "startvm", hostname, "--type", interface_type]
        result = subprocess.check_output(cmd, stderr=subprocess.STDOUT)
        LOG.info(result)

    # Wait for VM to start
    tmout = 20
    while tmout:
        tmout -= 1
        running_vms = vboxmanage_list(option="runningvms")
        if hostname.encode("utf-8") in running_vms:
            break
        time.sleep(1)
    else:
        raise f"Failed to start VM: {hostname}"
    LOG.info("VM '%s' started.", hostname)


def vboxmanage_controlvms(hosts=None, action=None):
    """
    This allows you to control a VM, e.g. pause, resume, etc.
    """

    assert hosts, "Hostname is required"
    assert action, "Need to provide an action to execute"

    for host in hosts:
        LOG.info("Executing %s action on VM %s", action, host)
        subprocess.call(
            ["vboxmanage", "controlvm", host, action], stderr=subprocess.STDOUT
        )
    time.sleep(1)


def vboxmanage_takesnapshot(hosts=None, name=None):
    """
    This allows you to take snapshot of VMs.
    """

    assert hosts, "Hostname is required"
    assert name, "Need to provide a name for the snapshot"

    for host in hosts:
        LOG.info("Taking snapshot %s on VM %s", name, host)
        subprocess.call(
            ["vboxmanage", "snapshot", host, "take", name], stderr=subprocess.STDOUT
        )


def vboxmanage_restoresnapshot(host=None, name=None):
    """
    This allows you to restore snapshot of a VM.
    """

    assert host, "Hostname is required"
    assert name, "Need to provide the snapshot to restore"

    LOG.info("Restoring snapshot %s on VM %s", name, host)
    subprocess.call(
        ["vboxmanage", "snapshot", host, "restore", name], stderr=subprocess.STDOUT
    )

    LOG.info("Waiting 10s")
    time.sleep(10)


def vboxmanage_getrulename(network, local_port):
    """
    Get port-forwarding rule for given NAT network and local port in VirtualBox.

    Args:
        network (str): Name of the NAT network.
        local_port (str): The local port number.

    Returns:
        (str): Name of rule or empty
    """

    # List information about all nat networks in VirtualBox
    cmd = ["vboxmanage", "list", "natnets"]
    result = subprocess.check_output(cmd, stderr=subprocess.STDOUT)
    natpattern = r"NetworkName:(.*?)loopback mappings \(ipv4\)"
    natnetworks = re.findall(natpattern,result.decode(),re.DOTALL)

    # Get the rule name of the given local port in the given natnetwork
    for natnetwork in natnetworks:
        natinfo = natnetwork.strip().split('\n')
        if natinfo[0] == network:
            try:
                startindex = natinfo.index("Port-forwarding (ipv4)")
            except ValueError:
                # If no index is found the function return an empty string
                return ""
            for index in range (startindex+1,len(natinfo)):
                rule = natinfo[index].strip()

                parsed_rule = rule.split(':')
                if int(parsed_rule[3]) == int(local_port):
                    return parsed_rule[0]
    return ""


def vboxmanage_addportforward(rule_name, local_port, guest_ip, guest_port, network):
    """
    Add port-forwarding rule for a NAT network in VirtualBox.

    Args:
        rule_name (str): Name of the port-forward rule to be added.
        local_port (str): The local port number to forward.
        guest_ip (str): The IP address of the guest to forward to.
        guest_port (str): The port number on the guest to forward to.
        network (str): Name of the NAT network.

    Returns:
        True if the port was added
        False if an error occurred when trying to add the port-forward rule.
    """

    rule = f"{rule_name}:tcp:[]:{local_port}:[{guest_ip}]:{guest_port}"

    LOG.info("Creating port-forwarding rule to: %s", rule)
    cmd = [
        "vboxmanage",
        "natnetwork",
        "modify",
        "--netname",
        network,
        "--port-forward-4",
        rule,
    ]
    try:
        subprocess.check_output(cmd, stderr=subprocess.STDOUT)
    except subprocess.CalledProcessError:
        LOG.info("Error while trying to create port-forwarding rule. Continuing installation!")
        return False
    return True


def vboxmanage_deleteportforward(rule_name, network):
    """
    Delete port-forwarding rule for a NAT network in VirtualBox.

    Args:
        rule_name (str): Name of the port-forward rule to be deleted.
        network (str): Name of the NAT network.

    Returns:
        None
    """

    LOG.info(
        "Removing previous forwarding rule '%s' from NAT network '%s'",
        rule_name,
        network,
    )
    cmd = [
        "vboxmanage",
        "natnetwork",
        "modify",
        "--netname",
        network,
        "--port-forward-4",
        "delete",
        rule_name,
    ]
    try:
        subprocess.check_output(cmd, stderr=subprocess.STDOUT)
    except subprocess.CalledProcessError:
        LOG.info("Error while trying to delete port-forwarding rule. Continuing installation!")