Eliminate parameter redundancies

-stx_config.ini_centos file and all
 code related to it deleted.
-localhost.yml updates ansible_become_pass
 value based on password parameter.
-Ip's used in the VM are now set from the
 localhost.yml
-ruamel.yaml added to requirementes.txt

Test Plan:
PASS: localhost.yml is succesfully overriten
PASS: ip variables are set from localhost.yml

Story: 2005051
Task: 47953

Change-Id: Ieb5fe5d6c0a7da5ef1b6aa7488d6edf48f4cc527
Signed-off-by: Daniel Caires <daniel.caires@encora.com>

virtualbox/pybox/Parser.py

@@ -64,7 +64,8 @@ def parse_setup_config(parser: ArgumentParser):
                             STORAGE
                         """,
                         choices=['AIO-SX', 'AIO-DX', 'STANDARD', 'STORAGE'],
-                        type=str)
+                        type=str,
+                        required=True)
     parser.add_argument("--controllers", help=
                         """
                         Number of controllers:
@@ -149,7 +150,8 @@ def parse_config_location(parser: ArgumentParser):
                         Location of ISO including the filename:
                             /folk/myousaf/bootimage.ISO
                         """,
-                        type=str)
+                        type=str,
+                        required=True)
     parser.add_argument("--config-files-dir", help=
                         """
                         Directory with config files, scripts, images (i.e.
@@ -169,20 +171,10 @@ def parse_config_location(parser: ArgumentParser):
                         Same as --config-files-dir but keep symbolic link as is.
                         """,
                         type=str)
-    parser.add_argument("--config-controller-ini", help=
-                        """
-                        Path to the local config_controller .ini. This
-                        file is transferred to the controller.  NOTE: OAM
-                        configuration in this ini is updated dynamically
-                        based on networking related args.
-                        (e.g. stx_config.ini_centos,
-                        ~/stx_config.ini_centos, /home/myousaf ...).
-                        """,
-                        type=str)
     parser.add_argument("--ansible-controller-config", help=
                         """
                         Path to a local YAML file to be copied as localhost.yml
-                        to the home directory of the controller-0. 
+                        to the home directory of the controller-0. NOTE: System password value is updated dynamically with user defined --password.
                         """,
                         type=str)
     parser.add_argument("--vbox-home-dir", help=
@@ -250,8 +242,7 @@ def parse_networking(parser: ArgumentParser):
     parser.add_argument("--vboxnet-ip", help=
                         """
                         The IP address of the host only adapter as it
-                        is configured on the host (i.e. gateway). This is also used to
+                        is configured on the host (i.e. gateway).
-                        update GATEWAY_IP in [OAM_NETWORK] of config_controller config file.
                         """,
                         type=str)
     parser.add_argument("--add-nat-interface", help=
@@ -259,28 +250,6 @@ def parse_networking(parser: ArgumentParser):
                         Add a new NAT interface to hosts.
                         """,
                         action='store_true')
-    parser.add_argument("--controller-floating-ip", help=
-                        """
-                        OAM floating IP.
-                        """,
-                        type=str,
-                        default="10.10.10.5")
-    parser.add_argument("--controller0-ip", help=
-                        """
-                        OAM IP of controller-0. This is also used to
-                        update IP_ADDRESS in [OAM_NETWORK] of
-                        config_controller config file of an AIO SX setup.
-                        This should not be the floating IP.
-                        """,
-                        type=str,
-                        default="10.10.10.3")
-    parser.add_argument("--controller1-ip", help=
-                        """
-                        OAM IP of controller-1.
-                        This should not be the floating IP.
-                        """,
-                        type=str,
-                        default="10.10.10.4")
     parser.add_argument("--vboxnet-type", help=
                         """
                         Type of vbox network, either hostonly on nat
@@ -315,30 +284,6 @@ def parse_networking(parser: ArgumentParser):
                         installed.
                         """,
                         type=str)
-    parser.add_argument("--ini-oam-cidr", help=
-                        """
-                        The IP network and mask for the oam net, used to
-                        update CIDR value in [OAM_NETWORK] of
-                        config_controller config file.  Default is
-                        10.10.10.0/24
-                        """,
-                        type=str)
-    parser.add_argument("--ini-oam-ip-start-address", help=
-                        """
-                        The start for the oam net allocation, used to
-                        update IP_START_ADDRESS value in [OAM_NETWORK] of
-                        config_controller config file.  Not needed for AIO
-                        SX setups.
-                        """,
-                        type=str)
-    parser.add_argument("--ini-oam-ip-end-address", help=
-                        """
-                        The end for the oam net allocation, used to update
-                        IP_END_ADDRESS value in [OAM_NETWORK] of
-                        config_controller config file.  Not needed for AIO
-                        SX setups.
-                        """,
-                        type=str)
 
 
 def parse_custom_scripts(parser: ArgumentParser):
@@ -429,7 +374,8 @@ def parse_other(parser: ArgumentParser):
                         """
                         The name of the lab to be created.
                         """,
-                        type=str)
+                        type=str,
+                        required=True)
     parser.add_argument("--userid", help=
                         """
                         Unique user id to differentiate vbox machine

virtualbox/pybox/README.md

@@ -35,7 +35,7 @@ Example stages:
 - create-lab           # Create VMs in vbox: controller-0, controller-1...
 - install-controller-0 # Install controller-0 from --iso-location
 - config-controller    # Run config controller using the
-- config-controller-ini updated based on --ini-* options.
+- ansible-controller-config updated based on args options.
 - rsync-config         # Rsync all files from --config-files-dir and
                          --config-files-dir* to /home/sysadmin.
 - lab-setup1           # Run lab_setup with one or more --lab-setup-conf
@@ -164,13 +164,10 @@ running it):
       --iso-location "$HOME/Downloads/stx-8.iso" \
       --labname StarlingX --install-mode serial \
       --config-files-dir ./configs/aio-sx/ \
-      --config-controller-ini ./configs/aio-sx/stx_config.ini_centos \
       --ansible-controller-config ./configs/aio-sx/localhost.yml \
       --vboxnet-type nat \
       --vboxnet-name NatNetwork \
       --nat-controller0-local-ssh-port 3122 \
-      --controller0-ip 10.10.10.3 \
-      --ini-oam-cidr '10.10.10.0/24' \
       --password $STX_INSTALL_PASSWORD \
       --snapshot
     ```

virtualbox/pybox/configs/aio-sx/localhost.yml

@@ -10,4 +10,4 @@ external_oam_floating_address: 10.10.10.3
 
 admin_username: admin
 admin_password: Li69nux*
-ansible_become_pass: Li69nux*
+ansible_become_pass: <system-password>

virtualbox/pybox/configs/aio-sx/stx_config.ini_centos

@@ -1,19 +0,0 @@
-[LOGICAL_INTERFACE_2]
-LAG_INTERFACE=N
-INTERFACE_MTU=1500
-INTERFACE_PORTS=enp0s3
-
-[OAM_NETWORK]
-IP_ADDRESS = 10.10.10.2
-CIDR=10.10.10.0/24
-GATEWAY=10.10.10.1
-LOGICAL_INTERFACE=LOGICAL_INTERFACE_2
-
-[AUTHENTICATION]
-ADMIN_PASSWORD=Li69nux*
-
-[VERSION]
-RELEASE = 19.01
-
-[SYSTEM]
-SYSTEM_MODE=simplex

virtualbox/pybox/install_vbox.py

@@ -10,7 +10,6 @@ StarlingX on VirtualBox.
 """
 
 import subprocess
-import configparser
 import getpass
 import time
 import re
@@ -20,6 +19,7 @@ import sys
 from sys import platform
 import paramiko
 import streamexpect
+import ruamel.yaml
 
 from utils import kpi, serial
 from utils.install_log import init_logging, get_log_dir, LOG
@@ -457,7 +457,44 @@ def create_lab(m_vboxoptions):
     )
 
 
-def get_hostnames(ignore=None, personalities=('controller', 'storage', 'worker')):
+def override_ansible_become_pass():
+    """
+    Override the ansible_become_pass value in the localhost.yml
+    with the password passed via terminal in the python call
+    """
+
+    file = V_BOX_OPTIONS.ansible_controller_config
+    new_file = "/tmp/localhost.yml"
+
+    #Load Ansible config file
+    try:
+        with open(file, encoding="utf-8") as stream:
+            yaml = ruamel.yaml.YAML()
+            yaml.preserve_quotes = True
+            yaml.explicit_start = True
+            loaded = yaml.load(stream)
+    except FileNotFoundError:
+        print(f'\n Ansible configuration file not found in {file} \n')
+        sys.exit(1)
+    except ruamel.yaml.YAMLError:
+        print("\n Error while parsing YAML file \n")
+        sys.exit(1)
+
+    # modify the password with the one passed on the python call
+    loaded['ansible_become_pass'] = V_BOX_OPTIONS.password
+
+    #Save it again
+    try:
+        with open(new_file, mode='w', encoding="utf-8") as stream:
+            yaml.dump(loaded, stream)
+    except ruamel.yaml.YAMLError as exc:
+        print(exc)
+
+    return new_file
+
+
+# pylint: disable=W0102
+def get_hostnames(ignore=None, personalities=['controller', 'storage', 'worker']):
     """
     Based on the number of nodes defined on the command line, construct
     the hostnames of each node.
@@ -899,61 +936,16 @@ def stage_config_controller(stream):  # pylint: disable=too-many-locals
 
     ip_addr, port = get_ssh_ip_and_port(
         'controller-0')  # Floating ip is not yet configured
-    # if True:
-    # Updated config file
-    LOG.info("#### Updating config_controller ini file networking" \
-             "settings and uploading it to controller.")
-    destination = "/home/" + \
-                  V_BOX_OPTIONS.username + "/stx_config.ini_centos"
-    configini = configparser.ConfigParser()
-    configini.optionxform = str
-    configini.read(V_BOX_OPTIONS.config_controller_ini)
-    old_cidr = configini['OAM_NETWORK']['CIDR']
-    new_cidr = V_BOX_OPTIONS.ini_oam_cidr
-    LOG.info("Replacing OAM_NETWORK/CIDR from %s to %s", old_cidr, new_cidr)
-    configini['OAM_NETWORK']['CIDR'] = new_cidr
-    old_gateway = configini['OAM_NETWORK']['GATEWAY']
-    new_gateway = V_BOX_OPTIONS.vboxnet_ip
-    LOG.info("Replacing OAM_NETWORK/GATEWAY from %s to %s", old_gateway, new_gateway)
-    configini['OAM_NETWORK']['GATEWAY'] = new_gateway
-    if V_BOX_OPTIONS.setup_type == AIO_SX:
-        old_ip_address = configini['OAM_NETWORK']['IP_ADDRESS']
-        new_ip_address = V_BOX_OPTIONS.controller0_ip
-        LOG.info("Replacing OAM_NETWORK/IP_ADDRESS from %s to %s",
-                 old_ip_address, new_ip_address)
-        configini['OAM_NETWORK']['IP_ADDRESS'] = new_ip_address
-    else:
-        old_start_addr = configini['OAM_NETWORK']['IP_START_ADDRESS']
-        new_start_addr = V_BOX_OPTIONS.ini_oam_ip_start_address
-        LOG.info("Replacing OAM_NETWORK/IP_START_ADDRESS from %s to %s",
-                 old_start_addr, new_start_addr)
-        configini['OAM_NETWORK']['IP_START_ADDRESS'] = new_start_addr
-        old_end_addr = configini['OAM_NETWORK']['IP_END_ADDRESS']
-        new_end_addr = V_BOX_OPTIONS.ini_oam_ip_end_address
-        LOG.info("Replacing OAM_NETWORK/IP_END_ADDRESS from %s to %s",
-                 old_end_addr, new_end_addr)
-        configini['OAM_NETWORK']['IP_END_ADDRESS'] = new_end_addr
 
-    # Take updated config file and copy it to controller
+    #Update localhost.yml with system password
-    with tempfile.NamedTemporaryFile(mode='w') as file:
+    new_config_ansible = override_ansible_become_pass()
-        configini.write(file, space_around_delimiters=False)
-        file.flush()
 
-        sftp_send(
-            file.name,
-            destination,
-            {
-                "remote_host": ip_addr,
-                "remote_port": port,
-                "username": V_BOX_OPTIONS.username,
-                "password": V_BOX_OPTIONS.password
-            }
-        )
 
+    #Send Ansible configuration file to VM
     LOG.info("Copying Ansible configuration file")
     destination_ansible = f'/home/{V_BOX_OPTIONS.username}/localhost.yml'
     sftp_send(
-        V_BOX_OPTIONS.ansible_controller_config,
+        new_config_ansible,
         destination_ansible,
         {
             "remote_host": ip_addr,
@@ -1584,7 +1576,7 @@ STAGE_CALLBACKS = {
          HELP: "Install controller-0 from --iso-location"},
     STG_CONFIG_CONTROLLER:
         {CALLBACK: stage_config_controller,
-         HELP: "Run config controller using the --config-controller-ini" \
+         HELP: "Run config controller using the --ansible-controller-config" \
                "updated based on --ini-* options."},
     STG_RSYNC_CONFIG:
         {CALLBACK: stage_rsync_config,
@@ -1758,6 +1750,32 @@ def load_config():
     else:
         V_BOX_OPTIONS.lab_setup_conf = V_BOX_OPTIONS.lab_setup_conf
 
+    try:
+        with open(V_BOX_OPTIONS.ansible_controller_config, encoding="utf-8") as stream:
+            loaded = ruamel.yaml.safe_load(stream)
+            if V_BOX_OPTIONS.setup_type != AIO_SX:
+                V_BOX_OPTIONS.controller_floating_ip = loaded.get('external_oam_floating_address')
+                V_BOX_OPTIONS.controller0_ip = loaded.get('external_oam_node_0_address')
+                V_BOX_OPTIONS.controller1_ip = loaded.get('external_oam_node_1_address')
+
+                assert V_BOX_OPTIONS.controller_floating_ip, "Missing external_oam_floating_address from ansible config file"
+                assert V_BOX_OPTIONS.controller0_ip, "Missing external_oam_node_0_address from ansible config file"
+                assert V_BOX_OPTIONS.controller1_ip, "Missing external_oam_node_1_address from ansible config file"
+            else:
+                V_BOX_OPTIONS.controller_floating_ip = None
+                # In a AIO-SX configuration the ip of controller-0 must be the same as the floating defined in ansible config file.
+                V_BOX_OPTIONS.controller0_ip = loaded.get('external_oam_floating_address')
+                V_BOX_OPTIONS.controller1_ip = None
+
+                assert V_BOX_OPTIONS.controller0_ip, "Missing external_oam_floating_address from ansible config file"
+    except FileNotFoundError:
+        print (f' \n Ansible configuration file not found in {V_BOX_OPTIONS.ansible_controller_config} \n')
+        sys.exit(1)
+    except ruamel.yaml.YAMLError:
+        print("\n Error while parsing YAML file \n")
+        sys.exit()
+
+
     if V_BOX_OPTIONS.setup_type == AIO_SX:
         V_BOX_OPTIONS.controllers = 1
         V_BOX_OPTIONS.workers = 0
@@ -1770,27 +1788,6 @@ def load_config():
         V_BOX_OPTIONS.storages = 0
 
 
-def pre_validate(m_vboxoptions):
-    """
-    Checks that required options have been set and prints an error message and exits
-    with an error code if any of them are missing.
-    """
-
-    err = False
-    if not m_vboxoptions.setup_type:
-        print("Please set --setup-type")
-        err = True
-    if not m_vboxoptions.labname:
-        print("Please set --labname")
-        err = True
-    if not m_vboxoptions.config_controller_ini:
-        print("Please set --iso-location")
-        err = True
-    if err:
-        print("\nMissing arguments. Please check --help and --list-stages for usage.")
-        sys.exit(5)
-
-
 def validate(v_box_opt, m_stages):
     """
     Validates the values of the configuration options based on the stages that are going
@@ -1824,8 +1821,8 @@ def validate(v_box_opt, m_stages):
             print("Second controller is configured, please set --controller1-ip")
             err = True
     if STG_CONFIG_CONTROLLER in m_stages:
-        if not v_box_opt.config_controller_ini:
+        if not v_box_opt.ansible_controller_config:
-            print(f"Please set --config-controller-ini as needed by stage {STG_CONFIG_CONTROLLER}")
+            print(f"Please set --ansible-controller-config as needed by stage {STG_CONFIG_CONTROLLER}")
             err = True
     if STG_RSYNC_CONFIG in m_stages:
         if not v_box_opt.config_files_dir and not v_box_opt.config_files_dir_dont_follow_links:
@@ -1907,7 +1904,6 @@ if __name__ == "__main__":
             print(wrap_stage_help(stage, STAGE_CALLBACKS[stage][HELP]))
         sys.exit(0)
 
-    pre_validate(V_BOX_OPTIONS)
 
     init_logging(V_BOX_OPTIONS.labname, V_BOX_OPTIONS.logpath)
     LOG.info("Logging to directory: %s", (get_log_dir() + "/"))

virtualbox/pybox/requirements.txt

@@ -4,4 +4,3 @@ pytest
 git+https://github.com/digidotcom/python-streamexpect#egg=streamexpect
 pexpect
 ruamel.yaml
-