From 15d64e84badfbde16a12e81b30ad6f602f8c4bdf Mon Sep 17 00:00:00 2001
From: "Anderson, Craig (ca846m)" <ca846m@att.com>
Date: Thu, 11 Feb 2021 14:15:04 -0800
Subject: [PATCH] Read inputs from stdin

Airshipctl implementation will provide container inputs via stdin. This
patchset allows the image-builder container to accept inputs of this
kind. It maintains backwards compatibility with previous verison until
it is no longer used by airshipctl.

Add control to enable or disable qcow compression.

Add wget package to image defaults.

Change-Id: I835c183be7ef6e06f2e3550de2a726df6f9d0e3f
---
 .zuul.yaml                                    |  10 +-
 image-builder/assets/entrypoint.sh            |  11 +-
 image-builder/assets/functions.sh             |   4 +
 image-builder/assets/functions_v2.sh          | 149 ++++++++++++++++++
 .../roles/multistrap/defaults/main.yaml       |   1 +
 .../playbooks/roles/qcow/defaults/main.yaml   |   1 +
 .../qcow/tasks/qcow-detach-n-compress.yaml    |   6 +
 image-builder/tools/cut_image.sh              |  22 ++-
 8 files changed, 193 insertions(+), 11 deletions(-)
 create mode 100755 image-builder/assets/functions_v2.sh

diff --git a/.zuul.yaml b/.zuul.yaml
index 4873297..0e375be 100644
--- a/.zuul.yaml
+++ b/.zuul.yaml
@@ -42,7 +42,7 @@
 
 - job:
     name: airship-images-build
-    nodeset: airship-images-single-node
+    nodeset: airship-images-single-node-32GB
     timeout: 7200
     post-timeout: 7200
     pre-run: playbooks/airship-images-deploy-docker.yaml
@@ -51,7 +51,7 @@
 
 - job:
     name: airship-images-publish
-    nodeset: airship-images-single-node
+    nodeset: airship-images-single-node-32GB
     timeout: 7200
     post-timeout: 7200
     pre-run: playbooks/airship-images-deploy-docker.yaml
@@ -73,6 +73,12 @@
 
 - nodeset:
     name: airship-images-single-node
+    nodes:
+      - name: primary
+        label: ubuntu-bionic
+
+- nodeset:
+    name: airship-images-single-node-32GB
     nodes:
       - name: primary
         label: ubuntu-bionic-32GB
diff --git a/image-builder/assets/entrypoint.sh b/image-builder/assets/entrypoint.sh
index 0d587a9..e7de6a8 100755
--- a/image-builder/assets/entrypoint.sh
+++ b/image-builder/assets/entrypoint.sh
@@ -1,5 +1,5 @@
 #!/bin/bash
-set -ex
+set -e
 
 SOURCE="${BASH_SOURCE[0]}"
 while [ -h "$SOURCE" ]; do # resolve $SOURCE until the file is no longer a symlink
@@ -11,7 +11,11 @@ BASEDIR="$( cd -P "$( dirname "$SOURCE" )" >/dev/null 2>&1 && pwd )"
 cd "$BASEDIR"
 
 BASEDIR="$(dirname "$(realpath "$0")")"
-source "${BASEDIR}/functions.sh"
+if [ "${VERSION}" = "v2" ]; then
+  source "${BASEDIR}/functions_v2.sh"
+else
+  source "${BASEDIR}/functions.sh"
+fi
 
 export http_proxy
 export https_proxy
@@ -57,8 +61,5 @@ else
   exit 1
 fi
 
-# Write metadata output file containing host path to image and md5sum
-_make_metadata "${IMG_NAME}"
-
 echo "All Ansible plays completed successfully"
 
diff --git a/image-builder/assets/functions.sh b/image-builder/assets/functions.sh
index a5829f5..ebe7580 100755
--- a/image-builder/assets/functions.sh
+++ b/image-builder/assets/functions.sh
@@ -1,4 +1,8 @@
 #!/bin/bash
+# NOTE: These functions are deprecated. It is only left here
+# for backwards compatibility until airshipctl is migrated
+# away from using it.
+set -x
 
 # Defaults
 OUTPUT_METADATA_FILE_NAME_DEFAULT='output-metadata.yaml'
diff --git a/image-builder/assets/functions_v2.sh b/image-builder/assets/functions_v2.sh
new file mode 100755
index 0000000..de00473
--- /dev/null
+++ b/image-builder/assets/functions_v2.sh
@@ -0,0 +1,149 @@
+#!/bin/bash
+
+# Defaults
+ISO_NAME_DEFAULT='ephemeral.iso'
+
+_validate_param(){
+  PARAM_VAL="$1"
+  PARAM_NAME="$2"
+  # Validate that a paramter is defined (default) or that
+  # it is defined and represents the path of a file or
+  # directory that is found on the filesystem (VAL_TYPE=file)
+  VAL_TYPE="$3"
+  NO_NULL_EXIT="$4"
+  echo "${PARAM_VAL:?}"
+  # yq will return the 'null' string if a key is either undefined or defined with no value
+  if [[ "${PARAM_VAL}" =~ null$ ]]
+  then
+    echo "variable ${PARAM_NAME} is not present in user-supplied config."
+    if [[ "${NO_NULL_EXIT}" == 'no_null_exit' ]]; then
+      echo "Using defaults"
+    else
+      exit 1
+    fi
+  else
+    if [[ ${VAL_TYPE} == 'file' ]]; then
+      if [[ ! -e "${PARAM_VAL}" ]]
+      then
+        echo "${PARAM_VAL} not exist"
+        exit 1
+      fi
+    fi
+  fi
+}
+
+# Capture stdin
+stdin=$(cat)
+
+yaml_dir=/tmp
+echo "$stdin" > ${yaml_dir}/builder_config
+
+OSCONFIG_FILE=osconfig
+USER_DATA_FILE=user_data
+NET_CONFIG_FILE=network_config
+QCOW_CONFIG_FILE=qcow
+
+file_list="${OSCONFIG_FILE}
+${USER_DATA_FILE}
+${NET_CONFIG_FILE}
+${QCOW_CONFIG_FILE}"
+
+IFS=$'\n'
+for f in $file_list; do
+  found_file=no
+  for l in $stdin; do
+    if [ "${l:0:1}" != " " ]; then
+      found_file=no
+    fi
+    if [ "$found_file" = "yes" ]; then
+      echo "$l" | sed 's/^    //g' >> ${yaml_dir}/${f}
+    fi
+    if [ "$l" = "${f}:" ]; then
+      found_file=yes
+    fi
+  done
+done
+unset IFS
+
+# Turn on -x after stdin is finished
+set -x
+
+# Output images to the first root-level mounted volume
+for f in $(ls / | grep -v 'proc\|sys\|dev'); do mountpoint /$f >& /dev/null && VOLUME=/$f; done
+if [ -z "$VOLUME" ]; then
+  echo "Error: Could not find a root-level volume mount to output images. Exiting."
+  exit 1
+fi
+
+# Read IMAGE_TYPE from the builder config yaml if not supplied as an env var
+if [[ -z "${IMAGE_TYPE}" ]]; then
+  # Make iso builds the default for backwards compatibility
+  echo "NOTE: No IMAGE_TYPE specified. Assuming 'iso'."
+  IMAGE_TYPE='iso'
+fi
+
+OUTPUT_FILE_NAME="$(yq r ${yaml_dir}/builder_config outputFileName)"
+
+_process_input_data_set_vars_osconfig(){
+  OSCONFIG_FILE="${yaml_dir}/${OSCONFIG_FILE}"
+  # Optional user-supplied playbook vars
+  if [[ -f "${OSCONFIG_FILE}" ]]; then
+    cp "${OSCONFIG_FILE}" /opt/assets/playbooks/roles/osconfig/vars/main.yaml
+  fi
+}
+
+_process_input_data_set_vars_iso(){
+  # Required user provided input
+  USER_DATA_FILE="${yaml_dir}/${USER_DATA_FILE}"
+  if [ ! -e $USER_DATA_FILE ]; then
+    echo "No user_data file supplied! Exiting."
+    exit 1
+  fi
+
+  # Required user provided input
+  NET_CONFIG_FILE="${yaml_dir}/${NET_CONFIG_FILE}"
+  if [ ! -e $USER_DATA_FILE ]; then
+    echo "No net_config file supplied! Exiting."
+    exit 1
+  fi
+  # cloud-init expects net confing specifically in json format
+  NET_CONFIG_JSON_FILE=/tmp/network_data.json
+  yq r -j "${NET_CONFIG_FILE}" > "${NET_CONFIG_JSON_FILE}"
+
+  # Optional user provided input
+  if [[ ${OUTPUT_FILE_NAME} != null ]]; then
+    IMG_NAME="${OUTPUT_FILE_NAME}"
+  else
+    IMG_NAME="${ISO_NAME_DEFAULT}"
+  fi
+  cat << EOF > /opt/assets/playbooks/roles/iso/vars/main.yaml
+meta_data_file: ${BASEDIR}/meta_data.json
+user_data_file: ${USER_DATA_FILE}
+network_data_file: ${NET_CONFIG_JSON_FILE}
+EOF
+}
+
+_process_input_data_set_vars_qcow(){
+  IMG_NAME=null
+  QCOW_CONFIG_FILE="${yaml_dir}/${QCOW_CONFIG_FILE}"
+  # Optional user-supplied playbook vars
+  if [[ -f "${QCOW_CONFIG_FILE}" ]]; then
+    cp "${QCOW_CONFIG_FILE}" /opt/assets/playbooks/roles/qcow/vars/main.yaml
+
+    # Extract the image output name in the ansible vars file provided
+    IMG_NAME="$(yq r "${QCOW_CONFIG_FILE}" img_name)"
+  fi
+
+  # Retrieve from playbook defaults if not provided in user input
+  if [[ "${IMG_NAME}" == 'null' ]]; then
+    IMG_NAME="$(yq r /opt/assets/playbooks/roles/qcow/defaults/main.yaml img_name)"
+  fi
+
+  # User-supplied image output name in builder-config takes precedence
+  if [[ ${OUTPUT_FILE_NAME} != null ]]; then
+    IMG_NAME="${OUTPUT_FILE_NAME}"
+  else
+    _validate_param "${IMG_NAME}" img_name
+  fi
+}
+
diff --git a/image-builder/assets/playbooks/roles/multistrap/defaults/main.yaml b/image-builder/assets/playbooks/roles/multistrap/defaults/main.yaml
index 6045234..6ebe8c0 100644
--- a/image-builder/assets/playbooks/roles/multistrap/defaults/main.yaml
+++ b/image-builder/assets/playbooks/roles/multistrap/defaults/main.yaml
@@ -68,6 +68,7 @@ ubuntu_packages:
     - traceroute
     - vim
     - vlan
+    - wget
     - xfsprogs
     - xz-utils
 repos:
diff --git a/image-builder/assets/playbooks/roles/qcow/defaults/main.yaml b/image-builder/assets/playbooks/roles/qcow/defaults/main.yaml
index fbc901b..4efe3ae 100644
--- a/image-builder/assets/playbooks/roles/qcow/defaults/main.yaml
+++ b/image-builder/assets/playbooks/roles/qcow/defaults/main.yaml
@@ -4,6 +4,7 @@ nbd_build_dir: /tmp/nbd_build_dir
 img_output_dir: /config
 img_name: airship-ubuntu.qcow2
 qcow_capacity: 5G
+qcow_compress: true
 partitions:
     # Partition numbering is according to list ordering.
     # Ironic default cloud-init configdrive injection requires
diff --git a/image-builder/assets/playbooks/roles/qcow/tasks/qcow-detach-n-compress.yaml b/image-builder/assets/playbooks/roles/qcow/tasks/qcow-detach-n-compress.yaml
index cdb0ae0..1b48240 100644
--- a/image-builder/assets/playbooks/roles/qcow/tasks/qcow-detach-n-compress.yaml
+++ b/image-builder/assets/playbooks/roles/qcow/tasks/qcow-detach-n-compress.yaml
@@ -6,3 +6,9 @@
 - name: "QCOW | Compressing QCoW and writing out to {{ img_output_dir }}/{{ img_name }}"
   shell: |
         qemu-img convert -p -O qcow2 -c {{ nbd_build_dir }}/{{ img_name }} {{ img_output_dir }}/{{ img_name }}
+  when: qcow_compress
+
+- name: "QCOW | Writing QCoW to {{ img_output_dir }}/{{ img_name }}"
+  shell: |
+        qemu-img convert -p -O qcow2 {{ nbd_build_dir }}/{{ img_name }} {{ img_output_dir }}/{{ img_name }}
+  when: not qcow_compress
diff --git a/image-builder/tools/cut_image.sh b/image-builder/tools/cut_image.sh
index 772907f..1e440d8 100755
--- a/image-builder/tools/cut_image.sh
+++ b/image-builder/tools/cut_image.sh
@@ -57,22 +57,35 @@ fi
 workdir="$(realpath ${host_mount_directory})"
 
 if [[ $build_type = iso ]]; then
-  sudo -E docker run -t --rm \
+  iso_config=/tmp/iso_config
+  echo "user_data:
+$(cat $host_mount_directory/user_data | sed 's/^/    /g')
+network_config:
+$(cat $host_mount_directory/network_data.json | sed 's/^/    /g')
+outputFileName: ephemeral.iso" > ${iso_config}
+  sudo -E docker run -i --rm \
    --volume $workdir:/config \
    --env BUILDER_CONFIG=/config/${build_type}.yaml \
    --env IMAGE_TYPE="iso" \
+   --env VERSION="v2" \
    --env http_proxy=$proxy \
    --env https_proxy=$proxy \
    --env HTTP_PROXY=$proxy \
    --env HTTPS_PROXY=$proxy \
    --env no_proxy=$noproxy \
    --env NO_PROXY=$noproxy \
-   ${image}
+   ${image} < ${iso_config}
   disk1="--disk path=${workdir}/ephemeral.iso,device=cdrom"
 elif [[ $build_type == qcow ]]; then
   sudo -E modprobe nbd
+  qcow_config=/tmp/qcow_config
+  echo "osconfig:
+$(cat $host_mount_directory/osconfig-control-plane-vars.yaml | sed 's/^/    /g')
+qcow:
+$(cat $host_mount_directory/qcow-control-plane-vars.yaml | sed 's/^/    /g')
+outputFileName: control-plane.qcow2" > ${qcow_config}
   echo "Note: This step can be slow if you don't have an SSD."
-  sudo -E docker run -t --rm \
+  sudo -E docker run -i --rm \
    --privileged \
    --volume /dev:/dev:rw  \
    --volume /dev/pts:/dev/pts:rw \
@@ -83,13 +96,14 @@ elif [[ $build_type == qcow ]]; then
    ${uefi_mount} \
    --env BUILDER_CONFIG=/config/${build_type}.yaml \
    --env IMAGE_TYPE="qcow" \
+   --env VERSION="v2" \
    --env http_proxy=$proxy \
    --env https_proxy=$proxy \
    --env HTTP_PROXY=$proxy \
    --env HTTPS_PROXY=$proxy \
    --env no_proxy=$noproxy \
    --env NO_PROXY=$noproxy \
-   ${image}
+   ${image} < ${qcow_config}
   cloud_init_config_dir='assets/tests/qcow/cloud-init'
   sudo -E cloud-localds -v --network-config="${cloud_init_config_dir}/network-config" "${workdir}/airship-ubuntu_config.iso" "${cloud_init_config_dir}/user-data" "${cloud_init_config_dir}/meta-data"
   disk1="--disk path=${workdir}/control-plane.qcow2"