diff --git a/.zuul.yaml b/.zuul.yaml index 4873297..0e375be 100644 --- a/.zuul.yaml +++ b/.zuul.yaml @@ -42,7 +42,7 @@ - job: name: airship-images-build - nodeset: airship-images-single-node + nodeset: airship-images-single-node-32GB timeout: 7200 post-timeout: 7200 pre-run: playbooks/airship-images-deploy-docker.yaml @@ -51,7 +51,7 @@ - job: name: airship-images-publish - nodeset: airship-images-single-node + nodeset: airship-images-single-node-32GB timeout: 7200 post-timeout: 7200 pre-run: playbooks/airship-images-deploy-docker.yaml @@ -73,6 +73,12 @@ - nodeset: name: airship-images-single-node + nodes: + - name: primary + label: ubuntu-bionic + +- nodeset: + name: airship-images-single-node-32GB nodes: - name: primary label: ubuntu-bionic-32GB diff --git a/image-builder/assets/entrypoint.sh b/image-builder/assets/entrypoint.sh index 0d587a9..e7de6a8 100755 --- a/image-builder/assets/entrypoint.sh +++ b/image-builder/assets/entrypoint.sh @@ -1,5 +1,5 @@ #!/bin/bash -set -ex +set -e SOURCE="${BASH_SOURCE[0]}" while [ -h "$SOURCE" ]; do # resolve $SOURCE until the file is no longer a symlink @@ -11,7 +11,11 @@ BASEDIR="$( cd -P "$( dirname "$SOURCE" )" >/dev/null 2>&1 && pwd )" cd "$BASEDIR" BASEDIR="$(dirname "$(realpath "$0")")" -source "${BASEDIR}/functions.sh" +if [ "${VERSION}" = "v2" ]; then + source "${BASEDIR}/functions_v2.sh" +else + source "${BASEDIR}/functions.sh" +fi export http_proxy export https_proxy @@ -57,8 +61,5 @@ else exit 1 fi -# Write metadata output file containing host path to image and md5sum -_make_metadata "${IMG_NAME}" - echo "All Ansible plays completed successfully" diff --git a/image-builder/assets/functions.sh b/image-builder/assets/functions.sh index a5829f5..ebe7580 100755 --- a/image-builder/assets/functions.sh +++ b/image-builder/assets/functions.sh @@ -1,4 +1,8 @@ #!/bin/bash +# NOTE: These functions are deprecated. It is only left here +# for backwards compatibility until airshipctl is migrated +# away from using it. +set -x # Defaults OUTPUT_METADATA_FILE_NAME_DEFAULT='output-metadata.yaml' diff --git a/image-builder/assets/functions_v2.sh b/image-builder/assets/functions_v2.sh new file mode 100755 index 0000000..de00473 --- /dev/null +++ b/image-builder/assets/functions_v2.sh @@ -0,0 +1,149 @@ +#!/bin/bash + +# Defaults +ISO_NAME_DEFAULT='ephemeral.iso' + +_validate_param(){ + PARAM_VAL="$1" + PARAM_NAME="$2" + # Validate that a paramter is defined (default) or that + # it is defined and represents the path of a file or + # directory that is found on the filesystem (VAL_TYPE=file) + VAL_TYPE="$3" + NO_NULL_EXIT="$4" + echo "${PARAM_VAL:?}" + # yq will return the 'null' string if a key is either undefined or defined with no value + if [[ "${PARAM_VAL}" =~ null$ ]] + then + echo "variable ${PARAM_NAME} is not present in user-supplied config." + if [[ "${NO_NULL_EXIT}" == 'no_null_exit' ]]; then + echo "Using defaults" + else + exit 1 + fi + else + if [[ ${VAL_TYPE} == 'file' ]]; then + if [[ ! -e "${PARAM_VAL}" ]] + then + echo "${PARAM_VAL} not exist" + exit 1 + fi + fi + fi +} + +# Capture stdin +stdin=$(cat) + +yaml_dir=/tmp +echo "$stdin" > ${yaml_dir}/builder_config + +OSCONFIG_FILE=osconfig +USER_DATA_FILE=user_data +NET_CONFIG_FILE=network_config +QCOW_CONFIG_FILE=qcow + +file_list="${OSCONFIG_FILE} +${USER_DATA_FILE} +${NET_CONFIG_FILE} +${QCOW_CONFIG_FILE}" + +IFS=$'\n' +for f in $file_list; do + found_file=no + for l in $stdin; do + if [ "${l:0:1}" != " " ]; then + found_file=no + fi + if [ "$found_file" = "yes" ]; then + echo "$l" | sed 's/^ //g' >> ${yaml_dir}/${f} + fi + if [ "$l" = "${f}:" ]; then + found_file=yes + fi + done +done +unset IFS + +# Turn on -x after stdin is finished +set -x + +# Output images to the first root-level mounted volume +for f in $(ls / | grep -v 'proc\|sys\|dev'); do mountpoint /$f >& /dev/null && VOLUME=/$f; done +if [ -z "$VOLUME" ]; then + echo "Error: Could not find a root-level volume mount to output images. Exiting." + exit 1 +fi + +# Read IMAGE_TYPE from the builder config yaml if not supplied as an env var +if [[ -z "${IMAGE_TYPE}" ]]; then + # Make iso builds the default for backwards compatibility + echo "NOTE: No IMAGE_TYPE specified. Assuming 'iso'." + IMAGE_TYPE='iso' +fi + +OUTPUT_FILE_NAME="$(yq r ${yaml_dir}/builder_config outputFileName)" + +_process_input_data_set_vars_osconfig(){ + OSCONFIG_FILE="${yaml_dir}/${OSCONFIG_FILE}" + # Optional user-supplied playbook vars + if [[ -f "${OSCONFIG_FILE}" ]]; then + cp "${OSCONFIG_FILE}" /opt/assets/playbooks/roles/osconfig/vars/main.yaml + fi +} + +_process_input_data_set_vars_iso(){ + # Required user provided input + USER_DATA_FILE="${yaml_dir}/${USER_DATA_FILE}" + if [ ! -e $USER_DATA_FILE ]; then + echo "No user_data file supplied! Exiting." + exit 1 + fi + + # Required user provided input + NET_CONFIG_FILE="${yaml_dir}/${NET_CONFIG_FILE}" + if [ ! -e $USER_DATA_FILE ]; then + echo "No net_config file supplied! Exiting." + exit 1 + fi + # cloud-init expects net confing specifically in json format + NET_CONFIG_JSON_FILE=/tmp/network_data.json + yq r -j "${NET_CONFIG_FILE}" > "${NET_CONFIG_JSON_FILE}" + + # Optional user provided input + if [[ ${OUTPUT_FILE_NAME} != null ]]; then + IMG_NAME="${OUTPUT_FILE_NAME}" + else + IMG_NAME="${ISO_NAME_DEFAULT}" + fi + cat << EOF > /opt/assets/playbooks/roles/iso/vars/main.yaml +meta_data_file: ${BASEDIR}/meta_data.json +user_data_file: ${USER_DATA_FILE} +network_data_file: ${NET_CONFIG_JSON_FILE} +EOF +} + +_process_input_data_set_vars_qcow(){ + IMG_NAME=null + QCOW_CONFIG_FILE="${yaml_dir}/${QCOW_CONFIG_FILE}" + # Optional user-supplied playbook vars + if [[ -f "${QCOW_CONFIG_FILE}" ]]; then + cp "${QCOW_CONFIG_FILE}" /opt/assets/playbooks/roles/qcow/vars/main.yaml + + # Extract the image output name in the ansible vars file provided + IMG_NAME="$(yq r "${QCOW_CONFIG_FILE}" img_name)" + fi + + # Retrieve from playbook defaults if not provided in user input + if [[ "${IMG_NAME}" == 'null' ]]; then + IMG_NAME="$(yq r /opt/assets/playbooks/roles/qcow/defaults/main.yaml img_name)" + fi + + # User-supplied image output name in builder-config takes precedence + if [[ ${OUTPUT_FILE_NAME} != null ]]; then + IMG_NAME="${OUTPUT_FILE_NAME}" + else + _validate_param "${IMG_NAME}" img_name + fi +} + diff --git a/image-builder/assets/playbooks/roles/multistrap/defaults/main.yaml b/image-builder/assets/playbooks/roles/multistrap/defaults/main.yaml index 6045234..6ebe8c0 100644 --- a/image-builder/assets/playbooks/roles/multistrap/defaults/main.yaml +++ b/image-builder/assets/playbooks/roles/multistrap/defaults/main.yaml @@ -68,6 +68,7 @@ ubuntu_packages: - traceroute - vim - vlan + - wget - xfsprogs - xz-utils repos: diff --git a/image-builder/assets/playbooks/roles/qcow/defaults/main.yaml b/image-builder/assets/playbooks/roles/qcow/defaults/main.yaml index fbc901b..4efe3ae 100644 --- a/image-builder/assets/playbooks/roles/qcow/defaults/main.yaml +++ b/image-builder/assets/playbooks/roles/qcow/defaults/main.yaml @@ -4,6 +4,7 @@ nbd_build_dir: /tmp/nbd_build_dir img_output_dir: /config img_name: airship-ubuntu.qcow2 qcow_capacity: 5G +qcow_compress: true partitions: # Partition numbering is according to list ordering. # Ironic default cloud-init configdrive injection requires diff --git a/image-builder/assets/playbooks/roles/qcow/tasks/qcow-detach-n-compress.yaml b/image-builder/assets/playbooks/roles/qcow/tasks/qcow-detach-n-compress.yaml index cdb0ae0..1b48240 100644 --- a/image-builder/assets/playbooks/roles/qcow/tasks/qcow-detach-n-compress.yaml +++ b/image-builder/assets/playbooks/roles/qcow/tasks/qcow-detach-n-compress.yaml @@ -6,3 +6,9 @@ - name: "QCOW | Compressing QCoW and writing out to {{ img_output_dir }}/{{ img_name }}" shell: | qemu-img convert -p -O qcow2 -c {{ nbd_build_dir }}/{{ img_name }} {{ img_output_dir }}/{{ img_name }} + when: qcow_compress + +- name: "QCOW | Writing QCoW to {{ img_output_dir }}/{{ img_name }}" + shell: | + qemu-img convert -p -O qcow2 {{ nbd_build_dir }}/{{ img_name }} {{ img_output_dir }}/{{ img_name }} + when: not qcow_compress diff --git a/image-builder/tools/cut_image.sh b/image-builder/tools/cut_image.sh index 772907f..1e440d8 100755 --- a/image-builder/tools/cut_image.sh +++ b/image-builder/tools/cut_image.sh @@ -57,22 +57,35 @@ fi workdir="$(realpath ${host_mount_directory})" if [[ $build_type = iso ]]; then - sudo -E docker run -t --rm \ + iso_config=/tmp/iso_config + echo "user_data: +$(cat $host_mount_directory/user_data | sed 's/^/ /g') +network_config: +$(cat $host_mount_directory/network_data.json | sed 's/^/ /g') +outputFileName: ephemeral.iso" > ${iso_config} + sudo -E docker run -i --rm \ --volume $workdir:/config \ --env BUILDER_CONFIG=/config/${build_type}.yaml \ --env IMAGE_TYPE="iso" \ + --env VERSION="v2" \ --env http_proxy=$proxy \ --env https_proxy=$proxy \ --env HTTP_PROXY=$proxy \ --env HTTPS_PROXY=$proxy \ --env no_proxy=$noproxy \ --env NO_PROXY=$noproxy \ - ${image} + ${image} < ${iso_config} disk1="--disk path=${workdir}/ephemeral.iso,device=cdrom" elif [[ $build_type == qcow ]]; then sudo -E modprobe nbd + qcow_config=/tmp/qcow_config + echo "osconfig: +$(cat $host_mount_directory/osconfig-control-plane-vars.yaml | sed 's/^/ /g') +qcow: +$(cat $host_mount_directory/qcow-control-plane-vars.yaml | sed 's/^/ /g') +outputFileName: control-plane.qcow2" > ${qcow_config} echo "Note: This step can be slow if you don't have an SSD." - sudo -E docker run -t --rm \ + sudo -E docker run -i --rm \ --privileged \ --volume /dev:/dev:rw \ --volume /dev/pts:/dev/pts:rw \ @@ -83,13 +96,14 @@ elif [[ $build_type == qcow ]]; then ${uefi_mount} \ --env BUILDER_CONFIG=/config/${build_type}.yaml \ --env IMAGE_TYPE="qcow" \ + --env VERSION="v2" \ --env http_proxy=$proxy \ --env https_proxy=$proxy \ --env HTTP_PROXY=$proxy \ --env HTTPS_PROXY=$proxy \ --env no_proxy=$noproxy \ --env NO_PROXY=$noproxy \ - ${image} + ${image} < ${qcow_config} cloud_init_config_dir='assets/tests/qcow/cloud-init' sudo -E cloud-localds -v --network-config="${cloud_init_config_dir}/network-config" "${workdir}/airship-ubuntu_config.iso" "${cloud_init_config_dir}/user-data" "${cloud_init_config_dir}/meta-data" disk1="--disk path=${workdir}/control-plane.qcow2"