osconfig cleanup

This patchset allows users to select the context in which certain
osconfig tasks run, rather than a hard-coded assumption.

It also adds scripts directories which are run in these contexts.

Change-Id: I695c3078b4d883a506c243bc715a509f6844c126

image-builder/Makefile

@@ -24,7 +24,12 @@ IMAGE_TYPE	?= iso # iso | qcow
 PUSH_IMAGE	?= false
 DISTRO          ?= ubuntu_focal
 WORKDIR         ?= ./config
+# Specify if you want to specify qcow config directories manually instead of using default naming conventions
 QCOW_CONF_DIRS  ?=
+# Set to true if you want to skip multistrap build step for testing
+SKIP_MULTISTRAP ?=
+# Set to true if you want to skip the other playsbooks except osconfig for osconfig playbook testing
+OSCONFIG_TEST   ?=
 IMAGE           ?= ${DOCKER_REGISTRY}/${IMAGE_PREFIX}/${IMAGE_NAME}:${IMAGE_TAG}-${DISTRO}
 QCOW_IMAGE      ?= ${DOCKER_REGISTRY}/${IMAGE_PREFIX}/${QCOW_IMAGE_NAME}:${IMAGE_TAG}-${DISTRO}
 PROXY           ?=
@@ -52,7 +57,9 @@ ifneq ($(PROXY), )
 	export HTTP_PROXY=$(PROXY)
 	export HTTPS_PROXY=$(PROXY)
 	export NO_PROXY=$(NO_PROXY)
-	sudo -E ./tools/multistrap.sh
+ifneq ($(SKIP_MULTISTRAP), true)
+	sudo -E ./tools/multistrap.sh $(WORKDIR)
+endif
 	sudo -E docker -D -l debug build --tag $(IMAGE) -f Dockerfile.$(DISTRO) . \
 	--label $(LABEL) \
 	--label "org.opencontainers.image.revision=$(COMMIT)" \
@@ -66,7 +73,9 @@ ifneq ($(PROXY), )
 	--build-arg no_proxy=$(NO_PROXY) \
 	--build-arg NO_PROXY=$(NO_PROXY)
 else
-	sudo -E ./tools/multistrap.sh
+ifneq ($(SKIP_MULTISTRAP), true)
+	sudo -E ./tools/multistrap.sh $(WORKDIR)
+endif
 	sudo -E docker -D -l debug build --tag $(IMAGE) -f Dockerfile.$(DISTRO) . \
 	--label $(LABEL) \
 	--label "org.opencontainers.image.revision=$(COMMIT)" \
@@ -93,7 +102,7 @@ ifneq ($(EXPLICIT_DIRS), )
 	iterDirs="$(EXPLICIT_DIRS)"
 else
 	# Assemble all images based on configs defined in each $(IMAGE_TYPE)* subdirectory
-	iterDirs=`find $(WORKDIR) -name "$(IMAGE_TYPE)*" -type d -exec basename {} \;`
+	iterDirs=`find $(WORKDIR) -maxdepth 1 -name "$(IMAGE_TYPE)*" -type d -exec basename {} \;`
 endif
 	for subdir in $$iterDirs; do
 	    # ISO configs
@@ -133,6 +142,6 @@ tests:
 clean:
 	set -ex
 	sudo -E tools/multistrap.sh clean
-	rm $(WORKDIR)/*.iso
-	rm $(WORKDIR)/*.qcow2
-	rm $(WORKDIR)/*.md5sum
+	if ls $(WORKDIR)/*.iso >& /dev/null; then sudo rm $(WORKDIR)/*.iso; fi
+	if ls $(WORKDIR)/*.qcow2 >& /dev/null; then sudo rm $(WORKDIR)/*.qcow2; fi
+	if ls $(WORKDIR)/*.md5sum >& /dev/null; then sudo rm $(WORKDIR)/*.md5sum; fi

image-builder/assets/entrypoint.sh

@@ -45,14 +45,13 @@ elif [[ "${IMAGE_TYPE}" == "qcow" ]]; then
   _process_input_data_set_vars_osconfig
 
   # Instruct ansible how to name image output artifact
-  extra_vars="$extra_vars img_name=${IMG_NAME}"
+  extra_vars="$extra_vars img_name=${IMG_NAME} run_context=qcow"
 
   echo "Executing Step 1: Create qcow2 partitions and filesystems"
   ansible-playbook -i /opt/assets/playbooks/inventory.yaml /opt/assets/playbooks/qcow.yaml --extra-vars "$extra_vars" --tags "prep_img" -vv
 
   echo "Executing Step 2: Applying changes from base-osconfig playbook"
-  ansible-playbook -i /opt/assets/playbooks/inventory.yaml /opt/assets/playbooks/base-osconfig.yaml --extra-vars "$extra_vars" --tags "runtime_and_buildtime" -vv
-  ansible-playbook -i /opt/assets/playbooks/inventory.yaml /opt/assets/playbooks/base-osconfig.yaml --extra-vars "$extra_vars" --tags "runtime_only" -vv
+  ansible-playbook -i /opt/assets/playbooks/inventory.yaml /opt/assets/playbooks/base-osconfig.yaml --extra-vars "$extra_vars" -vv
 
   echo "Executing Step 3: Close image and write qcow2"
   ansible-playbook -i /opt/assets/playbooks/inventory.yaml /opt/assets/playbooks/qcow.yaml --extra-vars "$extra_vars" --tags "close_img" -vv

image-builder/assets/playbooks/roles/osconfig/defaults/main.yaml

@@ -1,4 +1,8 @@
 rootfs_root: build
+default_run_context: common
+qcow_run_context: qcow
+user_scripts_dir_default: "/config/scripts/{{ default_run_context }}"
+user_scripts_dir_qcow: "/config/scripts/{{ qcow_run_context }}"
 
 kernel:
     modules:
@@ -134,6 +138,12 @@ directories:
       permissions: '0755'
       owner: root
       group: root
+      # The contexts where this operation is performed
+      # {{ default_run_context }} = part of shared base image
+      # qcow = is performed for QCOW but not ephemeral (unless
+      # combined with previous item)
+      run_contexts:
+          - "{{ default_run_context }}"
 
 # Any files to write to disk can be defined here
 files:
@@ -145,6 +155,12 @@ files:
       permissions: '0755'
       owner: root
       group: root
+      # The contexts where this operation is performed
+      # {{ default_run_context }} = part of shared base image
+      # qcow = is performed for QCOW but not ephemeral (unless
+      # combined with previous item)
+      run_contexts:
+          - "{{ default_run_context }}"
 
 systemd:
       # Full name, including systemd suffix. sample.service. sample.mount, sample.timer, etc.
@@ -164,20 +180,24 @@ systemd:
       # whether to override existing symlinks (e.g. name collision).
       # Use only if you are intenting to overwrite an existing systemd unit
       force: no
+      # The contexts where this operation is performed
+      # {{ default_run_context }} = part of shared base image
+      # qcow = is performed for QCOW but not ephemeral (unless
+      # combined with previous item)
+      run_contexts:
+          - "{{ default_run_context }}"
 
-# If any custom shell scripts are needed for qcow building for image building,
-# they can be added here.
-buildtime_user_scripts:
-    - file_content: |
-          #!/bin/bash
-          echo "custom container buildtime script"
-
-# Custom user shell scripts to be run during container execution, right before
-# final QCOW image is created.
-runtime_user_scripts:
+# If any custom shell scripts are needed for image building, they can be added here.
+user_scripts:
     - file_content: |
           #!/bin/bash
           echo "custom container buildtime script"
+      # The contexts where this operation is performed
+      # {{ default_run_context }} = part of shared base image
+      # qcow = is performed for QCOW but not ephemeral (unless
+      # combined with previous item)
+      run_contexts:
+          - "{{ default_run_context }}"
 
 # Any other adjustments to file or directory permissions, for files that already exist.
 file_permissions:
@@ -186,3 +206,9 @@ file_permissions:
       permissions: '0700'
       owner: root
       group: root
+      # The contexts where this operation is performed
+      # {{ default_run_context }} = part of shared base image
+      # qcow = is performed for QCOW but not ephemeral (unless
+      # combined with previous item)
+      run_contexts:
+          - "{{ default_run_context }}"

image-builder/assets/playbooks/roles/osconfig/tasks/banners.yaml

@@ -12,7 +12,7 @@
     owner: root
     group: root
     mode: '0755'
-  with_items:
+  loop:
     - absent
     - directory
 - name: "Finalise | Remove MOTD News config"
@@ -26,4 +26,3 @@
     owner: root
     group: root
     mode: '0755'
-

image-builder/assets/playbooks/roles/osconfig/tasks/buildtime-user-scripts.yaml

@@ -1,3 +0,0 @@
-- name: "user-scripts | running user-defined scripts"
-  shell: "{{ item.file_content }}"
-  with_items: "{{ buildtime_user_scripts }}"

image-builder/assets/playbooks/roles/osconfig/tasks/file-permissions.yaml

@@ -5,4 +5,5 @@
     mode: "{{ item.permissions }}"
     owner: "{{ item.owner }}"
     group: "{{ item.group }}"
-  with_items: "{{ file_permissions }}"
+  loop: "{{ file_permissions }}"
+  when: run_context in item.run_contexts

image-builder/assets/playbooks/roles/osconfig/tasks/finalise-rootfs.yaml

@@ -2,4 +2,4 @@
   shell:
     cmd: |
         find "/usr/" "/var/" \( -name "*.pyc" -o -name "__pycache__" \) -delete
-
+        apt -y clean

image-builder/assets/playbooks/roles/osconfig/tasks/limits.yaml

@@ -3,4 +3,4 @@
     src: limits.j2
     dest: "/etc/security/limits.d/99-{{ item.name }}.conf"
     mode: 0644
-  with_items: "{{ limits }}"
+  loop: "{{ limits }}"

image-builder/assets/playbooks/roles/osconfig/tasks/main.yaml

@@ -3,45 +3,54 @@
     file: main.yaml
     name: user-vars
 
-- block:
-  - name: "write user-provided files"
-    include_tasks: write-user-files.yaml
-  - name: "configure cloud-init"
-    include_tasks: cloud-init.yaml
-  - name: "configure modules"
-    include_tasks: modules.yaml
-  - name: "configure limits"
-    include_tasks: limits.yaml
-  - name: "configure sysctl"
-    include_tasks: sysctl.yaml
-  - name: "configure grub"
-    include_tasks: grub.yaml
-  - name: "configure kubernetes"
-    include_tasks: kubernetes.yaml
-  - name: "configure locale"
-    include_tasks: locale.yaml
-  - name: "configure hostname and hosts"
-    include_tasks: hostname-hosts.yaml
-  - name: "configure banners"
-    include_tasks: banners.yaml
-  - name: "unattended upgrades"
-    include_tasks: unattended-upgrades.yaml
-  - name: "configure base systemd"
-    include_tasks: systemd.yaml
-  - name: "configure user-defined systemd"
-    include_tasks: systemd-user.yaml
-  - name: "configure file permissions"
-    include_tasks: file-permissions.yaml
-  - name: "finalise rootfs"
-    include_tasks: finalise-rootfs.yaml
-  tags: runtime_and_buildtime
-- block:
-  - name: "run user-defined scripts"
-    include_tasks: buildtime-user-scripts.yaml
-  tags: buildtime_only
-- block:
-  - name: "run system-defined scripts"
-    include_tasks: runtime-system-scripts.yaml
-  - name: "run user-defined scripts"
-    include_tasks: runtime-user-scripts.yaml
-  tags: runtime_only
+# Tasks run when building image-builder container
+- name: "configure cloud-init"
+  include_tasks: cloud-init.yaml
+  when: run_context == default_run_context
+- name: "configure modules"
+  include_tasks: modules.yaml
+  when: run_context == default_run_context
+- name: "configure limits"
+  include_tasks: limits.yaml
+  when: run_context == default_run_context
+- name: "configure sysctl"
+  include_tasks: sysctl.yaml
+  when: run_context == default_run_context
+- name: "configure grub"
+  include_tasks: grub.yaml
+  when: run_context == default_run_context
+- name: "configure kubernetes"
+  include_tasks: kubernetes.yaml
+  when: run_context == default_run_context
+- name: "configure locale"
+  include_tasks: locale.yaml
+  when: run_context == default_run_context
+- name: "configure hostname and hosts"
+  include_tasks: hostname-hosts.yaml
+  when: run_context == default_run_context
+- name: "configure banners"
+  include_tasks: banners.yaml
+  when: run_context == default_run_context
+- name: "unattended upgrades"
+  include_tasks: unattended-upgrades.yaml
+  when: run_context == default_run_context
+- name: "configure base systemd"
+  include_tasks: systemd.yaml
+  when: run_context == default_run_context
+
+# Context-dependent tasks
+- name: "write user-provided files"
+  include_tasks: write-user-files.yaml
+- name: "configure user-defined systemd"
+  include_tasks: systemd-user.yaml
+- name: "run system-defined scripts for qcow"
+  include_tasks: runtime-system-scripts.yaml
+  when: run_context == qcow_run_context
+- name: "run user-defined scripts"
+  include_tasks: user-scripts.yaml
+- name: "configure file permissions"
+  include_tasks: file-permissions.yaml
+
+# Context-independent cleanup tasks
+- name: "finalise rootfs"
+  include_tasks: finalise-rootfs.yaml

image-builder/assets/playbooks/roles/osconfig/tasks/modules.yaml

@@ -3,10 +3,10 @@
     src: kernelmodules.j2
     dest: "/etc/modules-load.d/99-{{ item.name }}.conf"
     mode: 0644
-  with_items: "{{ kernel.modules.load }}"
+  loop: "{{ kernel.modules.load }}"
 
 - name: "Modules | Defining modules to blacklist"
   kernel_blacklist:
     name: "{{ item.name }}"
     state: present
-  with_items: "{{ kernel.modules.blacklist }}"
+  loop: "{{ kernel.modules.blacklist }}"

image-builder/assets/playbooks/roles/osconfig/tasks/runtime-user-scripts.yaml

@@ -1,3 +0,0 @@
-- name: "user-scripts | running user-defined scripts"
-  shell: "{{ item.file_content }}"
-  with_items: "{{ runtime_user_scripts }}"

image-builder/assets/playbooks/roles/osconfig/tasks/sysctl.yaml

@@ -3,4 +3,4 @@
     value: "{{ item.value }}"
     state: present
     reload: no
-  with_items: "{{ sysctl }}"
+  loop: "{{ sysctl }}"

image-builder/assets/playbooks/roles/osconfig/tasks/systemd-user.yaml

@@ -2,12 +2,13 @@
   template:
     src: generic-file-writer.j2
     dest: "/etc/systemd/system/{{ item.name }}"
-  with_items: "{{ systemd }}"
+  loop: "{{ systemd }}"
+  when: run_context in item.run_contexts
 
 - name: "Systemd | Configuring user-provided systemd unit {{ item.name }}"
   systemd:
     name: "{{ item.name }}"
     enabled: "{{ item.enabled }}"
     force: "{{ item.force }}"
-  with_items: "{{ systemd }}"
-
+  loop: "{{ systemd }}"
+  when: run_context in item.run_contexts

image-builder/assets/playbooks/roles/osconfig/tasks/user-scripts.yaml

@@ -0,0 +1,25 @@
+# Execute scripts defined in the playbook
+- name: "user-scripts | running user-defined scripts"
+  shell: "{{ item.file_content }}"
+  loop: "{{ user_scripts }}"
+  when: run_context in item.run_contexts
+
+- name: "user-scripts | check for common scripts dir"
+  stat:
+    path: "{{ user_scripts_dir_default }}"
+  register: common_stat_result
+
+- name: "user-scripts | check for qcow scripts dir"
+  stat:
+    path: "{{ user_scripts_dir_qcow }}"
+  register: qcow_stat_result
+
+# Bulk-execute scripts in the scripts directory
+- name: "user-scripts | running additional user-defined scripts"
+  shell: for s in $(find "{{ user_scripts_dir_default }}" -maxdepth 1 -type f | grep -v README.md | sort); do chmod 755 $s; eval $s; done
+  when: run_context == default_run_context and common_stat_result.stat.exists
+
+# Bulk-execute scripts in the scripts directory
+- name: "user-scripts | running additional user-defined scripts"
+  shell: for s in $(find "{{ user_scripts_dir_qcow }}" -maxdepth 1 -type f | grep -v README.md | sort); do chmod 755 $s; eval $s; done
+  when: run_context == qcow_run_context and qcow_stat_result.stat.exists

image-builder/assets/playbooks/roles/osconfig/tasks/write-user-files.yaml

@@ -5,7 +5,8 @@
     mode: "{{ item.permissions }}"
     owner: "{{ item.owner }}"
     group: "{{ item.group }}"
-  with_items: "{{ directories }}"
+  loop: "{{ directories }}"
+  when: run_context in item.run_contexts
 
 - name: "User Files | Writing user-provided file {{ item.name }}"
   template:
@@ -14,4 +15,5 @@
     mode: "{{ item.permissions }}"
     owner: "{{ item.owner }}"
     group: "{{ item.group }}"
-  with_items: "{{ files }}"
+  loop: "{{ files }}"
+  when: run_context in item.run_contexts

image-builder/assets/playbooks/roles/qcow/tasks/main.yaml

@@ -17,7 +17,7 @@
   - name: "QCOW | Mounting filesystems"
     include_tasks:
       file: mount-helper.yaml
-    with_items: "{{ partitions | sort( case_sensitive=True, attribute='mount_order' ) }}"
+    loop: "{{ partitions | sort( case_sensitive=True, attribute='mount_order' ) }}"
     vars:
       mount_offset: "{{ dst }}"
       state: mounted
@@ -46,7 +46,7 @@
   - name: "QCOW | Unmounting filesystems"
     include_tasks:
       file: mount-helper.yaml
-    with_items: "{{ partitions | sort( reverse=True, case_sensitive=True, attribute='mount' ) }}"
+    loop: "{{ partitions | sort( reverse=True, case_sensitive=True, attribute='mount' ) }}"
     vars:
       mount_offset: "{{ dst }}"
       state: unmounted

image-builder/assets/playbooks/roles/qcow/tasks/writing-image-content.yaml

@@ -1,6 +1,6 @@
 - name: "QCOW | Writing out fstab"
   include_tasks: mount-helper.yaml
-  with_items: "{{ partitions | sort( case_sensitive=True, attribute='mount' ) }}"
+  loop: "{{ partitions | sort( case_sensitive=True, attribute='mount' ) }}"
   vars:
     mount_offset: null
     state: present

image-builder/config/iso/README.md

@@ -0,0 +1,3 @@
+Data used when running the `iso` playbook.
+
+This is just for testing. The ISO is not published anywhere.

image-builder/config/rootfs/README.md

@@ -0,0 +1 @@
+Playbook variable overrides used for building the image-builder container image.

image-builder/config/scripts/common/README.md

@@ -0,0 +1,9 @@
+Scripts placed in this directory will be run by the `osconfig` playbook when
+building the shared/common image-builder container, which both the ephemeral
+ISO and target QCOW will inherit.
+
+This is a useful alternative to keep them separate instead of putting them all
+in the same overrides file.
+
+Scripts execution ordering is based on the sorted filenames of the scripts. Ex,
+005-script1.sh runs before 050-script2.sh.

image-builder/config/scripts/qcow/README.md

@@ -0,0 +1,9 @@
+Scripts placed in this directory will be run by the `osconfig` playbook when
+building target QCOWs. This is useful if you have some build instructions you
+only want to run on the target images, but not for the ephemeral ISO.
+
+This is a useful alternative to keep them separate instead of putting them all
+in the same overrides file.
+
+Scripts execution ordering is based on the sorted filenames of the scripts. Ex,
+005-script1.sh runs before 050-script2.sh.

image-builder/tools/cut_image.sh

@@ -89,7 +89,7 @@ outputFileName: $img_name" > ${iso_config}
   disk1="--disk path=${workdir}/${img_name},device=cdrom"
   network='--network network=default,mac=52:54:00:6c:99:85'
 elif [[ $build_type == qcow ]]; then
-  : ${img_name:=airship-ubuntu.qcow2}
+  : ${img_name:=$(cat $(dirname ${osconfig_params})/img_name)}
   if sudo virsh list | grep ${img_name}; then
     sudo virsh destroy ${img_name}
   fi

image-builder/tools/multistrap.sh

@@ -1,8 +1,19 @@
 #!/bin/bash
 
 set -e
+
+SOURCE="${BASH_SOURCE[0]}"
+while [ -h "$SOURCE" ]; do # resolve $SOURCE until the file is no longer a symlink
+  DIR="$( cd -P "$( dirname "$SOURCE" )" >/dev/null 2>&1 && pwd )"
+  SOURCE="$(readlink "$SOURCE")"
+  [[ $SOURCE != /* ]] && SOURCE="$DIR/$SOURCE" # if $SOURCE was a relative symlink, we need to resolve it relative to the path where the symlink file was located
+done
+BASEDIR="$( cd -P "$( dirname "$SOURCE" )" >/dev/null 2>&1 && pwd )"
+
 build_dir=assets/playbooks/build
 osconfig_build_dir=$(basename $build_dir)
+host_mount_directory="${1:-$BASEDIR/../config}"
+workdir="$(realpath ${host_mount_directory})"
 
 install_pkg(){
     dpkg -l $1 2> /dev/null | grep ^ii > /dev/null || sudo -E apt-get -y install $1
@@ -102,18 +113,28 @@ cp assets/playbooks/inventory.yaml $build_dir/opt/assets/playbooks/inventory.yam
 cp assets/playbooks/base-chroot.yaml $build_dir/opt/assets/playbooks/base-chroot.yaml
 cp -r assets/playbooks/roles/multistrap $build_dir/opt/assets/playbooks/roles
 # Run multistrap
-sudo -E ansible-playbook -i assets/playbooks/inventory.yaml assets/playbooks/base-chroot.yaml -vv
+if [ -z $OSCONFIG_TEST ]; then
+  sudo -E ansible-playbook -i assets/playbooks/inventory.yaml assets/playbooks/base-chroot.yaml -vv
+fi
 
 cp assets/playbooks/base-osconfig.yaml $build_dir/opt/assets/playbooks/base-osconfig.yaml
 cp -r assets/playbooks/roles/osconfig $build_dir/opt/assets/playbooks/roles
-sudo -E ansible-playbook -i assets/playbooks/inventory.yaml assets/playbooks/base-osconfig.yaml --tags "runtime_and_buildtime" -vv
-sudo -E ansible-playbook -i assets/playbooks/inventory.yaml assets/playbooks/base-osconfig.yaml --tags "buildtime_only" -vv
+if [ -d $build_dir/config ]; then
+  sudo rm -r $build_dir/config
+  mkdir -p $build_dir/config
+else
+  mkdir -p $build_dir/config
+fi
+cp -r $workdir/scripts $build_dir/config/
+sudo -E ansible-playbook -i assets/playbooks/inventory.yaml assets/playbooks/base-osconfig.yaml --extra-vars "run_context=common" -vv
 
 umount_chroot
 
 cp assets/playbooks/base-livecdcontent.yaml $build_dir/opt/assets/playbooks/base-livecdcontent.yaml
 cp -r assets/playbooks/roles/livecdcontent $build_dir/opt/assets/playbooks/roles
-sudo -E ansible-playbook -i assets/playbooks/inventory.yaml assets/playbooks/base-livecdcontent.yaml -vv
+if [ -z $OSCONFIG_TEST ]; then
+  sudo -E ansible-playbook -i assets/playbooks/inventory.yaml assets/playbooks/base-livecdcontent.yaml -vv
+fi
 
 cp assets/playbooks/iso.yaml $build_dir/opt/assets/playbooks/iso.yaml
 cp -r assets/playbooks/roles/iso $build_dir/opt/assets/playbooks/roles