From 0064db95fe9770ea3a7c35a4091c83cfbad24bee Mon Sep 17 00:00:00 2001
From: "Anderson, Craig (ca846m)" <ca846m@att.com>
Date: Thu, 11 Feb 2021 02:01:54 -0800
Subject: [PATCH] image-builder refactor

The original image-builder approach had an entirely containerized
approach for building target images.

This approach was flawed because:
1. There are a number of debian packages which will not install without
/sys, /proc, /dev, or /dev/pts mountpoints, and
2. Container build process does not support building with privileges
needed to bind-mount these directories into the chroot build space
3. It is a requirement for all packages to be installed in the container
image in order to avoid deployment risk of missing mirror resources

This patchset addresses this problem by performing necessary privileged
steps outside of a containerized build process. At the end of this
process, the root filesystem is packaged into a docker container when
elevated permissions are no longer required.

Change-Id: I5f8dc972f67c5649bf5f9403a5a512d06c948720
---
 .zuul.yaml                                    |   5 +-
 image-builder/Dockerfile.ubuntu_focal         | 108 ++++----------
 image-builder/Makefile                        |  32 ++---
 image-builder/assets/entrypoint.sh            |  19 ++-
 .../assets/playbooks/base-osconfig.yaml       |   2 +-
 image-builder/assets/playbooks/inventory.yaml |   2 +-
 .../playbooks/roles/iso/defaults/main.yaml    |   2 +-
 .../assets/playbooks/roles/iso/tasks/iso.yaml |  25 ++++
 .../roles/livecdcontent/defaults/main.yaml    |   5 +-
 .../roles/livecdcontent/tasks/livecd.yaml     |  26 +---
 .../roles/livecdcontent/tasks/squashfs.yaml   |   5 +
 .../templates/grub-livecd.cfg.j2              |   6 +-
 .../roles/multistrap/defaults/main.yaml       |  19 ++-
 .../roles/multistrap/tasks/main.yaml          |  42 +++---
 .../roles/multistrap/templates/kdump-tools.j2 |  75 ++++++++++
 .../roles/osconfig/defaults/main.yaml         |  36 ++---
 .../tasks/buildtime-user-scripts.yaml         |   3 +
 .../playbooks/roles/osconfig/tasks/main.yaml  |  14 +-
 .../roles/osconfig/tasks/post-install.yaml    | 108 --------------
 .../tasks/runtime-system-scripts.yaml         |  34 +++++
 .../osconfig/tasks/runtime-user-scripts.yaml  |   3 +
 .../playbooks/roles/osconfig/vars/main.yaml   |  14 --
 .../playbooks/roles/qcow/defaults/main.yaml   |  13 +-
 .../roles/qcow/tasks/boot-syslinux.yaml       |  12 +-
 .../roles/qcow/tasks/chroot-cleanup.yaml      |  10 +-
 .../roles/qcow/tasks/chroot-prep.yaml         |  29 +---
 .../roles/qcow/tasks/copy-files.yaml          |   5 +
 .../playbooks/roles/qcow/tasks/main.yaml      |   8 +-
 .../qcow/tasks/writing-image-content.yaml     |   9 +-
 .../qcow/templates/generic-file-writer.j2     |   1 +
 .../examples/osconfig-control-plane-vars.yaml |  18 +--
 ...ow-control-plane-vars-full-partitions.yaml |   3 -
 .../examples/qcow-control-plane-vars.yaml     |   1 -
 image-builder/tools/cut_image.sh              |  31 ++--
 .../tools/install_prereqs.ubuntu_focal        |  15 --
 image-builder/tools/multistrap.sh             | 136 ++++++++++++++++++
 36 files changed, 464 insertions(+), 412 deletions(-)
 create mode 100755 image-builder/assets/playbooks/roles/multistrap/templates/kdump-tools.j2
 create mode 100644 image-builder/assets/playbooks/roles/osconfig/tasks/buildtime-user-scripts.yaml
 delete mode 100644 image-builder/assets/playbooks/roles/osconfig/tasks/post-install.yaml
 create mode 100644 image-builder/assets/playbooks/roles/osconfig/tasks/runtime-system-scripts.yaml
 create mode 100644 image-builder/assets/playbooks/roles/osconfig/tasks/runtime-user-scripts.yaml
 create mode 100644 image-builder/assets/playbooks/roles/qcow/tasks/copy-files.yaml
 create mode 100644 image-builder/assets/playbooks/roles/qcow/templates/generic-file-writer.j2
 delete mode 100755 image-builder/tools/install_prereqs.ubuntu_focal
 create mode 100755 image-builder/tools/multistrap.sh

diff --git a/.zuul.yaml b/.zuul.yaml
index f1021f9..4873297 100644
--- a/.zuul.yaml
+++ b/.zuul.yaml
@@ -43,7 +43,8 @@
 - job:
     name: airship-images-build
     nodeset: airship-images-single-node
-    timeout: 3600
+    timeout: 7200
+    post-timeout: 7200
     pre-run: playbooks/airship-images-deploy-docker.yaml
     run: playbooks/airship-images-build.yaml
     post-run: playbooks/airship-collect-logs.yaml
@@ -74,7 +75,7 @@
     name: airship-images-single-node
     nodes:
       - name: primary
-        label: ubuntu-bionic
+        label: ubuntu-bionic-32GB
 
 - secret:
     name: images_airshipit_github_secret
diff --git a/image-builder/Dockerfile.ubuntu_focal b/image-builder/Dockerfile.ubuntu_focal
index 86a91f8..def0c3f 100644
--- a/image-builder/Dockerfile.ubuntu_focal
+++ b/image-builder/Dockerfile.ubuntu_focal
@@ -10,92 +10,21 @@ LABEL org.opencontainers.image.authors='airship-discuss@lists.airshipit.org, irc
 SHELL ["bash", "-exc"]
 ENV DEBIAN_FRONTEND noninteractive
 
-# Update distro and install ansible
-RUN apt-get update ;\
-    apt-get dist-upgrade -y ;\
-    apt-get install -y --no-install-recommends \
-        python3-minimal \
-        python3-pip \
-        python3-apt \
-        python3-setuptools ;\
-    pip3 install --upgrade wheel ;\
-    pip3 install --upgrade ansible ;\
-    rm -rf /var/lib/apt/lists/*
 
-
-
-FROM base-image as rootfs-builder
-# install requirements for building chroot
 RUN apt-get update ;\
     apt-get install -y --no-install-recommends  \
+        ca-certificates \
         multistrap \
         equivs \
-        curl \
-        ca-certificates \
         build-essential \
         gnupg2 \
-        dosfstools;\
-    rm -rf /var/lib/apt/lists/*
-
-COPY assets/playbooks/inventory.yaml /opt/assets/playbooks/inventory.yaml
-
-COPY assets/playbooks/base-chroot.yaml /opt/assets/playbooks/base-chroot.yaml
-COPY assets/playbooks/roles/multistrap /opt/assets/playbooks/roles/multistrap
-RUN ansible-playbook -i /opt/assets/playbooks/inventory.yaml /opt/assets/playbooks/base-chroot.yaml
-
-COPY assets/playbooks/base-osconfig.yaml /opt/assets/playbooks/base-osconfig.yaml
-COPY assets/playbooks/roles/osconfig /opt/assets/playbooks/roles/osconfig
-RUN ansible-playbook -i /opt/assets/playbooks/inventory.yaml /opt/assets/playbooks/base-osconfig.yaml --tags "pre_install"
-
-
-
-FROM base-image as squashfs-builder
-ENV root_chroot /mnt/rootfs
-ENV root_image /mnt/image
-ENV boot_src="/opt/grub"
-
-RUN apt-get update ;\
-    apt-get install -y --no-install-recommends  \
-        dosfstools \
-        mtools \
-        squashfs-tools \
-        grub-common \
-        grub2-common \
-        grub-pc-bin \
-        grub-efi-amd64-signed;\
-    rm -rf /var/lib/apt/lists/*
-
-COPY --from=rootfs-builder ${root_chroot} ${root_chroot}
-
-COPY assets/playbooks/inventory.yaml /opt/assets/playbooks/inventory.yaml
-COPY assets/playbooks/base-livecdcontent.yaml /opt/assets/playbooks/base-livecdcontent.yaml
-COPY assets/playbooks/roles/livecdcontent /opt/assets/playbooks/roles/livecdcontent
-RUN ansible-playbook -i /opt/assets/playbooks/inventory.yaml /opt/assets/playbooks/base-livecdcontent.yaml
-
-
-
-FROM base-image as image-builder
-ENV boot_src="/opt/grub"
-ENV root_image /mnt/image
-
-RUN apt-get update ;\
-    apt-get install -y --no-install-recommends  \
         xorriso \
-        grub-pc-bin \
         python3-minimal \
-        python3-yaml ;\
-    rm -rf /var/lib/apt/lists/*
-
-COPY --from=squashfs-builder ${root_image} ${root_image}
-
-COPY assets/playbooks/inventory.yaml /opt/assets/playbooks/inventory.yaml
-COPY assets/playbooks/iso.yaml /opt/assets/playbooks/iso.yaml
-COPY assets/playbooks/roles/iso /opt/assets/playbooks/roles/iso
-
-
-
-RUN apt-get update ;\
-    apt-get install -y --no-install-recommends  \
+        python3-yaml \
+        python3-pip \
+        python3-setuptools \
+        python3-apt \
+        grub-pc-bin \
         coreutils \
         curl \
         qemu-utils \
@@ -107,19 +36,30 @@ RUN apt-get update ;\
         vim \
         kmod \
         efivar \
+        rsync \
         dosfstools ;\
+    pip3 install --upgrade pip ;\
+    pip3 install --upgrade wheel ;\
+    pip3 install --upgrade ansible ;\
     rm -rf /var/lib/apt/lists/*
 
-
-COPY assets/playbooks/base-osconfig.yaml /opt/assets/playbooks/base-osconfig.yaml
-COPY assets/playbooks/roles/osconfig /opt/assets/playbooks/roles/osconfig
-
-COPY assets/playbooks/qcow.yaml /opt/assets/playbooks/qcow.yaml
-COPY assets/playbooks/roles/qcow /opt/assets/playbooks/roles/qcow
-
 RUN curl -L https://github.com/mikefarah/yq/releases/download/2.4.0/yq_linux_amd64 -o /bin/yq \
     && chmod +x /bin/yq
 
+COPY assets/playbooks/inventory.yaml /opt/assets/playbooks/inventory.yaml
+COPY assets/playbooks/base-chroot.yaml /opt/assets/playbooks/base-chroot.yaml
+COPY assets/playbooks/roles/multistrap /opt/assets/playbooks/roles/multistrap
+COPY assets/playbooks/base-osconfig.yaml /opt/assets/playbooks/base-osconfig.yaml
+COPY assets/playbooks/roles/osconfig /opt/assets/playbooks/roles/osconfig
+COPY assets/playbooks/base-livecdcontent.yaml /opt/assets/playbooks/base-livecdcontent.yaml
+COPY assets/playbooks/roles/livecdcontent /opt/assets/playbooks/roles/livecdcontent
+COPY assets/playbooks/iso.yaml /opt/assets/playbooks/iso.yaml
+COPY assets/playbooks/roles/iso /opt/assets/playbooks/roles/iso
+COPY assets/playbooks/qcow.yaml /opt/assets/playbooks/qcow.yaml
+COPY assets/playbooks/roles/qcow /opt/assets/playbooks/roles/qcow
+
+COPY assets/playbooks/build /build
+
 COPY assets/*.sh /usr/bin/local/
 COPY assets/*.json /usr/bin/local/
 CMD /usr/bin/local/entrypoint.sh
diff --git a/image-builder/Makefile b/image-builder/Makefile
index 648a992..33cc7cc 100644
--- a/image-builder/Makefile
+++ b/image-builder/Makefile
@@ -24,11 +24,10 @@ PUSH_IMAGE	?= false
 DISTRO          ?= ubuntu_focal
 IMAGE           ?= ${DOCKER_REGISTRY}/${IMAGE_PREFIX}/${IMAGE_NAME}:${IMAGE_TAG}-${DISTRO}
 IMAGE_ALIAS	?= $(DOCKER_REGISTRY)-$(IMAGE_NAME)-$(IMAGE_TAG)-${DISTRO}-${IMAGE_TYPE}
-UEFI_BOOT	?= true
 PROXY           ?=
 NO_PROXY        ?= localhost,127.0.0.1
 
-.PHONY: help build images install_prereqs cut_image run
+.PHONY: help build images cut_image run clean
 
 .ONESHELL:
 
@@ -38,17 +37,6 @@ help: ## This help.
 # Make target name that zuul expects for each project in this repo
 images: build
 
-install_prereqs:
-ifneq ($(PROXY), )
-	export http_proxy=$(PROXY)
-	export https_proxy=$(PROXY)
-	export no_proxy=$(NO_PROXY)
-	export HTTP_PROXY=$(PROXY)
-	export HTTPS_PROXY=$(PROXY)
-	export NO_PROXY=$(NO_PROXY)
-endif
-	sudo -E tools/install_prereqs.$(DISTRO)
-
 build:
 ifneq ($(PROXY), )
 	sudo -E ./tools/docker_proxy.sh $(PROXY) $(NO_PROXY)
@@ -58,7 +46,8 @@ ifneq ($(PROXY), )
 	export HTTP_PROXY=$(PROXY)
 	export HTTPS_PROXY=$(PROXY)
 	export NO_PROXY=$(NO_PROXY)
-	sudo -E docker build --tag $(IMAGE) -f Dockerfile.$(DISTRO) . \
+	sudo -E ./tools/multistrap.sh
+	sudo -E docker -D -l debug build --tag $(IMAGE) -f Dockerfile.$(DISTRO) . \
 	--label $(LABEL) \
 	--label "org.opencontainers.image.revision=$(COMMIT)" \
 	--label "org.opencontainers.image.created=\
@@ -69,22 +58,21 @@ ifneq ($(PROXY), )
 	--build-arg HTTP_PROXY=$(PROXY) \
 	--build-arg HTTPS_PROXY=$(PROXY) \
 	--build-arg no_proxy=$(NO_PROXY) \
-	--build-arg NO_PROXY=$(NO_PROXY) \
-	--build-arg UEFI_BOOT=$(UEFI_BOOT) || exit 1
+	--build-arg NO_PROXY=$(NO_PROXY) || exit 1
 else
-	sudo -E docker build --tag $(IMAGE) -f Dockerfile.$(DISTRO) . \
+	sudo -E ./tools/multistrap.sh
+	sudo -E docker -D -l debug build --tag $(IMAGE) -f Dockerfile.$(DISTRO) . \
 	--label $(LABEL) \
 	--label "org.opencontainers.image.revision=$(COMMIT)" \
 	--label "org.opencontainers.image.created=\
 	$(shell date --rfc-3339=seconds --utc)" \
-	--label "org.opencontainers.image.title=$(IMAGE_NAME)" \
-	--build-arg UEFI_BOOT=$(UEFI_BOOT) || exit 1
+	--label "org.opencontainers.image.title=$(IMAGE_NAME)" || exit 1
 endif
 ifeq ($(PUSH_IMAGE), true)
 	sudo -E docker push $(IMAGE)
 endif
 
-cut_image: install_prereqs
+cut_image:
 ifneq ($(PROXY), )
 	sudo -E ./tools/docker_proxy.sh $(PROXY) $(NO_PROXY)
 	export http_proxy=$(PROXY)
@@ -94,7 +82,7 @@ ifneq ($(PROXY), )
 	export HTTPS_PROXY=$(PROXY)
 	export NO_PROXY=$(NO_PROXY)
 endif
-	sudo -E tools/cut_image.sh $(IMAGE_TYPE) ./examples $(IMAGE) $(IMAGE_ALIAS) "$(UEFI_BOOT)" "$(PROXY)" "$(NO_PROXY)"
+	sudo -E tools/cut_image.sh $(IMAGE_TYPE) ./examples $(IMAGE) $(IMAGE_ALIAS) "$(PROXY)" "$(NO_PROXY)"
 
 run: ## Run the iso in kvm for testing
 	virsh start $(IMAGE_ALIAS)
@@ -102,3 +90,5 @@ run: ## Run the iso in kvm for testing
 tests:
 	true
 
+clean:
+	sudo -E tools/multistrap.sh clean
diff --git a/image-builder/assets/entrypoint.sh b/image-builder/assets/entrypoint.sh
index 78b88ec..0d587a9 100755
--- a/image-builder/assets/entrypoint.sh
+++ b/image-builder/assets/entrypoint.sh
@@ -13,12 +13,6 @@ cd "$BASEDIR"
 BASEDIR="$(dirname "$(realpath "$0")")"
 source "${BASEDIR}/functions.sh"
 
-: "${uefi_boot:=}"
-
-if [[ -n $uefi_boot ]]; then
-  extra_vars="uefi=$uefi_boot"
-fi
-
 export http_proxy
 export https_proxy
 export HTTP_PROXY
@@ -26,6 +20,10 @@ export HTTPS_PROXY
 export no_proxy
 export NO_PROXY
 
+if [ ! -e build ]; then
+  ln -s /chroot build
+fi
+
 # Instruct ansible to output the image artifact to the container's host mount
 extra_vars="$extra_vars img_output_dir=${VOLUME}"
 
@@ -37,7 +35,7 @@ if [[ "${IMAGE_TYPE}" == "iso" ]]; then
   extra_vars="$extra_vars img_name=${IMG_NAME}"
 
   echo "Executing Step 1"
-  ansible-playbook -i /opt/assets/playbooks/inventory.yaml /opt/assets/playbooks/iso.yaml --extra-vars "$extra_vars" -vvvv
+  ansible-playbook -i /opt/assets/playbooks/inventory.yaml /opt/assets/playbooks/iso.yaml --extra-vars "$extra_vars" -vv
 elif [[ "${IMAGE_TYPE}" == "qcow" ]]; then
   _process_input_data_set_vars_qcow
   _process_input_data_set_vars_osconfig
@@ -46,13 +44,14 @@ elif [[ "${IMAGE_TYPE}" == "qcow" ]]; then
   extra_vars="$extra_vars img_name=${IMG_NAME}"
 
   echo "Executing Step 1: Create qcow2 partitions and filesystems"
-  ansible-playbook -i /opt/assets/playbooks/inventory.yaml /opt/assets/playbooks/qcow.yaml --extra-vars "$extra_vars" --tags "prep_img" -vvvv
+  ansible-playbook -i /opt/assets/playbooks/inventory.yaml /opt/assets/playbooks/qcow.yaml --extra-vars "$extra_vars" --tags "prep_img" -vv
 
   echo "Executing Step 2: Applying changes from base-osconfig playbook"
-  ansible-playbook -i /opt/assets/playbooks/inventory.yaml /opt/assets/playbooks/base-osconfig.yaml --extra-vars "$extra_vars" -vvvv
+  ansible-playbook -i /opt/assets/playbooks/inventory.yaml /opt/assets/playbooks/base-osconfig.yaml --extra-vars "$extra_vars" --tags "runtime_and_buildtime" -vv
+  ansible-playbook -i /opt/assets/playbooks/inventory.yaml /opt/assets/playbooks/base-osconfig.yaml --extra-vars "$extra_vars" --tags "runtime_only" -vv
 
   echo "Executing Step 3: Close image and write qcow2"
-  ansible-playbook -i /opt/assets/playbooks/inventory.yaml /opt/assets/playbooks/qcow.yaml --extra-vars "$extra_vars" --tags "close_img" -vvvv
+  ansible-playbook -i /opt/assets/playbooks/inventory.yaml /opt/assets/playbooks/qcow.yaml --extra-vars "$extra_vars" --tags "close_img" -vv
 else
   echo "\${IMAGE_TYPE} value '${IMAGE_TYPE}' does not match an expected value: [ 'iso', 'qcow' ]"
   exit 1
diff --git a/image-builder/assets/playbooks/base-osconfig.yaml b/image-builder/assets/playbooks/base-osconfig.yaml
index a3c4884..782a52a 100644
--- a/image-builder/assets/playbooks/base-osconfig.yaml
+++ b/image-builder/assets/playbooks/base-osconfig.yaml
@@ -1,5 +1,5 @@
 ---
-- hosts: /mnt/rootfs
+- hosts: build
   gather_facts: false
   roles:
      - osconfig
diff --git a/image-builder/assets/playbooks/inventory.yaml b/image-builder/assets/playbooks/inventory.yaml
index 74ccf56..51cf12a 100644
--- a/image-builder/assets/playbooks/inventory.yaml
+++ b/image-builder/assets/playbooks/inventory.yaml
@@ -5,6 +5,6 @@ all:
             ansible_python_interpreter: /usr/bin/python3
 chroots:
     hosts:
-        /mnt/rootfs:
+        build:
             ansible_connection: chroot
             ansible_python_interpreter: /usr/bin/python3
diff --git a/image-builder/assets/playbooks/roles/iso/defaults/main.yaml b/image-builder/assets/playbooks/roles/iso/defaults/main.yaml
index 8ff0bcf..3d71f9e 100644
--- a/image-builder/assets/playbooks/roles/iso/defaults/main.yaml
+++ b/image-builder/assets/playbooks/roles/iso/defaults/main.yaml
@@ -1,6 +1,6 @@
 img_output_dir: /config
 img_name: ephemeral.iso
-root_image: /mnt/image
+root_image: /build
 
 meta_data_file: /config/meta_data.json
 user_data_file: /config/user_data
diff --git a/image-builder/assets/playbooks/roles/iso/tasks/iso.yaml b/image-builder/assets/playbooks/roles/iso/tasks/iso.yaml
index bcb6bce..36e2507 100644
--- a/image-builder/assets/playbooks/roles/iso/tasks/iso.yaml
+++ b/image-builder/assets/playbooks/roles/iso/tasks/iso.yaml
@@ -1,3 +1,28 @@
+- name: "ISO | Reduce image size"
+  file:
+    state: absent
+    path: "{{ root_image }}/lib"
+- name: "ISO | Reduce image size"
+  file:
+    state: absent
+    path: "{{ root_image }}/usr"
+- name: "ISO | Reduce image size"
+  file:
+    state: absent
+    path: "{{ root_image }}/bin"
+- name: "ISO | Reduce image size"
+  file:
+    state: absent
+    path: "{{ root_image }}/sbin"
+- name: "ISO | Reduce image size"
+  file:
+    state: absent
+    path: "{{ root_image }}/var"
+- name: "ISO | Reduce image size"
+  file:
+    state: absent
+    path: "{{ root_image }}/opt"
+
 - name: "ISO | Ensure any old iso image at target location is removed"
   file:
     state: absent
diff --git a/image-builder/assets/playbooks/roles/livecdcontent/defaults/main.yaml b/image-builder/assets/playbooks/roles/livecdcontent/defaults/main.yaml
index 7752a1a..00098ad 100644
--- a/image-builder/assets/playbooks/roles/livecdcontent/defaults/main.yaml
+++ b/image-builder/assets/playbooks/roles/livecdcontent/defaults/main.yaml
@@ -1,3 +1,2 @@
-root_chroot: /mnt/rootfs
-root_image: /mnt/image
-boot_src: /opt/grub
+root_chroot: build
+root_image: build
diff --git a/image-builder/assets/playbooks/roles/livecdcontent/tasks/livecd.yaml b/image-builder/assets/playbooks/roles/livecdcontent/tasks/livecd.yaml
index d0b045f..94844a4 100644
--- a/image-builder/assets/playbooks/roles/livecdcontent/tasks/livecd.yaml
+++ b/image-builder/assets/playbooks/roles/livecdcontent/tasks/livecd.yaml
@@ -1,29 +1,7 @@
-- name: ansible copy file locally - vmlinuz.
-  copy:
-    src: "{{ item }}"
-    dest: "{{ root_image }}/vmlinuz"
-    remote_src: yes
-  with_fileglob: "{{ root_chroot }}/boot/vmlinuz-*"
-
-- name: ansible copy file locally - initrd.
-  copy:
-    src: "{{ item }}"
-    dest: "{{ root_image }}/initrd"
-    remote_src: yes
-  with_fileglob: "{{ root_chroot }}/boot/initrd.img-*"
-
-- name: ansible copy file locally - config.
-  copy:
-    src: "{{ item }}"
-    dest: "{{ root_image }}/config"
-    remote_src: yes
-  with_fileglob: "{{ root_chroot }}/boot/config-*"
-
 - name: "Stamp out a marker file for grub to use when identifying the desired boot volume"
   copy:
-    #TODO: populate this with meaningful content
     content: "{{ ansible_date_time.date }}"
-    dest: "{{ root_image }}/AIRSHIP_EPHEMERAL"
+    dest: "{{ root_image }}/AIRSHIP"
 
 - name: "create directory for boot image assembly"
   tempfile:
@@ -77,4 +55,4 @@
   shell:
     cmd: |
         cat /usr/lib/grub/i386-pc/cdboot.img {{ bootimg_builddir.path }}/core.img > {{ root_image }}/boot/grub/bios.img
-        cp {{ bootimg_builddir.path }}/efiboot.img {{ root_image }}/boot/grub/
+        cp {{ bootimg_builddir.path }}/efiboot.img {{ root_image }}/boot/grub/efiboot.img
diff --git a/image-builder/assets/playbooks/roles/livecdcontent/tasks/squashfs.yaml b/image-builder/assets/playbooks/roles/livecdcontent/tasks/squashfs.yaml
index 26f034b..9dee4e0 100644
--- a/image-builder/assets/playbooks/roles/livecdcontent/tasks/squashfs.yaml
+++ b/image-builder/assets/playbooks/roles/livecdcontent/tasks/squashfs.yaml
@@ -4,6 +4,11 @@
     state: directory
     mode: '0755'
 
+- name: "ensure no previous squashfs file"
+  file:
+    path: "{{ root_image }}/live/filesystem.squashfs"
+    state: absent
+
 - name: "Building squashfs"
   shell:
     cmd: |
diff --git a/image-builder/assets/playbooks/roles/livecdcontent/templates/grub-livecd.cfg.j2 b/image-builder/assets/playbooks/roles/livecdcontent/templates/grub-livecd.cfg.j2
index c977b2c..4427677 100644
--- a/image-builder/assets/playbooks/roles/livecdcontent/templates/grub-livecd.cfg.j2
+++ b/image-builder/assets/playbooks/roles/livecdcontent/templates/grub-livecd.cfg.j2
@@ -1,4 +1,4 @@
-search --set=root --file /AIRSHIP_EPHEMERAL
+search --set=root --file /AIRSHIP
 
 insmod all_video
 
@@ -6,6 +6,6 @@ set default="0"
 set timeout=1
 
 menuentry "Airship Ephemeral" {
-    linux /vmlinuz boot=live quiet nomodeset overlay-size=70% systemd.unified_cgroup_hierarchy=0 ds=ConfigDrive
-    initrd /initrd
+    linux /boot/vmlinuz boot=live quiet nomodeset overlay-size=70% systemd.unified_cgroup_hierarchy=0 ds=ConfigDrive
+    initrd /boot/initrd.img
 }
diff --git a/image-builder/assets/playbooks/roles/multistrap/defaults/main.yaml b/image-builder/assets/playbooks/roles/multistrap/defaults/main.yaml
index 0410718..6045234 100644
--- a/image-builder/assets/playbooks/roles/multistrap/defaults/main.yaml
+++ b/image-builder/assets/playbooks/roles/multistrap/defaults/main.yaml
@@ -1,7 +1,10 @@
-rootfs_root: /mnt/rootfs
+rootfs_root: build
 rootfs_arch: amd64
 k8s_version: 1.18.6-00
+kernel_base_pkg: linux-image-generic
+kernel_headers_pkg: linux-headers-generic
 ubuntu_packages:
+    - apparmor
     - apt-file
     - apt-utils
     - apt-transport-https
@@ -13,15 +16,19 @@ ubuntu_packages:
     - cloud-init
     - conntrack
     - curl
+    - dbus
     - dnsutils
     - dosfstools
     - e2fsprogs
     - ebtables
+    - efivar
     - ethtool
     - file
+    - gawk
     - gettext-base
     - gnupg2
-    #- grub2 # cannot install until after boot partition is available
+    - grub2
+    - grub-efi-amd64-signed
     - ifenslave
     - isc-dhcp-client
     - iproute2
@@ -30,14 +37,19 @@ ubuntu_packages:
     - iputils-ping
     - iputils-tracepath
     - ipvsadm
+    - kdump-tools
+    - "{{ kernel_base_pkg }}"
+    - "{{ kernel_headers_pkg }}"
+    - kmod
     - less
-    - linux-image-generic # this will be reinstalled later when the boot partition is available
     - live-boot
     - locales
     - locales-all
+    - logrotate
     - lsb-release
     - lsof
     - man-db
+    - mawk
     - mbr
     - netplan.io
     - net-tools
@@ -46,6 +58,7 @@ ubuntu_packages:
     - passwd
     - python3
     - python3-apt
+    - rsyslog
     - socat
     - systemd
     - systemd-sysv
diff --git a/image-builder/assets/playbooks/roles/multistrap/tasks/main.yaml b/image-builder/assets/playbooks/roles/multistrap/tasks/main.yaml
index 963d99a..ddcc360 100644
--- a/image-builder/assets/playbooks/roles/multistrap/tasks/main.yaml
+++ b/image-builder/assets/playbooks/roles/multistrap/tasks/main.yaml
@@ -35,30 +35,36 @@
   include_tasks: apt-key-install.yaml
   loop: "{{ repos }}"
 
-- name: "ensuring directory {{ rootfs_root }}/dev exists for chroot"
-  file:
-    path: "{{ rootfs_root }}/dev"
-    state: directory
-    mode: '0755'
-
-- name: "Setting up devices for chroot"
+# kdump-tools does not install properly in multistrap environment. This fix allows kdump-tools
+# installation to succeed.
+- name: "kdump-tools fix - create directory"
   shell: |
-    mknod "{{ rootfs_root }}/dev/random" c 1 8
-    chmod 640 "{{ rootfs_root }}/dev/random"
-    chown 0:0 "{{ rootfs_root }}/dev/random"
-    mknod "{{ rootfs_root }}/dev/urandom" c 1 9
-    chmod 640 "{{ rootfs_root }}/dev/urandom"
-    chown 0:0 "{{ rootfs_root }}/dev/urandom"
-    mknod "{{ rootfs_root }}/dev/null" c 1 3
-    chmod 666 "{{ rootfs_root }}/dev/null"
-    chown 0:0 "{{ rootfs_root }}/dev/null"
+        set -e
+        mkdir -p "{{ rootfs_root }}/etc/kernel/postinst.d"
+- name: "kdump-tools fix - deploy build script"
+  template:
+    src: kdump-tools.j2
+    dest: "{{ rootfs_root }}/etc/kernel/postinst.d/kdump-tools"
+    mode: '0755'
+# kdump-tools deb package will overwrite script without write protection enabled
+- name: "kdump-tools fix - lock build script"
+  shell: |
+        set -e
+        chattr +i "{{ rootfs_root }}/etc/kernel/postinst.d/kdump-tools"
 
 - name: "Running multistrap"
   shell:
       cmd: "multistrap -f {{ multistrap_tempdir.path }}/multistrap.conf"
 
+#- name: "create grub.cfg"
+#  shell:
+#    cmd: |
+#        chroot {{ rootfs_root }} update-grub
+
 - name: "Lock sources.list to prevent conflict and duplicates with multistrap repo list"
   shell: |
         set -e
-        if [ -f {{ rootfs_root }}/etc/apt/sources.list ]; then rm {{ rootfs_root }}/etc/apt/sources.list; fi
-        ln -s /dev/null {{ rootfs_root }}/etc/apt/sources.list
+        if [ -f {{ rootfs_root }}/etc/apt/sources.list ] && [ ! -h {{ rootfs_root }}/etc/apt/sources.list ]; then
+          rm {{ rootfs_root }}/etc/apt/sources.list
+          ln -s /dev/null {{ rootfs_root }}/etc/apt/sources.list
+        fi
diff --git a/image-builder/assets/playbooks/roles/multistrap/templates/kdump-tools.j2 b/image-builder/assets/playbooks/roles/multistrap/templates/kdump-tools.j2
new file mode 100755
index 0000000..c34b128
--- /dev/null
+++ b/image-builder/assets/playbooks/roles/multistrap/templates/kdump-tools.j2
@@ -0,0 +1,75 @@
+#!/bin/sh -e
+
+version="$1"
+kdumpdir="/var/lib/kdump"
+
+[ -x /usr/sbin/mkinitramfs ] || exit 0
+
+# passing the kernel version is required
+if [ -z "${version}" ]; then
+	echo >&2 "W: kdump-tools: ${DPKG_MAINTSCRIPT_PACKAGE:-kdump-tools package} did not pass a version number"
+	exit 2
+fi
+
+if ! linux-version list | grep "${version}" > /dev/null ; then
+	exit 0
+fi
+
+# exit if kernel does not need an initramfs
+if [ "$INITRD" = 'No' ]; then
+	exit 0
+fi
+
+# avoid running multiple times
+if [ -n "$DEB_MAINT_PARAMS" ]; then
+	eval set -- "$DEB_MAINT_PARAMS"
+	if [ -z "$1" ] || [ "$1" != "configure" ]; then
+		exit 0
+	fi
+fi
+
+# We need a modified copy of initramfs-tools directory
+# with MODULES=dep in initramfs.conf
+if [ ! -d "$kdumpdir" ];then
+	mkdir "$kdumpdir" || true
+fi
+# Force re-creation of $kdumpdir/initramfs-tools
+# in case the source has changed since last time
+# we ran
+if [ -d "$kdumpdir/initramfs-tools" ];then
+	rm -Rf $kdumpdir/initramfs-tools || true
+fi
+cp -pr /etc/initramfs-tools "$kdumpdir" || true
+
+initramfsdir="$kdumpdir/initramfs-tools"
+
+# Add scsi_dh_* modules if in use otherwise
+# kexec reboot on multipath will fail
+# (LP: #1635597)
+for I in $(lsmod | grep scsi_dh | cut -d" " -f1);do
+	echo "${I}" >> $initramfsdir/modules
+done
+
+# canderson: This line needs to be commented out for kdump-tools to install with multistrap
+#sed -e 's/MODULES=.*/MODULES=dep/' /etc/initramfs-tools/initramfs.conf > "$initramfsdir/initramfs.conf" || true
+if ! [ -e "$initramfsdir/initramfs.conf" ];then
+	echo >&2 "W: kdump-tools: Unable to create $initramfsdir/initramfs.conf"
+	exit 2
+fi
+
+# Cleaning up existing initramfs with same version
+# as mkinitramfs do not have a force option
+if [ -e "$kdumpdir/initrd.img-${version}" ];then
+	rm -f "$kdumpdir/initrd.img-${version}" || true
+fi
+
+# we're good - create initramfs.
+echo "kdump-tools: Generating $kdumpdir/initrd.img-${version}"
+if mkinitramfs -d "$initramfsdir" -o "$kdumpdir/initrd.img-${version}.new" "${version}";then
+	mv "$kdumpdir/initrd.img-${version}.new" "$kdumpdir/initrd.img-${version}"
+else
+	mkinitramfs_return="$?"
+	rm -f "${initramfs}.new"
+	echo "update-initramfs: failed for ${initramfs} with $mkinitramfs_return." >&2
+        exit $mkinitramfs_return
+fi
diff --git a/image-builder/assets/playbooks/roles/osconfig/defaults/main.yaml b/image-builder/assets/playbooks/roles/osconfig/defaults/main.yaml
index 20f0c58..e996dcd 100644
--- a/image-builder/assets/playbooks/roles/osconfig/defaults/main.yaml
+++ b/image-builder/assets/playbooks/roles/osconfig/defaults/main.yaml
@@ -1,11 +1,6 @@
-rootfs_root: /mnt/rootfs
-
-cni_version: v0.8.2
-k8s_version: v1.18.6
+rootfs_root: build
 
 kernel:
-    base_pkg: linux-image-generic
-    headers_pkg: linux-headers-generic
     modules:
         load:
             - name: 8021q
@@ -92,26 +87,19 @@ systemd:
       # Use only if you are intenting to overwrite an existing systemd unit
       force: no
 
-# Note: You are encouraged to build your own image-builder container, where your desired
-# package list can be supplied to multistrap during the container build. However, this
-# option will allow you to layer additional packages (installed during container runtime,
-# instead of during the container build) where a customized container build is not possible
-# or not desired.
-# This is also needed for a specific subset of packages that fail to install successfully
-# with multistrap (e.g., kdump-tools).
-post_install_package_list:
-    - kdump-tools
-    - apparmor
-    - dbus
-    - rsyslog
-    - logrotate
-
-# If any other custom shell scripts are needed for bare-metal provisioning, they can be
-# added here.
-post_install_scripts:
+# If any custom shell scripts are needed for qcow building for image building,
+# they can be added here.
+buildtime_user_scripts:
     - file_content: |
           #!/bin/bash
-          echo "custom post-install script"
+          echo "custom container buildtime script"
+
+# Custom user shell scripts to be run during container execution, right before
+# final QCOW image is created.
+runtime_user_scripts:
+    - file_content: |
+          #!/bin/bash
+          echo "custom container buildtime script"
 
 # Any other adjustments to file or directory permissions, for files that already exist.
 file_permissions:
diff --git a/image-builder/assets/playbooks/roles/osconfig/tasks/buildtime-user-scripts.yaml b/image-builder/assets/playbooks/roles/osconfig/tasks/buildtime-user-scripts.yaml
new file mode 100644
index 0000000..8098981
--- /dev/null
+++ b/image-builder/assets/playbooks/roles/osconfig/tasks/buildtime-user-scripts.yaml
@@ -0,0 +1,3 @@
+- name: "user-scripts | running user-defined scripts"
+  shell: "{{ item.file_content }}"
+  with_items: "{{ buildtime_user_scripts }}"
diff --git a/image-builder/assets/playbooks/roles/osconfig/tasks/main.yaml b/image-builder/assets/playbooks/roles/osconfig/tasks/main.yaml
index 7fa474c..8f3c820 100644
--- a/image-builder/assets/playbooks/roles/osconfig/tasks/main.yaml
+++ b/image-builder/assets/playbooks/roles/osconfig/tasks/main.yaml
@@ -34,8 +34,14 @@
     include_tasks: file-permissions.yaml
   - name: "finalise rootfs"
     include_tasks: finalise-rootfs.yaml
-  tags: pre_install
+  tags: runtime_and_buildtime
 - block:
-  - name: "POST-INSTALL | Starting post-install"
-    include_tasks: post-install.yaml
-  tags: post_install
+  - name: "run user-defined scripts"
+    include_tasks: buildtime-user-scripts.yaml
+  tags: buildtime_only
+- block:
+  - name: "run system-defined scripts"
+    include_tasks: runtime-system-scripts.yaml
+  - name: "run user-defined scripts"
+    include_tasks: runtime-user-scripts.yaml
+  tags: runtime_only
diff --git a/image-builder/assets/playbooks/roles/osconfig/tasks/post-install.yaml b/image-builder/assets/playbooks/roles/osconfig/tasks/post-install.yaml
deleted file mode 100644
index f7928cd..0000000
--- a/image-builder/assets/playbooks/roles/osconfig/tasks/post-install.yaml
+++ /dev/null
@@ -1,108 +0,0 @@
-- name: "POST-INSTALL | Append any user-defined post-install pkgs to install list"
-  set_fact:
-    post_install_package_list: "{{ post_install_package_list + post_install_package_list_append }}"
-  when: post_install_package_list_append is defined
-
-- name: "POST-INSTALL | DNS sanity check"
-  shell:
-    executable: /bin/bash
-    cmd: |
-         set -e
-         proxy="{{ lookup('env', 'HTTP_PROXY') }}"
-         # Ensure proxy address is resolvable, if supplied as a domain name
-         if [[ -n $proxy ]]; then
-           # Extract proxy server address from url
-           proxy_address="$(echo "$proxy" | awk -F/ '{print $3}' | awk -F: '{print $1}')"
-           # If first letter of proxy address is a letter, verify that a DNS lookup is possible
-           if [[ $proxy_address == [a-zA-z]* ]]; then
-             echo "proxy check for '$proxy_address' ..."
-             nslookup $proxy_address > /dev/null || (
-               echo "Failed to resolve proxy '$proxy_address' with dns server '$(cat /etc/resolv.conf)'."
-               echo "Reconfigure DNS setting provided in the 'qcow' playbook to a DNS server that can resolve '$proxy_address'."
-               exit 1
-             )
-           fi
-         fi
-         echo "archive.ubuntu.com DNS check ..."
-         nslookup archive.ubuntu.com || (
-           echo "DNS lookup failure for archive.ubuntu.com with '$(cat /etc/resolv.conf)'"
-           exit 1
-         )
-
-- name: "POST-INSTALL | update source list"
-  apt:
-    update_cache: yes
-
-- name: "POST-INSTALL | generate locales"
-  shell: |
-        set -e
-        locale-gen en_US.UTF-8
-
-- name: "POST-INSTALL | Remove incomplete kernel install by multistrap"
-  shell: |
-        set -e
-        apt-get remove -y '^linux-image-.*'
-        apt-get remove -y '^linux-modules-.*'
-
-- name: "POST-INSTALL | install grub2 and kernel"
-  apt:
-    pkg:
-    - grub2
-    - grub-efi-amd64-signed
-    - efivar
-    - "{{ kernel.base_pkg }}"
-    - "{{ kernel.headers_pkg }}"
-    - kmod
-
-- name: "POST-INSTALL | grub-install LEGACY"
-  shell: |
-        set -e
-        grub-install --target=i386-pc --no-uefi-secure-boot --skip-fs-probe --force "{{ lookup('file', '/tmp/nbd') }}"
-        grub-install --target=i386-pc --no-uefi-secure-boot --skip-fs-probe --force --recheck "{{ lookup('file', '/tmp/nbd') }}"
-  when: uefi is not defined
-
-- name: "POST-INSTALL | grub-install UEFI"
-  shell: |
-        set -e
-        grub-install --target=i386-pc --uefi-secure-boot --skip-fs-probe --force "{{ lookup('file', '/tmp/nbd') }}"
-        grub-install --target=i386-pc --uefi-secure-boot --skip-fs-probe --force --recheck "{{ lookup('file', '/tmp/nbd') }}"
-        grub-install --target=x86_64-efi --uefi-secure-boot --skip-fs-probe --force "{{ lookup('file', '/tmp/nbd') }}"
-        grub-install --target=x86_64-efi --uefi-secure-boot --skip-fs-probe --force --recheck "{{ lookup('file', '/tmp/nbd') }}"
-  when: uefi is defined
-
-- name: "POST-INSTALL | generate grub cfg file"
-  shell: |
-        set -e
-        update-grub
-
-- name: "POST-INSTALL | install other user-requested packages, and kernel-dependent pkgs and ones that fail to install with multistrap"
-  apt:
-    pkg: "{{ post_install_package_list }}"
-
-- name: "POST-INSTALL | write root partition UUID to grub.cfg"
-  shell: |
-        set -e
-        cp -r /usr/lib/grub/* /boot/grub
-        blkid -s UUID -o value $(df -h | grep /$ | awk "{print \$1}") > /tmp/root_uuid
-        sed -i "s@root=/dev/nbd[0-9]p[0-9]@root=UUID=$(cat /tmp/root_uuid)@g" /boot/grub/grub.cfg
-        rm /tmp/root_uuid
-
-- name: "POST-INSTALL | write boot partition UUID to UEFI grub.cfg"
-  shell: |
-        set -e
-        blkid -s UUID -o value $(df -h | grep /boot$ | awk "{print \$1}") > /tmp/boot_uuid
-        echo "search.fs_uuid $(cat /tmp/boot_uuid) root hd0,gpt2" > /boot/efi/EFI/ubuntu/grub.cfg
-        echo "set prefix=(\$root)'/grub'" >> /boot/efi/EFI/ubuntu/grub.cfg
-        echo "configfile \$prefix/grub.cfg" >> /boot/efi/EFI/ubuntu/grub.cfg
-        rm /tmp/boot_uuid
-  when: uefi is defined
-
-- name: "POST-INSTALL | running user-defined post-scripts"
-  shell: "{{ item.file_content }}"
-  with_items: "{{ post_install_scripts }}"
-
-- name: "POST-INSTALL | cleanup deb cache"
-  shell: |
-        set -e
-        rm /var/cache/apt/archives/*.deb
-
diff --git a/image-builder/assets/playbooks/roles/osconfig/tasks/runtime-system-scripts.yaml b/image-builder/assets/playbooks/roles/osconfig/tasks/runtime-system-scripts.yaml
new file mode 100644
index 0000000..2fb7e50
--- /dev/null
+++ b/image-builder/assets/playbooks/roles/osconfig/tasks/runtime-system-scripts.yaml
@@ -0,0 +1,34 @@
+- name: "POST-INSTALL | generate locales"
+  shell: |
+        set -e
+        locale-gen en_US.UTF-8
+
+- name: "POST-INSTALL | grub-install"
+  shell: |
+        set -e
+        grub-install --target=i386-pc --skip-fs-probe --force "{{ lookup('file', '/tmp/nbd') }}"
+        grub-install --target=i386-pc --skip-fs-probe --force --recheck "{{ lookup('file', '/tmp/nbd') }}"
+        grub-install --target=x86_64-efi --skip-fs-probe --force "{{ lookup('file', '/tmp/nbd') }}"
+        grub-install --target=x86_64-efi --skip-fs-probe --force --recheck "{{ lookup('file', '/tmp/nbd') }}"
+
+- name: "POST-INSTALL | generate grub cfg file"
+  shell: |
+        set -e
+        update-grub
+
+- name: "POST-INSTALL | write root partition UUID to grub.cfg"
+  shell: |
+        set -e
+        cp -r /usr/lib/grub/* /boot/grub
+        blkid -s UUID -o value $(df -h | grep /$ | awk "{print \$1}") > /tmp/root_uuid
+        sed -i "s@root=/dev/nbd[0-9]p[0-9]@root=UUID=$(cat /tmp/root_uuid)@g" /boot/grub/grub.cfg
+        rm /tmp/root_uuid
+
+- name: "POST-INSTALL | write boot partition UUID to UEFI grub.cfg"
+  shell: |
+        set -e
+        blkid -s UUID -o value $(df -h | grep /boot$ | awk "{print \$1}") > /tmp/boot_uuid
+        echo "search.fs_uuid $(cat /tmp/boot_uuid) root hd0,gpt2" > /boot/efi/EFI/ubuntu/grub.cfg
+        echo "set prefix=(\$root)'/grub'" >> /boot/efi/EFI/ubuntu/grub.cfg
+        echo "configfile \$prefix/grub.cfg" >> /boot/efi/EFI/ubuntu/grub.cfg
+        rm /tmp/boot_uuid
diff --git a/image-builder/assets/playbooks/roles/osconfig/tasks/runtime-user-scripts.yaml b/image-builder/assets/playbooks/roles/osconfig/tasks/runtime-user-scripts.yaml
new file mode 100644
index 0000000..ff136e3
--- /dev/null
+++ b/image-builder/assets/playbooks/roles/osconfig/tasks/runtime-user-scripts.yaml
@@ -0,0 +1,3 @@
+- name: "user-scripts | running user-defined scripts"
+  shell: "{{ item.file_content }}"
+  with_items: "{{ runtime_user_scripts }}"
diff --git a/image-builder/assets/playbooks/roles/osconfig/vars/main.yaml b/image-builder/assets/playbooks/roles/osconfig/vars/main.yaml
index 8b508fb..077d22b 100644
--- a/image-builder/assets/playbooks/roles/osconfig/vars/main.yaml
+++ b/image-builder/assets/playbooks/roles/osconfig/vars/main.yaml
@@ -1,15 +1 @@
 # NOTE: This file will be *overwritten* by the container entrypoint with user-provided vars, if any are defined.
-#
-# The following are examples that show you how to override variables.
-#
-# Example 1: The following usage will *overwrite* the list of packages
-#            defined under defaults/main.yaml with the list here:
-#post_install_package_list:
-#     - package1
-#     - package2
-#
-# Example 2: The following usage will *append* to the list of default pkgs
-#            defined under defaults/main.yaml with the list here:
-#post_install_package_list_append:
-#     - package1
-#     - package2
diff --git a/image-builder/assets/playbooks/roles/qcow/defaults/main.yaml b/image-builder/assets/playbooks/roles/qcow/defaults/main.yaml
index da9e25c..fbc901b 100644
--- a/image-builder/assets/playbooks/roles/qcow/defaults/main.yaml
+++ b/image-builder/assets/playbooks/roles/qcow/defaults/main.yaml
@@ -1,8 +1,8 @@
-root_chroot: /mnt/rootfs
+src: /build
+dst: /chroot
 nbd_build_dir: /tmp/nbd_build_dir
 img_output_dir: /config
 img_name: airship-ubuntu.qcow2
-dns: 8.8.8.8
 qcow_capacity: 5G
 partitions:
     # Partition numbering is according to list ordering.
@@ -46,3 +46,12 @@ partitions:
           options: "defaults,errors=remount-ro,noatime"
           dump: 0
           fsck: 2
+# If any custom post-install shell scripts are needed for qcow building,
+# they can be added here. This should only be used if
+# osconfig_container_buildtime_scripts does not work in osconfig playbook.
+qcow_container_runtime_scripts:
+    - file_content: |
+          #!/bin/bash
+          echo "custom qcow post-install script"
+# This is only needed if you want DNS working when running qcow scripts above
+qcow_container_runtime_scripts_dns: 8.8.8.8
diff --git a/image-builder/assets/playbooks/roles/qcow/tasks/boot-syslinux.yaml b/image-builder/assets/playbooks/roles/qcow/tasks/boot-syslinux.yaml
index 3b9fb72..923d3e0 100644
--- a/image-builder/assets/playbooks/roles/qcow/tasks/boot-syslinux.yaml
+++ b/image-builder/assets/playbooks/roles/qcow/tasks/boot-syslinux.yaml
@@ -1,7 +1,7 @@
 - name: "QCOW | Installing extlinux"
   shell: |
-        mkdir -p "{{ root_chroot }}"/boot/syslinux
-        extlinux --install "{{ root_chroot }}"/boot/syslinux/ --device /dev/disk/by-partlabel/{{ ( partitions | selectattr('mount', 'equalto', '/boot') | list | first ).mount | hash('md5') }}
+        mkdir -p "{{ dst }}"/boot/syslinux
+        extlinux --install "{{ dst }}"/boot/syslinux/ --device /dev/disk/by-partlabel/{{ ( partitions | selectattr('mount', 'equalto', '/boot') | list | first ).mount | hash('md5') }}
 
 - name: "QCOW | Writing out syslinux config"
   copy:
@@ -11,12 +11,12 @@
             LABEL linux
             KERNEL /vmlinuz
             APPEND root=/dev/disk/by-partlabel/{{ ( partitions | selectattr('mount', 'equalto', '/') | list | first ).mount | hash('md5') }} initrd=/initrd.img
-    dest: ""{{ root_chroot }}/boot/syslinux/syslinux.cfg"
+    dest: ""{{ dst }}/boot/syslinux/syslinux.cfg"
 
 
 - name: "QCOW | Installing kernel and init ramdisk"
   shell: |
-        rm -rf "{{ root_chroot }}"/vmlinuz
-        cp -f /mnt/image/vmlinuz "{{ root_chroot }}"/boot/
+        rm -rf "{{ dst }}"/vmlinuz
+        cp -f /mnt/image/vmlinuz "{{ dst }}"/boot/
         rm -rf /tmp/mnt/initrd.img
-        cp -f /mnt/image/initrd "{{ root_chroot }}"/boot/initrd.img
+        cp -f /mnt/image/initrd "{{ dst }}"/boot/initrd.img
diff --git a/image-builder/assets/playbooks/roles/qcow/tasks/chroot-cleanup.yaml b/image-builder/assets/playbooks/roles/qcow/tasks/chroot-cleanup.yaml
index 8e9a806..44a1223 100644
--- a/image-builder/assets/playbooks/roles/qcow/tasks/chroot-cleanup.yaml
+++ b/image-builder/assets/playbooks/roles/qcow/tasks/chroot-cleanup.yaml
@@ -1,16 +1,18 @@
 - name: "QCOW | copy ansible playbooks to target image"
   shell: |
         set -e
-        cp -r /opt/assets "{{ root_chroot }}"/opt
+        cp -r /opt/assets "{{ dst }}"/opt
 
 - name: "QCOW | unmount target"
   shell: |
         set -e
         # restore resolv.conf
-        chroot "{{ root_chroot }}" /bin/bash -c 'rm /etc/resolv.conf; cd /etc; ln -s ../run/systemd/resolve/stub-resolv.conf resolv.conf'
-        cd "{{ root_chroot }}"
+        chroot "{{ dst }}" /bin/bash -c 'rm /etc/resolv.conf; cd /etc; ln -s ../run/systemd/resolve/stub-resolv.conf resolv.conf'
+        cd "{{ dst }}"
         mountpoint dev/pts > /dev/null && umount dev/pts
         mountpoint dev > /dev/null && umount dev
-        mountpoint sys/firmware/efi > /dev/null && umount sys/firmware/efi
+        if [ -d /sys/firmware/efi ]; then
+          mountpoint sys/firmware/efi > /dev/null && umount sys/firmware/efi
+        fi
         mountpoint sys > /dev/null && umount sys
         mountpoint proc > /dev/null && umount proc
diff --git a/image-builder/assets/playbooks/roles/qcow/tasks/chroot-prep.yaml b/image-builder/assets/playbooks/roles/qcow/tasks/chroot-prep.yaml
index eb13362..19362b5 100644
--- a/image-builder/assets/playbooks/roles/qcow/tasks/chroot-prep.yaml
+++ b/image-builder/assets/playbooks/roles/qcow/tasks/chroot-prep.yaml
@@ -1,30 +1,13 @@
-- name: "QCOW | mount sys LEGACY"
-  shell: |
-        set -e
-        mkdir -p "{{ root_chroot }}"
-        cd "{{ root_chroot }}"
-        mountpoint sys > /dev/null || mount -t sysfs /sys sys
-        # mount an empty dir to efi directory, otherwise grub will try to configure EFI boot for the target image, **iff** the build node was booted with EFI.
-        if [ -d sys/firmware/efi ]; then mountpoint sys/firmware/efi > /dev/null || mkdir /dummy; mount -o bind /dummy sys/firmware/efi; fi
-  when: uefi is not defined
-
-- name: "QCOW | mount sys UEFI"
-  shell: |
-        set -e
-        mkdir -p "{{ root_chroot }}"
-        cd "{{ root_chroot }}"
-        mountpoint sys > /dev/null || mount -t sysfs /sys sys
-        # Required for building UEFI targets
-        ls /sys/firmware/efi > /dev/null || (echo "efivars not present on build system. Build system must be booted into UEFI mode." && exit 1)
-        mountpoint sys/firmware/efi > /dev/null || mount -o bind /sys/firmware/efi sys/firmware/efi
-  when: uefi is defined
-
 - name: "QCOW | Mount remaining targets"
   shell: |
         set -e
-        cd "{{ root_chroot }}"
+        cd "{{ dst }}"
+        mountpoint sys > /dev/null || mount -t sysfs /sys sys
+        if [ -d /sys/firmware/efi ]; then
+            mountpoint sys/firmware/efi > /dev/null || mount -o bind /sys/firmware/efi sys/firmware/efi
+        fi
         mountpoint proc > /dev/null || mount -t proc /proc proc
         mountpoint dev > /dev/null || mount -o bind /dev dev
         mountpoint dev/pts > /dev/null || mount -t devpts /dev/pts dev/pts
         # temporarily override resolv.conf to working dns
-        chroot "{{ root_chroot }}" /bin/bash -c 'rm /etc/resolv.conf; echo "nameserver {{ dns }}" > /etc/resolv.conf'
+        chroot "{{ dst }}" /bin/bash -c 'rm /etc/resolv.conf; echo "nameserver {{ qcow_container_runtime_scripts_dns }}" > /etc/resolv.conf'
diff --git a/image-builder/assets/playbooks/roles/qcow/tasks/copy-files.yaml b/image-builder/assets/playbooks/roles/qcow/tasks/copy-files.yaml
new file mode 100644
index 0000000..7cac1e7
--- /dev/null
+++ b/image-builder/assets/playbooks/roles/qcow/tasks/copy-files.yaml
@@ -0,0 +1,5 @@
+# Copy files onto partitioned disk
+- name: "mount-helper | Copy files onto partition"
+  shell: |
+        set -e
+        rsync -ah {{ src }}/ {{ dst }}/ --exclude 'live'
diff --git a/image-builder/assets/playbooks/roles/qcow/tasks/main.yaml b/image-builder/assets/playbooks/roles/qcow/tasks/main.yaml
index 9ee1172..f4982bf 100644
--- a/image-builder/assets/playbooks/roles/qcow/tasks/main.yaml
+++ b/image-builder/assets/playbooks/roles/qcow/tasks/main.yaml
@@ -19,10 +19,14 @@
       file: mount-helper.yaml
     with_items: "{{ partitions | sort( case_sensitive=True, attribute='mount_order' ) }}"
     vars:
-      mount_offset: "{{ root_chroot }}"
+      mount_offset: "{{ dst }}"
       state: mounted
       fstab: /tmp/junkfstab
 
+  - name: "QCOW | Copy files to partition"
+    include_tasks:
+      file: copy-files.yaml
+
   - name: "QCOW | Writing image content"
     include_tasks:
       file: writing-image-content.yaml
@@ -44,7 +48,7 @@
       file: mount-helper.yaml
     with_items: "{{ partitions | sort( reverse=True, case_sensitive=True, attribute='mount' ) }}"
     vars:
-      mount_offset: "{{ root_chroot }}"
+      mount_offset: "{{ dst }}"
       state: unmounted
       fstab: /tmp/junkfstab
 
diff --git a/image-builder/assets/playbooks/roles/qcow/tasks/writing-image-content.yaml b/image-builder/assets/playbooks/roles/qcow/tasks/writing-image-content.yaml
index fc531a8..dc2f665 100644
--- a/image-builder/assets/playbooks/roles/qcow/tasks/writing-image-content.yaml
+++ b/image-builder/assets/playbooks/roles/qcow/tasks/writing-image-content.yaml
@@ -1,16 +1,11 @@
-- name: "QCOW | Writing out rootfs from squashfs"
-  shell: |
-        unsquashfs -d "{{ root_chroot }}" -f /mnt/image/live/filesystem.squashfs
-        exit 0
-
 - name: "QCOW | Writing out fstab"
   include_tasks: mount-helper.yaml
   with_items: "{{ partitions | sort( case_sensitive=True, attribute='mount' ) }}"
   vars:
     mount_offset: null
     state: present
-    fstab: "{{ root_chroot }}/etc/fstab"
+    fstab: "{{ dst }}/etc/fstab"
 
 - name: "QCOW | Setting debug password"
   shell: |
-        chroot "{{ root_chroot }}" sh -c "echo \"root:password\" | chpasswd"
+        chroot "{{ dst }}" sh -c "echo \"root:password\" | chpasswd"
diff --git a/image-builder/assets/playbooks/roles/qcow/templates/generic-file-writer.j2 b/image-builder/assets/playbooks/roles/qcow/templates/generic-file-writer.j2
new file mode 100644
index 0000000..e4e0664
--- /dev/null
+++ b/image-builder/assets/playbooks/roles/qcow/templates/generic-file-writer.j2
@@ -0,0 +1 @@
+{{ item.file_content }}
diff --git a/image-builder/examples/osconfig-control-plane-vars.yaml b/image-builder/examples/osconfig-control-plane-vars.yaml
index 5c3f630..9ee2092 100644
--- a/image-builder/examples/osconfig-control-plane-vars.yaml
+++ b/image-builder/examples/osconfig-control-plane-vars.yaml
@@ -1,11 +1,4 @@
-rootfs_root: /mnt/rootfs
-
-cni_version: v0.8.2
-k8s_version: v1.18.6
-
 kernel:
-    base_pkg: linux-image-generic
-    headers_pkg: linux-headers-generic
     modules:
         load:
             - name: 8021q
@@ -165,17 +158,10 @@ systemd:
       enabled: yes
       force: no
 
-post_install_package_list:
-    - kdump-tools
-    - apparmor
-    - dbus
-    - rsyslog
-    - logrotate
-
-post_install_scripts:
+buildtime_user_scripts:
     - file_content: |
           #!/bin/bash
-          echo "custom post-install script"
+          echo "custom container buildtime script"
 
 file_permissions:
       # Full path to file to create
diff --git a/image-builder/examples/qcow-control-plane-vars-full-partitions.yaml b/image-builder/examples/qcow-control-plane-vars-full-partitions.yaml
index 4672fde..7b1353c 100644
--- a/image-builder/examples/qcow-control-plane-vars-full-partitions.yaml
+++ b/image-builder/examples/qcow-control-plane-vars-full-partitions.yaml
@@ -1,6 +1,3 @@
-root_chroot: /mnt/rootfs
-nbd_build_dir: /tmp/nbd_build_dir
-dns: 8.8.8.8
 qcow_capacity: 412G
 partitions:
     # Partition numbering is according to list ordering.
diff --git a/image-builder/examples/qcow-control-plane-vars.yaml b/image-builder/examples/qcow-control-plane-vars.yaml
index 0d1efea..c35ee61 100644
--- a/image-builder/examples/qcow-control-plane-vars.yaml
+++ b/image-builder/examples/qcow-control-plane-vars.yaml
@@ -1,4 +1,3 @@
-dns: 8.8.8.8
 qcow_capacity: 5G
 partitions:
     # Partition numbering is according to list ordering.
diff --git a/image-builder/tools/cut_image.sh b/image-builder/tools/cut_image.sh
index 9288170..772907f 100755
--- a/image-builder/tools/cut_image.sh
+++ b/image-builder/tools/cut_image.sh
@@ -18,14 +18,10 @@ image="${3:-port/image-builder:latest-ubuntu_focal}"
 # Libvirt instance name to use for a new libvirt XML definition that
 # will be created to reference the newly created ISO or QCOW2 image.
 img_alias="${4:-port-image-builder-latest-ubuntu_focal-$build_type}"
-# Whether or not to build the image with UEFI support.
-# NOTE: Machines that are not booted with UEFI will be unable to create
-# UEFI images.
-uefi_boot="$5"
 # proxy to use, if applicable
-proxy="$6"
+proxy="$5"
 # noproxy to use, if applicable
-noproxy="$7"
+noproxy="$6"
 
 if [ -n "$proxy" ]; then
   export http_proxy=$proxy
@@ -39,8 +35,23 @@ if [ -n "$noproxy" ]; then
   export NO_PROXY=$noproxy
 fi
 
-if [ -n "$uefi_boot" ]; then
+# Install pre-requisites
+install_pkg(){
+  dpkg -l $1 2> /dev/null | grep ^ii > /dev/null || sudo -E apt-get -y install $1
+}
+
+sudo -E apt -y update
+
+install_pkg qemu-kvm
+install_pkg virtinst
+install_pkg libvirt-bin
+install_pkg cloud-image-utils
+install_pkg ovmf
+type docker >& /dev/null || install_pkg docker.io
+
+if [ -d /sys/firmware/efi ]; then
   uefi_mount='--volume /sys/firmware/efi:/sys/firmware/efi:rw'
+  uefi_boot_arg='--boot uefi'
 fi
 
 workdir="$(realpath ${host_mount_directory})"
@@ -58,9 +69,9 @@ if [[ $build_type = iso ]]; then
    --env NO_PROXY=$noproxy \
    ${image}
   disk1="--disk path=${workdir}/ephemeral.iso,device=cdrom"
-  uefi_boot_arg='--boot uefi'
 elif [[ $build_type == qcow ]]; then
   sudo -E modprobe nbd
+  echo "Note: This step can be slow if you don't have an SSD."
   sudo -E docker run -t --rm \
    --privileged \
    --volume /dev:/dev:rw  \
@@ -78,15 +89,11 @@ elif [[ $build_type == qcow ]]; then
    --env HTTPS_PROXY=$proxy \
    --env no_proxy=$noproxy \
    --env NO_PROXY=$noproxy \
-   --env uefi_boot=$uefi_boot \
    ${image}
   cloud_init_config_dir='assets/tests/qcow/cloud-init'
   sudo -E cloud-localds -v --network-config="${cloud_init_config_dir}/network-config" "${workdir}/airship-ubuntu_config.iso" "${cloud_init_config_dir}/user-data" "${cloud_init_config_dir}/meta-data"
   disk1="--disk path=${workdir}/control-plane.qcow2"
   disk2="--disk path=${workdir}/airship-ubuntu_config.iso,device=cdrom"
-  if [ -n "$uefi_boot" ]; then
-    uefi_boot_arg='--boot uefi'
-  fi
 else
   echo Unknown build type: $build_type, exiting.
   exit 1
diff --git a/image-builder/tools/install_prereqs.ubuntu_focal b/image-builder/tools/install_prereqs.ubuntu_focal
deleted file mode 100755
index 70e1452..0000000
--- a/image-builder/tools/install_prereqs.ubuntu_focal
+++ /dev/null
@@ -1,15 +0,0 @@
-#!/bin/bash
-
-install_pkg(){
-  dpkg -l $1 >& /dev/null || sudo -E apt-get -y install $1
-}
-
-install_pkg qemu-kvm
-install_pkg virtinst
-install_pkg libvirt-bin
-install_pkg cloud-image-utils
-install_pkg ovmf
-install_pkg efivar
-type docker >& /dev/null || install_pkg docker.io
-# required for building UEFI image
-sudo -E modprobe efivars
diff --git a/image-builder/tools/multistrap.sh b/image-builder/tools/multistrap.sh
new file mode 100755
index 0000000..598fc35
--- /dev/null
+++ b/image-builder/tools/multistrap.sh
@@ -0,0 +1,136 @@
+#!/bin/bash
+
+set -e
+build_dir=assets/playbooks/build
+osconfig_build_dir=$(basename $build_dir)
+
+install_pkg(){
+    dpkg -l $1 2> /dev/null | grep ^ii > /dev/null || sudo -E apt-get -y install $1
+}
+
+setup_chroot(){
+    # Idempotently setup chroot mounts
+    mkdir -p $build_dir
+    mkdir -p $build_dir/sys
+    mountpoint $build_dir/sys > /dev/null || sudo mount -t sysfs /sys $build_dir/sys
+    if [ -d /sys/firmware/efi ]; then
+        mountpoint $build_dir/sys/firmware/efi > /dev/null || sudo mount -o bind /sys/firmware/efi $build_dir/sys/firmware/efi
+    fi
+    mkdir -p $build_dir/proc
+    mountpoint $build_dir/proc > /dev/null || sudo mount -t proc /proc $build_dir/proc
+    mkdir -p $build_dir/dev
+    mountpoint $build_dir/dev > /dev/null || sudo mount -o bind /dev $build_dir/dev
+    mountpoint $build_dir/dev/pts > /dev/null || sudo mount -t devpts /dev/pts $build_dir/dev/pts
+    mkdir -p $osconfig_build_dir
+    mountpoint $osconfig_build_dir > /dev/null || sudo mount -o bind $build_dir $osconfig_build_dir
+}
+
+umount_helper(){
+    if [[ -d "$1" ]] && mountpoint "$1" > /devnull; then
+        sudo umount "$1"
+    fi
+}
+
+umount_chroot(){
+    # Idempotently teardown chroot mounts
+    umount_helper $build_dir/dev/pts
+    umount_helper $build_dir/dev
+    if [[ -d /sys/firmware/efi ]]; then
+	umount_helper $build_dir/sys/firmware/efi
+    fi
+    umount_helper $build_dir/sys
+    umount_helper $build_dir/proc
+    umount_helper $osconfig_build_dir
+}
+
+# Install pre-requisites
+sudo -E apt -y update
+
+install_pkg efivar
+# required for building UEFI image
+sudo -E modprobe efivars
+type docker >& /dev/null || install_pkg docker.io
+install_pkg equivs
+install_pkg ca-certificates
+install_pkg build-essential
+install_pkg gnupg2
+install_pkg multistrap
+install_pkg curl
+install_pkg grub-common
+install_pkg grub2-common
+install_pkg grub-pc-bin
+install_pkg grub-efi-amd64-signed
+install_pkg dosfstools
+install_pkg mtools
+install_pkg squashfs-tools
+install_pkg python3-minimal
+install_pkg python3-pip
+install_pkg python3-apt
+install_pkg python3-setuptools
+sudo -E pip3 install --upgrade pip
+pip3 show wheel >& /dev/null || sudo -E pip3 install --upgrade wheel
+pip3 show ansible >& /dev/null || sudo -E pip3 install --upgrade ansible
+
+if [[ $1 = clean ]]; then
+    umount_chroot
+    sudo chattr -i $build_dir/etc/kernel/postinst.d/kdump-tools
+    if [[ -d $build_dir ]]; then
+        sudo rm -rf $build_dir
+    fi
+    if [[ -d $osconfig_build_dir ]]; then
+        sudo rm -rf $osconfig_build_dir
+    fi
+    exit 0
+elif [[ $1 = umount ]]; then
+    umount_chroot
+    exit 0
+elif [[ $1 = mount ]]; then
+    setup_chroot
+    exit 0
+fi
+
+setup_chroot
+
+# Archive a copy of the ansible used to generate the image in the image itself
+mkdir -p $build_dir/opt/assets/playbooks/roles
+cp assets/playbooks/inventory.yaml $build_dir/opt/assets/playbooks/inventory.yaml
+cp assets/playbooks/base-chroot.yaml $build_dir/opt/assets/playbooks/base-chroot.yaml
+cp -r assets/playbooks/roles/multistrap $build_dir/opt/assets/playbooks/roles
+# Run multistrap
+sudo -E ansible-playbook -i assets/playbooks/inventory.yaml assets/playbooks/base-chroot.yaml -vv
+
+cp assets/playbooks/base-osconfig.yaml $build_dir/opt/assets/playbooks/base-osconfig.yaml
+cp -r assets/playbooks/roles/osconfig $build_dir/opt/assets/playbooks/roles
+sudo -E ansible-playbook -i assets/playbooks/inventory.yaml assets/playbooks/base-osconfig.yaml --tags "runtime_and_buildtime" -vv
+sudo -E ansible-playbook -i assets/playbooks/inventory.yaml assets/playbooks/base-osconfig.yaml --tags "buildtime_only" -vv
+
+umount_chroot
+
+cp assets/playbooks/base-livecdcontent.yaml $build_dir/opt/assets/playbooks/base-livecdcontent.yaml
+cp -r assets/playbooks/roles/livecdcontent $build_dir/opt/assets/playbooks/roles
+sudo -E ansible-playbook -i assets/playbooks/inventory.yaml assets/playbooks/base-livecdcontent.yaml -vv
+
+cp assets/playbooks/iso.yaml $build_dir/opt/assets/playbooks/iso.yaml
+cp -r assets/playbooks/roles/iso $build_dir/opt/assets/playbooks/roles
+cp assets/playbooks/qcow.yaml $build_dir/opt/assets/playbooks/qcow.yaml
+cp -r assets/playbooks/roles/qcow $build_dir/opt/assets/playbooks/roles
+
+if [ ! -e $build_dir/dev/random ]; then
+    sudo -E mknod $build_dir/dev/random c 1 8
+    sudo -E chmod 640 $build_dir/dev/random
+    sudo -E chown 0:0 $build_dir/dev/random
+fi
+if [ ! -e $build_dir/dev/urandom ]; then
+    sudo -E mknod $build_dir/dev/urandom c 1 9
+    sudo -E chmod 640 $build_Dir/dev/urandom
+    sudo -E chown 0:0 $build_Dir/dev/urandom
+fi
+if [ -f $build_dir/dev/null ]; then
+    sudo rm -f $build_dir/dev/null
+fi
+if [ ! -e $build_dir/dev/null ]; then
+    sudo -E mknod $build_dir/dev/null c 1 3
+    sudo -E chmod 666 $build_dir/dev/null
+    sudo -E chown 0:0 $build_dir/dev/null
+fi
+