Demonstrate how to upgrade specific components of image bundle

- folder for each version, i.e. k8s-1.19.14
  - job for each image being built 'airship-image-builder-build-k8s-1.19.14'
 implement profiles
  - Add a profiles folder that gets copied into the manifests folder
  - update readme with details
  - update zuul.d with additional pipelines to build the new profiles

Change-Id: Ia01516419e58a33b538b06cd31a536e8cbfc15c2

README.md

@@ -101,4 +101,4 @@ Configuration management of the base OS is divided into several realms, each wit
 # FAQ
 
 Q: Why is the build target slow?
-A: There is a `mksquashfs` command which runs as part of the build target, and performs slowly if your build environment lacks certain CPU flags which accelerate compression. Use "host-passthrough" or equivalent in your build environment to pass through these CPU flags. In libvirt domain XML, you would change your `cpu` mode element as follows: `<cpu mode='host-passthrough' check='none'/>`
+A: There is a `mksquashfs` command which runs as part of the build target, and performs slowly if your build environment lacks certain CPU flags which accelerate compression. Use "host-passthrough" or equivalent in your build environment to pass through these CPU flags. In libvirt domain XML, you would change your `cpu` mode element as follows: `<cpu mode='host-passthrough' check='none'/>`

image-builder/Makefile

@@ -23,6 +23,9 @@ IMAGE_TYPE	?= iso # iso | qcow
 PUSH_IMAGE	?= false
 DISTRO          ?= ubuntu_focal
 WORKDIR         ?= ./manifests
+# Specifiy if you want to use a different profile than the default.
+#	i.e. PROFILE ?= k8s-1.18
+PROFILE 		?=
 QCOW_BUNDLE     ?= ${WORKDIR}/qcow-bundle
 # Specify if you want to only build a certain subset of QCOW bundles
 QCOW_BUNDLE_DIRS  ?=
@@ -50,11 +53,17 @@ help: ## This help.
 images: build generate_iso package_qcow clean
 
 build:
+ifneq ($(PROFILE), )
+	set -ex
+	# Apply any user-defined profiles overrides to playbooks
+	rsync -rc ./../profiles/$(PROFILE)/manifests/ $(WORKDIR)/
+endif
 	set -ex
 	# Apply any user-defined rootfs overrides to playbooks
 	cp $(WORKDIR)/rootfs/multistrap-vars.yaml assets/playbooks/roles/multistrap/vars/main.yaml
 	cp $(WORKDIR)/rootfs/osconfig-vars.yaml assets/playbooks/roles/osconfig/vars/main.yaml
 	cp $(WORKDIR)/rootfs/livecdcontent-vars.yaml assets/playbooks/roles/livecdcontent/vars/main.yaml
+
 ifneq ($(PROXY), )
 	sudo -E ./tools/docker_proxy.sh $(PROXY) $(NO_PROXY)
 	export http_proxy=$(PROXY)
@@ -90,8 +99,8 @@ endif
 	--label "org.opencontainers.image.title=$(IMAGE_NAME)"
 endif
 	imgId=`sudo docker images | grep 'image-builder ' | awk '{print $$3}'`
-	sudo -E DOCKER_BUILDKIT=1 docker run $$imgId ls -ltra /build/usr/bin/sudo > /tmp/sticky_result
+	time sudo -E DOCKER_BUILDKIT=1 docker run $$imgId ls -ltra /build/usr/bin/sudo > /tmp/sticky_result
-	sudo grep '^-rws' /tmp/sticky_result >& /dev/null || \
+	time sudo grep '^-rws' /tmp/sticky_result >& /dev/null || \
 		(echo Could not find sticky bit set on target image sudo binary. Are you using buildkit? && \
 		sudo cat /tmp/sticky_result && exit 1)
 ifeq ($(PUSH_IMAGE), true)

image-builder/manifests/README.md

@@ -21,6 +21,10 @@
     |-- qcow
 ```
 
+## profiles
+
+To make modifications please consider using `profiles`.
+
 ## iso
 
 The image-builder `generate_iso` makefile target can be used to build the

playbooks/airship-image-builder-build.yaml

@@ -54,5 +54,6 @@
             PROXY: "{{ proxy.http }}"
             QCOW_CONF_DIRS: "{{ qcow_conf_dirs | default('') }}"
             USE_PROXY: "{{ proxy.enabled | lower }}"
-            WORKDIR: "{{ image_config_dir | default('manifests') }}"
+            WORKDIR: "{{ image_work_dir | default('manifests') }}"
+            PROFILE: "{{ image_profile_dir | default('') }}"
       become: True

playbooks/airship-image-builder-publish-latest.yaml

@@ -60,5 +60,6 @@
             PROXY: "{{ proxy.http }}"
             QCOW_CONF_DIRS: "{{ qcow_conf_dirs | default('') }}"
             USE_PROXY: "{{ proxy.enabled | lower }}"
-            WORKDIR: "{{ image_config_dir | default('manifests') }}"
+            WORKDIR: "{{ image_work_dir | default('manifests') }}"
+            PROFILE: "{{ image_profile_dir | default('') }}"
       become: True

profiles/README.md

@@ -0,0 +1,144 @@
+# Directory structure:
+
+```
+|-- profiles
+  |-- profile1
+    |-- manifests
+    |-- iso
+        +-- network_data.json
+        +-- user_data
+    |-- qcow-bundle-[bundle name]
+        |-- control-plane
+        +-- osconfig-vars.yaml
+        +-- qcow-vars.yaml
+        |-- data-plane
+        +-- osconfig-vars.yaml
+        +-- qcow-vars.yaml
+    |-- rootfs
+        |-- livecdcontent-vars.yaml
+        |-- multistrap-vars.yaml
+        |-- osconfig-vars.yaml
+    |-- scripts
+        |-- common
+        |-- qcow
+```
+
+## profiles
+This directory allows multiple bundle modifications to be created. It
+serves the same purpose as the `manifests` directory just expands on
+it's utility.
+
+The folder structure is the same as the `manifests` directory. Create
+a new profile folder and copy the contents of `manifests` into it. Make your
+modifications and add the `profile=profile1` flag to the `make images` command.
+
+The profile will copy only the modified files into the `manifests` folder.
+
+## iso
+
+The image-builder `generate_iso` makefile target can be used to build the
+ephemeral ISO using the test config data stored under the `manifests/iso`
+directory.
+
+This is *only* for testing. It is *not* an artifact promoted or published. The
+final ISO is built by airshipctl, where the network\_data and user\_data are
+sourced from airshipctl manifests.
+
+The following items are expected in the `manifests/iso` directory when using
+the `generate_iso` makefile target:
+- `user_data` - YAML file containing cloud-init user-data
+- `network_data.json` - JSON file containing cloud-init network data
+
+## qcow-bundles
+
+The image-builder `package_qcow` makefile target can be used to build the QCOW
+artifacts sourced from the manifests/qcow-bundle-\* directories.
+
+QCOWs are grouped into publishable "bundles", i.e. a container image where all
+QCOWs needed for a given deployment are stored. A bundle will be built for each
+`manifests/qcow-bundle*` directory. Each `manifests/qcow-bundle*` directory contains
+one subdirectory per QCOW that is part of that bundle, where overrides for
+those images can be placed.
+
+QCOWs expect the following files to be present in their directory:
+- `osconfig-vars.yaml` - YAML file containing `osconfig` playbook overrides
+- `qcow-vars.yaml` - YAML file containing `qcow` playboook overrides
+
+## rootfs
+
+This directory contains a number of image-builder ansible playbook overrides
+which are applied to base-image inherited by all ISO and QCOWs.
+
+`livecdcontent-vars.yaml` contains overrides to the livecdcontent playbook.
+
+`multistrap-vars.yaml` contains overrides to the `multistrap` playbook.
+
+`osconfig-vars.yaml` contains overrides to the `osconfig` playbook.
+NOTE: qcow-bundles contains another level of `osconfig-vars` overrides, which
+are applied on top of these common overrides. This common `osconfig-vars`
+overrides should be used for playbook overrides, except in cases where those
+overrides are actually unique to a particular QCOW variation (e.g., hugepages,
+cpu pinning, or other hardware-specific configs).
+
+## scripts
+
+This is a convenience directory for adding scripts that run when building images.
+These scripts run in the chroot of the target image. For example, a script that
+writes 'hello world' to `/hello-world.txt` will appear in the same path on the
+target image.
+
+Use the `manifests/scripts/qcow` directory for scripts that should only run
+when building the QCOWs. Use the `manifests/scripts/common` directory for
+scripts that are applied to the base container image, which is inherited both by
+the QCOWs as well as by the ephemeral ISO.
+
+No additional configuration is needed for these scripts to run. Just add your
+script(s) to these directories as needed.
+
+# Customizing images in your environment
+
+Keep in mind that some tasks could also be accomplished by cloud-init or by
+the hostconfig operator instead. Refer to the parent image-builder README to
+understand the different use-cases for each and to determine the best option
+for your use-case. These are lower-effort paths if they support your use-case.
+
+If you determine that you do require image customizations, start with a manual
+image build to reduce complexity:
+
+1. Clone this repository in your environment.
+1. Make any desired changes to the `manifests` directory to customize the
+   image, as described in prior sections.
+1. Perform a `docker login` to the docker registry you will publish image
+   artifacts to. This should be a registry you have credentials for and that
+   is accessible by the environment which you plan to consume these artifacts,
+   (e.g., airshipctl).
+1. Run the `make images` target to generate image artifacts. Ensure that the
+   `PUSH_IMAGE` environment variable is set to `true`, and that the
+   `DOCKER_REGISTRY` environment variable is set to the container image
+   repository you performed the login for in the previous step.
+
+Perform an end-to-end to deployment (e.g., with airshipctl) to verify your
+customized image performs as you expect and works properly.
+
+Now after getting this working, there are several options to proceed depending
+on the nature of the changes:
+1. Some set of changes to defaults could be proposed upstream (e.g., package
+   install list). This may be appropriate for changes that are useful for
+   everyone. In this case, you don't need a custom image because the changes
+   will be reflected in the image produced upstream.
+1. Some enhancements or additions to ansible playbooks to configure some other
+   aspects of the image, which are useful for everyone and proposed upstream.
+   In this case, you would be able to leverage ansible overrides to customize
+   your image with ansible playbooks that are maintained/adopted upstream.
+1. Some change to image configuration that is specific to your needs and not
+   appropriate to be upstreamed.
+
+In the case of #2 or #3 where you have some portion of image config changes that
+are specific to your use-case (i.e. not part of the default upstream image),
+and you want to perform regular rebuilds with the latest upstream image-builder
+plus your customized changes on top, then you can setup a Zuul child-job that
+interfaces with the image-builder parent-job to accomplish this.
+
+By overriding the `image_config_dir` zuul variable in your child-job, the
+image-builder Makefile will use use your customized manifests in place of the
+`manifests` directory that is present in upstream image-builder.

profiles/k8s-1.18/manifests/rootfs/README.md

@@ -0,0 +1 @@
+Playbook variable overrides used for building the image-builder container image.

profiles/k8s-1.18/manifests/rootfs/multistrap-vars.yaml

@@ -0,0 +1,50 @@
+# Overrides to the multistrap playbook defaults may be defined in this file.
+#
+# The following are examples that show you how to override variables.
+#
+# Example 1: The following usage will *overwrite* the list of repos & packages
+#            defined under multistrap/defaults/main.yaml with the list here:
+#repos:
+#  - register_repo_with_rootfs: true
+#    name: Ubuntu
+#    packages:
+#      - package1
+#      - package2
+#    source: http://archive.ubuntu.com/ubuntu/
+#    keyring_pkg: ubuntu-keyring
+#    suite: focal
+#    components: main restricted universe
+#
+# You would do the above in the event you had a local or other controlled mirror
+# you wanted to build the image from. In this case, you will have to redefine
+# everything, including the package list(s) for each mirror.
+#
+# Example 2: The following usage will *append* to the list of default repos
+#            defined under multistrap/defaults/main.yaml with the list here:
+#repos_append:
+#  - register_repo_with_rootfs: true
+#    name: DellUtilsRepo
+#    packages:
+#      - package1
+#      - package2
+#    source: http://dell.utils.example.com
+#    keyring_pkg: ubuntu-keyring
+#    suite: focal
+#    components: main restricted universe
+#
+# You would do the above in the event you are happy with the default mirrors, but
+# need to add another mirror that has additional third-party packages you require,
+# such as may be needed for HW manufacturer utilities that are not published to
+# the community Ubuntu mirrors.
+#
+# Example 3: The following usage will *append* to the list of default packages
+#            installed from the default Ubuntu mirror defined in
+#            multistrap/defaults/main.yaml:
+#ubuntu_packages_append:
+#  - package1
+#  - package2
+#
+# You would do the above if you are happy with the default mirrors, but need to
+# install some additional packages from the same mirror, for example your preferred
+# text editor, troubleshooting utilities, etc.
+k8s_version: 1.18.20-00

profiles/k8s-1.19/manifests/rootfs/README.md

@@ -0,0 +1 @@
+Playbook variable overrides used for building the image-builder container image.

profiles/k8s-1.19/manifests/rootfs/multistrap-vars.yaml

@@ -0,0 +1,50 @@
+# Overrides to the multistrap playbook defaults may be defined in this file.
+#
+# The following are examples that show you how to override variables.
+#
+# Example 1: The following usage will *overwrite* the list of repos & packages
+#            defined under multistrap/defaults/main.yaml with the list here:
+#repos:
+#  - register_repo_with_rootfs: true
+#    name: Ubuntu
+#    packages:
+#      - package1
+#      - package2
+#    source: http://archive.ubuntu.com/ubuntu/
+#    keyring_pkg: ubuntu-keyring
+#    suite: focal
+#    components: main restricted universe
+#
+# You would do the above in the event you had a local or other controlled mirror
+# you wanted to build the image from. In this case, you will have to redefine
+# everything, including the package list(s) for each mirror.
+#
+# Example 2: The following usage will *append* to the list of default repos
+#            defined under multistrap/defaults/main.yaml with the list here:
+#repos_append:
+#  - register_repo_with_rootfs: true
+#    name: DellUtilsRepo
+#    packages:
+#      - package1
+#      - package2
+#    source: http://dell.utils.example.com
+#    keyring_pkg: ubuntu-keyring
+#    suite: focal
+#    components: main restricted universe
+#
+# You would do the above in the event you are happy with the default mirrors, but
+# need to add another mirror that has additional third-party packages you require,
+# such as may be needed for HW manufacturer utilities that are not published to
+# the community Ubuntu mirrors.
+#
+# Example 3: The following usage will *append* to the list of default packages
+#            installed from the default Ubuntu mirror defined in
+#            multistrap/defaults/main.yaml:
+#ubuntu_packages_append:
+#  - package1
+#  - package2
+#
+# You would do the above if you are happy with the default mirrors, but need to
+# install some additional packages from the same mirror, for example your preferred
+# text editor, troubleshooting utilities, etc.
+k8s_version: 1.19.14-00

zuul.d/jobs.yaml

@@ -9,8 +9,9 @@
         secret: airship_image_builder_airshipit_github_key
         pass-to-parent: true
 
+# Run the default build job 1.18
 - job:
-    name: airship-image-builder-build
+    name: airship-image-builder-build-k8s-1.18
     nodeset: airship-image-builder-single-node
     timeout: 7200
     post-timeout: 7200
@@ -18,8 +19,22 @@
     run: playbooks/airship-image-builder-build.yaml
     post-run: playbooks/airship-image-builder-collect-logs.yaml
 
+# Run the build job 1.19
 - job:
-    name: airship-image-builder-publish-latest
+    name: airship-image-builder-build-k8s-1.19
+    nodeset: airship-image-builder-single-node
+    timeout: 7200
+    post-timeout: 7200
+    pre-run: playbooks/airship-image-builder-deploy-docker.yaml
+    run: playbooks/airship-image-builder-build.yaml
+    post-run: playbooks/airship-image-builder-collect-logs.yaml
+    vars:
+      image_profile_dir: k8s-1.19
+
+# Tag default job with 1.18
+- job:
+    name: airship-image-builder-publish-k8s-1.18
+    parent: airship-image-builder-build-k8s-1.18
     nodeset: airship-image-builder-single-node
     timeout: 7200
     post-timeout: 7200
@@ -28,12 +43,37 @@
     secrets:
       - name: image_builder_quay_creds
         secret: image_builder_quay_creds
-
-- job:
-    name: airship-image-builder-publish-commit
-    parent: airship-image-builder-publish-latest
     vars:
-      image_tag: "{{ zuul.newrev }}"
+      image_tag: "k8s-1.18-latest"
+
+# Tag the 1.19 job with 1.19
+- job:
+    name: airship-image-builder-publish-k8s-1.19
+    parent: airship-image-builder-build-k8s-1.19
+    nodeset: airship-image-builder-single-node
+    timeout: 7200
+    post-timeout: 7200
+    pre-run: playbooks/airship-image-builder-deploy-docker.yaml
+    run: playbooks/airship-image-builder-publish-latest.yaml
+    secrets:
+      - name: image_builder_quay_creds
+        secret: image_builder_quay_creds
+    vars:
+      image_tag: "k8s-1.19-latest"
+
+# Tag the 1.18 job with zuul rev tag
+- job:
+    name: airship-image-builder-publish-commit-k8s-1.18
+    parent: airship-image-builder-publish-k8s-1.18
+    vars:
+      image_tag: "k8s-1.18-{{ zuul.newrev }}"
+
+# Tag the 1.19 job with zuul rev tag
+- job:
+    name: airship-image-builder-publish-commit-k8s-1.19
+    parent: airship-image-builder-publish-k8s-1.19
+    vars:
+      image_tag: "k8s-1.19-{{ zuul.newrev }}"
 
 - nodeset:
     name: airship-image-builder-single-node

zuul.d/projects.yaml

@@ -15,15 +15,19 @@
       image_tag: latest
     check:
       jobs:
-        - airship-image-builder-build
+        - airship-image-builder-build-k8s-1.18
+        - airship-image-builder-build-k8s-1.19
     gate:
       jobs:
-        - airship-image-builder-build
+        - airship-image-builder-build-k8s-1.18
+        - airship-image-builder-build-k8s-1.19
     post:
       jobs:
         - airship-image-builder-upload-git-mirror
-        - airship-image-builder-publish-latest
+        - airship-image-builder-publish-k8s-1.18
-        - airship-image-builder-publish-commit
+        - airship-image-builder-publish-k8s-1.19
+        - airship-image-builder-publish-commit-k8s-1.18
+        - airship-image-builder-publish-commit-k8s-1.19
     tag:
       jobs:
         - airship-image-builder-upload-git-mirror