Adding fetch from url option

Adding fetch from url option to osconfig so that any resource can
be downloaded from given url and installed it during image build
process. Typical scenario in which user has to install custom package
in user defined script. Eg. Installing kernel driver 'i40e' etc.

Change-Id: Ic7d18f24e6faa4666b9d913cf360fe0bf11f2ef4

image-builder/assets/playbooks/roles/osconfig/defaults/main.yaml

@@ -212,3 +212,10 @@ file_permissions:
       # combined with previous item)
       run_contexts:
           - "{{ default_run_context }}"
+# If any required resources need to be fetched from URL for image build customization, they can be added here.
+# Downloaded resources can be found in /tmp/url_resources directory.
+# Example:-
+# fetch_from_url:
+#     -  url: https://www.example.com/resource.tar.gz
+#        use_proxy: no
+fetch_from_url: []

image-builder/assets/playbooks/roles/osconfig/tasks/fetch-from-url.yaml

@@ -0,0 +1,18 @@
+- name: "Append any user-defined custom urls"
+  set_fact:
+    fetch_from_url: "{{ fetch_from_url + fetch_from_url_append }}"
+  when: fetch_from_url_append is defined
+- when: fetch_from_url is defined
+  block:
+    - name: "ensuring directory /tmp/url_resources exists"
+      file:
+        path: "/tmp/url_resources"
+        state: directory
+        mode: '0755'
+    - name: "Download from url {{ item.url }}"
+      get_url:
+        url: "{{ item.url }}"
+        dest: "/tmp/url_resources/{{ item.url | basename }}"
+        mode: '0755'
+        use_proxy: "{{ item.use_proxy }}"
+      loop: "{{ fetch_from_url }}"

image-builder/assets/playbooks/roles/osconfig/tasks/main.yaml

@@ -37,6 +37,9 @@
 - name: "configure base systemd"
   include_tasks: systemd.yaml
   when: run_context == default_run_context
+- name: "fetch url resource"
+  include_tasks: fetch-from-url.yaml
+  when: run_context == default_run_context
 
 # Context-dependent tasks
 - name: "write user-provided files"

image-builder/assets/playbooks/roles/qcow/defaults/main.yaml

@@ -54,5 +54,3 @@ qcow_container_runtime_scripts:
     - file_content: |
           #!/bin/bash
           echo "custom qcow post-install script"
-# This is only needed if you want DNS working when running qcow scripts above
-qcow_container_runtime_scripts_dns: 8.8.8.8

image-builder/assets/playbooks/roles/qcow/tasks/chroot-cleanup.yaml

@@ -6,8 +6,6 @@
 - name: "QCOW | unmount target"
   shell: |
         set -e
-        # restore resolv.conf
-        chroot "{{ dst }}" /bin/bash -c 'rm /etc/resolv.conf; cd /etc; ln -s ../run/systemd/resolve/stub-resolv.conf resolv.conf'
         cd "{{ dst }}"
         mountpoint dev/pts > /dev/null && umount dev/pts
         mountpoint dev > /dev/null && umount dev
@@ -16,3 +14,6 @@
         fi
         mountpoint sys > /dev/null && umount sys
         mountpoint proc > /dev/null && umount proc
+        if [ -d "/run/systemd/resolve" ]; then
+          mountpoint run/systemd/resolve > /dev/null && umount -l run/systemd/resolve
+        fi

image-builder/assets/playbooks/roles/qcow/tasks/chroot-prep.yaml

@@ -9,5 +9,6 @@
         mountpoint proc > /dev/null || mount -t proc /proc proc
         mountpoint dev > /dev/null || mount -o bind /dev dev
         mountpoint dev/pts > /dev/null || mount -t devpts /dev/pts dev/pts
-        # temporarily override resolv.conf to working dns
+        if [ -d "/run/systemd/resolve" ]; then
-        chroot "{{ dst }}" /bin/bash -c 'rm /etc/resolv.conf; echo "nameserver {{ qcow_container_runtime_scripts_dns }}" > /etc/resolv.conf'
+            mountpoint run/systemd/resolve > /dev/null || mount -o bind /run/systemd/resolve run/systemd/resolve
+        fi

image-builder/tools/cut_image.sh

@@ -108,6 +108,7 @@ outputFileName: $img_name" > ${qcow_config}
    --volume /proc:/proc:rw \
    --volume /sys:/sys:rw \
    --volume /lib/modules:/lib/modules:rw \
+   --volume /run/systemd/resolve:/run/systemd/resolve:rw \
    --volume $workdir:/config \
    ${uefi_mount} \
    --env BUILDER_CONFIG=/config/${build_type}.yaml \

image-builder/tools/multistrap.sh

@@ -34,11 +34,16 @@ setup_chroot(){
     mountpoint $build_dir/dev/pts > /dev/null || sudo mount -t devpts /dev/pts $build_dir/dev/pts
     mkdir -p $osconfig_build_dir
     mountpoint $osconfig_build_dir > /dev/null || sudo mount -o bind $build_dir $osconfig_build_dir
+    if [ -d "/run/systemd/resolve" ]; then
+      mkdir -p $build_dir/run/systemd/resolve
+      mountpoint $build_dir/run/systemd/resolve > /dev/null || sudo mount -o bind /run/systemd/resolve $build_dir/run/systemd/resolve
+    fi
 }
 
 umount_helper(){
-    if [[ -d "$1" ]] && mountpoint "$1" > /devnull; then
+    if [[ -d "$1" ]] && mountpoint "$1" > /dev/null; then
-        sudo umount "$1"
+        # if umount fails in first attempt, we try to use lazy umount
+        sudo umount "$1" || sudo umount -l "$1"
     fi
 }
 
@@ -52,6 +57,9 @@ umount_chroot(){
     umount_helper $build_dir/sys
     umount_helper $build_dir/proc
     umount_helper $osconfig_build_dir
+    if [ -d "/run/systemd/resolve" ]; then
+      umount_helper $build_dir/run/systemd/resolve
+    fi
 }
 
 # Install pre-requisites
@@ -121,10 +129,8 @@ cp assets/playbooks/base-osconfig.yaml $build_dir/opt/assets/playbooks/base-osco
 cp -r assets/playbooks/roles/osconfig $build_dir/opt/assets/playbooks/roles
 if [ -d $build_dir/config ]; then
   sudo rm -r $build_dir/config
-  mkdir -p $build_dir/config
-else
-  mkdir -p $build_dir/config
 fi
+mkdir -p $build_dir/config
 cp -r $workdir/scripts $build_dir/config/
 if [ -z "$SKIP_OSCONFIG_ROLE" ]; then
   sudo -E ansible-playbook -i assets/playbooks/inventory.yaml assets/playbooks/base-osconfig.yaml --extra-vars "run_context=common" -vv