From dcc91451ea8b95d0c91c30878de7dc84f08462dc Mon Sep 17 00:00:00 2001 From: Pallav Gupta Date: Fri, 26 Feb 2021 18:40:25 +0000 Subject: [PATCH] Adding fetch from url option Adding fetch from url option to osconfig so that any resource can be downloaded from given url and installed it during image build process. Typical scenario in which user has to install custom package in user defined script. Eg. Installing kernel driver 'i40e' etc. Change-Id: Ic7d18f24e6faa4666b9d913cf360fe0bf11f2ef4 --- .../roles/osconfig/defaults/main.yaml | 7 +++++++ .../roles/osconfig/tasks/fetch-from-url.yaml | 18 ++++++++++++++++++ .../playbooks/roles/osconfig/tasks/main.yaml | 3 +++ .../playbooks/roles/qcow/defaults/main.yaml | 2 -- .../roles/qcow/tasks/chroot-cleanup.yaml | 5 +++-- .../roles/qcow/tasks/chroot-prep.yaml | 5 +++-- image-builder/tools/cut_image.sh | 1 + image-builder/tools/multistrap.sh | 16 +++++++++++----- 8 files changed, 46 insertions(+), 11 deletions(-) create mode 100644 image-builder/assets/playbooks/roles/osconfig/tasks/fetch-from-url.yaml diff --git a/image-builder/assets/playbooks/roles/osconfig/defaults/main.yaml b/image-builder/assets/playbooks/roles/osconfig/defaults/main.yaml index 75a6085..0b4f382 100644 --- a/image-builder/assets/playbooks/roles/osconfig/defaults/main.yaml +++ b/image-builder/assets/playbooks/roles/osconfig/defaults/main.yaml @@ -212,3 +212,10 @@ file_permissions: # combined with previous item) run_contexts: - "{{ default_run_context }}" +# If any required resources need to be fetched from URL for image build customization, they can be added here. +# Downloaded resources can be found in /tmp/url_resources directory. +# Example:- +# fetch_from_url: +# - url: https://www.example.com/resource.tar.gz +# use_proxy: no +fetch_from_url: [] diff --git a/image-builder/assets/playbooks/roles/osconfig/tasks/fetch-from-url.yaml b/image-builder/assets/playbooks/roles/osconfig/tasks/fetch-from-url.yaml new file mode 100644 index 0000000..eafb5b2 --- /dev/null +++ b/image-builder/assets/playbooks/roles/osconfig/tasks/fetch-from-url.yaml @@ -0,0 +1,18 @@ +- name: "Append any user-defined custom urls" + set_fact: + fetch_from_url: "{{ fetch_from_url + fetch_from_url_append }}" + when: fetch_from_url_append is defined +- when: fetch_from_url is defined + block: + - name: "ensuring directory /tmp/url_resources exists" + file: + path: "/tmp/url_resources" + state: directory + mode: '0755' + - name: "Download from url {{ item.url }}" + get_url: + url: "{{ item.url }}" + dest: "/tmp/url_resources/{{ item.url | basename }}" + mode: '0755' + use_proxy: "{{ item.use_proxy }}" + loop: "{{ fetch_from_url }}" diff --git a/image-builder/assets/playbooks/roles/osconfig/tasks/main.yaml b/image-builder/assets/playbooks/roles/osconfig/tasks/main.yaml index ad0d7a7..22e683d 100644 --- a/image-builder/assets/playbooks/roles/osconfig/tasks/main.yaml +++ b/image-builder/assets/playbooks/roles/osconfig/tasks/main.yaml @@ -37,6 +37,9 @@ - name: "configure base systemd" include_tasks: systemd.yaml when: run_context == default_run_context +- name: "fetch url resource" + include_tasks: fetch-from-url.yaml + when: run_context == default_run_context # Context-dependent tasks - name: "write user-provided files" diff --git a/image-builder/assets/playbooks/roles/qcow/defaults/main.yaml b/image-builder/assets/playbooks/roles/qcow/defaults/main.yaml index f5b48c4..3078313 100644 --- a/image-builder/assets/playbooks/roles/qcow/defaults/main.yaml +++ b/image-builder/assets/playbooks/roles/qcow/defaults/main.yaml @@ -54,5 +54,3 @@ qcow_container_runtime_scripts: - file_content: | #!/bin/bash echo "custom qcow post-install script" -# This is only needed if you want DNS working when running qcow scripts above -qcow_container_runtime_scripts_dns: 8.8.8.8 diff --git a/image-builder/assets/playbooks/roles/qcow/tasks/chroot-cleanup.yaml b/image-builder/assets/playbooks/roles/qcow/tasks/chroot-cleanup.yaml index 44a1223..d1e5523 100644 --- a/image-builder/assets/playbooks/roles/qcow/tasks/chroot-cleanup.yaml +++ b/image-builder/assets/playbooks/roles/qcow/tasks/chroot-cleanup.yaml @@ -6,8 +6,6 @@ - name: "QCOW | unmount target" shell: | set -e - # restore resolv.conf - chroot "{{ dst }}" /bin/bash -c 'rm /etc/resolv.conf; cd /etc; ln -s ../run/systemd/resolve/stub-resolv.conf resolv.conf' cd "{{ dst }}" mountpoint dev/pts > /dev/null && umount dev/pts mountpoint dev > /dev/null && umount dev @@ -16,3 +14,6 @@ fi mountpoint sys > /dev/null && umount sys mountpoint proc > /dev/null && umount proc + if [ -d "/run/systemd/resolve" ]; then + mountpoint run/systemd/resolve > /dev/null && umount -l run/systemd/resolve + fi diff --git a/image-builder/assets/playbooks/roles/qcow/tasks/chroot-prep.yaml b/image-builder/assets/playbooks/roles/qcow/tasks/chroot-prep.yaml index 19362b5..2b3fe52 100644 --- a/image-builder/assets/playbooks/roles/qcow/tasks/chroot-prep.yaml +++ b/image-builder/assets/playbooks/roles/qcow/tasks/chroot-prep.yaml @@ -9,5 +9,6 @@ mountpoint proc > /dev/null || mount -t proc /proc proc mountpoint dev > /dev/null || mount -o bind /dev dev mountpoint dev/pts > /dev/null || mount -t devpts /dev/pts dev/pts - # temporarily override resolv.conf to working dns - chroot "{{ dst }}" /bin/bash -c 'rm /etc/resolv.conf; echo "nameserver {{ qcow_container_runtime_scripts_dns }}" > /etc/resolv.conf' + if [ -d "/run/systemd/resolve" ]; then + mountpoint run/systemd/resolve > /dev/null || mount -o bind /run/systemd/resolve run/systemd/resolve + fi diff --git a/image-builder/tools/cut_image.sh b/image-builder/tools/cut_image.sh index 705a360..adf3cc1 100755 --- a/image-builder/tools/cut_image.sh +++ b/image-builder/tools/cut_image.sh @@ -108,6 +108,7 @@ outputFileName: $img_name" > ${qcow_config} --volume /proc:/proc:rw \ --volume /sys:/sys:rw \ --volume /lib/modules:/lib/modules:rw \ + --volume /run/systemd/resolve:/run/systemd/resolve:rw \ --volume $workdir:/config \ ${uefi_mount} \ --env BUILDER_CONFIG=/config/${build_type}.yaml \ diff --git a/image-builder/tools/multistrap.sh b/image-builder/tools/multistrap.sh index 3bff65e..ad49ddf 100755 --- a/image-builder/tools/multistrap.sh +++ b/image-builder/tools/multistrap.sh @@ -34,11 +34,16 @@ setup_chroot(){ mountpoint $build_dir/dev/pts > /dev/null || sudo mount -t devpts /dev/pts $build_dir/dev/pts mkdir -p $osconfig_build_dir mountpoint $osconfig_build_dir > /dev/null || sudo mount -o bind $build_dir $osconfig_build_dir + if [ -d "/run/systemd/resolve" ]; then + mkdir -p $build_dir/run/systemd/resolve + mountpoint $build_dir/run/systemd/resolve > /dev/null || sudo mount -o bind /run/systemd/resolve $build_dir/run/systemd/resolve + fi } umount_helper(){ - if [[ -d "$1" ]] && mountpoint "$1" > /devnull; then - sudo umount "$1" + if [[ -d "$1" ]] && mountpoint "$1" > /dev/null; then + # if umount fails in first attempt, we try to use lazy umount + sudo umount "$1" || sudo umount -l "$1" fi } @@ -52,6 +57,9 @@ umount_chroot(){ umount_helper $build_dir/sys umount_helper $build_dir/proc umount_helper $osconfig_build_dir + if [ -d "/run/systemd/resolve" ]; then + umount_helper $build_dir/run/systemd/resolve + fi } # Install pre-requisites @@ -121,10 +129,8 @@ cp assets/playbooks/base-osconfig.yaml $build_dir/opt/assets/playbooks/base-osco cp -r assets/playbooks/roles/osconfig $build_dir/opt/assets/playbooks/roles if [ -d $build_dir/config ]; then sudo rm -r $build_dir/config - mkdir -p $build_dir/config -else - mkdir -p $build_dir/config fi +mkdir -p $build_dir/config cp -r $workdir/scripts $build_dir/config/ if [ -z "$SKIP_OSCONFIG_ROLE" ]; then sudo -E ansible-playbook -i assets/playbooks/inventory.yaml assets/playbooks/base-osconfig.yaml --extra-vars "run_context=common" -vv