Shipyard updates

This PS makes the following changes: - makes sure the airskiff-deploy playbook is using 80Gb partition if available - adds available security updates to docker images Change-Id: Ibcb8fcddb319c3032483fdb73b55b725ea7737d4
2023-08-24 14:47:04 +00:00 · 2023-08-24 14:47:04 +00:00 · f0a24b11a0
commit f0a24b11a0
parent 7ea83c25ef
7 changed files with 82 additions and 11 deletions
--- a/.gitignore
+++ b/.gitignore
@ -118,3 +118,12 @@ AUTHORS

 # vscode
 .vscode/
+
+# tests
+airship-ucp-shipyard.values.yaml
+airflow-webserver.pid
+airflow.cfg
+airflow.db
+latest
+src/bin/shipyard_airflow/shipyard_airflow/config
+src/bin/shipyard_airflow/shipyard_airflow/webserver_config.py
--- a/images/airflow/Dockerfile.ubuntu_bionic
+++ b/images/airflow/Dockerfile.ubuntu_bionic
@ -54,7 +54,7 @@ ARG KUBECTL_VERSION=1.24.6
 ENV AIRFLOW_HOME=${AIRFLOW_HOME}

 RUN set -ex && \
-    apt-get -qq update && \
+    apt-get update && apt-get upgrade -y && \
    apt-get -y install \
    automake \
    ca-certificates \
--- a/images/airflow/Dockerfile.ubuntu_focal
+++ b/images/airflow/Dockerfile.ubuntu_focal
@ -54,7 +54,7 @@ ARG KUBECTL_VERSION=1.26.3
 ENV AIRFLOW_HOME=${AIRFLOW_HOME}

 RUN set -ex \
-    && apt-get update \
+    && apt-get update && apt-get upgrade -y \
    && apt-get install -y wget curl \
        apt-transport-https ca-certificates gnupg \
    && echo "deb http://apt.postgresql.org/pub/repos/apt focal-pgdg main" > /etc/apt/sources.list.d/pgdg.list \
--- a/images/shipyard/Dockerfile.ubuntu_bionic
+++ b/images/shipyard/Dockerfile.ubuntu_bionic
@ -36,7 +36,7 @@ ARG ctx_base=src/bin
 EXPOSE $PORT

 RUN set -ex && \
-    apt-get -qq update && \
+    apt-get update && apt-get upgrade -y && \
    apt-get -y install \
    automake \
    ca-certificates \
--- a/images/shipyard/Dockerfile.ubuntu_focal
+++ b/images/shipyard/Dockerfile.ubuntu_focal
@ -36,7 +36,7 @@ ARG ctx_base=src/bin
 EXPOSE $PORT

 RUN set -ex && \
-    apt-get -qq update && \
+    apt-get update && apt-get upgrade -y && \
    apt-get -y install \
    automake \
    ca-certificates \
--- a/tools/gate/playbooks/airskiff-deploy.yaml
+++ b/tools/gate/playbooks/airskiff-deploy.yaml
@ -23,6 +23,13 @@

  tasks:

+    - name: Install Packaging python module for airship
+      block:
+        - pip:
+            name: packaging
+            executable: pip3
+      become: True
+
    - name: Clone Required Repositories
      shell: |
        export CLONE_SHIPYARD={{ CLONE_SHIPYARD }}
@ -39,25 +46,61 @@

    - name: Deploy Kubernetes with Minikube
      shell: |
+        set -ex
+        sudo fdisk --list
+        df -h
+        sudo mkdir -p /opt/ext_vol
+        BIG_VOLUME=$(sudo fdisk -l 2>&1 | grep  -E 80G | grep Linux | awk '{print $1}')
+        if ! mount | grep "${BIG_VOLUME}"
+        then
+          sudo mkfs.ext4 "${BIG_VOLUME}"
+          sudo mount "${BIG_VOLUME}" /opt/ext_vol
+          df -h
+        fi
        ./tools/deployment/airskiff/developer/010-deploy-k8s.sh
      args:
        chdir: "{{ zuul.projects['opendev.org/airship/treasuremap'].src_dir }}"

+    - name: Setup openstack client
+      shell: |
+        set -ex
+        ./tools/deployment/airskiff/developer/020-setup-client.sh
+      args:
+        chdir: "{{ zuul.projects['opendev.org/airship/treasuremap'].src_dir }}"
+      become: yes
+
    - name: Build Shipyard and Airflow with submitted changes
      shell: |
        set -ex
        export DISTRO={{ SHIPYARD_IMAGE_DISTRO }}
        make images
-        if test "${DISTRO}" = 'ubuntu_bionic'
-        then
-          # this trick is needed to use bionic image instead of focal in airskiff deployment test
-          docker tag quay.io/airshipit/airflow:latest-ubuntu_bionic quay.io/airshipit/airflow:latest-ubuntu_focal
-          docker tag quay.io/airshipit/shipyard:latest-ubuntu_bionic quay.io/airshipit/shipyard:latest-ubuntu_focal
-        fi
+        docker system prune --force
      args:
        chdir: "{{ zuul.project.src_dir }}"
      become: yes

+    - name: Use locally built images in manifests
+      shell: |
+        set -ex
+        export DISTRO={{ SHIPYARD_IMAGE_DISTRO }}
+        docker rm registry --force || true
+        docker run -d -p 5000:5000 --restart=always --name registry registry:2
+        if test "${DISTRO}" = 'ubuntu_bionic'
+        then
+          docker tag quay.io/airshipit/shipyard:latest-ubuntu_bionic localhost:5000/shipyard:latest-ubuntu_bionic
+          docker push localhost:5000/shipyard:latest-ubuntu_bionic
+          sed -i "s#quay.io/airshipit/shipyard:latest-ubuntu_focal#localhost:5000/shipyard:latest-ubuntu_bionic#g" ./site/airskiff/software/config/versions.yaml
+          sed -i "s#quay.io/airshipit/shipyard:latest-ubuntu_focal#localhost:5000/shipyard:latest-ubuntu_bionic#g" ./global/software/config/versions.yaml
+        else
+          docker tag quay.io/airshipit/shipyard:latest-ubuntu_focal localhost:5000/shipyard:latest-ubuntu_focal
+          docker push localhost:5000/shipyard:latest-ubuntu_focal
+          sed -i "s#quay.io/airshipit/shipyard:latest-ubuntu_focal#localhost:5000/shipyard:latest-ubuntu_focal#g" ./site/airskiff/software/config/versions.yaml
+          sed -i "s#quay.io/airshipit/shipyard:latest-ubuntu_focal#localhost:5000/shipyard:latest-ubuntu_focal#g" ./global/software/config/versions.yaml
+        fi
+      args:
+        chdir: "{{ zuul.projects['opendev.org/airship/treasuremap'].src_dir }}"
+      become: yes
+
    - name: Build all charts locally
      shell: |
        set -ex
@ -66,6 +109,18 @@
        chdir: "{{ zuul.projects['opendev.org/airship/treasuremap'].src_dir }}"
      become: yes

+    - name: Start artifactory
+      shell: |
+        set -ex
+        # start http server with artifacts
+        docker rm artifacts --force || true
+        docker run --name artifacts -p 8282:80 -v $(pwd)/../artifacts:/usr/share/nginx/html -d nginx
+        sleep 10
+        curl --verbose -I http://control-plane.minikube.internal:8282/memcached.tgz
+      args:
+        chdir: "{{ zuul.projects['opendev.org/airship/treasuremap'].src_dir }}"
+      become: yes
+
    - name: Deploy Airship components using Armada
      shell: |
        mkdir -p ~/.kube
--- a/tools/gate/playbooks/docker-image-build.yaml
+++ b/tools/gate/playbooks/docker-image-build.yaml
@ -4,7 +4,6 @@
    - ensure-docker
    - ensure-python
    - ensure-pip
-    - ensure-tox

  tasks:
    - include_vars: vars.yaml
@ -43,6 +42,14 @@
            executable: pip3
      become: True

+    - name: Install tox python module for ansible docker login
+      block:
+        - pip:
+            name: tox
+            version: 3.28.0
+            executable: pip3
+      become: True
+
    - name: Make images - verbosive
      when: not publish
      shell: |