[airflow] Fix upgrade_airflow_worker.sh

This PS applies a fix for airflow dags state
command output filter.

Also applied several zuul gates fixes and bumps up
kubectl to 1.29.5 due to CVE vulnerability

Change-Id: Ic623f27ef8c0f856f2a80ea73fe48c992cd8936b

.zuul.yaml

@@ -147,6 +147,7 @@
     required-projects:
       - name: openstack/openstack-helm
       - name: openstack/openstack-helm-infra
+      - name: openstack/openstack-helm-plugin
       - name: airship/treasuremap
         override-checkout: v1.9
     irrelevant-files: *irrelevant-files
@@ -171,6 +172,7 @@
       kubeadm:
         pod_network_cidr: "10.244.0.0/24"
         service_cidr: "10.96.0.0/16"
+      osh_plugin_repo: "{{ zuul.project.src_dir }}/../../openstack/openstack-helm-plugin"
       loopback_setup: true
       loopback_device: /dev/loop100
       loopback_image: "/opt/ext_vol/openstack-helm/ceph-loop.img"
@@ -178,10 +180,11 @@
       kube_version_repo: "v1.29"
       # the list of k8s package versions are available here
       # https://pkgs.k8s.io/core:/stable:/{{ kube_version_repo }}/deb/Packages
-      kube_version: "1.29.2-1.1"
+      kube_version: "1.29.5-1.1"
+      calico_setup: true
       calico_version: "v3.27.0"
-      coredns_version: "v1.11.1"
-      helm_version: "v3.13.2"
+      cilium_setup: false
+      helm_version: "v3.14.2"
       yq_version: "v4.6.0"
       crictl_version: "v1.26.1"
       zuul_osh_relative_path: ../../openstack/openstack-helm
@@ -198,10 +201,10 @@
     parent: shipyard-base
     vars:
       site: airskiff
-      HELM_ARTIFACT_URL: https://get.helm.sh/helm-v3.13.2-linux-amd64.tar.gz
-      HTK_COMMIT: cfff60ec10a6c386f38db79bb9f59a552c2b032f
-      OSH_INFRA_COMMIT: cfff60ec10a6c386f38db79bb9f59a552c2b032f
-      OSH_COMMIT: 2d9457e34ca4200ed631466bd87569b0214c92e7
+      HELM_ARTIFACT_URL: https://get.helm.sh/helm-v3.14.2-linux-amd64.tar.gz
+      HTK_COMMIT: 05f2f45971abcf483189358d663e2b46c3fc2fe8
+      OSH_INFRA_COMMIT: 05f2f45971abcf483189358d663e2b46c3fc2fe8
+      OSH_COMMIT: 049e679939fbd3b0c659dd0977911b8dc3b5a015
       CLONE_SHIPYARD: false
       DISTRO: ubuntu_focal
       DOCKER_REGISTRY: localhost:5000

images/airflow/Dockerfile.ubuntu_focal

@@ -49,7 +49,7 @@ ARG DEBIAN_FRONTEND=noninteractive
 ARG ctx_base=src/bin
 
 # Kubectl version
-ARG KUBECTL_VERSION=1.29.2
+ARG KUBECTL_VERSION=1.29.5
 
 # Needed from apache-airflow 1.10.2, since core.airflow_home config is deprecated
 ENV AIRFLOW_HOME=${AIRFLOW_HOME}

images/airflow/script/upgrade_airflow_worker.sh

@@ -69,7 +69,7 @@ do
 
     # We will need to extract the last word in the 'check_dag_state'
     # string variable as that will contain the status of the dag run
-    dag_state=$(echo ${check_dag_state} | awk '{print $NF}')
+    dag_state=$(echo ${check_dag_state} | awk -F ',' '{print $1}')
     echo -e ${dag_state} >> /usr/local/airflow/upgrade_airflow_worker.log
 
     if [[ $dag_state == "success" ]]; then

src/bin/shipyard_airflow/shipyard_airflow/policy.py

@@ -300,7 +300,7 @@ class ShipyardPolicy(object):
         policy.DocumentedRuleDefault(
             ACTION_TEST_SITE,
             RULE_ADMIN_REQUIRED,
-            'Create a workflow action to invoke Helm tests on all releases ' \
+            'Create a workflow action to invoke Helm tests on all releases '
             'or a targeted release',
             [{
                 'path': '/api/v1.0/actions',

tools/gate/playbooks/deploy-env.yaml

@@ -18,11 +18,11 @@
   roles:
     - ensure-python
     - ensure-pip
-    - ensure-tox
     - clear-firewall
     - deploy-apparmor
     - deploy-selenium
     - deploy-env
+    - ensure-tox
 
   tasks:
     - name: Install Packaging python module for tools/airship