Reduce airship-porthole check jobs.

1) Zuul gates takes two hours to merge the change.
    This Reduces repeating of same Jobs.
 2) Consolidated deploy and apparmor scripts to Single Job.
 3) Consolidated unit and feature tests to Single Job.

Change-Id: I97eec8140a4e10fd1c70a0b732b3b3a28c45ab70
Signed-off-by: diwakar thyagaraj <diwakar.chitoor.thyagaraj@att.com>

tools/deployment/000-install-packages.sh

@@ -5,4 +5,4 @@ cd ${INSTALL_PATH}
 
 # Clone dependencies
 git clone https://opendev.org/openstack/openstack-helm-infra.git
-bash -c "./openstack-helm-infra/tools/deployment/common/000-install-packages.sh"
+bash -c "./openstack-helm-infra/tools/deployment/common/000-install-packages.sh"

tools/deployment/001-setup-apparmor-profiles.sh

@@ -0,0 +1,2 @@
+#!/bin/bash
+bash -c "../openstack-helm-infra/tools/deployment/common/001-setup-apparmor-profiles.sh"

tools/deployment/005-calicoctl-utility.sh

@@ -35,4 +35,4 @@ echo "Profile running: $profile"
       echo "$profile is the WRONG PROFILE!!"
       return 1
     fi
-  fi
+  fi

tools/deployment/utilities/000-install-packages.sh

@@ -1 +0,0 @@
-../apparmor/000-install-packages.sh

tools/deployment/utilities/001-setup-apparmor-profiles.sh

@@ -1 +0,0 @@
-../apparmor/001-setup-apparmor-profiles.sh

tools/deployment/utilities/002-deploy-k8s.sh

@@ -1 +0,0 @@
-../apparmor/002-deploy-k8s.sh

tools/deployment/utilities/005-calicoctl-utility.sh

@@ -1,16 +0,0 @@
-#!/bin/bash
-set -xe
-namespace=utility
-helm dependency update charts/calicoctl-utility
-helm  upgrade --install calicoctl-utility ./charts/calicoctl-utility --namespace=$namespace
-
-
-# Wait for Deployment
-: "${OSH_INFRA_PATH:="../../openstack-helm-infra"}"
-cd "${OSH_INFRA_PATH}"
-./tools/deployment/common/wait-for-pods.sh $namespace
-
-#NOTE: Validate Deployment info
-kubectl get -n $namespace secrets
-kubectl get -n $namespace configmaps
-kubectl get pods -n $namespace | grep calicoctl-utility

tools/deployment/utilities/010-ceph-utility.sh

@@ -1,243 +0,0 @@
-#!/bin/bash
-set -xe
-namespace="utility"
-CURRENT_DIR="$(pwd)"
-
-: ${OSH_INFRA_PATH:="../../openstack-helm-infra"}
-cd "${OSH_INFRA_PATH}"
-
-for CHART in ceph-mon ceph-client ceph-provisioners; do
-  make "${CHART}"
-done
-
-#NOTE: Deploy command
-: ${OSH_EXTRA_HELM_ARGS:=""}
-[ -s /tmp/ceph-fs-uuid.txt ] || uuidgen > /tmp/ceph-fs-uuid.txt
-CEPH_FS_ID="$(cat /tmp/ceph-fs-uuid.txt)"
-#NOTE(portdirect): to use RBD devices with Ubuntu kernels < 4.5 this
-# should be set to 'hammer'
-. /etc/os-release
-if [ "x${ID}" == "xubuntu" ] && \
-   [ "$(uname -r | awk -F "." '{ print $2 }')" -lt "5" ]; then
-  CRUSH_TUNABLES=hammer
-else
-  CRUSH_TUNABLES=null
-fi
-tee /tmp/ceph.yaml <<EOF
-endpoints:
-  ceph_mon:
-    namespace: ceph
-    port:
-      mon:
-        default: 6789
-  ceph_mgr:
-    namespace: ceph
-    port:
-      mgr:
-        default: 7000
-      metrics:
-        default: 9283
-network:
-  public: 172.17.0.1/16
-  cluster: 172.17.0.1/16
-  port:
-    mon: 6789
-    rgw: 8088
-    mgr: 7000
-deployment:
-  storage_secrets: true
-  ceph: true
-  rbd_provisioner: true
-  cephfs_provisioner: true
-  client_secrets: false
-  rgw_keystone_user_and_endpoints: false
-bootstrap:
-  enabled: true
-conf:
-  rgw_ks:
-    enabled: false
-  ceph:
-    global:
-      fsid: ${CEPH_FS_ID}
-      mon_addr: :6789
-      osd_pool_default_size: 1
-    osd:
-      osd_crush_chooseleaf_type: 0
-  pool:
-    crush:
-      tunables: ${CRUSH_TUNABLES}
-    target:
-      osd: 1
-      pg_per_osd: 100
-    default:
-      crush_rule: same_host
-    spec:
-      # RBD pool
-      - name: rbd
-        application: rbd
-        replication: 1
-        percent_total_data: 40
-      # CephFS pools
-      - name: cephfs_metadata
-        application: cephfs
-        replication: 1
-        percent_total_data: 5
-      - name: cephfs_data
-        application: cephfs
-        replication: 1
-        percent_total_data: 10
-      # RadosGW pools
-      - name: .rgw.root
-        application: rgw
-        replication: 1
-        percent_total_data: 0.1
-      - name: default.rgw.control
-        application: rgw
-        replication: 1
-        percent_total_data: 0.1
-      - name: default.rgw.data.root
-        application: rgw
-        replication: 1
-        percent_total_data: 0.1
-      - name: default.rgw.gc
-        application: rgw
-        replication: 1
-        percent_total_data: 0.1
-      - name: default.rgw.log
-        application: rgw
-        replication: 1
-        percent_total_data: 0.1
-      - name: default.rgw.intent-log
-        application: rgw
-        replication: 1
-        percent_total_data: 0.1
-      - name: default.rgw.meta
-        application: rgw
-        replication: 1
-        percent_total_data: 0.1
-      - name: default.rgw.usage
-        application: rgw
-        replication: 1
-        percent_total_data: 0.1
-      - name: default.rgw.users.keys
-        application: rgw
-        replication: 1
-        percent_total_data: 0.1
-      - name: default.rgw.users.email
-        application: rgw
-        replication: 1
-        percent_total_data: 0.1
-      - name: default.rgw.users.swift
-        application: rgw
-        replication: 1
-        percent_total_data: 0.1
-      - name: default.rgw.users.uid
-        application: rgw
-        replication: 1
-        percent_total_data: 0.1
-      - name: default.rgw.buckets.extra
-        application: rgw
-        replication: 1
-        percent_total_data: 0.1
-      - name: default.rgw.buckets.index
-        application: rgw
-        replication: 1
-        percent_total_data: 3
-      - name: default.rgw.buckets.data
-        application: rgw
-        replication: 1
-        percent_total_data: 34.8
-  storage:
-    osd:
-      - data:
-          type: directory
-          location: /var/lib/openstack-helm/ceph/osd/osd-one
-        journal:
-          type: directory
-          location: /var/lib/openstack-helm/ceph/osd/journal-one
-pod:
-  replicas:
-    mds: 1
-    mgr: 1
-    rgw: 1
-jobs:
-  ceph_defragosds:
-    # Execute every 15 minutes for gates
-    cron: "*/15 * * * *"
-    history:
-      # Number of successful job to keep
-      successJob: 1
-      # Number of failed job to keep
-      failJob: 1
-    concurrency:
-      # Skip new job if previous job still active
-      execPolicy: Forbid
-    startingDeadlineSecs: 60
-manifests:
-  cronjob_defragosds: true
-  job_bootstrap: false
-EOF
-
-for CHART in ceph-mon ceph-client ceph-provisioners; do
-  helm upgrade --install ${CHART} ./${CHART} \
-    --namespace=ceph \
-    --values=/tmp/ceph.yaml \
-    ${OSH_INFRA_EXTRA_HELM_ARGS} \
-    ${OSH_INFRA_EXTRA_HELM_ARGS_CEPH_DEPLOY}
-done
-  helm upgrade --install ceph-osd ./ceph-osd \
-    --namespace=ceph \
-    --values=/tmp/ceph.yaml
-
-  #NOTE: Wait for deploy
-  ./tools/deployment/common/wait-for-pods.sh ceph
-
-  #NOTE: Validate deploy
-  MON_POD=$(kubectl get pods \
-    --namespace=ceph \
-    --selector="application=ceph" \
-    --selector="component=mon" \
-    --no-headers | awk '{ print $1; exit }')
-  kubectl exec -n ceph ${MON_POD} -- ceph -s
-
-#NOTE: Deploy command
-: ${OSH_EXTRA_HELM_ARGS:=""}
-tee /tmp/ceph-utility-config.yaml <<EOF
-endpoints:
-  identity:
-    namespace: openstack
-  object_store:
-    namespace: ceph
-  ceph_mon:
-    namespace: ceph
-network:
-  public: 172.17.0.1/16
-  cluster: 172.17.0.1/16
-deployment:
-  storage_secrets: false
-  ceph: false
-  rbd_provisioner: false
-  cephfs_provisioner: false
-  client_secrets: true
-  rgw_keystone_user_and_endpoints: false
-bootstrap:
-  enabled: false
-conf:
-  rgw_ks:
-    enabled: true
-EOF
-
-helm upgrade --install ceph-utility-config ./ceph-provisioners \
-  --namespace=$namespace \
-  --values=/tmp/ceph-utility-config.yaml \
-  ${OSH_EXTRA_HELM_ARGS} \
-  ${OSH_EXTRA_HELM_ARGS_CEPH_NS_ACTIVATE}
-
-#Deploy Ceph-Utility
-cd ${CURRENT_DIR}
-helm dependency update charts/ceph-utility
-helm upgrade --install ceph-utility ./charts/ceph-utility --namespace=$namespace
-
-# Wait for Deployment
-cd "${OSH_INFRA_PATH}"
-./tools/deployment/common/wait-for-pods.sh $namespace

tools/deployment/utilities/020-compute-utility.sh

@@ -1,10 +0,0 @@
-#!/bin/bash
-set -xe
-namespace=utility
-helm dependency update charts/compute-utility
-helm  upgrade --install compute-utility ./charts/compute-utility --namespace=$namespace
-
-# Wait for Deployment
-: "${OSH_INFRA_PATH:="../../openstack-helm-infra"}"
-cd "${OSH_INFRA_PATH}"
-./tools/deployment/common/wait-for-pods.sh $namespace

tools/deployment/utilities/030-etcdctl-utility.sh

@@ -1,10 +0,0 @@
-#!/bin/bash
-set -xe
-namespace=utility
-helm dependency update charts/etcdctl-utility
-helm  upgrade --install etcdctl-utility ./charts/etcdctl-utility --namespace=$namespace
-
-# Wait for Deployment
-: "${OSH_INFRA_PATH:="../../openstack-helm-infra"}"
-cd "${OSH_INFRA_PATH}"
-./tools/deployment/common/wait-for-pods.sh $namespace

tools/deployment/utilities/040-mysqlclient-utility.sh

@@ -1,10 +0,0 @@
-#!/bin/bash
-set -xe
-namespace=utility
-helm dependency update charts/mysqlclient-utility
-helm  upgrade --install mysqlclient-utility ./charts/mysqlclient-utility  --namespace=$namespace
-
-# Wait for Deployment
-: "${OSH_INFRA_PATH:="../../openstack-helm-infra"}"
-cd "${OSH_INFRA_PATH}"
-./tools/deployment/common/wait-for-pods.sh $namespace

tools/deployment/utilities/050-openstack-utility.sh

@@ -1,13 +0,0 @@
-#!/bin/bash
-set -xe
-namespace=utility
-helm dependency update charts/openstack-utility
-helm  upgrade --install openstack-utility ./charts/openstack-utility --namespace=$namespace
-
-# Wait for Deployment
-: "${OSH_INFRA_PATH:="../../openstack-helm-infra"}"
-cd "${OSH_INFRA_PATH}"
-./tools/deployment/common/wait-for-pods.sh $namespace
-
-#NOTE: Validate Deployment info
-helm status openstack-utility

tools/deployment/utilities/060-postgresql-utility.sh

@@ -1,10 +0,0 @@
-#!/bin/bash
-set -xe
-namespace=utility
-helm dependency update charts/postgresql-utility
-helm upgrade --install postgresql-utility ./charts/postgresql-utility --namespace=$namespace
-
-# Wait for Deployment
-: "${OSH_INFRA_PATH:="../../openstack-helm-infra"}"
-cd "${OSH_INFRA_PATH}"
-./tools/deployment/common/wait-for-pods.sh $namespace

zuul.d/base.yaml

@@ -24,9 +24,7 @@
         - airship-porthole-images-build-gate-mysqlclient-utility
         - airship-porthole-images-build-gate-openstack-utility
         - airship-porthole-images-build-gate-postgresql-utility
-        - airship-porthole-deploy
+        - airship-porthole-deploy-feature-test
-        - airship-porthole-unit-tests
-        - airship-porthole-feature-tests
 
     gate:
       jobs:
@@ -38,16 +36,7 @@
         - airship-porthole-images-build-gate-mysqlclient-utility
         - airship-porthole-images-build-gate-openstack-utility
         - airship-porthole-images-build-gate-postgresql-utility
-        - airship-porthole-deploy
+        - airship-porthole-deploy-feature-test
-        - airship-porthole-apparmor:
-            voting: false
-        - airship-porthole-unit-tests
-        - airship-porthole-feature-tests
-
-    experimental:
-      jobs:
-        - airship-porthole-linter
-        - airship-porthole-apparmor
 
     post:
       jobs:
@@ -58,8 +47,8 @@
         - airship-porthole-images-publish-mysqlclient-utility
         - airship-porthole-images-publish-openstack-utility
         - airship-porthole-images-publish-postgresql-utility
-        - airship-porthole-apparmor:
+        - airship-porthole-deploy-feature-test
-            voting: false
+
 - nodeset:
     name: airship-porthole-single-node
     nodes:
@@ -90,97 +79,29 @@
     timeout: 300
     nodeset: airship-porthole-single-node
 
-- job:
-    name: airship-porthole-deploy
-    description: Deploys All Utility Containers
-    timeout: 7200
-    run: tools/gate/playbooks/airship-porthole-gate-runner.yaml
-    post-run: tools/gate/playbooks/airship-porthole-collect-logs.yaml
-    nodeset: airship-porthole-single-node
-    vars:
-      gate_scripts:
-         - ./tools/deployment/utilities/000-install-packages.sh
-         - ./tools/deployment/utilities/001-setup-apparmor-profiles.sh
-         - ./tools/deployment/utilities/002-deploy-k8s.sh
-         - ./tools/deployment/utilities/005-calicoctl-utility.sh
-         - ./tools/deployment/utilities/010-ceph-utility.sh
-         - ./tools/deployment/utilities/020-compute-utility.sh
-         - ./tools/deployment/utilities/030-etcdctl-utility.sh
-         - ./tools/deployment/utilities/040-mysqlclient-utility.sh
-         - ./tools/deployment/utilities/050-openstack-utility.sh
-         - ./tools/deployment/utilities/060-postgresql-utility.sh
-      args:
-        chdir: "{{ zuul.project.src_dir }}"
 
 - job:
-    name: airship-porthole-apparmor
+    name: airship-porthole-deploy-feature-test
-    description: Checks Pods for Apparmor profile
-    timeout: 7200
-    run: tools/gate/playbooks/airship-porthole-gate-runner.yaml
-    post-run: tools/gate/playbooks/airship-porthole-collect-logs.yaml
-    nodeset: airship-porthole-single-node
-    vars:
-      gate_scripts:
-         - ./tools/deployment/apparmor/000-install-packages.sh
-         - ./tools/deployment/apparmor/001-setup-apparmor-profiles.sh
-         - ./tools/deployment/apparmor/002-deploy-k8s.sh
-         - ./tools/deployment/apparmor/005-calicoctl-utility.sh
-         - ./tools/deployment/apparmor/010-ceph-utility.sh
-         - ./tools/deployment/apparmor/020-compute-utility.sh
-         - ./tools/deployment/apparmor/030-etcdctl-utility.sh
-         - ./tools/deployment/apparmor/040-mysqlclient-utility.sh
-         - ./tools/deployment/apparmor/050-openstack-utility.sh
-         - ./tools/deployment/apparmor/060-postgresql-utility.sh
-      args:
-        chdir: "{{ zuul.project.src_dir }}"
-
-- job:
-    name: airship-porthole-unit-tests
     description: |
-      Executes unit tests
+       Deploys all UC's and Executes unit and feature tests.
-    dependencies:
-      - airship-porthole-deploy
     run: tools/gate/playbooks/airship-porthole-gate-runner.yaml
     nodeset: airship-porthole-single-node
     timeout: 7200
-    post-run: tools/gate/playbooks/make-unit-tests.yaml
+    post-run:
+       - tools/gate/playbooks/make-unit-tests.yaml
+       - tools/gate/playbooks/make-feature-tests.yaml
     vars:
       gate_scripts:
-         - ./tools/deployment/utilities/000-install-packages.sh
+         - ./tools/deployment/000-install-packages.sh
-         - ./tools/deployment/utilities/001-setup-apparmor-profiles.sh
+         - ./tools/deployment/001-setup-apparmor-profiles.sh
-         - ./tools/deployment/utilities/002-deploy-k8s.sh
+         - ./tools/deployment/002-deploy-k8s.sh
-         - ./tools/deployment/utilities/005-calicoctl-utility.sh
+         - ./tools/deployment/005-calicoctl-utility.sh
-         - ./tools/deployment/utilities/010-ceph-utility.sh
+         - ./tools/deployment/010-ceph-utility.sh
-         - ./tools/deployment/utilities/020-compute-utility.sh
+         - ./tools/deployment/020-compute-utility.sh
-         - ./tools/deployment/utilities/030-etcdctl-utility.sh
+         - ./tools/deployment/030-etcdctl-utility.sh
-         - ./tools/deployment/utilities/040-mysqlclient-utility.sh
+         - ./tools/deployment/040-mysqlclient-utility.sh
-         - ./tools/deployment/utilities/050-openstack-utility.sh
+         - ./tools/deployment/050-openstack-utility.sh
-         - ./tools/deployment/utilities/060-postgresql-utility.sh
+         - ./tools/deployment/060-postgresql-utility.sh
-      args:
-        chdir: "{{ zuul.project.src_dir }}"
-
-- job:
-    name: airship-porthole-feature-tests
-    description: |
-      Executes feature tests
-    dependencies:
-      - airship-porthole-deploy
-    run: tools/gate/playbooks/airship-porthole-gate-runner.yaml
-    nodeset: airship-porthole-single-node
-    timeout: 7200
-    post-run: tools/gate/playbooks/make-feature-tests.yaml
-    vars:
-      gate_scripts:
-         - ./tools/deployment/utilities/000-install-packages.sh
-         - ./tools/deployment/utilities/001-setup-apparmor-profiles.sh
-         - ./tools/deployment/utilities/002-deploy-k8s.sh
-         - ./tools/deployment/utilities/005-calicoctl-utility.sh
-         - ./tools/deployment/utilities/010-ceph-utility.sh
-         - ./tools/deployment/utilities/020-compute-utility.sh
-         - ./tools/deployment/utilities/030-etcdctl-utility.sh
-         - ./tools/deployment/utilities/040-mysqlclient-utility.sh
-         - ./tools/deployment/utilities/050-openstack-utility.sh
-         - ./tools/deployment/utilities/060-postgresql-utility.sh
       args:
         chdir: "{{ zuul.project.src_dir }}"