Gate: Add publishing metadata for JUNit files

Includes:

* JSON-schema to validate gate manifests.
* `tox -e gate-lint` job to check gate manifests.
* More robust handling of conformance test results.
* Bufix in `tools/g2/lib/kube.sh` where a missing `fail` method was
  called.

Change-Id: I90740f659a691cb6ee92144205976ef97133f7bf

tools/g2/lib/kube.sh

@@ -30,13 +30,13 @@ kubectl_wait_for_pod() {
         elif [[ $POD_PHASE = "Failed" ]]; then
             log Pod ${POD_NAME} failed.
             kubectl_cmd ${VIA} --request-timeout 10s --namespace ${NAMESPACE} get -o yaml pod ${POD_NAME} 1>&2
-            fail
+            exit 1
         else
             now=$(date +%s)
             if [ $now -gt $end ]; then
                 log Pod did not terminate before timeout.
                 kubectl_cmd ${VIA} --request-timeout 10s --namespace ${NAMESPACE} get -o yaml pod ${POD_NAME} 1>&2
-                fail
+                exit 1
             fi
             sleep 1
         fi

tools/g2/manifest-schema.json

@@ -0,0 +1,101 @@
+{
+  "$schema": "http://json-schema.org/schema#",
+
+  "definitions": {
+    "publish": {
+      "type": "object",
+      "properties": {
+        "junit": {
+          "type": "array",
+          "items": {
+            "$ref": "#/definitions/relativePath"
+          }
+        }
+      },
+      "additionalProperties": false
+    },
+    "relativePath": {
+      "type": "string",
+      "pattern": "^[A-Za-z0-9][A-Za-z0-9_\\.-]*(/[A-Za-z0-9_\\.-]+)*[A-Za-z0-9_-]$"
+    }
+  },
+
+  "type": "object",
+  "properties": {
+    "configuration": {
+      "type": "array",
+      "items": {
+        "$ref": "#/definitions/relativePath"
+      }
+    },
+    "publish": {
+      "$ref": "#/definitions/publish"
+    },
+    "stages": {
+      "type": "array",
+      "items": {
+        "type": "object",
+        "properties": {
+          "arguments": {
+            "type": "array",
+            "items": {
+              "type": "string"
+            }
+          },
+          "name": {
+            "type": "string"
+          },
+          "publish": {
+            "$ref": "#/definitions/publish"
+          },
+          "script": {
+            "$ref": "#/definitions/relativePath"
+          }
+        },
+        "required": [
+          "name",
+          "script"
+        ],
+        "additionalProperties": false
+      },
+      "minItems": 1
+    },
+    "vm": {
+      "type": "object",
+      "properties": {
+        "memory": {
+          "type": "integer",
+          "minimum": 1024
+        },
+        "names": {
+          "type": "array",
+          "items": {
+            "type": "string",
+            "enum": [
+              "n0",
+              "n1",
+              "n2",
+              "n3"
+            ]
+          },
+          "uniqueItems": true
+        },
+        "vcpus": {
+          "type": "integer",
+          "minimum": 1,
+          "maximum": 8
+        }
+      },
+      "required": [
+        "memory",
+        "names",
+        "vcpus"
+      ],
+      "additionalProperties": false
+    }
+  },
+  "required": [
+    "stages"
+  ],
+  "additionalProperties": false
+}

tools/g2/manifests/conformance.json

@@ -45,7 +45,12 @@
     },
     {
       "name": "Check Conformance",
-      "script": "conformance.sh"
+      "script": "conformance.sh",
+      "publish": {
+        "junit": [
+          "conformance/plugins/e2e/results/junit_01.xml"
+        ]
+      }
     }
   ],
   "vm": {

tools/g2/stages/conformance.sh

@@ -11,10 +11,26 @@ rsync_cmd ${WORKSPACE}/tools/g2/sonobuoy.yaml ${GENESIS_NAME}:/root/
 ssh_cmd ${GENESIS_NAME} mkdir -p /mnt/sonobuoy
 kubectl_apply ${GENESIS_NAME} /root/sonobuoy.yaml
 
-kubectl_wait_for_pod ${GENESIS_NAME} heptio-sonobuoy sonobuoy 7200
+if kubectl_wait_for_pod ${GENESIS_NAME} heptio-sonobuoy sonobuoy 7200; then
+    log Pod succeeded
+    SUCCESS=1
+else
+    log Pod failed
+    SUCCESS=0
+fi
 
-FILENAME=$(ssh_cmd ${GENESIS_NAME} ls /mnt/sonobuoy)
-rsync_cmd ${GENESIS_NAME}:/mnt/sonobuoy/${FILENAME} ${WORKSPACE}/conformance/sonobuoy.tgz
-tar xf ${WORKSPACE}/conformance/sonobuoy.tgz -C ${WORKSPACE}/conformance
+FILENAME=$(ssh_cmd ${GENESIS_NAME} ls /mnt/sonobuoy || echo "")
+if [[ ! -z ${FILENAME} ]]; then
+    if rsync_cmd ${GENESIS_NAME}:/mnt/sonobuoy/${FILENAME} ${WORKSPACE}/conformance/sonobuoy.tgz; then
+        tar xf ${WORKSPACE}/conformance/sonobuoy.tgz -C ${WORKSPACE}/conformance
+    fi
+fi
 
-tail -n 1 conformance/plugins/e2e/results/e2e.log | grep '^SUCCESS!'
+if [[ ${SUCCESS} = "1" ]]; then
+    tail -n 1 conformance/plugins/e2e/results/e2e.log | grep '^SUCCESS!'
+else
+    if [[ -s conformance/plugins/e2e/results/e2e.log ]]; then
+        tail -n 50 conformance/plugins/e2e/results/e2e.log
+        exit 1
+    fi
+fi

tools/lint_gate.sh

@@ -0,0 +1,11 @@
+#!/usr/bin/env bash
+
+set -e
+
+SCRIPT_DIR=$(realpath "$(dirname "${0}")")
+WORKSPACE=$(realpath "${SCRIPT_DIR}/..")
+
+for manifest in $(find "${WORKSPACE}/tools/g2/manifests" -type f | sort); do
+    echo Checking "${manifest}"
+    python -m jsonschema "${WORKSPACE}/tools/g2/manifest-schema.json" -i "${manifest}"
+done

tox.ini

@@ -33,6 +33,13 @@ whitelist_externals=sh
 commands=
     sh -c "pip freeze | grep -v '^promenade' > {toxinidir}/requirements-frozen.txt"
 
+[testenv:gate-lint]
+deps =
+    jsonschema
+whitelist_externals = sh
+commands =
+    {toxinidir}/tools/lint_gate.sh
+
 [testenv:lint]
 deps =
     yapf