Add conformance test

Also fixes the two conformance issues exposed by Sonobuoy! Change-Id: I0b6bba2a47c7474983414399406d0a68657abb8d
2017-10-25 12:07:09 -05:00 · 2017-10-25 12:07:09 -05:00 · 9cf8123474
commit 9cf8123474
parent e56ad622c3
13 changed files with 448 additions and 17 deletions
--- a/.gitignore
+++ b/.gitignore
@ -2,6 +2,7 @@ __pycache__
 /*.log
 /.python-version
 /build
+/conformance
 /promenade.egg-info
 /tmp
 .tox/
--- a/charts/coredns/templates/bin/_anchor.tpl
+++ b/charts/coredns/templates/bin/_anchor.tpl
@ -24,21 +24,22 @@ promenade {
    log stdout
 }

-{{ .Values.coredns.cluster_domain }} {
-    kubernetes {
+. {
+    kubernetes{{- range .Values.coredns.kubernetes_zones }} {{ . -}}{{- end }} {
        endpoint https://{{ .Values.network.kubernetes_netloc }}
        tls /etc/coredns/coredns.pem /etc/coredns/coredns-key.pem /etc/coredns/cluster-ca.pem
+
+        pods insecure
    }
+    {{- if .Values.coredns.upstream_nameservers }}
+    {{ range .Values.coredns.upstream_nameservers }}
+    proxy . {{ . }}
+    {{- end }}
+    {{- end }}
+
    loadbalance
    cache {{ .Values.coredns.cache.ttl }}
-    errors stdout
-    log stdout
-}

-. {
-    {{- if .Values.coredns.upstream_nameservers }}
-    proxy .  {{- range .Values.coredns.upstream_nameservers }} {{ . -}}{{- end }}
-    {{- end }}
    errors stdout
    log stdout
 }
--- a/charts/coredns/values.yaml
+++ b/charts/coredns/values.yaml
@ -10,7 +10,8 @@ tls:
  key: placeholder

 coredns:
-  cluster_domain: cluster.local
+  kubernetes_zones:
+    - cluster.local
  cache:
    ttl: 60
  host_etc_path: /etc/coredns
@ -28,7 +29,7 @@ coredns:

 images:
  anchor: gcr.io/google_containers/hyperkube-amd64:v1.8.0
-  coredns: coredns/coredns:011
+  coredns: coredns/coredns:0.9.9

 network:
  kubernetes_netloc: 10.96.0.1
--- a/examples/basic/KubernetesNetwork.yaml
+++ b/examples/basic/KubernetesNetwork.yaml
@ -12,6 +12,7 @@ data:
    service_ip: 10.96.0.10
    bootstrap_validation_checks:
      - calico-etcd.kube-system.svc.cluster.local
+      - google.com
      - kubernetes-etcd.kube-system.svc.cluster.local
      - kubernetes.default.svc.cluster.local
    upstream_servers:
--- a/examples/basic/armada-resources.yaml
+++ b/examples/basic/armada-resources.yaml
@ -471,7 +471,10 @@ data:
    no_hooks: true
  values:
    coredns:
-      cluster_domain: cluster.local
+      kubernetes_zones:
+        - cluster.local
+        - 10.96.0.0/16
+        - 10.97.0.0/16
      upstream_nameservers:
        - 8.8.8.8
        - 8.8.4.4
@ -493,7 +496,7 @@ data:
                namespace: kube-system
    images:
      anchor: gcr.io/google_containers/hyperkube-amd64:v1.8.0
-      coredns: coredns/coredns:011
+      coredns: coredns/coredns:0.9.9
    tls:
      ca: placeholder
      cert: placeholder
--- a/examples/complete/KubernetesNetwork.yaml
+++ b/examples/complete/KubernetesNetwork.yaml
@ -12,6 +12,7 @@ data:
    service_ip: 10.96.0.10
    bootstrap_validation_checks:
      - calico-etcd.kube-system.svc.cluster.local
+      - google.com
      - kubernetes-etcd.kube-system.svc.cluster.local
      - kubernetes.default.svc.cluster.local
    upstream_servers:
--- a/examples/complete/armada-resources.yaml
+++ b/examples/complete/armada-resources.yaml
@ -500,7 +500,10 @@ data:
    no_hooks: true
  values:
    coredns:
-      cluster_domain: cluster.local
+      kubernetes_zones:
+        - cluster.local
+        - 10.96.0.0/16
+        - 10.97.0.0/16
      upstream_nameservers:
        - 8.8.8.8
        - 8.8.4.4
@ -522,7 +525,7 @@ data:
                namespace: kube-system
    images:
      anchor: gcr.io/google_containers/hyperkube-amd64:v1.8.0
-      coredns: coredns/coredns:011
+      coredns: coredns/coredns:0.9.9
    tls:
      ca: placeholder
      cert: placeholder
--- a/tools/g2/lib/kube.sh
+++ b/tools/g2/lib/kube.sh
@ -1,3 +1,9 @@
+kubectl_apply() {
+    VIA=${1}
+    FILE=${2}
+    ssh_cmd ${VIA} "cat ${FILE} | kubectl apply -f -"
+}
+
 kubectl_cmd() {
    VIA=${1}

@ -5,3 +11,34 @@ kubectl_cmd() {

    ssh_cmd ${VIA} kubectl ${@}
 }
+
+kubectl_wait_for_pod() {
+    VIA=${1}
+    NAMESPACE=${2}
+    POD_NAME=${3}
+    SEC=${4:-600}
+    log Waiting ${SEC} seconds for termination of pod ${POD_NAME}
+
+    POD_PHASE_JSONPATH='{.status.phase}'
+
+    end=$(($(date +%s) + $SEC))
+    while true; do
+        POD_PHASE=$(kubectl_cmd ${VIA} --request-timeout 10s --namespace ${NAMESPACE} get -o jsonpath="${POD_PHASE_JSONPATH}" pod ${POD_NAME})
+        if [[ ${POD_PHASE} = "Succeeded" ]]; then
+            log Pod ${POD_NAME} succeeded.
+            break
+        elif [[ $POD_PHASE = "Failed" ]]; then
+            log Pod ${POD_NAME} failed.
+            kubectl_cmd ${VIA} --request-timeout 10s --namespace ${NAMESPACE} get -o yaml pod ${POD_NAME} 1>&2
+            fail
+        else
+            now=$(date +%s)
+            if [ $now -gt $end ]; then
+                log Pod did not terminate before timeout.
+                kubectl_cmd ${VIA} --request-timeout 10s --namespace ${NAMESPACE} get -o yaml pod ${POD_NAME} 1>&2
+                fail
+            fi
+            sleep 1
+        fi
+    done
+}
--- a/tools/g2/manifests/conformance.json
+++ b/tools/g2/manifests/conformance.json
@ -0,0 +1,61 @@
+{
+  "configuration": [
+    "examples/basic"
+  ],
+  "stages": [
+    {
+      "name": "Gate Setup",
+      "script": "gate-setup.sh"
+    },
+    {
+      "name": "Build Image",
+      "script": "build-image.sh"
+    },
+    {
+      "name": "Generate Certificates",
+      "script": "generate-certificates.sh"
+    },
+    {
+      "name": "Build Scripts",
+      "script": "build-scripts.sh"
+    },
+    {
+      "name": "Create VMs",
+      "script": "create-vms.sh"
+    },
+    {
+      "name": "Genesis",
+      "script": "genesis.sh"
+    },
+    {
+      "name": "Join Masters",
+      "script": "join-masters.sh",
+      "arguments": [
+        "n1",
+        "n2",
+        "n3"
+      ]
+    },
+    {
+      "name": "Reprovision Genesis",
+      "script": "reprovision-genesis.sh",
+      "arguments": [
+        "n1 n2 n3"
+      ]
+    },
+    {
+      "name": "Check Conformance",
+      "script": "conformance.sh"
+    }
+  ],
+  "vm": {
+    "memory": 2048,
+    "names": [
+      "n0",
+      "n1",
+      "n2",
+      "n3"
+    ],
+    "vcpus": 2
+  }
+}
--- a/tools/g2/sonobuoy.yaml
+++ b/tools/g2/sonobuoy.yaml
@ -0,0 +1,302 @@
+---
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: heptio-sonobuoy
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  labels:
+    component: sonobuoy
+  name: sonobuoy-serviceaccount
+  namespace: heptio-sonobuoy
+---
+apiVersion: rbac.authorization.k8s.io/v1beta1
+kind: ClusterRoleBinding
+metadata:
+  labels:
+    component: sonobuoy
+  name: sonobuoy-serviceaccount
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: sonobuoy-serviceaccount
+subjects:
+- kind: ServiceAccount
+  name: sonobuoy-serviceaccount
+  namespace: heptio-sonobuoy
+---
+apiVersion: rbac.authorization.k8s.io/v1beta1
+kind: ClusterRole
+metadata:
+  labels:
+    component: sonobuoy
+  name: sonobuoy-serviceaccount
+  namespace: heptio-sonobuoy
+rules:
+- apiGroups:
+  - '*'
+  resources:
+  - '*'
+  verbs:
+  - '*'
+---
+apiVersion: v1
+data:
+  config.json: |
+    {
+        "Description": "EXAMPLE",
+        "Filters": {
+            "LabelSelector": "",
+            "Namespaces": ".*"
+        },
+        "PluginNamespace": "heptio-sonobuoy",
+        "Plugins": [
+            {
+                "name": "e2e"
+            }
+        ],
+        "Resources": [
+            "CertificateSigningRequests",
+            "ClusterRoleBindings",
+            "ClusterRoles",
+            "ComponentStatuses",
+            "CustomResourceDefinitions",
+            "Nodes",
+            "PersistentVolumes",
+            "PodSecurityPolicies",
+            "ServerVersion",
+            "StorageClasses",
+            "ConfigMaps",
+            "DaemonSets",
+            "Deployments",
+            "Endpoints",
+            "Events",
+            "HorizontalPodAutoscalers",
+            "Ingresses",
+            "Jobs",
+            "LimitRanges",
+            "PersistentVolumeClaims",
+            "Pods",
+            "PodDisruptionBudgets",
+            "PodTemplates",
+            "ReplicaSets",
+            "ReplicationControllers",
+            "ResourceQuotas",
+            "RoleBindings",
+            "Roles",
+            "ServerGroups",
+            "ServiceAccounts",
+            "Services",
+            "StatefulSets"
+        ],
+        "ResultsDir": "/tmp/sonobuoy",
+        "Server": {
+            "advertiseaddress": "sonobuoy-master:8080",
+            "bindaddress": "0.0.0.0",
+            "bindport": 8080,
+            "timeoutseconds": 5400
+        },
+        "Version": "v0.9.0"
+    }
+kind: ConfigMap
+metadata:
+  labels:
+    component: sonobuoy
+  name: sonobuoy-config-cm
+  namespace: heptio-sonobuoy
+---
+apiVersion: v1
+data:
+  e2e.yaml: |
+    driver: Job
+    name: e2e
+    resultType: e2e
+    spec:
+      containers:
+      - env:
+        - name: E2E_FOCUS
+          value: Conformance
+        image: gcr.io/heptio-images/kube-conformance:v1.8
+        imagePullPolicy: Always
+        name: e2e
+        volumeMounts:
+        - mountPath: /tmp/results
+          name: results
+      - command:
+        - sh
+        - -c
+        - /sonobuoy worker global -v 5 --logtostderr
+        env:
+        - name: NODE_NAME
+          valueFrom:
+            fieldRef:
+              apiVersion: v1
+              fieldPath: spec.nodeName
+        - name: RESULTS_DIR
+          value: /tmp/results
+        image: gcr.io/heptio-images/sonobuoy:master
+        imagePullPolicy: Always
+        name: sonobuoy-worker
+        volumeMounts:
+        - mountPath: /etc/sonobuoy
+          name: config
+        - mountPath: /tmp/results
+          name: results
+      restartPolicy: Never
+      serviceAccountName: sonobuoy-serviceaccount
+      tolerations:
+      - effect: NoSchedule
+        key: node-role.kubernetes.io/master
+        operator: Exists
+      - key: CriticalAddonsOnly
+        operator: Exists
+      volumes:
+      - emptyDir: {}
+        name: results
+      - configMap:
+          name: __SONOBUOY_CONFIGMAP__
+        name: config
+  systemdlogs.yaml: |
+    driver: DaemonSet
+    name: systemd_logs
+    resultType: systemd_logs
+    spec:
+      containers:
+      - command:
+        - sh
+        - -c
+        - /get_systemd_logs.sh && sleep 3600
+        env:
+        - name: NODE_NAME
+          valueFrom:
+            fieldRef:
+              apiVersion: v1
+              fieldPath: spec.nodeName
+        - name: RESULTS_DIR
+          value: /tmp/results
+        - name: CHROOT_DIR
+          value: /node
+        image: gcr.io/heptio-images/sonobuoy-plugin-systemd-logs:latest
+        imagePullPolicy: Always
+        name: systemd-logs
+        securityContext:
+          privileged: true
+        volumeMounts:
+        - mountPath: /node
+          name: root
+        - mountPath: /tmp/results
+          name: results
+        - mountPath: /etc/sonobuoy
+          name: config
+      - command:
+        - sh
+        - -c
+        - /sonobuoy worker single-node -v 5 --logtostderr && sleep 3600
+        env:
+        - name: NODE_NAME
+          valueFrom:
+            fieldRef:
+              apiVersion: v1
+              fieldPath: spec.nodeName
+        - name: RESULTS_DIR
+          value: /tmp/results
+        image: gcr.io/heptio-images/sonobuoy:master
+        imagePullPolicy: Always
+        name: sonobuoy-worker
+        securityContext:
+          privileged: true
+        volumeMounts:
+        - mountPath: /tmp/results
+          name: results
+        - mountPath: /etc/sonobuoy
+          name: config
+      dnsPolicy: ClusterFirstWithHostNet
+      hostIPC: true
+      hostNetwork: true
+      hostPID: true
+      tolerations:
+      - effect: NoSchedule
+        key: node-role.kubernetes.io/master
+        operator: Exists
+      - key: CriticalAddonsOnly
+        operator: Exists
+      volumes:
+      - hostPath:
+          path: /
+        name: root
+      - emptyDir: {}
+        name: results
+      - configMap:
+          name: __SONOBUOY_CONFIGMAP__
+        name: config
+kind: ConfigMap
+metadata:
+  labels:
+    component: sonobuoy
+  name: sonobuoy-plugins-cm
+  namespace: heptio-sonobuoy
+---
+apiVersion: v1
+kind: Pod
+metadata:
+  labels:
+    component: sonobuoy
+    run: sonobuoy-master
+    tier: analysis
+  name: sonobuoy
+  namespace: heptio-sonobuoy
+spec:
+  nodeSelector:
+    kubernetes.io/hostname: n0
+  containers:
+  - command:
+    - /bin/bash
+    - -c
+    - /sonobuoy master -v 3 --logtostderr
+    env:
+    - name: SONOBUOY_ADVERTISE_IP
+      valueFrom:
+        fieldRef:
+          fieldPath: status.podIP
+    image: gcr.io/heptio-images/sonobuoy:master
+    imagePullPolicy: Always
+    name: kube-sonobuoy
+    volumeMounts:
+    - mountPath: /etc/sonobuoy
+      name: sonobuoy-config-volume
+    - mountPath: /plugins.d
+      name: sonobuoy-plugins-volume
+    - mountPath: /tmp/sonobuoy
+      name: output-volume
+  restartPolicy: Never
+  serviceAccountName: sonobuoy-serviceaccount
+  volumes:
+  - configMap:
+      name: sonobuoy-config-cm
+    name: sonobuoy-config-volume
+  - configMap:
+      name: sonobuoy-plugins-cm
+    name: sonobuoy-plugins-volume
+  - hostPath:
+      path: /mnt/sonobuoy
+      type: Directory
+    name: output-volume
+---
+apiVersion: v1
+kind: Service
+metadata:
+  labels:
+    component: sonobuoy
+    run: sonobuoy-master
+  name: sonobuoy-master
+  namespace: heptio-sonobuoy
+spec:
+  ports:
+  - port: 8080
+    protocol: TCP
+    targetPort: 8080
+  selector:
+    run: sonobuoy-master
+  type: ClusterIP
--- a/tools/g2/stages/conformance.sh
+++ b/tools/g2/stages/conformance.sh
@ -0,0 +1,20 @@
+#!/usr/bin/env bash
+
+set -e
+
+source ${GATE_UTILS}
+
+rm -rf ${WORKSPACE}/conformance
+mkdir -p ${WORKSPACE}/conformance
+
+rsync_cmd ${WORKSPACE}/tools/g2/sonobuoy.yaml ${GENESIS_NAME}:/root/
+ssh_cmd ${GENESIS_NAME} mkdir -p /mnt/sonobuoy
+kubectl_apply ${GENESIS_NAME} /root/sonobuoy.yaml
+
+kubectl_wait_for_pod ${GENESIS_NAME} heptio-sonobuoy sonobuoy 7200
+
+FILENAME=$(ssh_cmd ${GENESIS_NAME} ls /mnt/sonobuoy)
+rsync_cmd ${GENESIS_NAME}:/mnt/sonobuoy/${FILENAME} ${WORKSPACE}/conformance/sonobuoy.tgz
+tar xf ${WORKSPACE}/conformance/sonobuoy.tgz -C ${WORKSPACE}/conformance
+
+tail -n 1 conformance/plugins/e2e/results/e2e.log | grep '^SUCCESS!'
--- a/tools/gate/default-config-env
+++ b/tools/gate/default-config-env
@ -3,7 +3,7 @@ IMAGE_CALICO_CNI=quay.io/calico/cni:v1.11.0
 IMAGE_CALICO_CTL=quay.io/calico/ctl:v1.6.1
 IMAGE_CALICO_KUBE_CONTROLLERS=quay.io/calico/kube-controllers:v1.0.0
 IMAGE_CALICO_NODE=quay.io/calico/node:v2.6.1
-IMAGE_COREDNS=coredns/coredns:011
+IMAGE_COREDNS=coredns/coredns:0.9.9
 IMAGE_ETCD=quay.io/coreos/etcd:v3.0.17
 IMAGE_HELM=lachlanevenson/k8s-helm:v2.5.1
 IMAGE_HYPERKUBE=gcr.io/google_containers/hyperkube-amd64:v1.8.0
--- a/tools/registry/IMAGES
+++ b/tools/registry/IMAGES
@ -1,5 +1,5 @@
 # source_name, tag, cache_name
-coredns/coredns,011,coredns
+coredns/coredns,0.9.9,coredns
 gcr.io/google_containers/hyperkube-amd64,v1.8.0,hyperkube
 gcr.io/google_containers/k8s-dns-dnsmasq-nanny-amd64,1.14.4,k8s-dns-dnsmasq-nanny-amd64
 gcr.io/google_containers/k8s-dns-kube-dns-amd64,1.14.4,k8s-dns-kube-dns-amd64