From 1e133ba51d4640d238c0fe268f465d11249ff81a Mon Sep 17 00:00:00 2001 From: Ian Wienand Date: Mon, 7 Nov 2022 16:22:31 +1100 Subject: [PATCH] enable-kubernetes: Fix jammy install, improve pod test This updates the ensure-kubernetes testing to check the pod is actually running. This was hiding some issues on Jammy where the installation succeeded but the pod was not ready. The essence of the problem seems to be that the containernetworking-plugins tools are coming from upstream packages on Ubuntu Jammy. This native package places the networking tools in a different location to those from the Opensuse kubic repo. We need to update the cri-o path and the docker path for our jobs. For cri-o this is just an update to the config file, which is separated out into the crio-Ubuntu-22.04 include file. For docker things are bit harder, because you need the cri-docker shim now to use a docker runtime with kubernetes. Per the note inline, this shim has some hard-coded assumptions which mean we need to override the way it overrides (!). This works but does all feel a bit fragile; we should probably consider our overall support for the docker backend. With ensure-kubernetes working now, we can revert the non-voting jobs from the eariler change Id6ee7ed38fec254493a2abbfa076b9671c907c83. Change-Id: I5f02f4e056a0e731d74d00ebafa96390c06175cf --- .../tasks/crio-Ubuntu-22.04.yaml | 41 ++++++++++ roles/ensure-kubernetes/tasks/minikube.yaml | 28 +++++++ .../templates/11-cri-docker-override.conf.j2 | 3 + test-playbooks/ensure-kubernetes/post.yaml | 80 +++++++++++-------- zuul-tests.d/container-roles-jobs.yaml | 8 +- 5 files changed, 119 insertions(+), 41 deletions(-) create mode 100644 roles/ensure-kubernetes/tasks/crio-Ubuntu-22.04.yaml create mode 100644 roles/ensure-kubernetes/templates/11-cri-docker-override.conf.j2 diff --git a/roles/ensure-kubernetes/tasks/crio-Ubuntu-22.04.yaml b/roles/ensure-kubernetes/tasks/crio-Ubuntu-22.04.yaml new file mode 100644 index 000000000..0e18d75a8 --- /dev/null +++ b/roles/ensure-kubernetes/tasks/crio-Ubuntu-22.04.yaml @@ -0,0 +1,41 @@ +- name: Add all repositories + include_role: + name: ensure-package-repositories + vars: + repositories_keys: + - url: "https://download.opensuse.org/repositories/devel:/kubic:/libcontainers:/stable/xUbuntu_{{ ansible_distribution_version }}/Release.key" + - url: "https://download.opensuse.org/repositories/devel:/kubic:/libcontainers:/stable:/cri-o:/1.24/xUbuntu_{{ ansible_distribution_version }}/Release.key" + repositories_list: + - repo: "deb https://download.opensuse.org/repositories/devel:/kubic:/libcontainers:/stable/xUbuntu_{{ ansible_distribution_version }}/ /" + - repo: "deb https://download.opensuse.org/repositories/devel:/kubic:/libcontainers:/stable:/cri-o:/1.24/xUbuntu_{{ ansible_distribution_version }}/ /" +- name: Install packages + package: + name: + - cri-o + - cri-o-runc + - containernetworking-plugins + - podman + - cri-tools + state: present + become: true + +- name: Find networking plugins + ini_file: + path: /etc/crio/crio.conf + section: crio.network + option: plugin_dirs + value: + - '/opt/cni/bin/' + - '/usr/lib/cni' + mode: 0644 + become: true + register: _crio_conf_updated + +# NOTE: want to restart here rather than notify and do it later, so +# that we don't go on without the config correct. +- name: Restart crio to pickup changes # noqa no-handler + service: + name: crio + state: restarted + become: yes + when: _crio_conf_updated.changed diff --git a/roles/ensure-kubernetes/tasks/minikube.yaml b/roles/ensure-kubernetes/tasks/minikube.yaml index 0126fdbaf..6db62c925 100644 --- a/roles/ensure-kubernetes/tasks/minikube.yaml +++ b/roles/ensure-kubernetes/tasks/minikube.yaml @@ -114,6 +114,34 @@ args: executable: '/bin/bash' + # minikube has a hard-coded cri-docker setup step that writes out + # /etc/systemd/system/cri-docker.service.d/10-cni.conf + # which overrides the ExecStart with CNI arguments. This seems to + # be written to assume different packages than we have on Ubuntu + # Jammy -- containernetworking-plugins is a native package and is + # in /usr/lib, whereas the OpenSuse kubic versions are in /opt. + # We thus add an 11-* config to override the override with + # something that works ... see + # https://github.com/kubernetes/minikube/issues/15320 + - name: Correct override for native packages + when: ansible_distribution_release == 'jammy' + block: + - name: Make override dir + file: + state: directory + path: /etc/systemd/system/cri-docker.service.d + owner: root + group: root + mode: '0755' + + - name: Override cri-docker + template: + src: 11-cri-docker-override.conf.j2 + dest: /etc/systemd/system/cri-docker.service.d/11-cri-docker-override.conf + owner: root + group: root + mode: '0644' + - name: Ensure cri-dockerd running service: name: cri-docker diff --git a/roles/ensure-kubernetes/templates/11-cri-docker-override.conf.j2 b/roles/ensure-kubernetes/templates/11-cri-docker-override.conf.j2 new file mode 100644 index 000000000..d48ca1a9a --- /dev/null +++ b/roles/ensure-kubernetes/templates/11-cri-docker-override.conf.j2 @@ -0,0 +1,3 @@ +[Service] +ExecStart= +ExecStart=/usr/local/bin/cri-dockerd --container-runtime-endpoint fd:// --network-plugin=cni --cni-bin-dir=/usr/lib/cni --hairpin-mode=promiscuous-bridge diff --git a/test-playbooks/ensure-kubernetes/post.yaml b/test-playbooks/ensure-kubernetes/post.yaml index 6fae592a6..441f27bcf 100644 --- a/test-playbooks/ensure-kubernetes/post.yaml +++ b/test-playbooks/ensure-kubernetes/post.yaml @@ -1,42 +1,52 @@ - hosts: all name: Post testing tasks: - # The default account is known to take a while to appear; see - # https://github.com/kubernetes/kubernetes/issues/66689 - - name: Ensure default account created - command: kubectl -n default get serviceaccount default -o name - retries: 5 - delay: 5 - register: result - until: result.rc == 0 + - name: Run functionality tests + block: + # The default account is known to take a while to appear; see + # https://github.com/kubernetes/kubernetes/issues/66689 + - name: Ensure default account created + command: kubectl -n default get serviceaccount default -o name + retries: 5 + delay: 5 + register: result + until: result.rc == 0 - - name: Create a test pod definition - copy: - dest: test-pod.yaml - content: | - apiVersion: v1 - kind: Pod - metadata: - name: test - spec: - restartPolicy: Never - containers: - - name: test - image: k8s.gcr.io/pause:3.1 + - name: Create a test pod definition + copy: + dest: test-pod.yaml + content: | + apiVersion: v1 + kind: Pod + metadata: + name: test + spec: + restartPolicy: Never + containers: + - name: test + image: k8s.gcr.io/pause:3.1 - - name: Start pod - command: kubectl apply -f test-pod.yaml + - name: Start pod + command: kubectl apply -f test-pod.yaml - - name: Check status - shell: sleep 5; kubectl get pods + - name: Ensure pod is running + shell: sleep 5; kubectl get pods + register: _get_pods_output + until: "'Running' in _get_pods_output.stdout" + retries: 3 + delay: 5 -- hosts: all - roles: - - collect-container-logs - - collect-kubernetes-logs - tasks: - - name: Get minikube logs - become: true - shell: "/tmp/minikube logs > {{ ansible_user_dir }}/zuul-output/logs/minikube.txt" - environment: - MINIKUBE_HOME: "{{ ansible_user_dir }}" + always: + - name: Collect container logs + import_role: + name: collect-container-logs + + - name: Collect kubernetes logs + import_role: + name: collect-kubernetes-logs + + - name: Get minikube logs + become: true + shell: "/tmp/minikube logs > {{ ansible_user_dir }}/zuul-output/logs/minikube.txt" + environment: + MINIKUBE_HOME: "{{ ansible_user_dir }}" diff --git a/zuul-tests.d/container-roles-jobs.yaml b/zuul-tests.d/container-roles-jobs.yaml index 23ad7a713..095bc2c7e 100644 --- a/zuul-tests.d/container-roles-jobs.yaml +++ b/zuul-tests.d/container-roles-jobs.yaml @@ -294,9 +294,6 @@ - job: name: zuul-jobs-test-registry-buildset-registry-k8s-docker - # NOTE(ianw) 2022-11-04 : This job is currently unhappy on Ubuntu - # Jammy, and needs full investigation. - voting: false dependencies: zuul-jobs-test-registry-buildset-registry description: | Test a buildset registry with kubernetes and docker @@ -322,9 +319,6 @@ - job: name: zuul-jobs-test-registry-buildset-registry-k8s-crio - # NOTE(ianw) 2022-11-04 : This job is currently unhappy on Ubuntu - # Jammy, and needs full investigation. - voting: false dependencies: zuul-jobs-test-registry-buildset-registry description: | Test a buildset registry with kubernetes and CRIO @@ -640,6 +634,8 @@ - zuul-jobs-test-registry-docker-multiarch - zuul-jobs-test-registry-podman - zuul-jobs-test-registry-buildset-registry + - zuul-jobs-test-registry-buildset-registry-k8s-docker + - zuul-jobs-test-registry-buildset-registry-k8s-crio - zuul-jobs-test-registry-buildset-registry-openshift-docker - zuul-jobs-test-ensure-kubernetes-docker-ubuntu-bionic - zuul-jobs-test-ensure-kubernetes-docker-ubuntu-focal