diff --git a/roles/ensure-kubernetes/tasks/crio-Ubuntu-22.04.yaml b/roles/ensure-kubernetes/tasks/crio-Ubuntu-22.04.yaml new file mode 100644 index 000000000..0e18d75a8 --- /dev/null +++ b/roles/ensure-kubernetes/tasks/crio-Ubuntu-22.04.yaml @@ -0,0 +1,41 @@ +- name: Add all repositories + include_role: + name: ensure-package-repositories + vars: + repositories_keys: + - url: "https://download.opensuse.org/repositories/devel:/kubic:/libcontainers:/stable/xUbuntu_{{ ansible_distribution_version }}/Release.key" + - url: "https://download.opensuse.org/repositories/devel:/kubic:/libcontainers:/stable:/cri-o:/1.24/xUbuntu_{{ ansible_distribution_version }}/Release.key" + repositories_list: + - repo: "deb https://download.opensuse.org/repositories/devel:/kubic:/libcontainers:/stable/xUbuntu_{{ ansible_distribution_version }}/ /" + - repo: "deb https://download.opensuse.org/repositories/devel:/kubic:/libcontainers:/stable:/cri-o:/1.24/xUbuntu_{{ ansible_distribution_version }}/ /" +- name: Install packages + package: + name: + - cri-o + - cri-o-runc + - containernetworking-plugins + - podman + - cri-tools + state: present + become: true + +- name: Find networking plugins + ini_file: + path: /etc/crio/crio.conf + section: crio.network + option: plugin_dirs + value: + - '/opt/cni/bin/' + - '/usr/lib/cni' + mode: 0644 + become: true + register: _crio_conf_updated + +# NOTE: want to restart here rather than notify and do it later, so +# that we don't go on without the config correct. +- name: Restart crio to pickup changes # noqa no-handler + service: + name: crio + state: restarted + become: yes + when: _crio_conf_updated.changed diff --git a/roles/ensure-kubernetes/tasks/minikube.yaml b/roles/ensure-kubernetes/tasks/minikube.yaml index 0126fdbaf..6db62c925 100644 --- a/roles/ensure-kubernetes/tasks/minikube.yaml +++ b/roles/ensure-kubernetes/tasks/minikube.yaml @@ -114,6 +114,34 @@ args: executable: '/bin/bash' + # minikube has a hard-coded cri-docker setup step that writes out + # /etc/systemd/system/cri-docker.service.d/10-cni.conf + # which overrides the ExecStart with CNI arguments. This seems to + # be written to assume different packages than we have on Ubuntu + # Jammy -- containernetworking-plugins is a native package and is + # in /usr/lib, whereas the OpenSuse kubic versions are in /opt. + # We thus add an 11-* config to override the override with + # something that works ... see + # https://github.com/kubernetes/minikube/issues/15320 + - name: Correct override for native packages + when: ansible_distribution_release == 'jammy' + block: + - name: Make override dir + file: + state: directory + path: /etc/systemd/system/cri-docker.service.d + owner: root + group: root + mode: '0755' + + - name: Override cri-docker + template: + src: 11-cri-docker-override.conf.j2 + dest: /etc/systemd/system/cri-docker.service.d/11-cri-docker-override.conf + owner: root + group: root + mode: '0644' + - name: Ensure cri-dockerd running service: name: cri-docker diff --git a/roles/ensure-kubernetes/templates/11-cri-docker-override.conf.j2 b/roles/ensure-kubernetes/templates/11-cri-docker-override.conf.j2 new file mode 100644 index 000000000..d48ca1a9a --- /dev/null +++ b/roles/ensure-kubernetes/templates/11-cri-docker-override.conf.j2 @@ -0,0 +1,3 @@ +[Service] +ExecStart= +ExecStart=/usr/local/bin/cri-dockerd --container-runtime-endpoint fd:// --network-plugin=cni --cni-bin-dir=/usr/lib/cni --hairpin-mode=promiscuous-bridge diff --git a/test-playbooks/ensure-kubernetes/post.yaml b/test-playbooks/ensure-kubernetes/post.yaml index 6fae592a6..441f27bcf 100644 --- a/test-playbooks/ensure-kubernetes/post.yaml +++ b/test-playbooks/ensure-kubernetes/post.yaml @@ -1,42 +1,52 @@ - hosts: all name: Post testing tasks: - # The default account is known to take a while to appear; see - # https://github.com/kubernetes/kubernetes/issues/66689 - - name: Ensure default account created - command: kubectl -n default get serviceaccount default -o name - retries: 5 - delay: 5 - register: result - until: result.rc == 0 + - name: Run functionality tests + block: + # The default account is known to take a while to appear; see + # https://github.com/kubernetes/kubernetes/issues/66689 + - name: Ensure default account created + command: kubectl -n default get serviceaccount default -o name + retries: 5 + delay: 5 + register: result + until: result.rc == 0 - - name: Create a test pod definition - copy: - dest: test-pod.yaml - content: | - apiVersion: v1 - kind: Pod - metadata: - name: test - spec: - restartPolicy: Never - containers: - - name: test - image: k8s.gcr.io/pause:3.1 + - name: Create a test pod definition + copy: + dest: test-pod.yaml + content: | + apiVersion: v1 + kind: Pod + metadata: + name: test + spec: + restartPolicy: Never + containers: + - name: test + image: k8s.gcr.io/pause:3.1 - - name: Start pod - command: kubectl apply -f test-pod.yaml + - name: Start pod + command: kubectl apply -f test-pod.yaml - - name: Check status - shell: sleep 5; kubectl get pods + - name: Ensure pod is running + shell: sleep 5; kubectl get pods + register: _get_pods_output + until: "'Running' in _get_pods_output.stdout" + retries: 3 + delay: 5 -- hosts: all - roles: - - collect-container-logs - - collect-kubernetes-logs - tasks: - - name: Get minikube logs - become: true - shell: "/tmp/minikube logs > {{ ansible_user_dir }}/zuul-output/logs/minikube.txt" - environment: - MINIKUBE_HOME: "{{ ansible_user_dir }}" + always: + - name: Collect container logs + import_role: + name: collect-container-logs + + - name: Collect kubernetes logs + import_role: + name: collect-kubernetes-logs + + - name: Get minikube logs + become: true + shell: "/tmp/minikube logs > {{ ansible_user_dir }}/zuul-output/logs/minikube.txt" + environment: + MINIKUBE_HOME: "{{ ansible_user_dir }}" diff --git a/zuul-tests.d/container-roles-jobs.yaml b/zuul-tests.d/container-roles-jobs.yaml index 23ad7a713..095bc2c7e 100644 --- a/zuul-tests.d/container-roles-jobs.yaml +++ b/zuul-tests.d/container-roles-jobs.yaml @@ -294,9 +294,6 @@ - job: name: zuul-jobs-test-registry-buildset-registry-k8s-docker - # NOTE(ianw) 2022-11-04 : This job is currently unhappy on Ubuntu - # Jammy, and needs full investigation. - voting: false dependencies: zuul-jobs-test-registry-buildset-registry description: | Test a buildset registry with kubernetes and docker @@ -322,9 +319,6 @@ - job: name: zuul-jobs-test-registry-buildset-registry-k8s-crio - # NOTE(ianw) 2022-11-04 : This job is currently unhappy on Ubuntu - # Jammy, and needs full investigation. - voting: false dependencies: zuul-jobs-test-registry-buildset-registry description: | Test a buildset registry with kubernetes and CRIO @@ -640,6 +634,8 @@ - zuul-jobs-test-registry-docker-multiarch - zuul-jobs-test-registry-podman - zuul-jobs-test-registry-buildset-registry + - zuul-jobs-test-registry-buildset-registry-k8s-docker + - zuul-jobs-test-registry-buildset-registry-k8s-crio - zuul-jobs-test-registry-buildset-registry-openshift-docker - zuul-jobs-test-ensure-kubernetes-docker-ubuntu-bionic - zuul-jobs-test-ensure-kubernetes-docker-ubuntu-focal