From fb440ce3d0d2749d5377266e64b765e6da172443 Mon Sep 17 00:00:00 2001 From: Andrew Bonney Date: Wed, 6 Jan 2021 14:19:01 +0000 Subject: [PATCH] Add support for kata container runtime This adds support for kata containers by installing and configuring the relevant runtime. The default remains as 'runc' but can be adjusted using the variable added to the defaults. Change-Id: Iea07012d092333c656b397f97b541a2f0a5f0e44 --- defaults/main.yml | 4 ++ handlers/main.yml | 84 +++++++++++++++-------- tasks/zun_compute.yml | 37 ++++++++-- templates/config.toml.j2 | 5 ++ templates/systemd-docker-override.conf.j2 | 2 +- templates/zun.conf.j2 | 2 +- vars/debian.yml | 12 +++- vars/redhat.yml | 11 ++- 8 files changed, 118 insertions(+), 39 deletions(-) create mode 100644 templates/config.toml.j2 diff --git a/defaults/main.yml b/defaults/main.yml index 98c5aaf..3d3b682 100644 --- a/defaults/main.yml +++ b/defaults/main.yml @@ -89,6 +89,7 @@ zun_kuryr_lock_path: "/var/lock/kuryr" ## Docker setup information zun_docker_package_version: "{{ _zun_docker_package_version }}" zun_containerd_package_version: "{{ _zun_containerd_package_version }}" +zun_kata_package_version: "{{ _zun_kata_package_version }}" # Set a list of users that are permitted to execute the docker binary. zun_docker_users: @@ -207,6 +208,9 @@ zun_service_endpoint_type: internalURL zun_recreate_keys: False ## General Zun configuration +# Select between the 'runc' or 'kata' runtime +zun_container_runtime: runc + # If ``zun_osapi_compute_workers`` is unset the system will use half the number of available VCPUS to # compute the number of api workers to use. # zun_osapi_compute_workers: 16 diff --git a/handlers/main.yml b/handlers/main.yml index f49075c..4f2a1a8 100644 --- a/handlers/main.yml +++ b/handlers/main.yml @@ -13,6 +13,60 @@ # See the License for the specific language governing permissions and # limitations under the License. +- name: Stop containerd + systemd: + name: "containerd" + enabled: yes + state: "stopped" + daemon_reload: yes + register: _stop + until: _stop is success + retries: 5 + delay: 2 + listen: "Restart containerd" + +- name: Start containerd + systemd: + name: "containerd" + enabled: yes + state: "started" + daemon_reload: yes + register: _start + until: _start is success + retries: 5 + delay: 2 + listen: "Restart containerd" + +- name: Stop docker + systemd: + name: "{{ item }}" + enabled: yes + state: "stopped" + daemon_reload: yes + with_items: + - docker + - kuryr-libnetwork + register: _stop + until: _stop is success + retries: 5 + delay: 2 + listen: "Restart docker" + +- name: Start docker + systemd: + name: "{{ item }}" + enabled: yes + state: "started" + daemon_reload: yes + with_items: + - kuryr-libnetwork + - docker + register: _start + until: _start is success + retries: 5 + delay: 2 + listen: "Restart docker" + - name: Stop services systemd: name: "{{ item.service_name }}" @@ -43,36 +97,6 @@ - "Restart zun services" - "venv changed" -- name: Stop docker - systemd: - name: "{{ item }}" - enabled: yes - state: "stopped" - daemon_reload: yes - with_items: - - docker - - kuryr-libnetwork - register: _stop - until: _stop is success - retries: 5 - delay: 2 - listen: "Restart kuryr services" - -- name: Start docker - systemd: - name: "{{ item }}" - enabled: yes - state: "started" - daemon_reload: yes - with_items: - - docker - - kuryr-libnetwork - register: _start - until: _start is success - retries: 5 - delay: 2 - listen: "Restart kuryr services" - - meta: noop listen: Manage LB when: false diff --git a/tasks/zun_compute.yml b/tasks/zun_compute.yml index c93aa8f..302eee3 100644 --- a/tasks/zun_compute.yml +++ b/tasks/zun_compute.yml @@ -42,6 +42,15 @@ - package: "containerd.io" version: "{{ zun_containerd_package_version }}" priority: 1000 + - package: "kata-runtime" + version: "{{ zun_kata_package_version }}" + priority: 1000 + - package: "kata-proxy" + version: "{{ zun_kata_package_version }}" + priority: 1000 + - package: "kata-shim" + version: "{{ zun_kata_package_version }}" + priority: 1000 - name: Get apt gpg key get_url: @@ -58,7 +67,7 @@ - name: Add apt repository apt_repository: - repo: deb [arch=amd64] {{ item.uri }} {{ ansible_distribution_release | lower }} stable + repo: "{{ item.repo }}" state: present filename: "{{ item.name }}" update_cache: no @@ -81,7 +90,7 @@ yum_repository: name: "{{ item.name }}" description: "{{ item.description }}" - baseurl: "{{ item.uri }}" + baseurl: "{{ item.repo }}" gpgkey: "{{ item.gpgkey | default(omit) }}" gpgcheck: yes enabled: yes @@ -113,6 +122,24 @@ cache_valid_time: "{{ (ansible_pkg_mgr == 'apt') | ternary(cache_timeout, omit) }}" enablerepo: "{{ (ansible_pkg_mgr in ['yum', 'dnf']) | ternary('extras', omit) }}" +- name: Ensure the containerd config directory exists + file: + path: "/etc/containerd" + state: "directory" + owner: "root" + group: "root" + mode: "0755" + +- name: Configure containerd + template: + src: "config.toml.j2" + dest: "/etc/containerd/config.toml" + mode: "0644" + owner: "root" + group: "root" + notify: + - Restart containerd + - name: Ensure the Docker config directory exists file: path: "/etc/docker" @@ -135,7 +162,7 @@ config_overrides: "{{ zun_docker_config_overrides }}" config_type: "json" notify: - - Restart kuryr services + - Restart docker - name: Create the kuryr system group group: @@ -214,7 +241,7 @@ config_type: "ini" notify: - Manage LB - - Restart kuryr services + - Restart docker tags: - zun-config - zun-post-install @@ -233,7 +260,7 @@ config_overrides: "{{ zun_kuryr_config_overrides }}" config_type: "json" notify: - - Restart kuryr services + - Restart docker - name: Ensure docker users are added to the docker group user: diff --git a/templates/config.toml.j2 b/templates/config.toml.j2 new file mode 100644 index 0000000..ad9d20d --- /dev/null +++ b/templates/config.toml.j2 @@ -0,0 +1,5 @@ +[plugins] + [plugins.cri] + [plugins.cri.containerd] + [plugins.cri.containerd.runtimes.kata] + runtime_type = "io.containerd.kata.v2" diff --git a/templates/systemd-docker-override.conf.j2 b/templates/systemd-docker-override.conf.j2 index f7d61a3..950c038 100644 --- a/templates/systemd-docker-override.conf.j2 +++ b/templates/systemd-docker-override.conf.j2 @@ -1,3 +1,3 @@ [Service] ExecStart= -ExecStart=/usr/bin/dockerd --group {{ zun_system_group_name }} -H tcp://127.0.0.1:2375 -H unix:///var/run/docker.sock --cluster-store etcd://{% for item in groups['zun_api'] %}{{ hostvars[item]['ansible_host'] }}:2379{% if not loop.last %},{% endif %}{% endfor %} +ExecStart=/usr/bin/dockerd --group {{ zun_system_group_name }} -H tcp://127.0.0.1:2375 -H unix:///var/run/docker.sock --cluster-store etcd://{% for item in groups['zun_api'] %}{{ hostvars[item]['ansible_host'] }}:2379{% if not loop.last %},{% endif %}{% endfor %} --add-runtime kata=/usr/bin/kata-runtime diff --git a/templates/zun.conf.j2 b/templates/zun.conf.j2 index 5b865c0..d44ac44 100644 --- a/templates/zun.conf.j2 +++ b/templates/zun.conf.j2 @@ -7,8 +7,8 @@ transport_url = {{ zun_oslomsg_rpc_transport }}://{% for host in zun_oslomsg_rpc use_sandbox = true image_driver_list = glance,docker default_image_driver = docker - capsule_driver = docker +container_runtime = {{ zun_container_runtime }} [api] diff --git a/vars/debian.yml b/vars/debian.yml index 26365f9..3b28f4c 100644 --- a/vars/debian.yml +++ b/vars/debian.yml @@ -15,11 +15,18 @@ _zun_docker_package_version: "5:19.03.14~*" _zun_containerd_package_version: "1.4.3-1" +_zun_kata_package_version: "{{ (ansible_distribution_release | lower == 'focal') | ternary('1.12.0-2', '1.11.5-9') }}" + +zun_kata_repo_version: "{{ (ansible_distribution_release | lower == 'focal') | ternary('stable-1.12', 'stable-1.11') }}" +zun_kata_repo_distro: "{{ (ansible_distribution | lower == 'ubuntu') | ternary('x', '') }}{{ ansible_distribution | capitalize }}" zun_docker_repo: - name: "docker-ce" - uri: "https://download.docker.com/linux/{{ ansible_distribution | lower }}" + repo: "deb [arch=amd64] https://download.docker.com/linux/{{ ansible_distribution | lower }} {{ ansible_distribution_release | lower }} stable" gpg_uri: "https://download.docker.com/linux/{{ ansible_distribution | lower }}/gpg" + - name: "kata" + repo: "deb https://download.opensuse.org/repositories/home:/katacontainers:/releases:/{{ ansible_architecture }}:/{{ zun_kata_repo_version }}/{{ zun_kata_repo_distro }}_{{ ansible_distribution_version }}/ /" + gpg_uri: "https://download.opensuse.org/repositories/home:/katacontainers:/releases:/{{ ansible_architecture }}:/{{ zun_kata_repo_version }}/{{ zun_kata_repo_distro }}_{{ ansible_distribution_version }}/Release.key" # Common apt packages zun_distro_packages: @@ -36,6 +43,9 @@ zun_distro_compute_packages: - pciutils - numactl - multipath-tools + - kata-runtime + - kata-proxy + - kata-shim zun_docker_groupname: docker diff --git a/vars/redhat.yml b/vars/redhat.yml index 32bafe1..f393e52 100644 --- a/vars/redhat.yml +++ b/vars/redhat.yml @@ -15,12 +15,17 @@ _zun_docker_package_version: "19.03.14-3" _zun_containerd_package_version: "1.4.3-3.1" +_zun_kata_package_version: "1.11.3-1" zun_docker_repo: - name: "docker-ce" description: Docker CE Stable - uri: "https://download.docker.com/linux/centos/{{ ansible_distribution_major_version }}/$basearch/stable" + repo: "https://download.docker.com/linux/centos/{{ ansible_distribution_major_version }}/$basearch/stable" gpgkey: "https://download.docker.com/linux/centos/gpg" + - name: "kata" + description: Kata runtime + repo: "http://mirror.centos.org/centos/{{ ansible_distribution_major_version }}/virt/$basearch/kata-containers" + gpgkey: "http://mirror.centos.org/centos/RPM-GPG-KEY-CentOS-Official" # Common yum packages zun_distro_packages: @@ -34,5 +39,9 @@ zun_distro_compute_packages: - pciutils - numactl - device-mapper-multipath + - "kata-runtime-{{ zun_kata_package_version }}.el{{ ansible_distribution_major_version }}.x86_64" + - "kata-shim-{{ zun_kata_package_version }}.el{{ ansible_distribution_major_version }}.x86_64" +# NOTE: This package is unavailable from the centos mirrors +# - "kata-proxy-{{ zun_kata_package_version }}.el{{ ansible_distribution_major_version }}.x86_64" zun_docker_groupname: docker