From c27aee882920f7eef9cb0314c1afe0d03b930871 Mon Sep 17 00:00:00 2001 From: Jonathan Rosser Date: Mon, 1 Jul 2024 17:28:22 +0100 Subject: [PATCH] Split large k8s_install playbook into more specific smaller playbooks Change-Id: I0569a323f12848cd100abf6e50ee7afdf08b37b3 --- .../openstack_deploy/user_variables_hooks.yml | 12 +- mcapi_vexxhost/playbooks/k8s_install.yml | 200 +----------------- .../playbooks/mcapi_control_plane_k8s.yml | 130 ++++++++++++ .../playbooks/mcapi_magnum_integration.yml | 57 +++++ mcapi_vexxhost/playbooks/mcapi_proxy.yml | 57 +++++ 5 files changed, 261 insertions(+), 195 deletions(-) create mode 100644 mcapi_vexxhost/playbooks/mcapi_control_plane_k8s.yml create mode 100644 mcapi_vexxhost/playbooks/mcapi_magnum_integration.yml create mode 100644 mcapi_vexxhost/playbooks/mcapi_proxy.yml diff --git a/mcapi_vexxhost/playbooks/files/openstack_deploy/user_variables_hooks.yml b/mcapi_vexxhost/playbooks/files/openstack_deploy/user_variables_hooks.yml index d207d131..304c8c6a 100644 --- a/mcapi_vexxhost/playbooks/files/openstack_deploy/user_variables_hooks.yml +++ b/mcapi_vexxhost/playbooks/files/openstack_deploy/user_variables_hooks.yml @@ -1 +1,11 @@ -post_setup_openstack_hook: osa_ops.mcapi_vexxhost.install_and_test +# This is used for a full AIO scenario + +# deploy the control plane k8s cluster as the last step in the "setup-ho +post_setup_infrastructure_hook: osa_ops.mcapi_vexxhost.mcapi_control_plane_k8s + +# drop the control plane k8s credentials and do helm setup in the magnum +# container before the openstack deployment +pre_setup_openstack_hook: osa_ops.mcapi_vexxhost.mcapi_magnum_integration + +# run the functional test after the openstack deployment +post_setup_openstack_hook: osa_ops.mcapi_vexxhost.functional_test diff --git a/mcapi_vexxhost/playbooks/k8s_install.yml b/mcapi_vexxhost/playbooks/k8s_install.yml index ff789d8e..bb25c35b 100644 --- a/mcapi_vexxhost/playbooks/k8s_install.yml +++ b/mcapi_vexxhost/playbooks/k8s_install.yml @@ -13,199 +13,11 @@ # See the License for the specific language governing permissions and # limitations under the License. -- name: Gather k8s facts - hosts: k8s_all - gather_facts: false - tags: - - always - tasks: - - name: Gather minimal facts for k8s - setup: - gather_subset: - - "!all" - - min - when: osa_gather_facts | default(True) +- name: Import mcapi control plane setup playbook + import_playbook: mcapi_control_plane_k8s.yml -- name: Create and configure k8s container - hosts: k8s_all - serial: "{{ k8s_serial | default('20%') }}" - gather_facts: true - user: root - pre_tasks: - - import_role: - name: openstack.osa.lxc_container_setup - vars: - list_of_bind_mounts: - - bind_dir_path: '/usr/lib/modules' - mount_path: '/usr/lib/modules' - - bind_dir_path: '/usr/src' - mount_path: '/usr/src' - - bind_dir_path: '/dev/kmsg' - mount_path: '/dev/kmsg' - create: file - extra_container_config: - - 'security.privileged=true' - - 'security.nested=true' - - 'raw.lxc="lxc.apparmor.profile=unconfined"' - - 'lxc.cap.drop=' - - 'lxc.cgroup.devices.allow=a' - - 'lxc.cgroup2.devices.allow=a' +- name: Import mcapi magnum integration playbook + import_playbook: mcapi_magnum_integration.yml - when: not is_metal - - - include_tasks: common-tasks/unbound-clients.yml - when: - - hostvars['localhost']['resolvconf_enabled'] | bool - - - name: ensure kernel headers are installed on host - package: - name: "linux-headers-{{ ansible_facts['kernel'] }}" - state: present - delegate_to: "{{ physical_host }}" - when: not is_metal - - roles: - - role: "openstack.osa.system_crontab_coordination" - - role: "systemd_service" - systemd_services: - - service_name: bpf-mount - execstarts: /usr/bin/bash -c '/usr/bin/mount bpffs -t bpf /sys/fs/bpf && /usr/bin/mount --make-shared /sys/fs/bpf' - - service_name: cilium-cgroup2-mount - execstarts: /usr/bin/bash -c 'mkdir -p /run/cilium/cgroupv2 && /usr/bin/mount -t cgroup2 none /run/cilium/cgroupv2 && /usr/bin/mount --make-shared /run/cilium/cgroupv2' - - # environment: "{{ deployment_environment_variables | default({}) }}" - - tasks: - #NOTE (jrosser) this task can be removed when https://github.com/vexxhost/ansible-collection-kubernetes/pull/118/ is merged and released - - ansible.builtin.package: - name: python3-cryptography - tags: - - k8s-container - - k8s - -- name: Configure haproxy services - import_playbook: openstack.osa.haproxy_service_config.yml - vars: - service_group: k8s_all - service_variable: "k8s_haproxy_services" - when: groups[service_group] | length > 0 - tags: - - haproxy-service-config - -- name: Install kubernetes - hosts: k8s_all - gather_facts: true - serial: "{{ k8s_serial | default('20%') }}" - user: root - vars: - k8s_node_labels: - openstack-control-plane: enabled - roles: - - role: "vexxhost.containers.containerd" - - role: "vexxhost.kubernetes.kubernetes" - - role: "vexxhost.kubernetes.helm" - - role: "vexxhost.kubernetes.cilium" - environment: "{{ deployment_environment_variables | default({}) }}" - tasks: - - kubernetes.core.k8s: - definition: - apiVersion: v1 - kind: Node - metadata: - name: "{{ inventory_hostname }}" - labels: "{{ k8s_node_labels }}" - tags: - - k8s - - k8s-install - -- name: Install cluster_api - hosts: k8s_all - gather_facts: true - user: root - roles: - - role: "vexxhost.kubernetes.cert_manager" - - role: "vexxhost.kubernetes.cluster_api" - # environment: "{{ deployment_environment_variables | default({}) }}" - tags: - - cluster-api - -- name: Set up helm and k8s credentials in magnum hosts - hosts: magnum_all - gather_facts: true - user: root - vars: - k8s_admin_conf_src: "/etc/kubernetes/admin.conf" - k8s_admin_conf_dest: "/var/lib/magnum/.kube/config" - tasks: - - name: Collect admin config from k8s cluster - slurp: - src: "{{ k8s_admin_conf_src }}" - register: k8s_admin_conf_slurp - delegate_to: "{{ groups['k8s_all'][0] }}" - run_once: true - - - name: Ensure target directory exists - file: - state: directory - path: "{{ k8s_admin_conf_dest | dirname }}" - owner: magnum - group: magnum - - - name: Write k8s admin config to magnum home dir - copy: - content: "{{ k8s_admin_conf_slurp.content | b64decode }}" - dest: "{{ k8s_admin_conf_dest }}" - owner: magnum - group: magnum - mode: '0600' - - - name: Install helm - include_role: - name: "vexxhost.kubernetes.helm" - - # environment: "{{ deployment_environment_variables | default({}) }}" - tags: - - magnum_k8s_conf - -# deploy the proxy service to communicate directly between magnum coe -# clusters and the capi control plane without going via a public floating -# IP -# For OVS/Linuxbridge this can be installed to compute nodes, but for -# OVN it must be installed to all computes. Override the target hosts -# for this play as required -- name: Install magnum-cluster-api-proxy - hosts: "{{ magnum_cluster_api_proxy_hosts | default('network_hosts') }}" - vars: - _venv_tag: "{{ venv_tag | default('untagged') }}" - _bin: "/openstack/venvs/magnum-cluster-api-proxy-{{ _venv_tag }}/bin" - magnum_cluster_api_proxy_system_group_name: 'capi_proxy' - magnum_cluster_api_proxy_system_user_name: 'capi_proxy' - magnum_cluster_api_proxy_system_user_comment: 'Magnum Cluster API Proxy System User' - magnum_cluster_api_proxy_system_user_home: '/var/lib/{{ magnum_cluster_api_proxy_system_user_name }}' - magnum_cluster_api_proxy_system_user_shell: '/bin/false' - magnum_cluster_api_proxy_etc_directory: '/etc/capi_proxy' - - k8s_admin_conf_src: "/etc/kubernetes/admin.conf" - k8s_admin_conf_dest: "{{ magnum_cluster_api_proxy_system_user_home }}/.kube/config" - environment: "{{ deployment_environment_variables | default({}) }}" - tasks: - - - name: Gather facts - setup: - gather_subset: - - "!all" - - min - when: osa_gather_facts | default(True) - tags: - - always - - - name: Setup installation variables - import_role: - name: openstack.osa.install_defaults - - - name: Install proxy service - include_role: - name: osa_ops.mcapi_vexxhost.proxy - - tags: - - magnum-cluster-api-proxy +- name: Import mcapi proxy installation playbook + import_playbook: mcapi_proxy.yml diff --git a/mcapi_vexxhost/playbooks/mcapi_control_plane_k8s.yml b/mcapi_vexxhost/playbooks/mcapi_control_plane_k8s.yml new file mode 100644 index 00000000..c007491d --- /dev/null +++ b/mcapi_vexxhost/playbooks/mcapi_control_plane_k8s.yml @@ -0,0 +1,130 @@ +--- +# Copyright 2023, BBC R&D +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Gather k8s facts + hosts: k8s_all + gather_facts: false + tags: + - always + tasks: + - name: Gather minimal facts for k8s + setup: + gather_subset: + - "!all" + - min + when: osa_gather_facts | default(True) + +- name: Create and configure k8s container + hosts: k8s_all + serial: "{{ k8s_serial | default('20%') }}" + gather_facts: true + user: root + pre_tasks: + - import_role: + name: openstack.osa.lxc_container_setup + vars: + list_of_bind_mounts: + - bind_dir_path: '/usr/lib/modules' + mount_path: '/usr/lib/modules' + - bind_dir_path: '/usr/src' + mount_path: '/usr/src' + - bind_dir_path: '/dev/kmsg' + mount_path: '/dev/kmsg' + create: file + extra_container_config: + - 'security.privileged=true' + - 'security.nested=true' + - 'raw.lxc="lxc.apparmor.profile=unconfined"' + - 'lxc.cap.drop=' + - 'lxc.cgroup.devices.allow=a' + - 'lxc.cgroup2.devices.allow=a' + + when: not is_metal + + - include_tasks: common-tasks/unbound-clients.yml + when: + - hostvars['localhost']['resolvconf_enabled'] | bool + + - name: ensure kernel headers are installed on host + package: + name: "linux-headers-{{ ansible_facts['kernel'] }}" + state: present + delegate_to: "{{ physical_host }}" + when: not is_metal + + roles: + - role: "openstack.osa.system_crontab_coordination" + - role: "systemd_service" + systemd_services: + - service_name: bpf-mount + execstarts: /usr/bin/bash -c '/usr/bin/mount bpffs -t bpf /sys/fs/bpf && /usr/bin/mount --make-shared /sys/fs/bpf' + - service_name: cilium-cgroup2-mount + execstarts: /usr/bin/bash -c 'mkdir -p /run/cilium/cgroupv2 && /usr/bin/mount -t cgroup2 none /run/cilium/cgroupv2 && /usr/bin/mount --make-shared /run/cilium/cgroupv2' + + # environment: "{{ deployment_environment_variables | default({}) }}" + + tasks: + #NOTE (jrosser) this task can be removed when https://github.com/vexxhost/ansible-collection-kubernetes/pull/118/ is merged and released + - ansible.builtin.package: + name: python3-cryptography + tags: + - k8s-container + - k8s + +- name: Configure haproxy services + import_playbook: openstack.osa.haproxy_service_config.yml + vars: + service_group: k8s_all + service_variable: "k8s_haproxy_services" + when: groups[service_group] | length > 0 + tags: + - haproxy-service-config + +- name: Install kubernetes + hosts: k8s_all + gather_facts: true + serial: "{{ k8s_serial | default('20%') }}" + user: root + vars: + k8s_node_labels: + openstack-control-plane: enabled + roles: + - role: "vexxhost.containers.containerd" + - role: "vexxhost.kubernetes.kubernetes" + - role: "vexxhost.kubernetes.helm" + - role: "vexxhost.kubernetes.cilium" + environment: "{{ deployment_environment_variables | default({}) }}" + tasks: + - kubernetes.core.k8s: + definition: + apiVersion: v1 + kind: Node + metadata: + name: "{{ inventory_hostname }}" + labels: "{{ k8s_node_labels }}" + tags: + - k8s + - k8s-install + +- name: Install cluster_api + hosts: k8s_all + gather_facts: true + user: root + roles: + - role: "vexxhost.kubernetes.cert_manager" + - role: "vexxhost.kubernetes.cluster_api" + # environment: "{{ deployment_environment_variables | default({}) }}" + tags: + - cluster-api diff --git a/mcapi_vexxhost/playbooks/mcapi_magnum_integration.yml b/mcapi_vexxhost/playbooks/mcapi_magnum_integration.yml new file mode 100644 index 00000000..2919d635 --- /dev/null +++ b/mcapi_vexxhost/playbooks/mcapi_magnum_integration.yml @@ -0,0 +1,57 @@ +--- +# Copyright 2023, BBC R&D +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Set up helm and k8s credentials in magnum hosts + hosts: magnum_all + gather_facts: true + user: root + vars: + k8s_admin_conf_src: "/etc/kubernetes/admin.conf" + k8s_admin_conf_dest: "/var/lib/magnum/.kube/config" + tasks: + - name: Run os_magnum role pre-tasks to create user/group/dirs + include_role: + name: os_magnum + tasks_from: magnum_pre_install.yml + + - name: Collect admin config from k8s cluster + slurp: + src: "{{ k8s_admin_conf_src }}" + register: k8s_admin_conf_slurp + delegate_to: "{{ groups['k8s_all'][0] }}" + run_once: true + + - name: Ensure target directory exists + file: + state: directory + path: "{{ k8s_admin_conf_dest | dirname }}" + owner: magnum + group: magnum + + - name: Write k8s admin config to magnum home dir + copy: + content: "{{ k8s_admin_conf_slurp.content | b64decode }}" + dest: "{{ k8s_admin_conf_dest }}" + owner: magnum + group: magnum + mode: '0600' + + - name: Install helm + include_role: + name: "vexxhost.kubernetes.helm" + + # environment: "{{ deployment_environment_variables | default({}) }}" + tags: + - magnum_k8s_conf diff --git a/mcapi_vexxhost/playbooks/mcapi_proxy.yml b/mcapi_vexxhost/playbooks/mcapi_proxy.yml new file mode 100644 index 00000000..5ff941f0 --- /dev/null +++ b/mcapi_vexxhost/playbooks/mcapi_proxy.yml @@ -0,0 +1,57 @@ +--- +# Copyright 2023, BBC R&D +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# deploy the proxy service to communicate directly between magnum coe +# clusters and the capi control plane without going via a public floating +# IP +# For OVS/Linuxbridge this can be installed to compute nodes, but for +# OVN it must be installed to all computes. Override the target hosts +# for this play as required +- name: Install magnum-cluster-api-proxy + hosts: "{{ magnum_cluster_api_proxy_hosts | default('network_hosts') }}" + vars: + _venv_tag: "{{ venv_tag | default('untagged') }}" + _bin: "/openstack/venvs/magnum-cluster-api-proxy-{{ _venv_tag }}/bin" + magnum_cluster_api_proxy_system_group_name: 'capi_proxy' + magnum_cluster_api_proxy_system_user_name: 'capi_proxy' + magnum_cluster_api_proxy_system_user_comment: 'Magnum Cluster API Proxy System User' + magnum_cluster_api_proxy_system_user_home: '/var/lib/{{ magnum_cluster_api_proxy_system_user_name }}' + magnum_cluster_api_proxy_system_user_shell: '/bin/false' + magnum_cluster_api_proxy_etc_directory: '/etc/capi_proxy' + + k8s_admin_conf_src: "/etc/kubernetes/admin.conf" + k8s_admin_conf_dest: "{{ magnum_cluster_api_proxy_system_user_home }}/.kube/config" + environment: "{{ deployment_environment_variables | default({}) }}" + tasks: + + - name: Gather facts + setup: + gather_subset: + - "!all" + - min + when: osa_gather_facts | default(True) + tags: + - always + + - name: Setup installation variables + import_role: + name: openstack.osa.install_defaults + + - name: Install proxy service + include_role: + name: osa_ops.mcapi_vexxhost.proxy + + tags: + - magnum-cluster-api-proxy