From d5a9f025b8d98fe14d3736c290c43cbdbf0646d7 Mon Sep 17 00:00:00 2001 From: Jesse Pretorius Date: Tue, 20 Mar 2018 12:12:46 +0000 Subject: [PATCH] Separate build and install stage In order to ensure that the build tasks are entirely skipped when a package venv is re-used, the build and install stages are split. The ability to re-use venvs is also now able to be toggled. Disabling this feature would set the build to always happen, catering to environments where a service venv is always deployed to the same folder (eg: stateless hypervisors with squashfs partitions). The ability to set constraints, etc is changed to a generalised set of arguments that can be passed to the pip install task. --- defaults/main.yml | 63 +++++++--- handlers/main.yml | 6 +- tasks/main.yml | 199 ++------------------------------ tasks/python_venv_build.yml | 111 ++++++++++++++++++ tasks/python_venv_install.yml | 53 +++++++++ tasks/python_venv_preflight.yml | 43 +++++++ tasks/python_venv_set_facts.yml | 34 ++++++ 7 files changed, 296 insertions(+), 213 deletions(-) create mode 100644 tasks/python_venv_build.yml create mode 100644 tasks/python_venv_install.yml create mode 100644 tasks/python_venv_preflight.yml create mode 100644 tasks/python_venv_set_facts.yml diff --git a/defaults/main.yml b/defaults/main.yml index 57ba1f1..442850d 100644 --- a/defaults/main.yml +++ b/defaults/main.yml @@ -13,7 +13,20 @@ # See the License for the specific language governing permissions and # limitations under the License. -# The list of distribution packages to install +# +# Required variables +# + +# The path where venvs are extracted to +# on the target host during an install, for example: +# venv_destination_path: "/openstack/venvs/myvenv" + +# +# Optional variables +# + +# Distribution packages which must be installed +# on the host for the purpose of building the venv. distro_package_list: [] # Set the package install state for packages @@ -22,34 +35,46 @@ distro_package_state: "latest" # The time in seconds that the distribution package # cache is valid for. This is only used by the apt -# package manager +# and zypper package managers. distro_cache_valid_time: 600 # Python packages which must be installed -# on to the host +# on to the host for the purpose of building +# the venv. host_pip_packages: [] +# Arguments to pass to the installation +# of pip packages on the host. +host_pip_install_args: "" + # Python packages which must be installed -# into the venv +# into the venv. venv_pip_packages: [] -# General pip install constraints -pip_install_constraints: "" +# Arguments to pass to the venv build +venv_pip_install_args: "" -# Specific constraints for the venv -pip_install_venv_constraints: "" +# Enable the reuse of venvs across multiple hosts. +# This sets the build process to copy the venv to +# the deployment host once it's built, then to +# re-use the venv in subsequent deployments. +venv_reuse_enable: yes -# General pip install extra options -# This is especially useful for proxy options -pip_install_options: "" +# The path where a built venv should be stored on the +# deployment host. +venv_reuse_download_path: "{{ lookup('env', 'HOME') | default('/opt', true) }}/cache" -# The path where venvs are stored on the -# deployment host -venv_download_path: "{{ lookup('env', 'HOME') | default('/opt', true) }}/cache/files" +# The owner of the venv_reuse_download_path +venv_reuse_download_path_owner: "{{ lookup('env', 'USER') | default('root', true) }}" -# The owner of the venv_download_path -venv_download_path_owner: "{{ lookup('env', 'USER') | default('root', true) }}" +# The facts to set when the venv changes during a +# build, or the installation of a venv. +# Eg: +# set_facts_when_changed: +# - section: glance +# option: venv_tag +# value: "{{ glance_venv_tag }}" +venv_facts_when_changed: [] -# The path where venvs are extracted to -# on the target host, for example: -# venv_destination_path: "/openstack/venvs/myvenv" +# The INI file name to use for the fact setting. +venv_facts_dest: "openstack_ansible" diff --git a/handlers/main.yml b/handlers/main.yml index c64ed45..eec181a 100644 --- a/handlers/main.yml +++ b/handlers/main.yml @@ -14,9 +14,5 @@ # limitations under the License. - meta: noop - listen: Manage LB - when: false - -- meta: noop - listen: Restart services + listen: venv changed when: false diff --git a/tasks/main.yml b/tasks/main.yml index 6c7841e..8868993 100644 --- a/tasks/main.yml +++ b/tasks/main.yml @@ -13,199 +13,20 @@ # See the License for the specific language governing permissions and # limitations under the License. -# TODO(odyssey4me): -# 1. Cater for Logan's model where we must just extract -# over the existing folder without wiping it out first. -# 2. Also, wiping the directory out from under it probably -# wrecks the service while it's running. We should figure -# out a better way of atomically replacing the venv without -# ripping the folder out from under it. - -- name: Install distro packages - package: - name: "{{ distro_package_list }}" - state: "{{ distro_package_state }}" - update_cache: "{{ (ansible_pkg_mgr in ['apt', 'zypper']) | ternary('yes', omit) }}" - cache_valid_time: "{{ (ansible_pkg_mgr == 'apt') | ternary(distro_cache_valid_time, omit) }}" - register: _install_distro_packages - until: _install_distro_packages | success - retries: 5 - delay: 2 - -- name: Install required pip packages on the host - pip: - name: "{{ host_pip_packages }}" - state: latest - extra_args: >- - {{ pip_install_constraints }} - {{ pip_install_options }} - register: _install_host_pip_packages - until: _install_host_pip_packages | success - retries: 5 - delay: 2 - -- name: Ensure that venv_download_path exists on the deployment host - file: - path: "{{ venv_download_path }}/{{ venv_destination_path | dirname }}" - state: directory - owner: "{{ venv_download_path_owner }}" - delegate_to: localhost - run_once: yes - -- name: Check if venv is present on the deployment host - stat: - path: "{{ venv_download_path }}/{{ venv_destination_path }}.tgz" - get_attributes: no - get_checksum: no - get_md5: no - get_mime: no - register: _venv_tgz - delegate_to: localhost - run_once: yes - -- name: Copy the venv checksum file to the target host - copy: - src: "{{ venv_download_path }}/{{ venv_destination_path }}.checksum" - dest: "{{ venv_destination_path | dirname }}" - register: _venv_checksum_copy - when: - - _venv_tgz.stat.exists | bool - -# TODO(odyssey4me): -# 1. Cater for Logan's model where we must just extract -# over the existing folder without wiping it out first. -# 2. Also, removing it like this probably wrecks the service -# while it's running. We should figure out a better way -# of atomically replacing the venv without ripping the -# folder out from under it. - -# Due to our Ansible strategy, a skipped task does not have -# a dictionary result. As such we validate that the register -# is a mapping (dict). -- name: Remove existing venv on target host if it is changing - file: - path: "{{ venv_destination_path }}" - state: absent - when: - - _venv_checksum_copy is mapping - - _venv_checksum_copy | changed - -- name: Create venv directory on the target host - file: - path: "{{ venv_destination_path }}" - state: directory - register: _create_venv_dir - -# Due to our Ansible strategy, a skipped task does not have -# a dictionary result. As such we validate that the register -# is a mapping (dict). -- name: Unarchive pre-built venv - unarchive: - src: "{{ venv_download_path }}/{{ venv_destination_path }}.tgz" - dest: "{{ venv_destination_path }}" - remote_src: no - when: - - _venv_checksum_copy is mapping - - _venv_checksum_copy | changed - notify: - - Manage LB - - Restart services - -#TODO(odyssey4me): -# Split the venv build into multiple parts: -# 1. Create the venv without pip, setuptools, wheel -# 2. Use get-pip.py to install the right versions -# of pip, setuptools, wheel into the venv -# 3. Install the packages into the venv - -- name: Build venv - pip: - name: "{{ venv_pip_packages }}" - state: latest - virtualenv: "{{ venv_destination_path }}" - virtualenv_site_packages: "no" - extra_args: >- - {{ pip_install_venv_constraints }} - {{ pip_install_constraints }} - {{ pip_install_options }} - register: _install_venv_pip_packages - until: _install_venv_pip_packages | success - retries: 5 - delay: 2 - when: - - not _venv_tgz.stat.exists | bool - notify: - - Manage LB - - Restart services - -# Due to our Ansible strategy, a skipped task does not have -# a dictionary result. As such we validate that the register -# is a mapping (dict). -- name: Update virtualenv python and paths - shell: | - sed -si '1s/^.*python.*$/#!{{ (venv_destination_path ~ '/bin') | replace ('/','\/') }}\/python/' {{ venv_destination_path }}/bin/* - virtualenv {{ venv_destination_path }} - when: - - _venv_checksum_copy is mapping - - _venv_checksum_copy | changed +- include_tasks: "python_venv_preflight.yml" tags: - - skip_ansible_lint + - always -# Due to our Ansible strategy, a skipped task does not have -# a dictionary result. As such we validate that the register -# is a mapping (dict). -- name: Clean up the virtualenv before packaging - shell: | - find {{ venv_destination_path }}/bin -type f -name '*.pyc' -delete +- include_tasks: "python_venv_build.yml" when: - - _install_venv_pip_packages is mapping - - _install_venv_pip_packages | changed + - (not _src_venv_present.stat.exists | bool) or + (not venv_reuse_enable | bool) -# Note(odyssey4me): -# We purposefully use shel instead of the archive module -# here. The archive module's output is far too verbose to -# be practical when debugging. -# -# Due to our Ansible strategy, a skipped task does not have -# a dictionary result. As such we validate that the register -# is a mapping (dict). -- name: Package venv - shell: | - tar czf '{{ venv_destination_path }}.tgz' -C '{{ venv_destination_path }}' . - args: - chdir: "{{ venv_destination_path }}" - executable: /bin/bash - warn: no - register: _venv_package_build +- include_tasks: "python_venv_install.yml" when: - - _install_venv_pip_packages is mapping - - _install_venv_pip_packages | changed + - venv_reuse_enable | bool + - _src_venv_present.stat.exists | bool -# Due to our Ansible strategy, a skipped task does not have -# a dictionary result. As such we validate that the register -# is a mapping (dict). -- name: Prepare checksum for packaged venv - shell: | - sha1sum '{{ venv_destination_path }}.tgz' | awk '{print $1}' > '{{ venv_destination_path }}.checksum' - args: - executable: /bin/bash +- include_tasks: "python_venv_set_facts.yml" when: - - _venv_package_build is mapping - - _venv_package_build | changed - -# Due to our Ansible strategy, a skipped task does not have -# a dictionary result. As such we validate that the register -# is a mapping (dict). -- name: Copy the packaged venv and checksum file to the deployment host - fetch: - src: "{{ item.src }}" - dest: "{{ item.dest }}" - flat: yes - with_items: - - src: "{{ venv_destination_path }}.tgz" - dest: "{{ venv_download_path }}/{{ venv_destination_path }}.tgz" - - src: "{{ venv_destination_path }}.checksum" - dest: "{{ venv_download_path }}/{{ venv_destination_path }}.checksum" - when: - - _venv_package_build is mapping - - _venv_package_build | changed + - venv_facts_when_changed != [] diff --git a/tasks/python_venv_build.yml b/tasks/python_venv_build.yml new file mode 100644 index 0000000..8ef52ce --- /dev/null +++ b/tasks/python_venv_build.yml @@ -0,0 +1,111 @@ +--- +# Copyright 2018, Rackspace US, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Install distro packages for venv build + package: + name: "{{ distro_package_list }}" + state: "{{ distro_package_state }}" + update_cache: "{{ (ansible_pkg_mgr in ['apt', 'zypper']) | ternary('yes', omit) }}" + cache_valid_time: "{{ (ansible_pkg_mgr == 'apt') | ternary(distro_cache_valid_time, omit) }}" + register: _install_distro_packages + until: _install_distro_packages | success + retries: 5 + delay: 2 + +- name: Install pip packages on the host for venv build + pip: + name: "{{ host_pip_packages }}" + state: latest + extra_args: "{{ host_pip_install_args }}" + register: _install_host_pip_packages + until: _install_host_pip_packages | success + retries: 5 + delay: 2 + +- name: Create venv directory on the target host + file: + path: "{{ venv_destination_path }}" + state: directory + +#TODO(odyssey4me): +# Split the venv build into multiple parts: +# 1. Create the venv without pip, setuptools, wheel +# 2. Use get-pip.py to install the right versions +# of pip, setuptools, wheel into the venv +# 3. Install the packages into the venv + +- name: Build venv + pip: + name: "{{ venv_pip_packages }}" + state: latest + virtualenv: "{{ venv_destination_path }}" + virtualenv_site_packages: "no" + extra_args: "{{ venv_pip_install_args }}" + register: _install_venv_pip_packages + until: _install_venv_pip_packages | success + retries: 5 + delay: 2 + notify: + - venv changed + +- name: Package the venv when venv_reuse_enable is enabled + when: venv_reuse_enable | bool + block: + + - name: Clean up the virtualenv before packaging + shell: | + find {{ venv_destination_path }}/bin -type f -name '*.pyc' -delete + when: + - _install_venv_pip_packages is mapping + - _install_venv_pip_packages | changed + + # Note(odyssey4me): + # We purposefully use shell instead of the archive module + # here. The archive module's output is far too verbose to + # be practical when debugging. + - name: Package venv + shell: | + tar czf '{{ venv_destination_path }}.tgz' -C '{{ venv_destination_path }}' . + args: + chdir: "{{ venv_destination_path }}" + executable: /bin/bash + warn: no + register: _venv_package_build + when: + - _install_venv_pip_packages is mapping + - _install_venv_pip_packages | changed + + - name: Prepare checksum for packaged venv + shell: | + sha1sum '{{ venv_destination_path }}.tgz' | awk '{print $1}' > '{{ venv_destination_path }}.checksum' + args: + executable: /bin/bash + when: + - _venv_package_build is mapping + - _venv_package_build | changed + + - name: Copy the packaged venv and checksum file to the deployment host + fetch: + src: "{{ item.src }}" + dest: "{{ item.dest }}" + flat: yes + with_items: + - src: "{{ venv_destination_path }}.tgz" + dest: "{{ venv_reuse_download_path }}/{{ venv_destination_path }}.tgz" + - src: "{{ venv_destination_path }}.checksum" + dest: "{{ venv_reuse_download_path }}/{{ venv_destination_path }}.checksum" + when: + - _venv_package_build is mapping + - _venv_package_build | changed diff --git a/tasks/python_venv_install.yml b/tasks/python_venv_install.yml new file mode 100644 index 0000000..db7abe5 --- /dev/null +++ b/tasks/python_venv_install.yml @@ -0,0 +1,53 @@ +--- +# Copyright 2018, Rackspace US, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Copy the venv checksum file to the target host + copy: + src: "{{ venv_reuse_download_path }}/{{ venv_destination_path }}.checksum" + dest: "{{ venv_destination_path | dirname }}" + register: _venv_checksum_copy + - _src_venv_present.stat.exists | bool + +- name: Remove existing venv on target host if it is changing + file: + path: "{{ venv_destination_path }}" + state: absent + - _venv_checksum_copy is mapping + - _venv_checksum_copy | changed + +- name: Create venv directory on the target host + file: + path: "{{ venv_destination_path }}" + state: directory + +- name: Unarchive pre-built venv + unarchive: + src: "{{ venv_reuse_download_path }}/{{ venv_destination_path }}.tgz" + dest: "{{ venv_destination_path }}" + remote_src: no + - _venv_checksum_copy is mapping + - _venv_checksum_copy | changed + notify: + - venv changed + +- name: Update virtualenv python and paths + shell: | + sed -si '1s/^.*python.*$/#!{{ (venv_destination_path ~ '/bin') | replace ('/','\/') }}\/python/' {{ venv_destination_path }}/bin/* + virtualenv {{ venv_destination_path }} + when: + - _venv_checksum_copy is mapping + - _venv_checksum_copy | changed + tags: + - skip_ansible_lint diff --git a/tasks/python_venv_preflight.yml b/tasks/python_venv_preflight.yml new file mode 100644 index 0000000..998ea0c --- /dev/null +++ b/tasks/python_venv_preflight.yml @@ -0,0 +1,43 @@ +--- +# Copyright 2018, Rackspace US, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Verify that venv_destination_path has been provided + fail: + msg: | + The variable venv_destination_path is required and + has not been set + when: + - venv_destination_path is not defined + +- name: Check if venv tgz is present on the deployment host + stat: + path: "{{ venv_reuse_download_path }}/{{ venv_destination_path }}.tgz" + get_attributes: no + get_checksum: no + get_md5: no + get_mime: no + register: _src_venv_present + delegate_to: localhost + run_once: yes + +- name: Ensure that venv_reuse_download_path exists on the deployment host + file: + path: "{{ venv_reuse_download_path }}/{{ venv_destination_path | dirname }}" + state: directory + owner: "{{ venv_reuse_download_path_owner }}" + delegate_to: localhost + run_once: yes + when: + - venv_reuse_enable | bool diff --git a/tasks/python_venv_set_facts.yml b/tasks/python_venv_set_facts.yml new file mode 100644 index 0000000..2188040 --- /dev/null +++ b/tasks/python_venv_set_facts.yml @@ -0,0 +1,34 @@ +--- +# Copyright 2018, Rackspace US, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Ensure local facts folder exists + file: + path: /etc/ansible/facts.d + state: directory + +- name: Record the necessary facts + ini_file: + dest: "/etc/ansible/facts.d/{{ venv_facts_dest }}.fact" + section: "{{ item.section }}" + option: "{{ item.option }}" + value: "{{ item.value }}" + with_items: "{{ venv_facts_when_changed }}" + when: + - (_venv_checksum_copy is defined and + _venv_checksum_copy is mapping and + _venv_checksum_copy | changed) or + (_install_venv_pip_packages is defined and + _install_venv_pip_packages is mapping and + _install_venv_pip_packages | changed)