openstack/ansible-role-python_venv_build
Code Issues Proposed changes

Separate build and install stage

Browse Source 
In order to ensure that the build tasks are entirely skipped
when a package venv is re-used, the build and install stages
are split.

The ability to re-use venvs is also now able to be toggled.
Disabling this feature would set the build to always happen,
catering to environments where a service venv is always
deployed to the same folder (eg: stateless hypervisors with
squashfs partitions).

The ability to set constraints, etc is changed to a generalised
set of arguments that can be passed to the pip install task.
This commit is contained in:
Jesse Pretorius 2018-03-20 12:12:46 +00:00
parent afafacfba5
commit d5a9f025b8
7 changed files with 296 additions and 213 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

63
defaults/main.yml
View File

@ -13,7 +13,20 @@
# See the License for the specific language governing permissions and
# limitations under the License.
# The list of distribution packages to install
#
# Required variables
#
# The path where venvs are extracted to
# on the target host during an install, for example:
# venv_destination_path: "/openstack/venvs/myvenv"
#
# Optional variables
#
# Distribution packages which must be installed
# on the host for the purpose of building the venv.
distro_package_list: []
# Set the package install state for packages
@ -22,34 +35,46 @@ distro_package_state: "latest"
# The time in seconds that the distribution package
# cache is valid for. This is only used by the apt
# package manager
# and zypper package managers.
distro_cache_valid_time: 600
# Python packages which must be installed
# on to the host
# on to the host for the purpose of building
# the venv.
host_pip_packages: []
# Arguments to pass to the installation
# of pip packages on the host.
host_pip_install_args: ""
# Python packages which must be installed
# into the venv
# into the venv.
venv_pip_packages: []
# General pip install constraints
pip_install_constraints: ""
# Arguments to pass to the venv build
venv_pip_install_args: ""
# Specific constraints for the venv
pip_install_venv_constraints: ""
# Enable the reuse of venvs across multiple hosts.
# This sets the build process to copy the venv to
# the deployment host once it's built, then to
# re-use the venv in subsequent deployments.
venv_reuse_enable: yes
# General pip install extra options
# This is especially useful for proxy options
pip_install_options: ""
# The path where a built venv should be stored on the
# deployment host.
venv_reuse_download_path: "{{ lookup('env', 'HOME') | default('/opt', true) }}/cache"
# The path where venvs are stored on the
# deployment host
venv_download_path: "{{ lookup('env', 'HOME') | default('/opt', true) }}/cache/files"
# The owner of the venv_reuse_download_path
venv_reuse_download_path_owner: "{{ lookup('env', 'USER') | default('root', true) }}"
# The owner of the venv_download_path
venv_download_path_owner: "{{ lookup('env', 'USER') | default('root', true) }}"
# The facts to set when the venv changes during a
# build, or the installation of a venv.
# Eg:
# set_facts_when_changed:
# - section: glance
# option: venv_tag
# value: "{{ glance_venv_tag }}"
venv_facts_when_changed: []
# The path where venvs are extracted to
# on the target host, for example:
# venv_destination_path: "/openstack/venvs/myvenv"
# The INI file name to use for the fact setting.
venv_facts_dest: "openstack_ansible"

6
handlers/main.yml
View File

@ -14,9 +14,5 @@
# limitations under the License.
- meta: noop
listen: Manage LB
when: false
- meta: noop
listen: Restart services
listen: venv changed
when: false

199
tasks/main.yml
View File

@ -13,199 +13,20 @@
# See the License for the specific language governing permissions and
# limitations under the License.
# TODO(odyssey4me):
# 1. Cater for Logan's model where we must just extract
# over the existing folder without wiping it out first.
# 2. Also, wiping the directory out from under it probably
# wrecks the service while it's running. We should figure
# out a better way of atomically replacing the venv without
# ripping the folder out from under it.
- name: Install distro packages
package:
name: "{{ distro_package_list }}"
state: "{{ distro_package_state }}"
update_cache: "{{ (ansible_pkg_mgr in ['apt', 'zypper']) | ternary('yes', omit) }}"
cache_valid_time: "{{ (ansible_pkg_mgr == 'apt') | ternary(distro_cache_valid_time, omit) }}"
register: _install_distro_packages
until: _install_distro_packages | success
retries: 5
delay: 2
- name: Install required pip packages on the host
pip:
name: "{{ host_pip_packages }}"
state: latest
extra_args: >-
{{ pip_install_constraints }}
{{ pip_install_options }}
register: _install_host_pip_packages
until: _install_host_pip_packages | success
retries: 5
delay: 2
- name: Ensure that venv_download_path exists on the deployment host
file:
path: "{{ venv_download_path }}/{{ venv_destination_path | dirname }}"
state: directory
owner: "{{ venv_download_path_owner }}"
delegate_to: localhost
run_once: yes
- name: Check if venv is present on the deployment host
stat:
path: "{{ venv_download_path }}/{{ venv_destination_path }}.tgz"
get_attributes: no
get_checksum: no
get_md5: no
get_mime: no
register: _venv_tgz
delegate_to: localhost
run_once: yes
- name: Copy the venv checksum file to the target host
copy:
src: "{{ venv_download_path }}/{{ venv_destination_path }}.checksum"
dest: "{{ venv_destination_path | dirname }}"
register: _venv_checksum_copy
when:
- _venv_tgz.stat.exists | bool
# TODO(odyssey4me):
# 1. Cater for Logan's model where we must just extract
# over the existing folder without wiping it out first.
# 2. Also, removing it like this probably wrecks the service
# while it's running. We should figure out a better way
# of atomically replacing the venv without ripping the
# folder out from under it.
# Due to our Ansible strategy, a skipped task does not have
# a dictionary result. As such we validate that the register
# is a mapping (dict).
- name: Remove existing venv on target host if it is changing
file:
path: "{{ venv_destination_path }}"
state: absent
when:
- _venv_checksum_copy is mapping
- _venv_checksum_copy | changed
- name: Create venv directory on the target host
file:
path: "{{ venv_destination_path }}"
state: directory
register: _create_venv_dir
# Due to our Ansible strategy, a skipped task does not have
# a dictionary result. As such we validate that the register
# is a mapping (dict).
- name: Unarchive pre-built venv
unarchive:
src: "{{ venv_download_path }}/{{ venv_destination_path }}.tgz"
dest: "{{ venv_destination_path }}"
remote_src: no
when:
- _venv_checksum_copy is mapping
- _venv_checksum_copy | changed
notify:
- Manage LB
- Restart services
#TODO(odyssey4me):
# Split the venv build into multiple parts:
# 1. Create the venv without pip, setuptools, wheel
# 2. Use get-pip.py to install the right versions
# of pip, setuptools, wheel into the venv
# 3. Install the packages into the venv
- name: Build venv
pip:
name: "{{ venv_pip_packages }}"
state: latest
virtualenv: "{{ venv_destination_path }}"
virtualenv_site_packages: "no"
extra_args: >-
{{ pip_install_venv_constraints }}
{{ pip_install_constraints }}
{{ pip_install_options }}
register: _install_venv_pip_packages
until: _install_venv_pip_packages | success
retries: 5
delay: 2
when:
- not _venv_tgz.stat.exists | bool
notify:
- Manage LB
- Restart services
# Due to our Ansible strategy, a skipped task does not have
# a dictionary result. As such we validate that the register
# is a mapping (dict).
- name: Update virtualenv python and paths
shell: |
sed -si '1s/^.*python.*$/#!{{ (venv_destination_path ~ '/bin') | replace ('/','\/') }}\/python/' {{ venv_destination_path }}/bin/*
virtualenv {{ venv_destination_path }}
when:
- _venv_checksum_copy is mapping
- _venv_checksum_copy | changed
- include_tasks: "python_venv_preflight.yml"
tags:
- skip_ansible_lint
- always
# Due to our Ansible strategy, a skipped task does not have
# a dictionary result. As such we validate that the register
# is a mapping (dict).
- name: Clean up the virtualenv before packaging
shell: |
find {{ venv_destination_path }}/bin -type f -name '*.pyc' -delete
- include_tasks: "python_venv_build.yml"
when:
- _install_venv_pip_packages is mapping
- _install_venv_pip_packages | changed
- (not _src_venv_present.stat.exists | bool) or
(not venv_reuse_enable | bool)
# Note(odyssey4me):
# We purposefully use shel instead of the archive module
# here. The archive module's output is far too verbose to
# be practical when debugging.
#
# Due to our Ansible strategy, a skipped task does not have
# a dictionary result. As such we validate that the register
# is a mapping (dict).
- name: Package venv
shell: |
tar czf '{{ venv_destination_path }}.tgz' -C '{{ venv_destination_path }}' .
args:
chdir: "{{ venv_destination_path }}"
executable: /bin/bash
warn: no
register: _venv_package_build
- include_tasks: "python_venv_install.yml"
when:
- _install_venv_pip_packages is mapping
- _install_venv_pip_packages | changed
- venv_reuse_enable | bool
- _src_venv_present.stat.exists | bool
# Due to our Ansible strategy, a skipped task does not have
# a dictionary result. As such we validate that the register
# is a mapping (dict).
- name: Prepare checksum for packaged venv
shell: |
sha1sum '{{ venv_destination_path }}.tgz' | awk '{print $1}' > '{{ venv_destination_path }}.checksum'
args:
executable: /bin/bash
- include_tasks: "python_venv_set_facts.yml"
when:
- _venv_package_build is mapping
- _venv_package_build | changed
# Due to our Ansible strategy, a skipped task does not have
# a dictionary result. As such we validate that the register
# is a mapping (dict).
- name: Copy the packaged venv and checksum file to the deployment host
fetch:
src: "{{ item.src }}"
dest: "{{ item.dest }}"
flat: yes
with_items:
- src: "{{ venv_destination_path }}.tgz"
dest: "{{ venv_download_path }}/{{ venv_destination_path }}.tgz"
- src: "{{ venv_destination_path }}.checksum"
dest: "{{ venv_download_path }}/{{ venv_destination_path }}.checksum"
when:
- _venv_package_build is mapping
- _venv_package_build | changed
- venv_facts_when_changed != []

111
tasks/python_venv_build.yml Normal file
View File

@ -0,0 +1,111 @@
---
# Copyright 2018, Rackspace US, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
- name: Install distro packages for venv build
package:
name: "{{ distro_package_list }}"
state: "{{ distro_package_state }}"
update_cache: "{{ (ansible_pkg_mgr in ['apt', 'zypper']) | ternary('yes', omit) }}"
cache_valid_time: "{{ (ansible_pkg_mgr == 'apt') | ternary(distro_cache_valid_time, omit) }}"
register: _install_distro_packages
until: _install_distro_packages | success
retries: 5
delay: 2
- name: Install pip packages on the host for venv build
pip:
name: "{{ host_pip_packages }}"
state: latest
extra_args: "{{ host_pip_install_args }}"
register: _install_host_pip_packages
until: _install_host_pip_packages | success
retries: 5
delay: 2
- name: Create venv directory on the target host
file:
path: "{{ venv_destination_path }}"
state: directory
#TODO(odyssey4me):
# Split the venv build into multiple parts:
# 1. Create the venv without pip, setuptools, wheel
# 2. Use get-pip.py to install the right versions
# of pip, setuptools, wheel into the venv
# 3. Install the packages into the venv
- name: Build venv
pip:
name: "{{ venv_pip_packages }}"
state: latest
virtualenv: "{{ venv_destination_path }}"
virtualenv_site_packages: "no"
extra_args: "{{ venv_pip_install_args }}"
register: _install_venv_pip_packages
until: _install_venv_pip_packages | success
retries: 5
delay: 2
notify:
- venv changed
- name: Package the venv when venv_reuse_enable is enabled
when: venv_reuse_enable | bool
block:
- name: Clean up the virtualenv before packaging
shell: |
find {{ venv_destination_path }}/bin -type f -name '*.pyc' -delete
when:
- _install_venv_pip_packages is mapping
- _install_venv_pip_packages | changed
# Note(odyssey4me):
# We purposefully use shell instead of the archive module
# here. The archive module's output is far too verbose to
# be practical when debugging.
- name: Package venv
shell: |
tar czf '{{ venv_destination_path }}.tgz' -C '{{ venv_destination_path }}' .
args:
chdir: "{{ venv_destination_path }}"
executable: /bin/bash
warn: no
register: _venv_package_build
when:
- _install_venv_pip_packages is mapping
- _install_venv_pip_packages | changed
- name: Prepare checksum for packaged venv
shell: |
sha1sum '{{ venv_destination_path }}.tgz' | awk '{print $1}' > '{{ venv_destination_path }}.checksum'
args:
executable: /bin/bash
when:
- _venv_package_build is mapping
- _venv_package_build | changed
- name: Copy the packaged venv and checksum file to the deployment host
fetch:
src: "{{ item.src }}"
dest: "{{ item.dest }}"
flat: yes
with_items:
- src: "{{ venv_destination_path }}.tgz"
dest: "{{ venv_reuse_download_path }}/{{ venv_destination_path }}.tgz"
- src: "{{ venv_destination_path }}.checksum"
dest: "{{ venv_reuse_download_path }}/{{ venv_destination_path }}.checksum"
when:
- _venv_package_build is mapping
- _venv_package_build | changed

53
tasks/python_venv_install.yml Normal file
View File

@ -0,0 +1,53 @@
---
# Copyright 2018, Rackspace US, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
- name: Copy the venv checksum file to the target host
copy:
src: "{{ venv_reuse_download_path }}/{{ venv_destination_path }}.checksum"
dest: "{{ venv_destination_path | dirname }}"
register: _venv_checksum_copy
- _src_venv_present.stat.exists | bool
- name: Remove existing venv on target host if it is changing
file:
path: "{{ venv_destination_path }}"
state: absent
- _venv_checksum_copy is mapping
- _venv_checksum_copy | changed
- name: Create venv directory on the target host
file:
path: "{{ venv_destination_path }}"
state: directory
- name: Unarchive pre-built venv
unarchive:
src: "{{ venv_reuse_download_path }}/{{ venv_destination_path }}.tgz"
dest: "{{ venv_destination_path }}"
remote_src: no
- _venv_checksum_copy is mapping
- _venv_checksum_copy | changed
notify:
- venv changed
- name: Update virtualenv python and paths
shell: |
sed -si '1s/^.*python.*$/#!{{ (venv_destination_path ~ '/bin') | replace ('/','\/') }}\/python/' {{ venv_destination_path }}/bin/*
virtualenv {{ venv_destination_path }}
when:
- _venv_checksum_copy is mapping
- _venv_checksum_copy | changed
tags:
- skip_ansible_lint

43
tasks/python_venv_preflight.yml Normal file
View File

@ -0,0 +1,43 @@
---
# Copyright 2018, Rackspace US, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
- name: Verify that venv_destination_path has been provided
fail:
msg: |
The variable venv_destination_path is required and
has not been set
when:
- venv_destination_path is not defined
- name: Check if venv tgz is present on the deployment host
stat:
path: "{{ venv_reuse_download_path }}/{{ venv_destination_path }}.tgz"
get_attributes: no
get_checksum: no
get_md5: no
get_mime: no
register: _src_venv_present
delegate_to: localhost
run_once: yes
- name: Ensure that venv_reuse_download_path exists on the deployment host
file:
path: "{{ venv_reuse_download_path }}/{{ venv_destination_path | dirname }}"
state: directory
owner: "{{ venv_reuse_download_path_owner }}"
delegate_to: localhost
run_once: yes
when:
- venv_reuse_enable | bool

34
tasks/python_venv_set_facts.yml Normal file
View File

@ -0,0 +1,34 @@
---
# Copyright 2018, Rackspace US, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
- name: Ensure local facts folder exists
file:
path: /etc/ansible/facts.d
state: directory
- name: Record the necessary facts
ini_file:
dest: "/etc/ansible/facts.d/{{ venv_facts_dest }}.fact"
section: "{{ item.section }}"
option: "{{ item.option }}"
value: "{{ item.value }}"
with_items: "{{ venv_facts_when_changed }}"
when:
- (_venv_checksum_copy is defined and
_venv_checksum_copy is mapping and
_venv_checksum_copy | changed) or
(_install_venv_pip_packages is defined and
_install_venv_pip_packages is mapping and
_install_venv_pip_packages | changed)