Separate build and install stage

In order to ensure that the build tasks are entirely skipped
when a package venv is re-used, the build and install stages
are split.

The ability to re-use venvs is also now able to be toggled.
Disabling this feature would set the build to always happen,
catering to environments where a service venv is always
deployed to the same folder (eg: stateless hypervisors with
squashfs partitions).

The ability to set constraints, etc is changed to a generalised
set of arguments that can be passed to the pip install task.

defaults/main.yml

@@ -13,7 +13,20 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-# The list of distribution packages to install
+#
+# Required variables
+#
+
+# The path where venvs are extracted to
+# on the target host during an install, for example:
+# venv_destination_path: "/openstack/venvs/myvenv"
+
+#
+# Optional variables
+#
+
+# Distribution packages which must be installed
+# on the host for the purpose of building the venv.
 distro_package_list: []
 
 # Set the package install state for packages
@@ -22,34 +35,46 @@ distro_package_state: "latest"
 
 # The time in seconds that the distribution package
 # cache is valid for. This is only used by the apt
-# package manager
+# and zypper package managers.
 distro_cache_valid_time: 600
 
 # Python packages which must be installed
-# on to the host
+# on to the host for the purpose of building
+# the venv.
 host_pip_packages: []
 
+# Arguments to pass to the installation
+# of pip packages on the host.
+host_pip_install_args: ""
+
 # Python packages which must be installed
-# into the venv
+# into the venv.
 venv_pip_packages: []
 
-# General pip install constraints
+# Arguments to pass to the venv build
-pip_install_constraints: ""
+venv_pip_install_args: ""
 
-# Specific constraints for the venv
+# Enable the reuse of venvs across multiple hosts.
-pip_install_venv_constraints: ""
+# This sets the build process to copy the venv to
+# the deployment host once it's built, then to
+# re-use the venv in subsequent deployments.
+venv_reuse_enable: yes
 
-# General pip install extra options
+# The path where a built venv should be stored on the
-# This is especially useful for proxy options
+# deployment host.
-pip_install_options: ""
+venv_reuse_download_path: "{{ lookup('env', 'HOME') | default('/opt', true) }}/cache"
 
-# The path where venvs are stored on the
+# The owner of the venv_reuse_download_path
-# deployment host
+venv_reuse_download_path_owner: "{{ lookup('env', 'USER') | default('root', true) }}"
-venv_download_path: "{{ lookup('env', 'HOME') | default('/opt', true) }}/cache/files"
 
-# The owner of the venv_download_path
+# The facts to set when the venv changes during a
-venv_download_path_owner: "{{ lookup('env', 'USER') | default('root', true) }}"
+# build, or the installation of a venv.
+# Eg:
+# set_facts_when_changed:
+#   - section: glance
+#     option: venv_tag
+#     value: "{{ glance_venv_tag }}"
+venv_facts_when_changed: []
 
-# The path where venvs are extracted to
+# The INI file name to use for the fact setting.
-# on the target host, for example:
+venv_facts_dest: "openstack_ansible"
-# venv_destination_path: "/openstack/venvs/myvenv"

handlers/main.yml

@@ -14,9 +14,5 @@
 # limitations under the License.
 
 - meta: noop
-  listen: Manage LB
+  listen: venv changed
-  when: false
-
-- meta: noop
-  listen: Restart services
   when: false

tasks/main.yml

@@ -13,199 +13,20 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-# TODO(odyssey4me):
+- include_tasks: "python_venv_preflight.yml"
-# 1. Cater for Logan's model where we must just extract
-#    over the existing folder without wiping it out first.
-# 2. Also, wiping the directory out from under it probably
-#    wrecks the service while it's running. We should figure
-#    out a better way of atomically replacing the venv without
-#    ripping the folder out from under it.
-
-- name: Install distro packages
-  package:
-    name: "{{ distro_package_list }}"
-    state: "{{ distro_package_state }}"
-    update_cache: "{{ (ansible_pkg_mgr in ['apt', 'zypper']) | ternary('yes', omit) }}"
-    cache_valid_time: "{{ (ansible_pkg_mgr == 'apt') | ternary(distro_cache_valid_time, omit) }}"
-  register: _install_distro_packages
-  until: _install_distro_packages | success
-  retries: 5
-  delay: 2
-
-- name: Install required pip packages on the host
-  pip:
-    name: "{{ host_pip_packages }}"
-    state: latest
-    extra_args: >-
-      {{ pip_install_constraints }}
-      {{ pip_install_options }}
-  register: _install_host_pip_packages
-  until: _install_host_pip_packages | success
-  retries: 5
-  delay: 2
-
-- name: Ensure that venv_download_path exists on the deployment host
-  file:
-    path: "{{ venv_download_path }}/{{ venv_destination_path | dirname }}"
-    state: directory
-    owner: "{{ venv_download_path_owner }}"
-  delegate_to: localhost
-  run_once: yes
-
-- name: Check if venv is present on the deployment host
-  stat:
-    path: "{{ venv_download_path }}/{{ venv_destination_path }}.tgz"
-    get_attributes: no
-    get_checksum: no
-    get_md5: no
-    get_mime: no
-  register: _venv_tgz
-  delegate_to: localhost
-  run_once: yes
-
-- name: Copy the venv checksum file to the target host
-  copy:
-    src: "{{ venv_download_path }}/{{ venv_destination_path }}.checksum"
-    dest: "{{ venv_destination_path | dirname }}"
-  register: _venv_checksum_copy
-  when:
-    - _venv_tgz.stat.exists | bool
-
-# TODO(odyssey4me):
-# 1. Cater for Logan's model where we must just extract
-#    over the existing folder without wiping it out first.
-# 2. Also, removing it like this probably wrecks the service
-#    while it's running. We should figure out a better way
-#    of atomically replacing the venv without ripping the
-#    folder out from under it.
-
-# Due to our Ansible strategy, a skipped task does not have
-# a dictionary result. As such we validate that the register
-# is a mapping (dict).
-- name: Remove existing venv on target host if it is changing
-  file:
-    path: "{{ venv_destination_path }}"
-    state: absent
-  when:
-    - _venv_checksum_copy is mapping
-    - _venv_checksum_copy | changed
-
-- name: Create venv directory on the target host
-  file:
-    path: "{{ venv_destination_path }}"
-    state: directory
-  register: _create_venv_dir
-
-# Due to our Ansible strategy, a skipped task does not have
-# a dictionary result. As such we validate that the register
-# is a mapping (dict).
-- name: Unarchive pre-built venv
-  unarchive:
-    src: "{{ venv_download_path }}/{{ venv_destination_path }}.tgz"
-    dest: "{{ venv_destination_path }}"
-    remote_src: no
-  when:
-    - _venv_checksum_copy is mapping
-    - _venv_checksum_copy | changed
-  notify:
-    - Manage LB
-    - Restart services
-
-#TODO(odyssey4me):
-# Split the venv build into multiple parts:
-# 1. Create the venv without pip, setuptools, wheel
-# 2. Use get-pip.py to install the right versions
-#    of pip, setuptools, wheel into the venv
-# 3. Install the packages into the venv
-
-- name: Build venv
-  pip:
-    name: "{{ venv_pip_packages }}"
-    state: latest
-    virtualenv: "{{ venv_destination_path }}"
-    virtualenv_site_packages: "no"
-    extra_args: >-
-      {{ pip_install_venv_constraints }}
-      {{ pip_install_constraints }}
-      {{ pip_install_options }}
-  register: _install_venv_pip_packages
-  until: _install_venv_pip_packages | success
-  retries: 5
-  delay: 2
-  when:
-    - not _venv_tgz.stat.exists | bool
-  notify:
-    - Manage LB
-    - Restart services
-
-# Due to our Ansible strategy, a skipped task does not have
-# a dictionary result. As such we validate that the register
-# is a mapping (dict).
-- name: Update virtualenv python and paths
-  shell: |
-    sed -si '1s/^.*python.*$/#!{{ (venv_destination_path ~ '/bin') | replace ('/','\/') }}\/python/' {{ venv_destination_path }}/bin/*
-    virtualenv {{ venv_destination_path }}
-  when:
-    - _venv_checksum_copy is mapping
-    - _venv_checksum_copy | changed
   tags:
-    - skip_ansible_lint
+    - always
 
-# Due to our Ansible strategy, a skipped task does not have
+- include_tasks: "python_venv_build.yml"
-# a dictionary result. As such we validate that the register
-# is a mapping (dict).
-- name: Clean up the virtualenv before packaging
-  shell: |
-    find {{ venv_destination_path }}/bin -type f -name '*.pyc' -delete
   when:
-    - _install_venv_pip_packages is mapping
+    - (not _src_venv_present.stat.exists | bool) or
-    - _install_venv_pip_packages | changed
+      (not venv_reuse_enable | bool)
 
-# Note(odyssey4me):
+- include_tasks: "python_venv_install.yml"
-# We purposefully use shel instead of the archive module
-# here. The archive module's output is far too verbose to
-# be practical when debugging.
-#
-# Due to our Ansible strategy, a skipped task does not have
-# a dictionary result. As such we validate that the register
-# is a mapping (dict).
-- name: Package venv
-  shell: |
-    tar czf '{{ venv_destination_path }}.tgz' -C '{{ venv_destination_path }}' .
-  args:
-    chdir: "{{ venv_destination_path }}"
-    executable: /bin/bash
-    warn: no
-  register: _venv_package_build
   when:
-    - _install_venv_pip_packages is mapping
+    - venv_reuse_enable | bool
-    - _install_venv_pip_packages | changed
+    - _src_venv_present.stat.exists | bool
 
-# Due to our Ansible strategy, a skipped task does not have
+- include_tasks: "python_venv_set_facts.yml"
-# a dictionary result. As such we validate that the register
-# is a mapping (dict).
-- name: Prepare checksum for packaged venv
-  shell: |
-    sha1sum '{{ venv_destination_path }}.tgz' | awk '{print $1}' > '{{ venv_destination_path }}.checksum'
-  args:
-    executable: /bin/bash
   when:
-    - _venv_package_build is mapping
+    - venv_facts_when_changed != []
-    - _venv_package_build | changed
-
-# Due to our Ansible strategy, a skipped task does not have
-# a dictionary result. As such we validate that the register
-# is a mapping (dict).
-- name: Copy the packaged venv and checksum file to the deployment host
-  fetch:
-    src: "{{ item.src }}"
-    dest: "{{ item.dest }}"
-    flat: yes
-  with_items:
-    - src: "{{ venv_destination_path }}.tgz"
-      dest: "{{ venv_download_path }}/{{ venv_destination_path }}.tgz"
-    - src: "{{ venv_destination_path }}.checksum"
-      dest: "{{ venv_download_path }}/{{ venv_destination_path }}.checksum"
-  when:
-    - _venv_package_build is mapping
-    - _venv_package_build | changed

tasks/python_venv_build.yml

@@ -0,0 +1,111 @@
+---
+# Copyright 2018, Rackspace US, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+- name: Install distro packages for venv build
+  package:
+    name: "{{ distro_package_list }}"
+    state: "{{ distro_package_state }}"
+    update_cache: "{{ (ansible_pkg_mgr in ['apt', 'zypper']) | ternary('yes', omit) }}"
+    cache_valid_time: "{{ (ansible_pkg_mgr == 'apt') | ternary(distro_cache_valid_time, omit) }}"
+  register: _install_distro_packages
+  until: _install_distro_packages | success
+  retries: 5
+  delay: 2
+
+- name: Install pip packages on the host for venv build
+  pip:
+    name: "{{ host_pip_packages }}"
+    state: latest
+    extra_args: "{{ host_pip_install_args }}"
+  register: _install_host_pip_packages
+  until: _install_host_pip_packages | success
+  retries: 5
+  delay: 2
+
+- name: Create venv directory on the target host
+  file:
+    path: "{{ venv_destination_path }}"
+    state: directory
+
+#TODO(odyssey4me):
+# Split the venv build into multiple parts:
+# 1. Create the venv without pip, setuptools, wheel
+# 2. Use get-pip.py to install the right versions
+#    of pip, setuptools, wheel into the venv
+# 3. Install the packages into the venv
+
+- name: Build venv
+  pip:
+    name: "{{ venv_pip_packages }}"
+    state: latest
+    virtualenv: "{{ venv_destination_path }}"
+    virtualenv_site_packages: "no"
+    extra_args: "{{ venv_pip_install_args }}"
+  register: _install_venv_pip_packages
+  until: _install_venv_pip_packages | success
+  retries: 5
+  delay: 2
+  notify:
+    - venv changed
+
+- name: Package the venv when venv_reuse_enable is enabled
+  when: venv_reuse_enable | bool
+  block:
+
+  - name: Clean up the virtualenv before packaging
+    shell: |
+      find {{ venv_destination_path }}/bin -type f -name '*.pyc' -delete
+    when:
+      - _install_venv_pip_packages is mapping
+      - _install_venv_pip_packages | changed
+
+  # Note(odyssey4me):
+  # We purposefully use shell instead of the archive module
+  # here. The archive module's output is far too verbose to
+  # be practical when debugging.
+  - name: Package venv
+    shell: |
+      tar czf '{{ venv_destination_path }}.tgz' -C '{{ venv_destination_path }}' .
+    args:
+      chdir: "{{ venv_destination_path }}"
+      executable: /bin/bash
+      warn: no
+    register: _venv_package_build
+    when:
+      - _install_venv_pip_packages is mapping
+      - _install_venv_pip_packages | changed
+
+  - name: Prepare checksum for packaged venv
+    shell: |
+      sha1sum '{{ venv_destination_path }}.tgz' | awk '{print $1}' > '{{ venv_destination_path }}.checksum'
+    args:
+      executable: /bin/bash
+    when:
+      - _venv_package_build is mapping
+      - _venv_package_build | changed
+
+  - name: Copy the packaged venv and checksum file to the deployment host
+    fetch:
+      src: "{{ item.src }}"
+      dest: "{{ item.dest }}"
+      flat: yes
+    with_items:
+      - src: "{{ venv_destination_path }}.tgz"
+        dest: "{{ venv_reuse_download_path }}/{{ venv_destination_path }}.tgz"
+      - src: "{{ venv_destination_path }}.checksum"
+        dest: "{{ venv_reuse_download_path }}/{{ venv_destination_path }}.checksum"
+    when:
+      - _venv_package_build is mapping
+      - _venv_package_build | changed

tasks/python_venv_install.yml

@@ -0,0 +1,53 @@
+---
+# Copyright 2018, Rackspace US, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+- name: Copy the venv checksum file to the target host
+  copy:
+    src: "{{ venv_reuse_download_path }}/{{ venv_destination_path }}.checksum"
+    dest: "{{ venv_destination_path | dirname }}"
+  register: _venv_checksum_copy
+    - _src_venv_present.stat.exists | bool
+
+- name: Remove existing venv on target host if it is changing
+  file:
+    path: "{{ venv_destination_path }}"
+    state: absent
+    - _venv_checksum_copy is mapping
+    - _venv_checksum_copy | changed
+
+- name: Create venv directory on the target host
+  file:
+    path: "{{ venv_destination_path }}"
+    state: directory
+
+- name: Unarchive pre-built venv
+  unarchive:
+    src: "{{ venv_reuse_download_path }}/{{ venv_destination_path }}.tgz"
+    dest: "{{ venv_destination_path }}"
+    remote_src: no
+    - _venv_checksum_copy is mapping
+    - _venv_checksum_copy | changed
+  notify:
+    - venv changed
+
+- name: Update virtualenv python and paths
+  shell: |
+    sed -si '1s/^.*python.*$/#!{{ (venv_destination_path ~ '/bin') | replace ('/','\/') }}\/python/' {{ venv_destination_path }}/bin/*
+    virtualenv {{ venv_destination_path }}
+  when:
+    - _venv_checksum_copy is mapping
+    - _venv_checksum_copy | changed
+  tags:
+    - skip_ansible_lint

tasks/python_venv_preflight.yml

@@ -0,0 +1,43 @@
+---
+# Copyright 2018, Rackspace US, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+- name: Verify that venv_destination_path has been provided
+  fail:
+    msg: |
+      The variable venv_destination_path is required and
+      has not been set
+  when:
+    - venv_destination_path is not defined
+
+- name: Check if venv tgz is present on the deployment host
+  stat:
+    path: "{{ venv_reuse_download_path }}/{{ venv_destination_path }}.tgz"
+    get_attributes: no
+    get_checksum: no
+    get_md5: no
+    get_mime: no
+  register: _src_venv_present
+  delegate_to: localhost
+  run_once: yes
+
+- name: Ensure that venv_reuse_download_path exists on the deployment host
+  file:
+    path: "{{ venv_reuse_download_path }}/{{ venv_destination_path | dirname }}"
+    state: directory
+    owner: "{{ venv_reuse_download_path_owner }}"
+  delegate_to: localhost
+  run_once: yes
+  when:
+    - venv_reuse_enable | bool

tasks/python_venv_set_facts.yml

@@ -0,0 +1,34 @@
+---
+# Copyright 2018, Rackspace US, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+- name: Ensure local facts folder exists
+  file:
+    path: /etc/ansible/facts.d
+    state: directory
+
+- name: Record the necessary facts
+  ini_file:
+    dest: "/etc/ansible/facts.d/{{ venv_facts_dest }}.fact"
+    section: "{{ item.section }}"
+    option: "{{ item.option }}"
+    value: "{{ item.value }}"
+  with_items: "{{ venv_facts_when_changed }}"
+  when:
+    - (_venv_checksum_copy is defined and
+       _venv_checksum_copy is mapping and
+       _venv_checksum_copy | changed) or
+      (_install_venv_pip_packages is defined and
+       _install_venv_pip_packages is mapping and
+       _install_venv_pip_packages | changed)