openstack/ansible-role-python_venv_build
Code Issues Proposed changes

Simplify builds/installs by eliminating venv packaging

Browse Source 
Python venvs are not particularly portable. To overcome this
problem we have a bunch of stuff implemented to fix them
after we unpack them. This ranges from relatively simple
things like re-implementing python in the venv and changing
the shebangs for everything in it, to more complex things
like having to package a venv per distro/architecture.

All of this can be eliminated by simplifying the mechanism
into just creating the venv on the target host, and installing
the python packages into it. To help speed up the build, we
can simply build wheels before hand and store them on a web
server somewhere.

This patch implements the changes to:

1. Do away with packaging the venv.
2. Keep it simple. We install into the venv, and that's that.
3. Add a toggle to rebuild the venv if you'd like to.
4. Use import_tasks with tags for each stage so that it's
   easy to skip a portion of what executes.

Change-Id: I708b5cf32e5cce6a18624d0b3be0cd4c828ad389
This commit is contained in:
Jesse Pretorius 2018-08-31 14:39:18 +01:00 committed by Jesse Pretorius (odyssey4me)
parent 54b46788b3
commit ac5e5e9283
10 changed files with 341 additions and 538 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

42
defaults/main.yml
View File

@ -53,33 +53,31 @@ venv_pip_install_args: ""
# The python executable to use for creating the venv
venv_python_executable: "python2"
# Enable the ability to *only* do the build, so that
# the build host can be pre-populated with the venvs.
venv_build_only: no
# Enable the recreation of the venv from scratch.
# This is useful if you think the venv may be corrupted
# or if you have changed options which means that packages
# should be removed from the venv.
# Under normal circumstances, the installs will be done
# into the existing venv over the top of any previously
# installed packages.
venv_rebuild: no
# Enable the packaging of venvs, so that they can
# be used for installation.
venv_build_package: yes
# Set the host where the wheels will be built.
# If this host is not the same as the target host, then
# python wheels will be built in order to speed up the
# subsequent venv builds on this host and others. When
# this is the same as the target host, then we will not
# bother building wheels.
venv_build_host: "{{ ((groups['repo_all'] is defined) and (groups['repo_all'] | length > 0)) | ternary(groups.get('repo_all')[0], inventory_hostname) }}"
# Enable the use of wheels to speed up the venv
# build process, especially when building multiple
# venvs.
venv_build_wheels: yes
# The path for the wheel build venv.
# This is the path where a venv will be created on the
# build host for the purpose of building the wheels.
venv_build_host_venv_path: "/openstack/venvs/wheel-builder"
# The path where the wheels are cached on the build host
# for speeding up the build process.
venv_build_wheel_path: "{{ lookup('env', 'HOME') }}/archive/wheels"
# The path where the venvs are archived on the build host
# to be reused for installations.
venv_build_archive_path: "{{ lookup('env', 'HOME') }}/archive/venvs"
# The path where a built venv is sourced from when
# installing. If the path set begins with https(s)://
# then the install process will recognise the source
# is a web server. Otherwise the path is assumed to
# be on the deployment host.
venv_install_source_path: "{{ lookup('env', 'HOME') }}/archive/venvs"
venv_build_host_wheel_path: "{{ repo_pypiserver_package_path | default('/var/www/repo/pools') }}"
# The facts to set when the venv changes during a
# build, or the installation of a venv.

36
tasks/main.yml
View File

@ -13,35 +13,19 @@
# See the License for the specific language governing permissions and
# limitations under the License.
- include_tasks: "python_venv_preflight.yml"
- import_tasks: "python_venv_preflight.yml"
tags:
- always
# To ensure that only the first host in every distro/architecture
# group does the build, and the others make use of that build,
# without forcing the use of serialisation, we use a dynamic
# group based on the venv_distro_arch_grouping which combines
# the distribution/architecture and is therefore unique for mixed
# environments.
- name: Add hosts to dynamic inventory group to ensure build/install serialization
group_by:
key: "{{ venv_distro_arch_grouping }}"
- import_tasks: "python_venv_wheel_build.yml"
run_once: yes
tags:
- always
- build
- include_tasks: "python_venv_build.yml"
when:
- (venv_build_only | bool) or
((_venv_source == 'file') and
(not _src_venv_present.stat.exists | bool))
- inventory_hostname == groups[venv_distro_arch_grouping][0]
- import_tasks: "python_venv_install.yml"
tags:
- install
- include_tasks: "python_venv_install.yml"
when:
- not venv_build_only | bool
- inventory_hostname != groups[venv_distro_arch_grouping][0]
- include_tasks: "python_venv_set_facts.yml"
when:
- not venv_build_only | bool
- venv_facts_when_changed != []
- import_tasks: "python_venv_set_facts.yml"
tags:
- install

164
tasks/python_venv_build.yml
View File

@ -1,164 +0,0 @@
---
# Copyright 2018, Rackspace US, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
- name: Install distro packages for venv build
package:
name: "{{ venv_build_distro_package_list + venv_install_distro_package_list }}"
state: "{{ venv_distro_package_state }}"
update_cache: "{{ (ansible_pkg_mgr in ['apt', 'zypper']) | ternary('yes', omit) }}"
cache_valid_time: "{{ (ansible_pkg_mgr == 'apt') | ternary(venv_distro_cache_valid_time, omit) }}"
when:
- (venv_build_distro_package_list | length > 0) or
(venv_install_distro_package_list | length > 0)
register: _install_distro_packages
until: _install_distro_packages is success
retries: 5
delay: 2
- name: Ensure a fresh venv_install_destination_path if venv_build_only is enabled
file:
path: "{{ venv_install_destination_path }}"
state: absent
when:
- venv_build_only | bool
- name: Create wheel/venv directories on the target host
file:
path: "{{ item }}"
state: directory
with_items:
- "{{ venv_build_archive_path }}"
- "{{ venv_build_wheel_path }}"
# NOTE(odyssey4me):
# Not using --always-copy for CentOS/SuSE due to
# https://github.com/pypa/virtualenv/issues/565
- name: Create the virtualenv (if it does not exist)
command: >-
virtualenv
{{ _venv_create_extra_options }}
--python={{ venv_python_executable }}
{{ (ansible_pkg_mgr == 'apt') | ternary('--always-copy', '') }}
{{ venv_install_destination_path }}
args:
creates: "{{ venv_install_destination_path }}/bin/activate"
- name: Upgrade pip/setuptools/wheel to the versions we want
pip:
name:
- pip
- setuptools
- wheel
state: "{{ venv_pip_package_state }}"
virtualenv: "{{ venv_install_destination_path }}"
extra_args: >-
--find-links {{ venv_build_wheel_path }}/
--log /var/log/python_venv_build.log
{{ venv_pip_install_args }}
register: _update_virtualenv_packages
until: _update_virtualenv_packages is success
retries: 5
delay: 2
- name: Build wheels for the packages to be installed into the venv
command: >-
{{ venv_install_destination_path }}/bin/pip wheel
--wheel-dir {{ venv_build_wheel_path }}/
--find-links {{ venv_build_wheel_path }}/
--log /var/log/python_wheel_build.log
{{ venv_pip_install_args }}
{{ venv_pip_packages | join(' ') }}
when:
- venv_build_wheels | bool
- name: Build venv
pip:
name: "{{ venv_pip_packages }}"
state: "{{ venv_pip_package_state }}"
virtualenv: "{{ venv_install_destination_path }}"
extra_args: >-
--find-links {{ venv_build_wheel_path }}/
--log /var/log/python_venv_build.log
{{ venv_pip_install_args }}
register: _install_venv_pip_packages
until: _install_venv_pip_packages is success
retries: 5
delay: 2
notify:
- venv changed
- name: Package the venv when venv_build_package is enabled
when:
- venv_build_package | bool
block:
- name: Clean up the virtualenv before packaging
shell: |
find {{ venv_install_destination_path }}/bin -type f -name '*.pyc' -delete
when:
- _install_venv_pip_packages is mapping
- _install_venv_pip_packages is changed
# Note(odyssey4me):
# We purposefully use shell instead of the archive module
# here. The archive module's output is far too verbose to
# be practical when debugging.
- name: Package venv
command: |
tar czf '{{ venv_build_archive_path }}/{{ venv_install_destination_path | basename }}.tgz' -C '{{ venv_install_destination_path }}' .
args:
chdir: "{{ venv_install_destination_path }}"
warn: no
register: _venv_package_build
when:
- _install_venv_pip_packages is mapping
- _install_venv_pip_packages is changed
- name: Prepare checksum for packaged venv
shell: |
sha1sum '{{ venv_build_archive_path }}/{{ venv_install_destination_path | basename }}.tgz' | awk '{print $1}' > '{{ venv_build_archive_path }}/{{ venv_install_destination_path | basename }}.checksum'
args:
executable: /bin/bash
when:
- _venv_package_build is mapping
- _venv_package_build is changed
- name: Ensure that venv_install_source_path exists on the deployment host
file:
path: "{{ venv_install_source_path }}"
state: directory
owner: "{{ lookup('env', 'USER') }}"
recurse: yes
delegate_to: localhost
run_once: yes
when:
- _venv_source == 'file'
- inventory_hostname != 'localhost'
- name: Copy the packaged venv and checksum file to the deployment host
fetch:
src: "{{ item.src }}"
dest: "{{ item.dest }}"
flat: yes
with_items:
- src: "{{ venv_build_archive_path }}/{{ venv_install_destination_path | basename }}.tgz"
dest: "{{ venv_install_source_path }}/{{ venv_install_destination_path | basename }}.tgz"
- src: "{{ venv_build_archive_path }}/{{ venv_install_destination_path | basename }}.checksum"
dest: "{{ venv_install_source_path }}/{{ venv_install_destination_path | basename }}.checksum"
when:
- _venv_source == 'file'
- _venv_package_build is mapping
- _venv_package_build is changed
- inventory_hostname != 'localhost'

137
tasks/python_venv_install.yml
View File

@ -13,103 +13,72 @@
# See the License for the specific language governing permissions and
# limitations under the License.
- name: Install distro packages for venv install
# TODO(odyssey4me):
# Set a fact for the selective inclusion of the build package list.
# Perhaps do this if the distro/architecture of the target host differs
# from the build host.
- name: Install distro packages for venv build
package:
name: "{{ venv_install_distro_package_list }}"
name: "{{ venv_build_distro_package_list + venv_install_distro_package_list }}"
state: "{{ venv_distro_package_state }}"
update_cache: "{{ (ansible_pkg_mgr in ['apt', 'zypper']) | ternary('yes', omit) }}"
cache_valid_time: "{{ (ansible_pkg_mgr == 'apt') | ternary(venv_distro_cache_valid_time, omit) }}"
when:
- venv_install_distro_package_list | length > 0
- (venv_build_distro_package_list | length > 0) or
(venv_install_distro_package_list | length > 0)
register: _install_distro_packages
until: _install_distro_packages is success
until: _install_distro_packages is success
retries: 5
delay: 2
- name: Create venv parent directories on the target host
file:
path: "{{ venv_install_destination_path | dirname }}"
state: directory
- name: Copy the venv archive checksum
copy:
src: "{{ venv_install_source_path }}/{{ venv_install_destination_path | basename }}.checksum"
dest: "{{ venv_install_destination_path | dirname }}/"
register: _venv_checksum_copy
when:
- _venv_source == 'file'
- name: Copy the venv archive
copy:
src: "{{ venv_install_source_path }}/{{ venv_install_destination_path | basename }}.tgz"
dest: "{{ venv_install_destination_path | dirname }}/"
when:
- _venv_source == 'file'
- name: Download the venv archive checksum
uri:
url: "{{ venv_install_source_path }}/{{ venv_install_destination_path | basename }}.checksum"
dest: "{{ venv_install_destination_path | dirname }}/"
return_content: yes
register: _venv_checksum_download
when:
- _venv_source == 'url'
- name: Download the venv archive
get_url:
url: "{{ venv_install_source_path }}/{{ venv_install_destination_path | basename }}.tgz"
dest: "{{ venv_install_destination_path | dirname }}/"
checksum: "sha1:{{ _venv_checksum_download.content | trim }}"
when:
- _venv_source == 'url'
- name: Remove existing venv on target host if it is changing
- name: Ensure a fresh venv_install_destination_path if venv_rebuild is enabled
file:
path: "{{ venv_install_destination_path }}"
state: absent
when:
- (_venv_checksum_copy is mapping and _venv_checksum_copy | changed) or
(_venv_checksum_download is mapping and _venv_checksum_download | changed)
- name: Create venv directory on the target host
file:
path: "{{ venv_install_destination_path }}"
state: directory
register: _venv_dir_create
- name: Unarchive pre-built venv
unarchive:
src: "{{ venv_install_destination_path }}.tgz"
dest: "{{ venv_install_destination_path }}"
remote_src: yes
when:
- (_venv_checksum_copy is mapping and _venv_checksum_copy | changed) or
(_venv_checksum_download is mapping and _venv_checksum_download | changed) or
_venv_dir_create is changed
notify:
- venv changed
- venv_rebuild | bool
# NOTE(odyssey4me):
# We reinitialize the venv to ensure that the right
# version of python is in the venv, but we do not
# want virtualenv to also replace pip, setuptools
# and wheel so we tell it not to.
# We do not use --always-copy for CentOS/SuSE due
# to https://github.com/pypa/virtualenv/issues/565
- name: Update virtualenv python and paths
shell: |
sed -si '1s/^.*python.*$/#!{{ (venv_install_destination_path ~ '/bin') | replace ('/','\/') }}\/python/' {{ venv_install_destination_path }}/bin/*
virtualenv {{ venv_install_destination_path }} \
{{ (ansible_pkg_mgr == 'apt') | ternary('--always-copy', '') }} \
--no-pip \
--no-setuptools \
--no-wheel
# Not using --always-copy for CentOS/SuSE due to
# https://github.com/pypa/virtualenv/issues/565
- name: Create the virtualenv (if it does not exist)
command: >-
virtualenv
{{ _venv_create_extra_options }}
--python={{ venv_python_executable }}
{{ (ansible_pkg_mgr == 'apt') | ternary('--always-copy', '') }}
{{ venv_install_destination_path }}
args:
executable: /bin/bash
warn: no
when:
- (_venv_checksum_copy is mapping and _venv_checksum_copy | changed) or
(_venv_checksum_download is mapping and _venv_checksum_download | changed) or
_venv_dir_create is changed
tags:
- skip_ansible_lint
creates: "{{ venv_install_destination_path }}/bin/activate"
- name: Upgrade pip/setuptools/wheel to the versions we want
pip:
name:
- pip
- setuptools
- wheel
state: "{{ venv_pip_package_state }}"
virtualenv: "{{ venv_install_destination_path }}"
extra_args: >-
--log /var/log/python_venv_build.log
{{ venv_pip_install_args }}
register: _update_virtualenv_packages
until: _update_virtualenv_packages is success
retries: 5
delay: 2
- name: Install python packages into the venv
pip:
name: "{{ venv_pip_packages }}"
state: "{{ venv_pip_package_state }}"
virtualenv: "{{ venv_install_destination_path }}"
extra_args: >-
--log /var/log/python_venv_build.log
{{ venv_pip_install_args }}
register: _install_venv_pip_packages
until: _install_venv_pip_packages is success
retries: 5
delay: 2
notify:
- venv changed

17
tasks/python_venv_preflight.yml
View File

@ -45,20 +45,3 @@
_venv_create_extra_options: >-
{{ ((_virtualenv_version.stdout | trim) is version_compare('14.0.0', '<')) | ternary('--never-download', '--no-download') }}
{{ ((_virtualenv_version.stdout | trim) is version_compare('1.7.0', '<')) | ternary('--no-site-packages', '') }}
- name: Check whether the venv_install_source_path is a URL or a file path
set_fact:
_venv_source: "{{ ((venv_install_source_path | trim) is match('^https?://.*')) | ternary('url', 'file') }}"
- name: Check if venv tgz is present on the deployment host
stat:
path: "{{ venv_install_source_path }}/{{ venv_install_destination_path | basename }}.tgz"
get_attributes: no
get_checksum: no
get_md5: no
get_mime: no
register: _src_venv_present
delegate_to: localhost
run_once: yes
when:
- _venv_source == 'file'

9
tasks/python_venv_set_facts.yml
View File

@ -26,12 +26,7 @@
value: "{{ item.value }}"
with_items: "{{ venv_facts_when_changed }}"
when:
- (_venv_checksum_copy is defined and
_venv_checksum_copy is mapping and
_venv_checksum_copy | changed) or
(_venv_checksum_download is defined and
_venv_checksum_download is mapping and
_venv_checksum_download | changed) or
(_install_venv_pip_packages is defined and
- venv_facts_when_changed != []
- (_install_venv_pip_packages is defined and
_install_venv_pip_packages is mapping and
_install_venv_pip_packages | changed)

85
tasks/python_venv_wheel_build.yml Normal file
View File

@ -0,0 +1,85 @@
---
# Copyright 2018, Rackspace US, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
- name: Build the wheels on the build host
delegate_to: "{{ venv_build_host }}"
become: "{{ venv_build_host == 'localhost' }}"
when:
- venv_build_host != inventory_hostname
block:
- name: Install distro packages for wheel build
package:
name: "{{ venv_build_distro_package_list + venv_install_distro_package_list }}"
state: "{{ venv_distro_package_state }}"
update_cache: "{{ (ansible_pkg_mgr in ['apt', 'zypper']) | ternary('yes', omit) }}"
cache_valid_time: "{{ (ansible_pkg_mgr == 'apt') | ternary(venv_distro_cache_valid_time, omit) }}"
when:
- (venv_build_distro_package_list | length > 0) or
(venv_install_distro_package_list | length > 0)
register: _install_build_distro_packages
until: _install_build_distro_packages is success
retries: 5
delay: 2
- name: Ensure a fresh venv_build_host_venv_path if venv_rebuild is enabled
file:
path: "{{ venv_build_host_venv_path }}"
state: absent
when:
- venv_rebuild | bool
- name: Create wheel directory on the build host
file:
path: "{{ venv_build_host_wheel_path }}"
state: directory
# NOTE(odyssey4me):
# Not using --always-copy for CentOS/SuSE due to
# https://github.com/pypa/virtualenv/issues/565
- name: Create the wheel build virtualenv (if it does not exist)
command: >-
virtualenv
{{ _venv_create_extra_options }}
--python={{ venv_python_executable }}
{{ (ansible_pkg_mgr == 'apt') | ternary('--always-copy', '') }}
{{ venv_build_host_venv_path }}
args:
creates: "{{ venv_build_host_venv_path }}/bin/activate"
- name: Upgrade the wheel build virtualenv pip/setuptools/wheel to the versions we want
pip:
name:
- pip
- setuptools
- wheel
state: "{{ venv_pip_package_state }}"
virtualenv: "{{ venv_build_host_venv_path }}"
extra_args: >-
--find-links {{ venv_build_host_wheel_path }}/
--log /var/log/python_venv_build.log
{{ venv_pip_install_args }}
register: _update_virtualenv_packages
until: _update_virtualenv_packages is success
retries: 5
delay: 2
- name: Build wheels for the packages to be installed into the venv
command: >-
{{ venv_build_host_venv_path }}/bin/pip wheel
--wheel-dir {{ venv_build_host_wheel_path }}/
--find-links {{ venv_build_host_wheel_path }}/
--log /var/log/python_wheel_build.log
{{ venv_pip_install_args }}
{{ venv_pip_packages | join(' ') }}

170
tests/test-functional.yml
View File

@ -1,170 +0,0 @@
---
# Copyright 2018, Rackspace US, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
- name: Clean up from previous tests
hosts: "{{ build_host }}:{{ install_hosts }}"
become: yes
any_errors_fatal: yes
tasks:
- name: Clean up files/folders from previous tests
file:
path: "{{ item.path }}"
state: absent
with_items:
- path: "{{ lookup('env', 'HOME') }}/archive"
- path: "{{ lookup('env', 'HOME') }}/venvs"
- name: Clean up facts from previous tests
ini_file:
path: "/etc/ansible/facts.d/openstack_ansible.fact"
section: "{{ item }}"
state: absent
with_items: "{{ build_host.split(':') + install_hosts.split(':') }}"
- name: Refresh the inventory to clear the added groups
meta: refresh_inventory
- name: Set venv_build_archive_path and venv_install_source_path
set_fact:
venv_build_archive_path: >-
{%- if build_host == "container1" %}
{%- if ansible_distribution == "Ubuntu" %}
{%- set _path = "/var/www/html" %}
{%- elif ansible_distribution == "CentOS" %}
{%- set _path = "/usr/share/nginx/html" %}
{%- else %}
{%- set _path = "/srv/www/htdocs" %}
{%- endif %}
{%- else -%}
{%- set _path = lookup('env', 'HOME') ~ "/archive/venvs" -%}
{%- endif -%}
{{- _path }}
venv_install_source_path: >-
{%- if build_host == "container1" -%}
http://{{ hostvars['container1'].ansible_default_ipv4.address }}
{%- else -%}
{{- lookup('env', 'HOME') }}/archive/venvs
{%- endif -%}
- name: Setup web server for url-based venv install
when:
- inventory_hostname == 'container1'
block:
- name: Install EPEL gpg keys
rpm_key:
key: "http://dl.fedoraproject.org/pub/epel/RPM-GPG-KEY-EPEL-7"
state: present
when:
- ansible_pkg_mgr in ['yum', 'dnf']
register: _add_yum_keys
until: _add_yum_keys is success
retries: 5
delay: 2
- name: Install the EPEL repository
yum_repository:
name: epel-nginx
baseurl: "{{ (centos_epel_mirror | default ('http://download.fedoraproject.org/pub/epel')) ~ '/' ~ ansible_distribution_major_version ~ '/' ~ ansible_architecture }}"
description: 'Extra Packages for Enterprise Linux 7 - $basearch'
gpgcheck: yes
enabled: yes
state: present
includepkgs: 'nginx*'
when:
- ansible_pkg_mgr in ['yum', 'dnf']
register: install_epel_repo
until: install_epel_repo is success
retries: 5
delay: 2
- name: Install distro packages
package:
name: "nginx"
update_cache: "{{ (ansible_pkg_mgr in ['apt', 'zypper']) | ternary('yes', omit) }}"
register: install
until: install is success
retries: 5
delay: 2
- name: Enable and start nginx
service:
name: nginx
enabled: yes
daemon_reload: yes
state: restarted
- name: Execute build
hosts: "{{ build_host }}"
become: yes
any_errors_fatal: yes
tasks:
- name: Execute venv build
include_role:
name: "python_venv_build"
private: yes
vars:
venv_build_only: yes
- name: Execute install
hosts: "{{ install_hosts }}"
become: yes
any_errors_fatal: yes
tasks:
- name: Execute venv install
include_role:
name: "python_venv_build"
private: yes
vars:
venv_facts_when_changed:
- section: "{{ inventory_hostname }}"
option: "test"
value: True
- name: refresh local facts
setup:
filter: ansible_local
gather_subset: "!all"
- name: Show the ansible_local facts
debug:
var: ansible_local
- name: Verify that the facts were set
assert:
that:
- ansible_local['openstack_ansible'][inventory_hostname]['test'] | bool
- name: Find files/folders on targets
find:
file_type: directory
get_checksum: no
recurse: no
paths:
- "{{ venv_install_destination_path | dirname }}"
register: _target_folders
- name: Compile the folder list from the targets
set_fact:
_target_folder_list: "{{ _target_folders['files'] | map(attribute='path') | list }}"
- name: Show the files/folder from the targets
debug:
var: _target_folder_list
- name: Verify the folder list from the targets
assert:
that:
- "{{ venv_install_destination_path in _target_folder_list }}"

200
tests/test.yml
View File

@ -16,38 +16,180 @@
- name: Prepare the host/containers
import_playbook: common/test-setup-host.yml
- name: Verify building on a web server, and installing using a URL
import_playbook: test-functional.yml
vars:
build_host: container1
install_hosts: "container2:container3"
venv_pip_packages:
- "Jinja2==2.10"
venv_install_destination_path: "{{ lookup('env', 'HOME') }}/venvs/test-venv"
- name: Prepare web server on localhost to serve python packages
hosts: localhost
connection: local
become: yes
any_errors_fatal: yes
tasks:
- name: Set venv_build_archive_path and venv_install_source_path
set_fact:
venv_build_host_wheel_path: >-
{%- if ansible_distribution == "Ubuntu" %}
{%- set _path = "/var/www/html" %}
{%- elif ansible_distribution == "CentOS" %}
{%- set _path = "/usr/share/nginx/html" %}
{%- else %}
{%- set _path = "/srv/www/htdocs" %}
{%- endif %}
{{- _path }}
- name: Verify building on localhost, and installing using a copy
import_playbook: test-functional.yml
vars:
build_host: localhost
install_hosts: "container1:container2:container3"
venv_pip_packages:
- "Jinja2==2.10"
venv_install_destination_path: "{{ lookup('env', 'HOME') }}/venvs/test-venv"
- name: Install EPEL gpg keys
rpm_key:
key: "http://dl.fedoraproject.org/pub/epel/RPM-GPG-KEY-EPEL-7"
state: present
when:
- ansible_pkg_mgr in ['yum', 'dnf']
register: _add_yum_keys
until: _add_yum_keys is success
retries: 5
delay: 2
- name: Verify building on a build host, and installing using a copy
import_playbook: test-functional.yml
vars:
build_host: container2
install_hosts: "container1:container3"
venv_pip_packages:
- "Jinja2==2.10"
venv_install_destination_path: "{{ lookup('env', 'HOME') }}/venvs/test-venv"
- name: Install the EPEL repository
yum_repository:
name: epel-nginx
baseurl: "{{ (centos_epel_mirror | default ('http://download.fedoraproject.org/pub/epel')) ~ '/' ~ ansible_distribution_major_version ~ '/' ~ ansible_architecture }}"
description: 'Extra Packages for Enterprise Linux 7 - $basearch'
gpgcheck: yes
enabled: yes
state: present
includepkgs: 'nginx*'
when:
- ansible_pkg_mgr in ['yum', 'dnf']
register: install_epel_repo
until: install_epel_repo is success
retries: 5
delay: 2
- name: Verify setting multiple build hosts behaves correctly
import_playbook: test-functional.yml
- name: Install distro packages
package:
name: "nginx"
update_cache: "{{ (ansible_pkg_mgr in ['apt', 'zypper']) | ternary('yes', omit) }}"
register: install
until: install is success
retries: 5
delay: 2
- name: Enable and start nginx
service:
name: nginx
enabled: yes
daemon_reload: yes
state: restarted
- name: Verify not using a build host
hosts: "container1"
remote_user: root
any_errors_fatal: yes
vars:
build_host: "localhost:container1"
install_hosts: "container2:container3"
venv_pip_packages:
- "Jinja2==2.10"
venv_install_destination_path: "{{ lookup('env', 'HOME') }}/venvs/test-venv"
venv_install_destination_path: "/openstack/venvs/test-venv"
tasks:
- name: Execute venv install
include_role:
name: "python_venv_build"
private: yes
vars:
venv_facts_when_changed:
- section: "{{ inventory_hostname }}"
option: "test"
value: True
- name: refresh local facts
setup:
filter: ansible_local
gather_subset: "!all"
- name: Show the ansible_local facts
debug:
var: ansible_local
- name: Verify that the facts were set
assert:
that:
- ansible_local['openstack_ansible'][inventory_hostname]['test'] | bool
- name: Find files/folders on targets
find:
file_type: directory
get_checksum: no
recurse: no
paths:
- "{{ venv_install_destination_path | dirname }}"
register: _target_folders
- name: Compile the folder list from the targets
set_fact:
_target_folder_list: "{{ _target_folders['files'] | map(attribute='path') | list }}"
- name: Show the files/folder from the targets
debug:
var: _target_folder_list
- name: Verify the folder list from the targets
assert:
that:
- "{{ venv_install_destination_path in _target_folder_list }}"
- name: Verify using a build host
hosts: "container2:container3"
remote_user: root
any_errors_fatal: yes
vars:
venv_pip_packages:
- "Jinja2==2.10"
venv_install_destination_path: "/openstack/venvs/test-venv"
venv_pip_install_args: >-
--find-links http://{{ hostvars['localhost'].ansible_default_ipv4.address }}
--trusted-host {{ hostvars['localhost'].ansible_default_ipv4.address }}
venv_build_host: localhost
venv_build_host_wheel_path: "{{ hostvars['localhost']['venv_build_host_wheel_path'] }}"
tasks:
- name: Execute venv install
include_role:
name: "python_venv_build"
private: yes
vars:
venv_facts_when_changed:
- section: "{{ inventory_hostname }}"
option: "test"
value: True
- name: refresh local facts
setup:
filter: ansible_local
gather_subset: "!all"
- name: Show the ansible_local facts
debug:
var: ansible_local
- name: Verify that the facts were set
assert:
that:
- ansible_local['openstack_ansible'][inventory_hostname]['test'] | bool
- name: Find files/folders on targets
find:
file_type: directory
get_checksum: no
recurse: no
paths:
- "{{ venv_install_destination_path | dirname }}"
register: _target_folders
- name: Compile the folder list from the targets
set_fact:
_target_folder_list: "{{ _target_folders['files'] | map(attribute='path') | list }}"
- name: Show the files/folder from the targets
debug:
var: _target_folder_list
- name: Verify the folder list from the targets
assert:
that:
- "{{ venv_install_destination_path in _target_folder_list }}"

19
vars/main.yml
View File

@ -1,19 +0,0 @@
---
# Copyright 2018, Rackspace US, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# NOTE(hwoarang): ansible_distribution may return a string with spaces
# such as "openSUSE Leap" so we need to replace the space with underscore
# in order to create a more sensible repo name for the distro.
venv_distro_arch_grouping: "{{ (ansible_distribution | lower) | replace(' ', '_') }}-{{ ansible_distribution_version.split('.')[:2] | join('.') }}-{{ ansible_architecture | lower }}"