Ensure build/install works in serial/parallel execution

When executing the role in parallel, there is a race condition
due to all targets trying to build simultaneously and then all
trying to copy their packages back to the same folder on the
deployment host.

In this patch we implement the use of a dynamic group based on
the venv_reuse_download_subfolder which, by default, combines
the architecture and distribution data. We then use this group
to ensure that only the first of each group does the build and
the others consume the first host's package.

The conditional in tasks/python_venv_install.yml checking
whether the src venv is present was unnecessary given the same
conditional is present in tasks/main.yml. It interferes with
the new parallel execution capability, so it's been removed.

To validate whether builds happen properly, whether in serial
or parallel, and to also implement a way to implement tests for
future validations, we implement a test system using multiple
containers while allowing localhost to be the deployment host
only.

Change-Id: I29d230c16bee3e2d7e434f98c05af4a6a53ff10a

tasks/main.yml

@@ -17,16 +17,28 @@
   tags:
     - always
 
+# To ensure that only the first host in every distro/architecture
+# group does the build, and the others make use of that build,
+# without forcing the use of serialisation, we use a dynamic
+# group based on the venv_reuse_download_subfolder which combines
+# the distribution/architecture and is therefore unique for mixed
+# environments.
+- name: Add hosts to dynamic inventory group to ensure build/install serialization
+  group_by:
+    key: "{{ venv_reuse_download_subfolder }}"
+
 - include_tasks: "python_venv_build.yml"
   when:
     - (not _src_venv_present.stat.exists | bool) or
       (not venv_reuse_enable | bool)
+    - inventory_hostname == groups[venv_reuse_download_subfolder][0]
 
 - include_tasks: "python_venv_install.yml"
   when:
     - venv_reuse_enable | bool
     - not venv_reuse_build_only | bool
-    - _src_venv_present.stat.exists | bool
+    - (_src_venv_present.stat.exists | bool) or
+      inventory_hostname != groups[venv_reuse_download_subfolder][0]
 
 - include_tasks: "python_venv_set_facts.yml"
   when:

tasks/python_venv_install.yml

@@ -18,8 +18,6 @@
     src: "{{ venv_reuse_download_path }}/{{ venv_destination_path | basename }}.checksum"
     dest: "{{ venv_destination_path | dirname }}/"
   register: _venv_checksum_copy
-  when:
-    - _src_venv_present.stat.exists | bool
 
 - name: Remove existing venv on target host if it is changing
   file:

tests/ansible-role-requirements.yml

@@ -0,0 +1,20 @@
+- name: apt_package_pinning
+  src: https://git.openstack.org/openstack/openstack-ansible-apt_package_pinning
+  scm: git
+  version: master
+- name: pip_install
+  src: https://git.openstack.org/openstack/openstack-ansible-pip_install
+  scm: git
+  version: master
+- name: openstack_hosts
+  src: https://git.openstack.org/openstack/openstack-ansible-openstack_hosts
+  scm: git
+  version: master
+- name: lxc_hosts
+  src: https://git.openstack.org/openstack/openstack-ansible-lxc_hosts
+  scm: git
+  version: master
+- name: lxc_container_create
+  src: https://git.openstack.org/openstack/openstack-ansible-lxc_container_create
+  scm: git
+  version: master

tests/group_vars/all_containers.yml

@@ -0,0 +1,31 @@
+---
+# Copyright 2018, Rackspace US, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+container_name: "{{ inventory_hostname }}"
+container_networks:
+  management_address:
+    address: "{{ ansible_host }}"
+    bridge: "br-mgmt"
+    interface: "eth1"
+    netmask: "255.255.252.0"
+    type: "veth"
+physical_host: localhost
+properties:
+  service_name: "{{ inventory_hostname }}"
+
+# NOTE(cloudnull): The lxc-openstack AA profile for is used to ensure general
+#                  container functionality typical to the integrated build.
+lxc_container_config_list:
+  - "lxc.aa_profile=lxc-openstack"

tests/host_vars/container1.yml

@@ -0,0 +1,18 @@
+---
+# Copyright 2018, Rackspace US, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+ansible_host: 10.1.0.2
+ansible_become: True
+ansible_user: root

tests/host_vars/container2.yml

@@ -0,0 +1,18 @@
+---
+# Copyright 2018, Rackspace US, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+ansible_host: 10.1.0.3
+ansible_become: True
+ansible_user: root

tests/host_vars/container3.yml

@@ -0,0 +1,18 @@
+---
+# Copyright 2018, Rackspace US, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+ansible_host: 10.1.0.4
+ansible_become: True
+ansible_user: root

tests/host_vars/container4.yml

@@ -0,0 +1,18 @@
+---
+# Copyright 2018, Rackspace US, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+ansible_host: 10.1.0.5
+ansible_become: True
+ansible_user: root

tests/host_vars/localhost.yml

@@ -1,5 +1,5 @@
 ---
-# Copyright 2016, Rackspace US, Inc.
+# Copyright 2018, Rackspace US, Inc.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -13,4 +13,8 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+bridges:
+  - name: "br-mgmt"
+    ip_addr: "10.1.0.1"
+
 ansible_python_interpreter: "/usr/bin/python2"

tests/inventory

@@ -1,2 +1,20 @@
+[all]
 localhost
+container1
+container2
+container3
+container4
 
+[all_containers]
+container1
+container2
+container3
+container4
+
+[test1]
+container1
+container2
+
+[test2]
+container3
+container4

tests/test-functional.yml

@@ -0,0 +1,138 @@
+---
+# Copyright 2018, Rackspace US, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+- name: Clean up from previous tests
+  hosts: "{{ target_host_group_name }}:localhost"
+  become: yes
+  any_errors_fatal: yes
+  tasks:
+    - name: Clean up files/folders from previous tests
+      file:
+        path: "{{ item }}"
+        state: absent
+      with_items:
+        - "{{ lookup('env', 'HOME') | default('/opt', true) }}/venvs"
+        - "{{ lookup('env', 'HOME') | default('/opt', true) }}/cache"
+
+    - name: Clean up facts from previous tests
+      ini_file:
+        path: "/etc/ansible/facts.d/openstack_ansible.fact"
+        section: "{{ target_host_group_name }}"
+        state: absent
+
+    - name: Refresh the inventory to clear the added groups
+      meta: refresh_inventory
+
+- name: Execute role test and verify target host content
+  hosts: "{{ target_host_group_name }}"
+  become: yes
+  any_errors_fatal: yes
+  serial: "{{ target_serial }}"
+  vars:
+    base_directory: "{{ lookup('env', 'HOME') | default('/opt', true) }}"
+  tasks:
+
+    - name: Execute role
+      include_role:
+        name: "python_venv_build"
+        private: yes
+      vars:
+        venv_pip_packages:
+          - "Jinja2==2.10"
+        venv_destination_path: "{{ base_directory }}/venvs/{{ target_host_group_name }}"
+        venv_facts_when_changed:
+          - section: "{{ target_host_group_name }}"
+            option: "test"
+            value: True
+
+    - name: refresh local facts
+      setup:
+        filter: ansible_local
+        gather_subset: "!all"
+
+    - name: Show the ansible_local facts
+      debug:
+        var: ansible_local
+
+    - name: Verify that the facts were set
+      assert:
+        that:
+          - ansible_local['openstack_ansible'][target_host_group_name]['test'] | bool
+
+    - name: Find files/folders on targets
+      find:
+        file_type: directory
+        get_checksum: no
+        recurse: no
+        paths:
+          - "{{ base_directory }}/venvs"
+      register: _target_folders
+
+    - name: Compile the folder list from the targets
+      set_fact:
+        _target_folder_list: >-
+          {%- set folder_list = [] %}
+          {%- for item in _target_folders['files'] %}
+          {%-   set _ = folder_list.append(item['path']) %}
+          {%- endfor %}
+          {{ folder_list }}
+
+    - name: Show the files/folder from the targets
+      debug:
+        var: _target_folder_list
+
+    - name: Verify the folder list from the targets
+      assert:
+        that:
+          - "{{ base_directory ~ '/venvs/' ~ target_host_group_name in _target_folder_list }}"
+
+- name: Verify deploy host content
+  hosts: localhost
+  connection: local
+  become: yes
+  any_errors_fatal: yes
+  vars:
+    base_directory: "{{ lookup('env', 'HOME') | default('/opt', true) }}"
+    sub_directory: "{{ (ansible_distribution | lower) | replace(' ', '_') }}-{{ ansible_distribution_version.split('.')[:2] | join('.') }}-{{ ansible_architecture | lower }}"
+    package_file_path: "{{ base_directory }}/cache/venvs/{{ sub_directory }}/{{ target_host_group_name }}"
+  tasks:
+
+    - name: Find files/folders on deploy host
+      find:
+        file_type: any
+        get_checksum: no
+        recurse: yes
+        paths:
+          - "{{ base_directory }}/cache"
+      register: _localhost_folders
+
+    - name: Compile the folder list from the deploy host
+      set_fact:
+        _localhost_folder_list: >-
+          {%- set folder_list = [] %}
+          {%- for item in _localhost_folders['files'] %}
+          {%-   set _ = folder_list.append(item['path']) %}
+          {%- endfor %}
+          {{ folder_list }}
+
+    - name: Show the files/folders from the deploy host
+      debug:
+        var: _localhost_folder_list
+
+    - name: Verify the folder list from the deploy host
+      assert:
+        that:
+          - "{{ package_file_path ~ '.tgz' in _localhost_folder_list }}"
+          - "{{ package_file_path ~ '.checksum' in _localhost_folder_list }}"

tests/test.yml

@@ -13,19 +13,17 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-- hosts: localhost
-  connection: local
-  user: root
-  become: true
-  tasks:
-    - name: Execute role
-      include_role:
-        name: "python_venv_build"
-        private: yes
-      vars:
-        venv_pip_packages: "jinja==1.2"
-        venv_destination_path: "{{ lookup('env', 'HOME') | default('/opt', true) }}/test_venv"
-        venv_facts_when_changed:
-          - section: "test"
-            option: "test_fact"
-            value: True
+- name: Prepare the host/containers
+  include: common/test-setup-host.yml
+
+- name: Verify that the role works with a serial playbook
+  include: test-functional.yml
+  vars:
+    target_host_group_name: test1
+    target_serial: 1
+
+- name: Verify that the role works with a parallel playbook
+  include: test-functional.yml
+  vars:
+    target_host_group_name: test2
+    target_serial: "100%"