Downloads amphora images from artifact storage

This will download a test amphora image per default form the
Open Stack artifact storage to speed up tests. Operators can
configure their won artifact storage to simplify deploys.

(cherry picked from commit 95eee6bc11e97105cb0356d7475bee699d404bda0)

Change-Id: I408d4128d35aab889dbe89fd9497d83a7830129b

defaults/main.yml

@@ -238,12 +238,21 @@ octavia_glance_image_tag: octavia-amphora-image
 octavia_amp_image_owner_id:
 # add here the glance image id if tagging is not used (not recommended for prod)
 octavia_amp_image_id:
-# add here the file name of the image if it should be uplaoded automatically
-# Note: This does not support image tags so don't use for prod
+# download the image from an artefact server
+# Note: The default is the Octavia test image so don't use that in prod
+octavia_download_artefact: True
+# The host to download images to if enabled
+# Options are ['deployment-host', 'target-host']
+octavia_image_downloader: "deployment-host"
+# The URL to downlaod from
+octavia_artefact_url: http://tarballs.openstack.org/octavia/test-images/test-only-amphora-x64-haproxy-ubuntu-xenial.qcow2
+# the directory to store the downloaded file to
+octavia_amp_image_path: "~/"
+# add here the file name of the image if it should be uploaded automatically
 octavia_amp_image_file_name:
 # enable uploading image to glance automatically
-# Note: This does not support image tags so don't use for prod
-octavia_amp_image_upload_enabled: False
+octavia_amp_image_upload_enabled: "{{ octavia_download_artefact }}"
+
 # Name of the Octavia security group
 octavia_security_group_name: octavia_sec_grp
 # Restrict access to only authorized hosts

doc/source/configure-octavia.rst

@@ -76,6 +76,16 @@ ips which overlap with ips assigned to hosts or containers (see the
 Building Octavia images
 ~~~~~~~~~~~~~~~~~~~~~~~
 
+.. note::
+    The default behavior is to download a test image from the OpenStack artifact
+    storage the Octavia team provides daily. Because this image doesn't apply
+    operating system security patches in a timely manner it is unsuited
+    for production use.
+
+    Some Operating System vendors might provide official amphora builds or an
+    organization might maintain their own artifact storage - for those cases the
+    automatic download can be leveraged, too.
+
 Images using the ``diskimage-builder`` must be built outside of a container.
 For this process, use one of the physical hosts within the environment.
 
@@ -115,6 +125,10 @@ For this process, use one of the physical hosts within the environment.
       glance image-create --name amphora-x64-haproxy --visibility private --disk-format qcow2 \
          --container-format bare --tags octavia-amphora-image </var/lib/octavia/amphora-x64-haproxy.qcow2
 
+   .. note::
+        Alternatively you can specify the new image in the appropriate settings and rerun the
+        ansible with an appropriate tag.
+
 You can find more information abpout the diskimage script and the process at
 https://github.com/openstack/octavia/tree/master/diskimage-create
 

releasenotes/notes/image_download-754d13e7df9b7891.yaml

@@ -0,0 +1,16 @@
+---
+features:
+  - |
+    This consolidates the amphora image tasks in a common file and adds a way
+    to download an amphora image from an artefact storage over http(s). With
+    the Octavia team providing test images the tests were modified to not
+    build images any longer but download them.
+security:
+  - |
+    It is commonly considered bad practice to downlaod random images from the
+    Internet expecially the test images the Octavia team provides which could
+    potentially include unpatched operating system packages - so for any
+    production deploy adjust the download url to an artifact storage your
+    organization controls. The system also does not authenticate the image
+    (e.g. with an md5) so should only be used on networks your organization
+    controls.

tasks/main.yml

@@ -87,6 +87,10 @@
     - octavia-install
     - octavia-config
 
+- include: octavia_amp_image.yml
+  tags:
+    - octavia-config
+
 
 - name: Flush handlers
   meta: flush_handlers

tasks/octavia_amp_image.yml

@@ -0,0 +1,105 @@
+---
+  # Copyright 2018, Rackspace US, Inc.
+  #
+  # Licensed under the Apache License, Version 2.0 (the "License");
+  # you may not use this file except in compliance with the License.
+  # You may obtain a copy of the License at
+  #
+  #     http://www.apache.org/licenses/LICENSE-2.0
+  #
+  # Unless required by applicable law or agreed to in writing, software
+  # distributed under the License is distributed on an "AS IS" BASIS,
+  # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  # See the License for the specific language governing permissions and
+  # limitations under the License.
+
+- name: Download image from artefact server
+  get_url:
+    url: "{{ octavia_artefact_url }}"
+    dest: "{{ octavia_amp_image_path }}"
+  retries: 10
+  delay: 10
+  register: octavia_download_result
+  when:
+    - octavia_download_artefact|bool
+  delegate_to: "{{ (octavia_image_downloader == 'deployment-host') | ternary('localhost', inventory_hostname) }}"
+
+- name: Set the filename fact
+  set_fact:
+    octavia_amp_image_file_name: "{{ octavia_download_result.dest }}"
+  when:
+    - octavia_image_downloader == "deployment-host"
+    - octavia_download_artefact|bool
+
+- name: Copy download images from deployment-host to target-host(s)
+  copy:
+    src: "{{ octavia_amp_image_file_name }}"
+    dest: "~/{{ octavia_amp_image_file_name|basename }}"
+  when:
+    - octavia_amp_image_upload_enabled
+    - octavia_image_downloader == "deployment-host"
+  register: octavia_amp_image_copy_result
+  until: octavia_amp_image_copy_result | success
+  retries: 6
+  delay: 5
+
+- name: Set if we need to upload an image
+  set_fact:
+    octavia_amp_image_needs_upload: "{{ (octavia_image_downloader != 'deployment-host')|ternary(octavia_download_result|changed, octavia_amp_image_copy_result|changed) }}"
+    octavia_dst_amp_image_path: "{{ ((octavia_image_downloader == 'deployment-host') and (octavia_download_artefact|bool))|ternary(octavia_amp_image_copy_result.dest, octavia_download_result.dest) }}"
+  when:
+    - octavia_amp_image_upload_enabled
+
+- name: Get curremt image id
+  os_image_facts:
+    auth:
+      auth_url: "{{ keystone_service_adminurl }}"
+      username: "{{ octavia_service_user_name }}"
+      password: "{{ octavia_service_password }}"
+      project_name: "{{ octavia_service_project_name }}"
+      user_domain_name: "{{ octavia_service_user_domain_id }}"
+      project_domain_name: "{{ octavia_service_project_domain_id }}"
+    endpoint_type: "{{ octavia_ansible_endpoint_type }}"
+    region_name: "{{ octavia_service_region }}"
+    validate_certs: "{{ keystone_service_adminuri_insecure }}"
+    auth_type: "{{ octavia_keystone_auth_plugin }}"
+    image: amphora-x64-haproxy
+  when:
+     - octavia_amp_image_needs_upload|default(False)
+
+# use shell since os_image doesn't support tags
+- name: Upload new image to glance
+  shell: |
+    . {{ ansible_env.HOME }}/openrc
+    openstack image create --file  {{ octavia_dst_amp_image_path }} --disk-format qcow2 \
+    --tag {{ octavia_glance_image_tag }}  --private --project service amphora-x64-haproxy
+  when:
+    - octavia_amp_image_needs_upload|default(False)
+  run_once: True
+  tags:
+    - skip_ansible_lint
+
+- name: Delete old image from glance
+  os_image:
+    auth:
+      auth_url: "{{ keystone_service_adminurl }}"
+      username: "{{ octavia_service_user_name }}"
+      password: "{{ octavia_service_password }}"
+      project_name: "{{ octavia_service_project_name }}"
+      user_domain_name: "{{ octavia_service_user_domain_id }}"
+      project_domain_name: "{{ octavia_service_project_domain_id }}"
+    endpoint_type: "{{ octavia_ansible_endpoint_type }}"
+    region_name: "{{ octavia_service_region }}"
+    validate_certs: "{{ keystone_service_adminuri_insecure }}"
+    auth_type: "{{ octavia_keystone_auth_plugin }}"
+    id: "{{ openstack.id }}"
+    state: absent
+  when:
+   - openstack is defined # result from os_image_facts
+
+
+
+
+
+
+

tasks/octavia_post_install.yml

@@ -56,54 +56,6 @@
     - src: "{{ octavia_ca_private_key }}"
       dest: "/etc/octavia/certs/ca_key.pem"
 
-- name: Copy image
-  copy:
-    src: "{{ octavia_amp_image_file_name}}"
-    dest: "{{ octavia_amp_image_file_name}}"
-  when: octavia_amp_image_upload_enabled
-
-- name: Upload image to glance
-  os_image:
-    auth:
-      auth_url: "{{ keystone_service_adminurl }}"
-      username: "{{ octavia_service_user_name }}"
-      password: "{{ octavia_service_password }}"
-      project_name: "{{ octavia_service_project_name }}"
-      user_domain_name: "{{ octavia_service_user_domain_id }}"
-      project_domain_name: "{{ octavia_service_project_domain_id }}"
-    endpoint_type: "{{ octavia_ansible_endpoint_type }}"
-    region_name: "{{ octavia_service_region }}"
-    validate_certs: "{{ keystone_service_adminuri_insecure }}"
-    auth_type: "{{ octavia_keystone_auth_plugin }}"
-    name: amphora-x64-haproxy
-    container_format: bare
-    disk_format: qcow2
-    state: present
-    filename: "{{ octavia_amp_image_file_name}}"
-    is_public:
-  when: octavia_amp_image_upload_enabled
-
-- name: Get image uuid
-  os_image_facts:
-    auth:
-      auth_url: "{{ keystone_service_adminurl }}"
-      username: "{{ octavia_service_user_name }}"
-      password: "{{ octavia_service_password }}"
-      project_name: "{{ octavia_service_project_name }}"
-      user_domain_name: "{{ octavia_service_user_domain_id }}"
-      project_domain_name: "{{ octavia_service_project_domain_id }}"
-    endpoint_type: "{{ octavia_ansible_endpoint_type }}"
-    region_name: "{{ octavia_service_region }}"
-    validate_certs: "{{ keystone_service_adminuri_insecure }}"
-    auth_type: "{{ octavia_keystone_auth_plugin }}"
-    image: amphora-x64-haproxy
-  when: octavia_amp_image_upload_enabled
-
-- name: Set image UUID fact
-  set_fact:
-     octavia_amp_image_id: "{{ openstack_image.id }}"
-  when: octavia_amp_image_upload_enabled
-
 - name: Copy user provided HAProxy templates
   copy:
     src: "{{ item.src }}"

tests/os_octavia-overrides.yml

@@ -15,6 +15,7 @@
 ## Octavia rewuires currently nested virtualization (VT-X) which only one jenkins
 ## gate cloud provides reliabley. To cut down on gate errors this will
 ## disbale the tests which spin up an amphora
+## As of 4/6/18 zuul runs out of memory testing with this enabled
 test_octavia_amphora: False
 
 test_octavia_api_group: "{{ ((groups['octavia_api'] is defined) and (groups['octavia_api'] | length > 0)) | ternary('octavia_api', 'all_containers') }}"

tests/test-configure-octavia.yml

@@ -28,42 +28,12 @@
       retries: 5
       delay: 2
       with_items:
-        - qemu
-        - uuid-runtime
-        - curl
-        - kpartx
         - git
-    - name: Install pip requirements
-      pip:
-        name: "{{ item }}"
-        state: "{{ octavia_pip_package_state }}"
-      register: install_packages
-      until: install_packages|success
-      retries: 5
-      delay: 2
-      with_items:
-        - argparse
-        - "Babel>=1.3"
-        - dib-utils
-        - PyYAML
     - name: Clone Octavia
       git:
         repo: "https://git.openstack.org/openstack/octavia"
         dest: "{{ octavia_system_home_folder }}/octavia"
         version: "{{ octavia_git_install_branch }}"
-    - name: Clone Diskimage-Builder
-      git:
-        repo: "https://git.openstack.org/openstack/diskimage-builder"
-        dest: "{{ octavia_system_home_folder }}/diskimage-builder"
-        version: "2.9.0"
-    - name: Create amphora image
-      shell: "./diskimage-create.sh -o {{ octavia_system_home_folder }}/amphora-x64-haproxy.qcow2"
-      args:
-        chdir: "{{ octavia_system_home_folder }}/octavia/diskimage-create"
-        creates: "{{ octavia_system_home_folder }}/amphora-x64-haproxy.qcow2"
-      when: test_octavia_amphora | bool
-      tags:
-        - skip_ansible_lint
     - name: Change permission
       file:
         path: "{{ octavia_system_home_folder }}/octavia/bin/create_certificates.sh"

tests/test-octavia.yml

@@ -73,17 +73,6 @@
         network_name: "public"
         name: "public-subnet"
         cidr: "10.1.3.0/24"
-    - name: Upload image to glance
-      shell: >-
-          glance image-create --name amphora-x64-haproxy --visibility private --disk-format qcow2 \
-            --container-format bare --tags octavia-amphora-image <{{ octavia_system_home_folder }}/amphora-x64-haproxy.qcow2 \
-            && touch {{ octavia_system_home_folder }}/image
-      args:
-        creates: "{{ octavia_system_home_folder }}/image"
-      environment: "{{ env }}"
-      when: test_octavia_amphora | bool
-      tags:
-        - skip_ansible_lint
     - name: Create ssh-key
       shell: >
           cat /dev/zero | ssh-keygen -q -N ""