From 51ea328b7c8289b3d491b2f9bb428e5740ab0795 Mon Sep 17 00:00:00 2001 From: Markos Chandras Date: Wed, 30 May 2018 12:31:50 +0100 Subject: [PATCH] Add support for using distribution packages for OpenStack services Distributions provide packages for the OpenStack services so we add support for using these instead of the pip ones. However, functional testing is not complete yet since it requires tempest to be in the swift virtual environment which doesn't exist for distro installs. As such, for functional testing to cover this method of installation it needs to be changed quite a bit. Change-Id: Id75e56440685df407a9991018dc07b5a75429082 Implements: blueprint openstack-distribution-packages --- defaults/main.yml | 5 +- ...istribution-packages-b1c9e1f488e53872.yaml | 7 + tasks/main.yml | 15 ++ tasks/swift_install.yml | 130 +++--------------- tasks/swift_install_source.yml | 128 +++++++++++++++++ tox.ini | 11 ++ vars/debian.yml | 11 ++ vars/distro_install.yml | 17 +++ vars/redhat-7.yml | 10 ++ vars/source_install.yml | 17 +++ vars/suse-42.yml | 10 ++ zuul.d/project.yaml | 7 + 12 files changed, 253 insertions(+), 115 deletions(-) create mode 100644 releasenotes/notes/openstack-distribution-packages-b1c9e1f488e53872.yaml create mode 100644 tasks/swift_install_source.yml create mode 100644 vars/distro_install.yml create mode 100644 vars/source_install.yml diff --git a/defaults/main.yml b/defaults/main.yml index 6180ce55..8021b04a 100644 --- a/defaults/main.yml +++ b/defaults/main.yml @@ -25,6 +25,9 @@ debug: False swift_package_state: "latest" swift_pip_package_state: "latest" +# Set installation method. +swift_install_method: "source" + # Git repo details for swift swift_git_repo: https://git.openstack.org/openstack/swift swift_git_install_branch: master @@ -41,7 +44,7 @@ swift_developer_constraints: # Name of the virtual env to deploy into swift_venv_tag: untagged -swift_bin: "/openstack/venvs/swift-{{ swift_venv_tag }}/bin" +swift_bin: "{{ _swift_bin }}" # venv_download, even when true, will use the fallback method of building the # venv from scratch if the venv download fails. diff --git a/releasenotes/notes/openstack-distribution-packages-b1c9e1f488e53872.yaml b/releasenotes/notes/openstack-distribution-packages-b1c9e1f488e53872.yaml new file mode 100644 index 00000000..970f3a04 --- /dev/null +++ b/releasenotes/notes/openstack-distribution-packages-b1c9e1f488e53872.yaml @@ -0,0 +1,7 @@ +--- +features: + - | + The role now supports using the distribution packages for the OpenStack + services instead of the pip ones. This feature is disabled by default + and can be enabled by simply setting the ``swift_install_method`` + variable to ``distro``. diff --git a/tasks/main.yml b/tasks/main.yml index 30c0bbca..c20ce08a 100644 --- a/tasks/main.yml +++ b/tasks/main.yml @@ -24,6 +24,21 @@ tags: - always +- name: Fail if service was deployed using a different installation method + fail: + msg: "Switching installation methods for OpenStack services is not supported" + when: + - ansible_local is defined + - ansible_local.openstack_ansible is defined + - ansible_local.openstack_ansible.swift is defined + - ansible_local.openstack_ansible.swift.install_method is defined + - ansible_local.openstack_ansible.swift.install_method != swift_install_method + +- name: Gather variables for installation method + include_vars: "{{ swift_install_method }}_install.yml" + tags: + - always + # Check the swift_hash_path_* variables haven't changed - include_tasks: swift_check_hashes.yml when: diff --git a/tasks/swift_install.yml b/tasks/swift_install.yml index b066f806..6de0935f 100644 --- a/tasks/swift_install.yml +++ b/tasks/swift_install.yml @@ -13,9 +13,21 @@ # See the License for the specific language governing permissions and # limitations under the License. +- name: Record the installation method + ini_file: + dest: "/etc/ansible/facts.d/openstack_ansible.fact" + section: "swift" + option: "install_method" + value: "{{ swift_install_method }}" + +- name: Refresh local facts to ensure the swift section is present + setup: + filter: ansible_local + gather_subset: "!all" + - name: Install distro packages package: - name: "{{ swift_distro_packages }}" + name: "{{ swift_package_list }}" state: "{{ swift_package_state }}" update_cache: "{{ (ansible_pkg_mgr in ['apt', 'zypper']) | ternary('yes', omit) }}" cache_valid_time: "{{ (ansible_pkg_mgr == 'apt') | ternary(cache_timeout, omit) }}" @@ -24,119 +36,9 @@ retries: 5 delay: 2 -- name: Create developer mode constraint file - copy: - dest: "/opt/developer-pip-constraints.txt" - content: | - {% for item in swift_developer_constraints %} - {{ item }} - {% endfor %} - when: - - swift_developer_mode | bool - -- name: Install requires pip packages - pip: - name: "{{ swift_requires_pip_packages }}" - state: "{{ swift_pip_package_state }}" - extra_args: >- - {{ swift_developer_mode | ternary(pip_install_developer_constraints | default('--constraint /opt/developer-pip-constraints.txt'), '') }} - {{ (pip_install_upper_constraints is defined) | ternary('--constraint ' + pip_install_upper_constraints | default(''),'') }} - {{ pip_install_options | default('') }} - register: install_packages - until: install_packages|success - retries: 5 - delay: 2 - -- name: Retrieve checksum for venv download - uri: - url: "{{ swift_venv_download_url | replace('tgz', 'checksum') }}" - return_content: yes - register: swift_venv_checksum - when: swift_venv_download | bool - -- name: Attempt venv download - get_url: - url: "{{ swift_venv_download_url }}" - dest: "/var/cache/{{ swift_venv_download_url | basename }}" - checksum: "sha1:{{ swift_venv_checksum.content | trim }}" - register: swift_get_venv - when: swift_venv_download | bool - -- name: Remove existing venv - file: - path: "{{ swift_bin | dirname }}" - state: absent - when: swift_get_venv | changed - -- name: Create swift venv dir - file: - path: "{{ swift_bin | dirname }}" - state: directory - mode: "0755" - register: swift_venv_dir - when: swift_get_venv | changed - -- name: Unarchive pre-built venv - unarchive: - src: "/var/cache/{{ swift_venv_download_url | basename }}" - dest: "{{ swift_bin | dirname }}" - copy: "no" - when: swift_get_venv | changed - notify: - - Restart swift services - -- name: Install pip packages - pip: - name: "{{ swift_pip_packages }}" - state: "{{ swift_pip_package_state }}" - virtualenv: "{{ swift_bin | dirname }}" - virtualenv_site_packages: "no" - extra_args: >- - {{ swift_developer_mode | ternary(pip_install_developer_constraints | default('--constraint /opt/developer-pip-constraints.txt'), '') }} - {{ (pip_install_upper_constraints is defined) | ternary('--constraint ' + pip_install_upper_constraints | default(''),'') }} - {{ pip_install_options | default('') }} - register: install_packages - until: install_packages|success - retries: 5 - delay: 2 - when: swift_get_venv | failed or swift_get_venv | skipped - notify: - - Restart swift services - -- name: Remove python from path first (CentOS, openSUSE) - file: - path: "{{ swift_bin | dirname }}/bin/python2.7" - state: "absent" - when: - - ansible_pkg_mgr in ['yum', 'dnf', 'zypper'] - - swift_get_venv | changed - -# NOTE(odyssey4me): -# We reinitialize the venv to ensure that the right -# version of python is in the venv, but we do not -# want virtualenv to also replace pip, setuptools -# and wheel so we tell it not to. -# We do not use --always-copy for CentOS/SuSE due -# to https://github.com/pypa/virtualenv/issues/565 -- name: Update virtualenv path - shell: | - find {{ swift_bin }} -name \*.pyc -delete - sed -si '1s/^.*python.*$/#!{{ swift_bin | replace ('/','\/') }}\/python/' {{ swift_bin }}/* - virtualenv {{ swift_bin | dirname }} \ - {{ (ansible_pkg_mgr == 'apt') | ternary('--always-copy', '') }} \ - --no-pip \ - --no-setuptools \ - --no-wheel - when: swift_get_venv | changed - tags: - - skip_ansible_lint +- name: Install swift packages from PIP + include_tasks: swift_install_source.yml + when: swift_install_method == 'source' - include_tasks: swift_pypy_setup.yml when: swift_pypy_enabled | bool - -- name: Record the venv tag deployed - ini_file: - dest: "/etc/ansible/facts.d/openstack_ansible.fact" - section: swift - option: venv_tag - value: "{{ swift_venv_tag }}" diff --git a/tasks/swift_install_source.yml b/tasks/swift_install_source.yml new file mode 100644 index 00000000..947cdb6e --- /dev/null +++ b/tasks/swift_install_source.yml @@ -0,0 +1,128 @@ +--- +# Copyright 2018, SUSE Linux GmbH. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Create developer mode constraint file + copy: + dest: "/opt/developer-pip-constraints.txt" + content: | + {% for item in swift_developer_constraints %} + {{ item }} + {% endfor %} + when: + - swift_developer_mode | bool + +- name: Install requires pip packages + pip: + name: "{{ swift_requires_pip_packages }}" + state: "{{ swift_pip_package_state }}" + extra_args: >- + {{ swift_developer_mode | ternary(pip_install_developer_constraints | default('--constraint /opt/developer-pip-constraints.txt'), '') }} + {{ (pip_install_upper_constraints is defined) | ternary('--constraint ' + pip_install_upper_constraints | default(''),'') }} + {{ pip_install_options | default('') }} + register: install_packages + until: install_packages|success + retries: 5 + delay: 2 + +- name: Retrieve checksum for venv download + uri: + url: "{{ swift_venv_download_url | replace('tgz', 'checksum') }}" + return_content: yes + register: swift_venv_checksum + when: swift_venv_download | bool + +- name: Attempt venv download + get_url: + url: "{{ swift_venv_download_url }}" + dest: "/var/cache/{{ swift_venv_download_url | basename }}" + checksum: "sha1:{{ swift_venv_checksum.content | trim }}" + register: swift_get_venv + when: swift_venv_download | bool + +- name: Remove existing venv + file: + path: "{{ swift_bin | dirname }}" + state: absent + when: swift_get_venv | changed + +- name: Create swift venv dir + file: + path: "{{ swift_bin | dirname }}" + state: directory + mode: "0755" + register: swift_venv_dir + when: swift_get_venv | changed + +- name: Unarchive pre-built venv + unarchive: + src: "/var/cache/{{ swift_venv_download_url | basename }}" + dest: "{{ swift_bin | dirname }}" + copy: "no" + when: swift_get_venv | changed + notify: + - Restart swift services + +- name: Install pip packages + pip: + name: "{{ swift_pip_packages }}" + state: "{{ swift_pip_package_state }}" + virtualenv: "{{ swift_bin | dirname }}" + virtualenv_site_packages: "no" + extra_args: >- + {{ swift_developer_mode | ternary(pip_install_developer_constraints | default('--constraint /opt/developer-pip-constraints.txt'), '') }} + {{ (pip_install_upper_constraints is defined) | ternary('--constraint ' + pip_install_upper_constraints | default(''),'') }} + {{ pip_install_options | default('') }} + register: install_packages + until: install_packages|success + retries: 5 + delay: 2 + when: swift_get_venv | failed or swift_get_venv | skipped + notify: + - Restart swift services + +- name: Remove python from path first (CentOS, openSUSE) + file: + path: "{{ swift_bin | dirname }}/bin/python2.7" + state: "absent" + when: + - ansible_pkg_mgr in ['yum', 'dnf', 'zypper'] + - swift_get_venv | changed + +# NOTE(odyssey4me): +# We reinitialize the venv to ensure that the right +# version of python is in the venv, but we do not +# want virtualenv to also replace pip, setuptools +# and wheel so we tell it not to. +# We do not use --always-copy for CentOS/SuSE due +# to https://github.com/pypa/virtualenv/issues/565 +- name: Update virtualenv path + shell: | + find {{ swift_bin }} -name \*.pyc -delete + sed -si '1s/^.*python.*$/#!{{ swift_bin | replace ('/','\/') }}\/python/' {{ swift_bin }}/* + virtualenv {{ swift_bin | dirname }} \ + {{ (ansible_pkg_mgr == 'apt') | ternary('--always-copy', '') }} \ + --no-pip \ + --no-setuptools \ + --no-wheel + when: swift_get_venv | changed + tags: + - skip_ansible_lint + +- name: Record the venv tag deployed + ini_file: + dest: "/etc/ansible/facts.d/openstack_ansible.fact" + section: swift + option: venv_tag + value: "{{ swift_venv_tag }}" diff --git a/tox.ini b/tox.ini index 3392ab2b..ad0c8816 100644 --- a/tox.ini +++ b/tox.ini @@ -101,6 +101,17 @@ deps = commands = bash -c "{toxinidir}/tests/common/test-ansible-functional.sh" +[testenv:distro_install] +deps = + {[testenv:ansible]deps} +setenv = + {[testenv]setenv} + # The test-swift-functional.yml requires tempest on the swift venv and + # the distro scenario is not ready for that yet. + TEST_PLAYBOOK={toxinidir}/tests/test.yml + ANSIBLE_PARAMETERS=-e swift_install_method=distro +commands = + bash -c "{toxinidir}/tests/common/test-ansible-functional.sh" [testenv:upgrade] deps = diff --git a/vars/debian.yml b/vars/debian.yml index 18ef5014..9b8fea31 100644 --- a/vars/debian.yml +++ b/vars/debian.yml @@ -27,4 +27,15 @@ swift_distro_packages: - rsync - libssl-dev +swift_service_distro_packages: + - python-keystoneclient # Keystoneclient needed to OSA keystone lib + - python-memcache + - python-swift + - swift + - swift-account + - swift-container + - swift-object + - swift-plugin-s3 + - swift-proxy + swift_rsync_service_name: rsync diff --git a/vars/distro_install.yml b/vars/distro_install.yml new file mode 100644 index 00000000..d6c90069 --- /dev/null +++ b/vars/distro_install.yml @@ -0,0 +1,17 @@ +--- +# Copyright 2018, SUSE Linux GmbH. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +swift_package_list: "{{ swift_distro_packages + swift_service_distro_packages }}" +_swift_bin: "/usr/bin" diff --git a/vars/redhat-7.yml b/vars/redhat-7.yml index d1e1300f..c64c2298 100644 --- a/vars/redhat-7.yml +++ b/vars/redhat-7.yml @@ -30,4 +30,14 @@ swift_distro_packages: - cronie-anacron - which +swift_service_distro_packages: + - openstack-swift-account + - openstack-swift-container + - openstack-swift-object + - openstack-swift-plugin-swift3 + - openstack-swift-proxy + - python-keystoneclient # Keystoneclient needed to OSA keystone lib + - python-memcached + - python-swift + swift_rsync_service_name: rsyncd diff --git a/vars/source_install.yml b/vars/source_install.yml new file mode 100644 index 00000000..ac48e245 --- /dev/null +++ b/vars/source_install.yml @@ -0,0 +1,17 @@ +--- +# Copyright 2018, SUSE Linux GmbH. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +swift_package_list: "{{ swift_distro_packages }}" +_swift_bin: "/openstack/venvs/swift-{{ swift_venv_tag }}/bin" diff --git a/vars/suse-42.yml b/vars/suse-42.yml index 201fa9ce..d716d71b 100644 --- a/vars/suse-42.yml +++ b/vars/suse-42.yml @@ -29,4 +29,14 @@ swift_distro_packages: - cronie-anacron - which +swift_service_distro_packages: + - openstack-swift + - openstack-swift-account + - openstack-swift-container + - openstack-swift-object + - openstack-swift-proxy + - python-keystoneclient # Keystoneclient needed to OSA keystone lib + - python-python-memcached + - python-swift + swift_rsync_service_name: rsyncd diff --git a/zuul.d/project.yaml b/zuul.d/project.yaml index 90cfdeb5..a127a00f 100644 --- a/zuul.d/project.yaml +++ b/zuul.d/project.yaml @@ -26,6 +26,11 @@ - openstack-ansible-multireg-ubuntu-xenial - openstack-ansible-multireg-centos-7-nv - openstack-ansible-swift3-ubuntu-xenial-nv + - openstack-ansible-functional-distro_install-ubuntu-xenial + # NOTE(hwoarang) Centos7 is having some troubles with repo dependencies + # so disabling until it's investigated. + - openstack-ansible-functional-distro_install-centos-7-nv + - openstack-ansible-functional-distro_install-opensuse-423 experimental: jobs: - openstack-ansible-pypy-ubuntu-xenial-nv @@ -40,3 +45,5 @@ - openstack-ansible-functional-ubuntu-xenial - openstack-ansible-upgrade-ubuntu-xenial - openstack-ansible-multireg-ubuntu-xenial + - openstack-ansible-functional-distro_install-ubuntu-xenial + - openstack-ansible-functional-distro_install-opensuse-423