From eaacd3f8413f594e195a8bb0500bb870bfe3f72d Mon Sep 17 00:00:00 2001 From: Tobias Urdin Date: Tue, 19 Jun 2018 17:32:27 +0200 Subject: [PATCH] Add support to select ceph version We are closing in on adding Ceph Mimic support and want to ensure we can select either Luminous or Mimic for running our tests. For example Ubuntu 18.04 must run Ceph Mimic because Luminous is not available (yet), Debian doesn't have Ceph Mimic packages so we must run Luminous there. Depends-On: https://review.openstack.org/#/c/576247/ Change-Id: If84ecbf86f3ebde8e69a1a26759ad967d061725a --- .zuul.yaml | 118 +++++++++++++++------------ configure_facts.sh | 22 +++-- functions | 14 +++- manifests/repos.pp | 9 +- playbooks/run-integration-tests.yaml | 3 +- run_tests.sh | 1 + 6 files changed, 103 insertions(+), 64 deletions(-) diff --git a/.zuul.yaml b/.zuul.yaml index e1e9291f2..01a050e61 100644 --- a/.zuul.yaml +++ b/.zuul.yaml @@ -71,6 +71,10 @@ - ^.gitignore$ roles: - zuul: openstack-infra/zuul-jobs + # NOTE(tobasco): The ceph variable must be provided with a default here + # or zuul will complain when it tries to render the run-integration-tests template. + vars: + ceph: luminous - job: name: puppet-openstack-integration-4 @@ -83,19 +87,20 @@ parent: puppet-openstack-integration-4 vars: scenario: scenario001 + ceph: luminous - job: - name: puppet-openstack-integration-4-scenario001-tempest-ubuntu-xenial + name: puppet-openstack-integration-4-scenario001-tempest-ubuntu-xenial-luminous parent: puppet-openstack-integration-4-scenario001 nodeset: ubuntu-xenial - job: - name: puppet-openstack-integration-4-scenario001-tempest-centos-7 + name: puppet-openstack-integration-4-scenario001-tempest-centos-7-luminous parent: puppet-openstack-integration-4-scenario001 nodeset: centos-7 - job: - name: puppet-openstack-integration-4-scenario001-tempest-debian-stable + name: puppet-openstack-integration-4-scenario001-tempest-debian-stable-luminous parent: puppet-openstack-integration-4-scenario001 nodeset: debian-stable @@ -147,21 +152,24 @@ branches: ^(?!stable/newton).*$ vars: scenario: scenario004 + ceph: mimic - job: - name: puppet-openstack-integration-4-scenario004-tempest-ubuntu-xenial + name: puppet-openstack-integration-4-scenario004-tempest-ubuntu-xenial-mimic parent: puppet-openstack-integration-4-scenario004 nodeset: ubuntu-xenial - job: - name: puppet-openstack-integration-4-scenario004-tempest-centos-7 + name: puppet-openstack-integration-4-scenario004-tempest-centos-7-mimic parent: puppet-openstack-integration-4-scenario004 nodeset: centos-7 - job: - name: puppet-openstack-integration-4-scenario004-tempest-debian-stable + name: puppet-openstack-integration-4-scenario004-tempest-debian-stable-luminous parent: puppet-openstack-integration-4-scenario004 nodeset: debian-stable + vars: + ceph: luminous - job: name: puppet-openstack-integration-5 @@ -175,19 +183,20 @@ parent: puppet-openstack-integration-5 vars: scenario: scenario001 + ceph: luminous - job: - name: puppet-openstack-integration-5-scenario001-tempest-ubuntu-xenial + name: puppet-openstack-integration-5-scenario001-tempest-ubuntu-xenial-luminous parent: puppet-openstack-integration-5-scenario001 nodeset: ubuntu-xenial - job: - name: puppet-openstack-integration-5-scenario001-tempest-centos-7 + name: puppet-openstack-integration-5-scenario001-tempest-centos-7-luminous parent: puppet-openstack-integration-5-scenario001 nodeset: centos-7 - job: - name: puppet-openstack-integration-5-scenario001-tempest-debian-stable + name: puppet-openstack-integration-5-scenario001-tempest-debian-stable-luminous parent: puppet-openstack-integration-5-scenario001 nodeset: debian-stable @@ -239,94 +248,97 @@ branches: ^(?!stable/newton).*$ vars: scenario: scenario004 + ceph: mimic - job: - name: puppet-openstack-integration-5-scenario004-tempest-ubuntu-xenial + name: puppet-openstack-integration-5-scenario004-tempest-ubuntu-xenial-mimic parent: puppet-openstack-integration-5-scenario004 nodeset: ubuntu-xenial - job: - name: puppet-openstack-integration-5-scenario004-tempest-centos-7 + name: puppet-openstack-integration-5-scenario004-tempest-centos-7-mimic parent: puppet-openstack-integration-5-scenario004 nodeset: centos-7 - job: - name: puppet-openstack-integration-5-scenario004-tempest-debian-stable - parent: puppet-openstack-integration-5-scenario001 + name: puppet-openstack-integration-5-scenario004-tempest-debian-stable-luminous + parent: puppet-openstack-integration-5-scenario004 nodeset: debian-stable + vars: + ceph: luminous - project-template: name: puppet-openstack-integration-jobs-all check: jobs: - - puppet-openstack-integration-4-scenario001-tempest-centos-7 + - puppet-openstack-integration-4-scenario001-tempest-centos-7-luminous - puppet-openstack-integration-4-scenario002-tempest-centos-7 - puppet-openstack-integration-4-scenario003-tempest-centos-7 - - puppet-openstack-integration-4-scenario004-tempest-centos-7 - - puppet-openstack-integration-5-scenario001-tempest-centos-7 + - puppet-openstack-integration-4-scenario004-tempest-centos-7-mimic + - puppet-openstack-integration-5-scenario001-tempest-centos-7-luminous - puppet-openstack-integration-5-scenario002-tempest-centos-7 - puppet-openstack-integration-5-scenario003-tempest-centos-7 - - puppet-openstack-integration-5-scenario004-tempest-centos-7 - - puppet-openstack-integration-4-scenario001-tempest-ubuntu-xenial + - puppet-openstack-integration-5-scenario004-tempest-centos-7-mimic + - puppet-openstack-integration-4-scenario001-tempest-ubuntu-xenial-luminous - puppet-openstack-integration-4-scenario002-tempest-ubuntu-xenial - puppet-openstack-integration-4-scenario003-tempest-ubuntu-xenial - - puppet-openstack-integration-4-scenario004-tempest-ubuntu-xenial - - puppet-openstack-integration-5-scenario001-tempest-ubuntu-xenial + - puppet-openstack-integration-4-scenario004-tempest-ubuntu-xenial-mimic + - puppet-openstack-integration-5-scenario001-tempest-ubuntu-xenial-luminous - puppet-openstack-integration-5-scenario002-tempest-ubuntu-xenial - puppet-openstack-integration-5-scenario003-tempest-ubuntu-xenial - - puppet-openstack-integration-5-scenario004-tempest-ubuntu-xenial - - puppet-openstack-integration-4-scenario001-tempest-debian-stable: + - puppet-openstack-integration-5-scenario004-tempest-ubuntu-xenial-mimic + - puppet-openstack-integration-4-scenario001-tempest-debian-stable-luminous: voting: false - puppet-openstack-integration-4-scenario002-tempest-debian-stable: voting: false - puppet-openstack-integration-4-scenario003-tempest-debian-stable: voting: false - - puppet-openstack-integration-4-scenario004-tempest-debian-stable: + - puppet-openstack-integration-4-scenario004-tempest-debian-stable-luminous: voting: false gate: jobs: - - puppet-openstack-integration-4-scenario001-tempest-centos-7 + - puppet-openstack-integration-4-scenario001-tempest-centos-7-luminous - puppet-openstack-integration-4-scenario002-tempest-centos-7 - puppet-openstack-integration-4-scenario003-tempest-centos-7 - - puppet-openstack-integration-4-scenario004-tempest-centos-7 - - puppet-openstack-integration-5-scenario001-tempest-centos-7 + - puppet-openstack-integration-4-scenario004-tempest-centos-7-mimic + - puppet-openstack-integration-5-scenario001-tempest-centos-7-luminous - puppet-openstack-integration-5-scenario002-tempest-centos-7 - puppet-openstack-integration-5-scenario003-tempest-centos-7 - - puppet-openstack-integration-5-scenario004-tempest-centos-7 - - puppet-openstack-integration-4-scenario001-tempest-ubuntu-xenial + - puppet-openstack-integration-5-scenario004-tempest-centos-7-mimic + - puppet-openstack-integration-4-scenario001-tempest-ubuntu-xenial-luminous - puppet-openstack-integration-4-scenario002-tempest-ubuntu-xenial - puppet-openstack-integration-4-scenario003-tempest-ubuntu-xenial - - puppet-openstack-integration-4-scenario004-tempest-ubuntu-xenial - - puppet-openstack-integration-5-scenario001-tempest-ubuntu-xenial + - puppet-openstack-integration-4-scenario004-tempest-ubuntu-xenial-mimic + - puppet-openstack-integration-5-scenario001-tempest-ubuntu-xenial-luminous - puppet-openstack-integration-5-scenario002-tempest-ubuntu-xenial - puppet-openstack-integration-5-scenario003-tempest-ubuntu-xenial - - puppet-openstack-integration-5-scenario004-tempest-ubuntu-xenial + - puppet-openstack-integration-5-scenario004-tempest-ubuntu-xenial-mimic experimental: jobs: - - puppet-openstack-integration-5-scenario001-tempest-debian-stable + - puppet-openstack-integration-5-scenario001-tempest-debian-stable-luminous - puppet-openstack-integration-5-scenario002-tempest-debian-stable - puppet-openstack-integration-5-scenario003-tempest-debian-stable - - puppet-openstack-integration-5-scenario004-tempest-debian-stable + - puppet-openstack-integration-5-scenario004-tempest-debian-stable-luminous - project-template: name: puppet-openstack-integration-jobs-scenario001 check: jobs: - - puppet-openstack-integration-4-scenario001-tempest-ubuntu-xenial - - puppet-openstack-integration-5-scenario001-tempest-ubuntu-xenial - - puppet-openstack-integration-4-scenario001-tempest-centos-7 - - puppet-openstack-integration-5-scenario001-tempest-centos-7 - - puppet-openstack-integration-4-scenario001-tempest-debian-stable: + - puppet-openstack-integration-4-scenario001-tempest-ubuntu-xenial-luminous + - puppet-openstack-integration-5-scenario001-tempest-ubuntu-xenial-luminous + - puppet-openstack-integration-4-scenario001-tempest-centos-7-luminous + - puppet-openstack-integration-5-scenario001-tempest-centos-7-luminous + - puppet-openstack-integration-4-scenario001-tempest-debian-stable-luminous: voting: false gate: jobs: - - puppet-openstack-integration-4-scenario001-tempest-ubuntu-xenial - - puppet-openstack-integration-5-scenario001-tempest-ubuntu-xenial - - puppet-openstack-integration-4-scenario001-tempest-centos-7 - - puppet-openstack-integration-5-scenario001-tempest-centos-7 + - puppet-openstack-integration-4-scenario001-tempest-ubuntu-xenial-luminous + - puppet-openstack-integration-5-scenario001-tempest-ubuntu-xenial-luminous + - puppet-openstack-integration-4-scenario001-tempest-centos-7-luminous + - puppet-openstack-integration-5-scenario001-tempest-centos-7-luminous experimental: jobs: - - puppet-openstack-integration-5-scenario001-tempest-debian-stable + - puppet-openstack-integration-5-scenario001-tempest-debian-stable-luminous - project-template: name: puppet-openstack-integration-jobs-scenario002 @@ -372,21 +384,21 @@ name: puppet-openstack-integration-jobs-scenario004 check: jobs: - - puppet-openstack-integration-4-scenario004-tempest-ubuntu-xenial - - puppet-openstack-integration-5-scenario004-tempest-ubuntu-xenial - - puppet-openstack-integration-4-scenario004-tempest-centos-7 - - puppet-openstack-integration-5-scenario004-tempest-centos-7 - - puppet-openstack-integration-4-scenario004-tempest-debian-stable: + - puppet-openstack-integration-4-scenario004-tempest-ubuntu-xenial-mimic + - puppet-openstack-integration-5-scenario004-tempest-ubuntu-xenial-mimic + - puppet-openstack-integration-4-scenario004-tempest-centos-7-mimic + - puppet-openstack-integration-5-scenario004-tempest-centos-7-mimic + - puppet-openstack-integration-4-scenario004-tempest-debian-stable-luminous: voting: false gate: jobs: - - puppet-openstack-integration-4-scenario004-tempest-ubuntu-xenial - - puppet-openstack-integration-5-scenario004-tempest-ubuntu-xenial - - puppet-openstack-integration-4-scenario004-tempest-centos-7 - - puppet-openstack-integration-5-scenario004-tempest-centos-7 + - puppet-openstack-integration-4-scenario004-tempest-ubuntu-xenial-mimic + - puppet-openstack-integration-5-scenario004-tempest-ubuntu-xenial-mimic + - puppet-openstack-integration-4-scenario004-tempest-centos-7-mimic + - puppet-openstack-integration-5-scenario004-tempest-centos-7-mimic experimental: jobs: - - puppet-openstack-integration-5-scenario004-tempest-debian-stable + - puppet-openstack-integration-5-scenario004-tempest-debian-stable-luminous - job: name: puppet-openstack-module-base diff --git a/configure_facts.sh b/configure_facts.sh index 224afc942..8e72ba7ac 100644 --- a/configure_facts.sh +++ b/configure_facts.sh @@ -24,10 +24,15 @@ if [ -f /etc/ci/mirror_info.sh ]; then CENTOS_MIRROR_HOST="http://${NODEPOOL_MIRROR_HOST}" DEPS_MIRROR_HOST="${NODEPOOL_RDO_PROXY}/centos7-master/deps/latest/" if uses_debs; then - CEPH_MIRROR_HOST="${CENTOS_MIRROR_HOST}/ceph-deb-luminous" + CEPH_MIRROR_HOST="${CENTOS_MIRROR_HOST}/ceph-deb-${CEPH_VERSION}" NODEPOOL_PUPPETLABS_MIRROR="http://${NODEPOOL_MIRROR_HOST}/apt-puppetlabs" else - CEPH_MIRROR_HOST="${NODEPOOL_BUILDLOGS_CENTOS_PROXY}/centos/7/storage/x86_64/ceph-luminous/" + # TODO(tobasco): Remove this CBS candidate repo for Mimic when Storage SIG release it. + if [ "$CEPH_VERSION" == "mimic" ]; then + CEPH_MIRROR_HOST='http://cbs.centos.org/repos/storage7-ceph-mimic-candidate/x86_64/os/' + else + CEPH_MIRROR_HOST="${NODEPOOL_BUILDLOGS_CENTOS_PROXY}/centos/7/storage/x86_64/ceph-${CEPH_VERSION}/" + fi NODEPOOL_PUPPETLABS_MIRROR="http://${NODEPOOL_MIRROR_HOST}/yum-puppetlabs" fi else @@ -36,10 +41,15 @@ else NODEPOOL_RDO_PROXY='https://trunk.rdoproject.org' NODEPOOL_UCA_MIRROR='http://ubuntu-cloud.archive.canonical.com/ubuntu' if uses_debs; then - CEPH_MIRROR_HOST='https://download.ceph.com/debian-luminous' + CEPH_MIRROR_HOST="https://download.ceph.com/debian-${CEPH_VERSION}" NODEPOOL_PUPPETLABS_MIRROR='https://apt.puppetlabs.com' else - CEPH_MIRROR_HOST='https://buildlogs.centos.org/centos/7/storage/x86_64/ceph-luminous/' + # TODO(tobasco): Remove this CBS candidate repo for Mimic when Storage SIG releases it. + if [ "$CEPH_VERSION" == "mimic" ]; then + CEPH_MIRROR_HOST='http://cbs.centos.org/repos/storage7-ceph-mimic-candidate/x86_64/os/' + else + CEPH_MIRROR_HOST="https://buildlogs.centos.org/centos/7/storage/x86_64/ceph-${CEPH_VERSION}/" + fi NODEPOOL_PUPPETLABS_MIRROR="https://yum.puppetlabs.com" fi fi @@ -56,13 +66,15 @@ export FACTER_uca_mirror_host=$NODEPOOL_UCA_MIRROR export FACTER_deps_mirror_host=$DEPS_MIRROR_HOST export FACTER_ceph_mirror_host=$CEPH_MIRROR_HOST export FACTER_rdo_mirror_host=$RDO_MIRROR_HOST +export FACTER_ceph_version=$CEPH_VERSION MIRROR_FACTS="\ centos_mirror_host=${FACTER_centos_mirror_host} uca_mirror_host=${FACTER_uca_mirror_host} deps_mirror_host=${FACTER_deps_mirror_host} ceph_mirror_host=${FACTER_ceph_mirror_host} -rdo_mirror_host=${FACTER_rdo_mirror_host}" +rdo_mirror_host=${FACTER_rdo_mirror_host} +ceph_version=${FACTER_ceph_version}" if [ "${WRITE_FACTS}" = true ]; then $SUDO mkdir -p /etc/facter/facts.d/ diff --git a/functions b/functions index 86d3d07b4..e8fc8196e 100644 --- a/functions +++ b/functions @@ -19,15 +19,23 @@ install_external() { # - ``SCRIPT_DIR`` must be set to script path # - ``ZUUL_BRANCH`` must be set to Zuul branch. Fallback to 'master'. install_openstack() { + # Periodic jobs run without ref on master + ZUUL_BRANCH=${ZUUL_BRANCH:-master} + + if [ $ZUUL_PROJECT != "openstack/puppet-ceph" ] && [ -n "$CEPH_VERSION" ]; then + if [ $CEPH_VERSION == "mimic" ]; then + ZUUL_BRANCH="master" + else + ZUUL_BRANCH="stable/$CEPH_VERSION" + fi + fi + cat > clonemap.yaml < 'http://cbs.centos.org/repos/storage7-ceph-mimic-candidate/x86_64/os/', + default => "https://buildlogs.centos.org/centos/7/storage/x86_64/ceph-${::ceph_version}/" + } + $ceph_mirror = pick($::ceph_mirror_host, $ceph_mirror_fallback) # On CentOS, deploy Ceph using SIG repository and get rid of EPEL. # https://wiki.centos.org/SpecialInterestGroup/Storage/ if $::operatingsystem == 'CentOS' { diff --git a/playbooks/run-integration-tests.yaml b/playbooks/run-integration-tests.yaml index 5fce421fa..570421749 100644 --- a/playbooks/run-integration-tests.yaml +++ b/playbooks/run-integration-tests.yaml @@ -4,10 +4,11 @@ cmd: | set -ex trap "./copy_logs.sh" EXIT + export CEPH_VERSION={{ ceph }} export PUPPET_MAJ_VERSION={{ puppet }} export SCENARIO={{ scenario }} export GEM_HOME=`pwd`/.bundled_gems ./run_tests.sh executable: /bin/bash chdir: '{{ ansible_user_dir }}/workspace/puppet-openstack-integration' - environment: '{{ zuul | zuul_legacy_vars }}' \ No newline at end of file + environment: '{{ zuul | zuul_legacy_vars }}' diff --git a/run_tests.sh b/run_tests.sh index bd2034ebc..f950cde48 100755 --- a/run_tests.sh +++ b/run_tests.sh @@ -18,6 +18,7 @@ source ${SCRIPT_DIR}/functions export WORKSPACE=${WORKSPACE:-/tmp} export PUPPET_URL_SUFFIX= +export CEPH_VERSION=${CEPH_VERSION:-luminous} export PUPPET_MAJ_VERSION=${PUPPET_MAJ_VERSION:-4} export SCENARIO=${SCENARIO:-scenario001} export MANAGE_PUPPET_MODULES=${MANAGE_PUPPET_MODULES:-true}