Merge "Add support to select ceph version"

.zuul.yaml

@@ -74,6 +74,10 @@
       - ^.gitignore$
     roles:
       - zuul: openstack-infra/zuul-jobs
+    # NOTE(tobasco): The ceph variable must be provided with a default here
+    # or zuul will complain when it tries to render the run-integration-tests template.
+    vars:
+      ceph: luminous
 
 - job:
     name: puppet-openstack-integration-4
@@ -88,21 +92,22 @@
     abstract: true
     vars:
       scenario: scenario001
+      ceph: luminous
 
 - job:
-    name: puppet-openstack-integration-4-scenario001-tempest-ubuntu-xenial
+    name: puppet-openstack-integration-4-scenario001-tempest-ubuntu-xenial-luminous
     parent: puppet-openstack-integration-4-scenario001
     branches: ^stable/(newton|ocata|pike|queens).*$
     nodeset: ubuntu-xenial
 
 - job:
-    name: puppet-openstack-integration-4-scenario001-tempest-centos-7
+    name: puppet-openstack-integration-4-scenario001-tempest-centos-7-luminous
     parent: puppet-openstack-integration-4-scenario001
     branches: ^stable/(newton|ocata|pike|queens).*$
     nodeset: centos-7
 
 - job:
-    name: puppet-openstack-integration-4-scenario001-tempest-debian-stable
+    name: puppet-openstack-integration-4-scenario001-tempest-debian-stable-luminous
     parent: puppet-openstack-integration-4-scenario001
     branches: ^(?!stable/(newton|ocata|pike|queens)).*$
     nodeset: debian-stable
@@ -166,22 +171,25 @@
     branches: ^(stable/(ocata|pike|queens)).*$
     vars:
       scenario: scenario004
+      ceph: mimic
 
 - job:
-    name: puppet-openstack-integration-4-scenario004-tempest-ubuntu-xenial
+    name: puppet-openstack-integration-4-scenario004-tempest-ubuntu-xenial-mimic
     parent: puppet-openstack-integration-4-scenario004
     nodeset: ubuntu-xenial
 
 - job:
-    name: puppet-openstack-integration-4-scenario004-tempest-centos-7
+    name: puppet-openstack-integration-4-scenario004-tempest-centos-7-mimic
     parent: puppet-openstack-integration-4-scenario004
     nodeset: centos-7
 
 - job:
-    name: puppet-openstack-integration-4-scenario004-tempest-debian-stable
+    name: puppet-openstack-integration-4-scenario004-tempest-debian-stable-luminous
     parent: puppet-openstack-integration-4-scenario004
     branches: ^(?!stable/(newton|ocata|pike|queens)).*$
     nodeset: debian-stable
+    vars:
+      ceph: luminous
 
 - job:
     name: puppet-openstack-integration-5
@@ -197,19 +205,20 @@
     abstract: true
     vars:
       scenario: scenario001
+      ceph: luminous
 
 - job:
-    name: puppet-openstack-integration-5-scenario001-tempest-ubuntu-xenial
+    name: puppet-openstack-integration-5-scenario001-tempest-ubuntu-xenial-luminous
     parent: puppet-openstack-integration-5-scenario001
     nodeset: ubuntu-xenial
 
 - job:
-    name: puppet-openstack-integration-5-scenario001-tempest-centos-7
+    name: puppet-openstack-integration-5-scenario001-tempest-centos-7-luminous
     parent: puppet-openstack-integration-5-scenario001
     nodeset: centos-7
 
 - job:
-    name: puppet-openstack-integration-5-scenario001-tempest-debian-stable
+    name: puppet-openstack-integration-5-scenario001-tempest-debian-stable-luminous
     parent: puppet-openstack-integration-5-scenario001
     nodeset: debian-stable
 
@@ -264,94 +273,97 @@
     abstract: true
     vars:
       scenario: scenario004
+      ceph: mimic
 
 - job:
-    name: puppet-openstack-integration-5-scenario004-tempest-ubuntu-xenial
+    name: puppet-openstack-integration-5-scenario004-tempest-ubuntu-xenial-mimic
     parent: puppet-openstack-integration-5-scenario004
     nodeset: ubuntu-xenial
 
 - job:
-    name: puppet-openstack-integration-5-scenario004-tempest-centos-7
+    name: puppet-openstack-integration-5-scenario004-tempest-centos-7-mimic
     parent: puppet-openstack-integration-5-scenario004
     nodeset: centos-7
 
 - job:
-    name: puppet-openstack-integration-5-scenario004-tempest-debian-stable
+    name: puppet-openstack-integration-5-scenario004-tempest-debian-stable-luminous
-    parent: puppet-openstack-integration-5-scenario001
+    parent: puppet-openstack-integration-5-scenario004
     nodeset: debian-stable
+    vars:
+      ceph: luminous
 
 - project-template:
     name: puppet-openstack-integration-jobs-all
     check:
       jobs:
-        - puppet-openstack-integration-4-scenario001-tempest-centos-7
+        - puppet-openstack-integration-4-scenario001-tempest-centos-7-luminous
         - puppet-openstack-integration-4-scenario002-tempest-centos-7
         - puppet-openstack-integration-4-scenario003-tempest-centos-7
-        - puppet-openstack-integration-4-scenario004-tempest-centos-7
+        - puppet-openstack-integration-4-scenario004-tempest-centos-7-mimic
-        - puppet-openstack-integration-5-scenario001-tempest-centos-7
+        - puppet-openstack-integration-5-scenario001-tempest-centos-7-luminous
         - puppet-openstack-integration-5-scenario002-tempest-centos-7
         - puppet-openstack-integration-5-scenario003-tempest-centos-7
-        - puppet-openstack-integration-5-scenario004-tempest-centos-7
+        - puppet-openstack-integration-5-scenario004-tempest-centos-7-mimic
-        - puppet-openstack-integration-4-scenario001-tempest-ubuntu-xenial
+        - puppet-openstack-integration-4-scenario001-tempest-ubuntu-xenial-luminous
         - puppet-openstack-integration-4-scenario002-tempest-ubuntu-xenial
         - puppet-openstack-integration-4-scenario003-tempest-ubuntu-xenial
-        - puppet-openstack-integration-4-scenario004-tempest-ubuntu-xenial
+        - puppet-openstack-integration-4-scenario004-tempest-ubuntu-xenial-mimic
-        - puppet-openstack-integration-5-scenario001-tempest-ubuntu-xenial
+        - puppet-openstack-integration-5-scenario001-tempest-ubuntu-xenial-luminous
         - puppet-openstack-integration-5-scenario002-tempest-ubuntu-xenial
         - puppet-openstack-integration-5-scenario003-tempest-ubuntu-xenial
-        - puppet-openstack-integration-5-scenario004-tempest-ubuntu-xenial
+        - puppet-openstack-integration-5-scenario004-tempest-ubuntu-xenial-mimic
-        - puppet-openstack-integration-4-scenario001-tempest-debian-stable:
+        - puppet-openstack-integration-4-scenario001-tempest-debian-stable-luminous:
             voting: false
         - puppet-openstack-integration-4-scenario002-tempest-debian-stable:
             voting: false
         - puppet-openstack-integration-4-scenario003-tempest-debian-stable:
             voting: false
-        - puppet-openstack-integration-4-scenario004-tempest-debian-stable:
+        - puppet-openstack-integration-4-scenario004-tempest-debian-stable-luminous:
             voting: false
     gate:
       jobs:
-        - puppet-openstack-integration-4-scenario001-tempest-centos-7
+        - puppet-openstack-integration-4-scenario001-tempest-centos-7-luminous
         - puppet-openstack-integration-4-scenario002-tempest-centos-7
         - puppet-openstack-integration-4-scenario003-tempest-centos-7
-        - puppet-openstack-integration-4-scenario004-tempest-centos-7
+        - puppet-openstack-integration-4-scenario004-tempest-centos-7-mimic
-        - puppet-openstack-integration-5-scenario001-tempest-centos-7
+        - puppet-openstack-integration-5-scenario001-tempest-centos-7-luminous
         - puppet-openstack-integration-5-scenario002-tempest-centos-7
         - puppet-openstack-integration-5-scenario003-tempest-centos-7
-        - puppet-openstack-integration-5-scenario004-tempest-centos-7
+        - puppet-openstack-integration-5-scenario004-tempest-centos-7-mimic
-        - puppet-openstack-integration-4-scenario001-tempest-ubuntu-xenial
+        - puppet-openstack-integration-4-scenario001-tempest-ubuntu-xenial-luminous
         - puppet-openstack-integration-4-scenario002-tempest-ubuntu-xenial
         - puppet-openstack-integration-4-scenario003-tempest-ubuntu-xenial
-        - puppet-openstack-integration-4-scenario004-tempest-ubuntu-xenial
+        - puppet-openstack-integration-4-scenario004-tempest-ubuntu-xenial-mimic
-        - puppet-openstack-integration-5-scenario001-tempest-ubuntu-xenial
+        - puppet-openstack-integration-5-scenario001-tempest-ubuntu-xenial-luminous
         - puppet-openstack-integration-5-scenario002-tempest-ubuntu-xenial
         - puppet-openstack-integration-5-scenario003-tempest-ubuntu-xenial
-        - puppet-openstack-integration-5-scenario004-tempest-ubuntu-xenial
+        - puppet-openstack-integration-5-scenario004-tempest-ubuntu-xenial-mimic
     experimental:
       jobs:
-        - puppet-openstack-integration-5-scenario001-tempest-debian-stable
+        - puppet-openstack-integration-5-scenario001-tempest-debian-stable-luminous
         - puppet-openstack-integration-5-scenario002-tempest-debian-stable
         - puppet-openstack-integration-5-scenario003-tempest-debian-stable
-        - puppet-openstack-integration-5-scenario004-tempest-debian-stable
+        - puppet-openstack-integration-5-scenario004-tempest-debian-stable-luminous
 
 - project-template:
     name: puppet-openstack-integration-jobs-scenario001
     check:
       jobs:
-        - puppet-openstack-integration-4-scenario001-tempest-ubuntu-xenial
+        - puppet-openstack-integration-4-scenario001-tempest-ubuntu-xenial-luminous
-        - puppet-openstack-integration-5-scenario001-tempest-ubuntu-xenial
+        - puppet-openstack-integration-5-scenario001-tempest-ubuntu-xenial-luminous
-        - puppet-openstack-integration-4-scenario001-tempest-centos-7
+        - puppet-openstack-integration-4-scenario001-tempest-centos-7-luminous
-        - puppet-openstack-integration-5-scenario001-tempest-centos-7
+        - puppet-openstack-integration-5-scenario001-tempest-centos-7-luminous
-        - puppet-openstack-integration-4-scenario001-tempest-debian-stable:
+        - puppet-openstack-integration-4-scenario001-tempest-debian-stable-luminous:
             voting: false
     gate:
       jobs:
-        - puppet-openstack-integration-4-scenario001-tempest-ubuntu-xenial
+        - puppet-openstack-integration-4-scenario001-tempest-ubuntu-xenial-luminous
-        - puppet-openstack-integration-5-scenario001-tempest-ubuntu-xenial
+        - puppet-openstack-integration-5-scenario001-tempest-ubuntu-xenial-luminous
-        - puppet-openstack-integration-4-scenario001-tempest-centos-7
+        - puppet-openstack-integration-4-scenario001-tempest-centos-7-luminous
-        - puppet-openstack-integration-5-scenario001-tempest-centos-7
+        - puppet-openstack-integration-5-scenario001-tempest-centos-7-luminous
     experimental:
       jobs:
-        - puppet-openstack-integration-5-scenario001-tempest-debian-stable
+        - puppet-openstack-integration-5-scenario001-tempest-debian-stable-luminous
 
 - project-template:
     name: puppet-openstack-integration-jobs-scenario002
@@ -397,21 +409,21 @@
     name: puppet-openstack-integration-jobs-scenario004
     check:
       jobs:
-        - puppet-openstack-integration-4-scenario004-tempest-ubuntu-xenial
+        - puppet-openstack-integration-4-scenario004-tempest-ubuntu-xenial-mimic
-        - puppet-openstack-integration-5-scenario004-tempest-ubuntu-xenial
+        - puppet-openstack-integration-5-scenario004-tempest-ubuntu-xenial-mimic
-        - puppet-openstack-integration-4-scenario004-tempest-centos-7
+        - puppet-openstack-integration-4-scenario004-tempest-centos-7-mimic
-        - puppet-openstack-integration-5-scenario004-tempest-centos-7
+        - puppet-openstack-integration-5-scenario004-tempest-centos-7-mimic
-        - puppet-openstack-integration-4-scenario004-tempest-debian-stable:
+        - puppet-openstack-integration-4-scenario004-tempest-debian-stable-luminous:
             voting: false
     gate:
       jobs:
-        - puppet-openstack-integration-4-scenario004-tempest-ubuntu-xenial
+        - puppet-openstack-integration-4-scenario004-tempest-ubuntu-xenial-mimic
-        - puppet-openstack-integration-5-scenario004-tempest-ubuntu-xenial
+        - puppet-openstack-integration-5-scenario004-tempest-ubuntu-xenial-mimic
-        - puppet-openstack-integration-4-scenario004-tempest-centos-7
+        - puppet-openstack-integration-4-scenario004-tempest-centos-7-mimic
-        - puppet-openstack-integration-5-scenario004-tempest-centos-7
+        - puppet-openstack-integration-5-scenario004-tempest-centos-7-mimic
     experimental:
       jobs:
-        - puppet-openstack-integration-5-scenario004-tempest-debian-stable
+        - puppet-openstack-integration-5-scenario004-tempest-debian-stable-luminous
 
 - job:
     name: puppet-openstack-module-base

configure_facts.sh

@@ -24,10 +24,15 @@ if [ -f /etc/ci/mirror_info.sh ]; then
     CENTOS_MIRROR_HOST="http://${NODEPOOL_MIRROR_HOST}"
     DEPS_MIRROR_HOST="${NODEPOOL_RDO_PROXY}/centos7-master/deps/latest/"
     if uses_debs; then
-        CEPH_MIRROR_HOST="${CENTOS_MIRROR_HOST}/ceph-deb-luminous"
+        CEPH_MIRROR_HOST="${CENTOS_MIRROR_HOST}/ceph-deb-${CEPH_VERSION}"
         NODEPOOL_PUPPETLABS_MIRROR="http://${NODEPOOL_MIRROR_HOST}/apt-puppetlabs"
     else
-        CEPH_MIRROR_HOST="${NODEPOOL_BUILDLOGS_CENTOS_PROXY}/centos/7/storage/x86_64/ceph-luminous/"
+        # TODO(tobasco): Remove this CBS candidate repo for Mimic when Storage SIG release it.
+        if [ "$CEPH_VERSION" == "mimic" ]; then
+            CEPH_MIRROR_HOST='http://cbs.centos.org/repos/storage7-ceph-mimic-candidate/x86_64/os/'
+        else
+            CEPH_MIRROR_HOST="${NODEPOOL_BUILDLOGS_CENTOS_PROXY}/centos/7/storage/x86_64/ceph-${CEPH_VERSION}/"
+        fi
         NODEPOOL_PUPPETLABS_MIRROR="http://${NODEPOOL_MIRROR_HOST}/yum-puppetlabs"
     fi
 else
@@ -36,10 +41,15 @@ else
     NODEPOOL_RDO_PROXY='https://trunk.rdoproject.org'
     NODEPOOL_UCA_MIRROR='http://ubuntu-cloud.archive.canonical.com/ubuntu'
     if uses_debs; then
-        CEPH_MIRROR_HOST='https://download.ceph.com/debian-luminous'
+        CEPH_MIRROR_HOST="https://download.ceph.com/debian-${CEPH_VERSION}"
         NODEPOOL_PUPPETLABS_MIRROR='https://apt.puppetlabs.com'
     else
-        CEPH_MIRROR_HOST='https://buildlogs.centos.org/centos/7/storage/x86_64/ceph-luminous/'
+        # TODO(tobasco): Remove this CBS candidate repo for Mimic when Storage SIG releases it.
+        if [ "$CEPH_VERSION" == "mimic" ]; then
+            CEPH_MIRROR_HOST='http://cbs.centos.org/repos/storage7-ceph-mimic-candidate/x86_64/os/'
+        else
+            CEPH_MIRROR_HOST="https://buildlogs.centos.org/centos/7/storage/x86_64/ceph-${CEPH_VERSION}/"
+        fi
         NODEPOOL_PUPPETLABS_MIRROR="https://yum.puppetlabs.com"
     fi
 fi
@@ -56,13 +66,15 @@ export FACTER_uca_mirror_host=$NODEPOOL_UCA_MIRROR
 export FACTER_deps_mirror_host=$DEPS_MIRROR_HOST
 export FACTER_ceph_mirror_host=$CEPH_MIRROR_HOST
 export FACTER_rdo_mirror_host=$RDO_MIRROR_HOST
+export FACTER_ceph_version=$CEPH_VERSION
 
 MIRROR_FACTS="\
 centos_mirror_host=${FACTER_centos_mirror_host}
 uca_mirror_host=${FACTER_uca_mirror_host}
 deps_mirror_host=${FACTER_deps_mirror_host}
 ceph_mirror_host=${FACTER_ceph_mirror_host}
-rdo_mirror_host=${FACTER_rdo_mirror_host}"
+rdo_mirror_host=${FACTER_rdo_mirror_host}
+ceph_version=${FACTER_ceph_version}"
 
 if [ "${WRITE_FACTS}" = true ]; then
     $SUDO mkdir -p /etc/facter/facts.d/

functions

@@ -19,15 +19,23 @@ install_external() {
 # - ``SCRIPT_DIR`` must be set to script path
 # - ``ZUUL_BRANCH`` must be set to Zuul branch. Fallback to 'master'.
 install_openstack() {
+  # Periodic jobs run without ref on master
+  ZUUL_BRANCH=${ZUUL_BRANCH:-master}
+
+  if [ $ZUUL_PROJECT != "openstack/puppet-ceph" ] && [ -n "$CEPH_VERSION" ]; then
+    if [ $CEPH_VERSION == "mimic" ]; then
+      ZUUL_BRANCH="master"
+    else
+      ZUUL_BRANCH="stable/$CEPH_VERSION"
+    fi
+  fi
+
   cat > clonemap.yaml <<EOF
 clonemap:
   - name: '(.*?)/puppet-(.*)'
     dest: '$PUPPETFILE_DIR/\2'
 EOF
 
-  # Periodic jobs run without ref on master
-  ZUUL_BRANCH=${ZUUL_BRANCH:-master}
-
   local project_names=$(awk '{ if ($1 == ":git") print $3 }' \
     ${SCRIPT_DIR}/Puppetfile0 | tr -d "'," | cut -d '/' -f 4- | xargs
   )

manifests/repos.pp

@@ -31,7 +31,7 @@ class openstack_integration::repos {
       }
       $enable_sig  = false
       $enable_epel = false
-      $ceph_mirror = pick($::ceph_mirror_host, 'http://download.ceph.com/debian-luminous/')
+      $ceph_mirror = pick($::ceph_mirror_host, "http://download.ceph.com/debian-${::ceph_version}/")
     }
     'RedHat': {
       class { '::openstack_extras::repo::redhat::redhat':
@@ -52,7 +52,12 @@ class openstack_integration::repos {
           },
         },
       }
-      $ceph_mirror = pick($::ceph_mirror_host, 'https://buildlogs.centos.org/centos/7/storage/x86_64/ceph-luminous/')
+      # TODO(tobasco): Remove this CBS candidate repo for Mimic when Storage SIG release it.
+      $ceph_mirror_fallback = $::ceph_version ? {
+        'mimic' => 'http://cbs.centos.org/repos/storage7-ceph-mimic-candidate/x86_64/os/',
+        default => "https://buildlogs.centos.org/centos/7/storage/x86_64/ceph-${::ceph_version}/"
+      }
+      $ceph_mirror = pick($::ceph_mirror_host, $ceph_mirror_fallback)
       # On CentOS, deploy Ceph using SIG repository and get rid of EPEL.
       # https://wiki.centos.org/SpecialInterestGroup/Storage/
       if $::operatingsystem == 'CentOS' {

playbooks/run-integration-tests.yaml

@@ -4,6 +4,7 @@
         cmd: |
           set -ex
           trap "./copy_logs.sh" EXIT
+          export CEPH_VERSION={{ ceph }}
           export PUPPET_MAJ_VERSION={{ puppet }}
           export SCENARIO={{ scenario }}
           export GEM_HOME=`pwd`/.bundled_gems

run_tests.sh

@@ -18,6 +18,7 @@ source ${SCRIPT_DIR}/functions
 
 export WORKSPACE=${WORKSPACE:-/tmp}
 export PUPPET_URL_SUFFIX=
+export CEPH_VERSION=${CEPH_VERSION:-luminous}
 export PUPPET_MAJ_VERSION=${PUPPET_MAJ_VERSION:-4}
 export SCENARIO=${SCENARIO:-scenario001}
 export MANAGE_PUPPET_MODULES=${MANAGE_PUPPET_MODULES:-true}