From b44d0bdd5a19661130b6ada02f9ae825783a78a1 Mon Sep 17 00:00:00 2001 From: yatinkarel Date: Tue, 10 Dec 2019 11:22:06 +0530 Subject: [PATCH] Add CentOS8 support and jobs - scenario001 and 004 will be running without ceph until ceph repos get available for CentOS8 - scenario003 will run not run with linuxbridge until next CentOS minor version as it has issues:- - https://bugzilla.redhat.com/show_bug.cgi?id=1720637 - Add Puppetfile_centos7 to have different puppet modules as compared to other distros. Currently only puppet-mysql needs to be kept pinned for centos7 as newere mariadb version is required to remove the pin, CentOS8 and ubuntu have required mariadb version so pin is removed. So automatic updates of puppet modules will not be applied for CentOS7 as those jobs are going to be removed in coming months. - check for rdo_dlrn url existence only for RedHat distros. Change-Id: I98fc088cd87c1412544f9590ce7a925b413297e2 --- Puppetfile | 2 +- Puppetfile_centos7 | 252 ++++++++++++++++++++++++++++++++++++++++ configure_facts.sh | 9 +- external_modules.txt | 2 +- fixtures/scenario001.pp | 23 +++- fixtures/scenario003.pp | 9 +- fixtures/scenario004.pp | 21 +++- functions | 16 ++- manifests/config.pp | 5 +- manifests/mysql.pp | 4 +- manifests/neutron.pp | 6 + manifests/rabbitmq.pp | 2 + zuul.d/integration.yaml | 34 ++++++ 13 files changed, 361 insertions(+), 24 deletions(-) create mode 100644 Puppetfile_centos7 diff --git a/Puppetfile b/Puppetfile index 84467f6e2..d9b19ce78 100644 --- a/Puppetfile +++ b/Puppetfile @@ -196,7 +196,7 @@ mod 'inifile', mod 'mysql', :git => 'https://github.com/puppetlabs/puppetlabs-mysql', - :ref => '6.2.0' + :ref => 'v10.3.0' mod 'postgresql', :git => 'https://github.com/puppetlabs/puppetlabs-postgresql', diff --git a/Puppetfile_centos7 b/Puppetfile_centos7 new file mode 100644 index 000000000..339878700 --- /dev/null +++ b/Puppetfile_centos7 @@ -0,0 +1,252 @@ +# Auto-generated Puppetfile for Puppet OpenStack project + +## OpenStack modules +mod 'aodh', + :git => 'https://opendev.org/openstack/puppet-aodh', + :ref => 'master' + +mod 'barbican', + :git => 'https://opendev.org/openstack/puppet-barbican', + :ref => 'master' + +mod 'ceilometer', + :git => 'https://opendev.org/openstack/puppet-ceilometer', + :ref => 'master' + +mod 'ceph', + :git => 'https://opendev.org/openstack/puppet-ceph', + :ref => 'master' + +mod 'cinder', + :git => 'https://opendev.org/openstack/puppet-cinder', + :ref => 'master' + +mod 'cloudkitty', + :git => 'https://opendev.org/openstack/puppet-cloudkitty', + :ref => 'master' + +mod 'congress', + :git => 'https://opendev.org/openstack/puppet-congress', + :ref => 'master' + +mod 'designate', + :git => 'https://opendev.org/openstack/puppet-designate', + :ref => 'master' + +mod 'ec2api', + :git => 'https://opendev.org/openstack/puppet-ec2api', + :ref => 'master' + +mod 'glance', + :git => 'https://opendev.org/openstack/puppet-glance', + :ref => 'master' + +mod 'gnocchi', + :git => 'https://opendev.org/openstack/puppet-gnocchi', + :ref => 'master' + +mod 'heat', + :git => 'https://opendev.org/openstack/puppet-heat', + :ref => 'master' + +mod 'horizon', + :git => 'https://opendev.org/openstack/puppet-horizon', + :ref => 'master' + +mod 'ironic', + :git => 'https://opendev.org/openstack/puppet-ironic', + :ref => 'master' + +mod 'keystone', + :git => 'https://opendev.org/openstack/puppet-keystone', + :ref => 'master' + +mod 'manila', + :git => 'https://opendev.org/openstack/puppet-manila', + :ref => 'master' + +mod 'mistral', + :git => 'https://opendev.org/openstack/puppet-mistral', + :ref => 'master' + +mod 'monasca', + :git => 'https://opendev.org/openstack/puppet-monasca', + :ref => 'master' + +mod 'murano', + :git => 'https://opendev.org/openstack/puppet-murano', + :ref => 'master' + +mod 'neutron', + :git => 'https://opendev.org/openstack/puppet-neutron', + :ref => 'master' + +mod 'nova', + :git => 'https://opendev.org/openstack/puppet-nova', + :ref => 'master' + +mod 'octavia', + :git => 'https://opendev.org/openstack/puppet-octavia', + :ref => 'master' + +mod 'openstack_extras', + :git => 'https://opendev.org/openstack/puppet-openstack_extras', + :ref => 'master' + +mod 'openstacklib', + :git => 'https://opendev.org/openstack/puppet-openstacklib', + :ref => 'master' + +mod 'oslo', + :git => 'https://opendev.org/openstack/puppet-oslo', + :ref => 'master' + +mod 'ovn', + :git => 'https://opendev.org/openstack/puppet-ovn', + :ref => 'master' + +mod 'panko', + :git => 'https://opendev.org/openstack/puppet-panko', + :ref => 'master' + +mod 'placement', + :git => 'https://opendev.org/openstack/puppet-placement', + :ref => 'master' + +mod 'qdr', + :git => 'https://opendev.org/openstack/puppet-qdr', + :ref => 'master' + +mod 'sahara', + :git => 'https://opendev.org/openstack/puppet-sahara', + :ref => 'master' + +mod 'swift', + :git => 'https://opendev.org/openstack/puppet-swift', + :ref => 'master' + +mod 'tacker', + :git => 'https://opendev.org/openstack/puppet-tacker', + :ref => 'master' + +mod 'tempest', + :git => 'https://opendev.org/openstack/puppet-tempest', + :ref => 'master' + +mod 'trove', + :git => 'https://opendev.org/openstack/puppet-trove', + :ref => 'master' + +mod 'vswitch', + :git => 'https://opendev.org/openstack/puppet-vswitch', + :ref => 'master' + +mod 'vitrage', + :git => 'https://opendev.org/openstack/puppet-vitrage', + :ref => 'master' + +mod 'watcher', + :git => 'https://opendev.org/openstack/puppet-watcher', + :ref => 'master' + +mod 'zaqar', + :git => 'https://opendev.org/openstack/puppet-zaqar', + :ref => 'master' + +## External modules +mod 'powerdns', + :git => 'https://github.com/antonlindstrom/puppet-powerdns', + :ref => '0.0.5' + +mod 'kmod', + :git => 'https://github.com/camptocamp/puppet-kmod', + :ref => '2.3.1' + +mod 'sysctl', + :git => 'https://github.com/duritong/puppet-sysctl', + :ref => 'v0.0.12' + +mod 'ipaclient', + :git => 'https://github.com/joshuabaird/puppet-ipaclient', + :ref => '2.5.2' + +mod 'staging', + :git => 'https://github.com/nanliu/puppet-staging', + :ref => '1.0.4' + +mod 'apache', + :git => 'https://github.com/puppetlabs/puppetlabs-apache', + :ref => 'v5.3.0' + +mod 'apt', + :git => 'https://github.com/puppetlabs/puppetlabs-apt', + :ref => '2.4.0' + +mod 'concat', + :git => 'https://github.com/puppetlabs/puppetlabs-concat', + :ref => '2.2.1' + +mod 'firewall', + :git => 'https://github.com/puppetlabs/puppetlabs-firewall', + :ref => 'v2.2.0' + +mod 'inifile', + :git => 'https://github.com/puppetlabs/puppetlabs-inifile', + :ref => '2.2.0' + +mod 'mysql', + :git => 'https://github.com/puppetlabs/puppetlabs-mysql', + :ref => '6.2.0' + +mod 'postgresql', + :git => 'https://github.com/puppetlabs/puppetlabs-postgresql', + :ref => 'v6.2.0' + +mod 'rabbitmq', + :git => 'https://github.com/voxpupuli/puppet-rabbitmq', + :ref => 'v10.0.0' + +mod 'rsync', + :git => 'https://github.com/puppetlabs/puppetlabs-rsync', + :ref => '1.1.1' + +mod 'stdlib', + :git => 'https://github.com/puppetlabs/puppetlabs-stdlib', + :ref => 'v6.2.0' + +mod 'vcsrepo', + :git => 'https://github.com/puppetlabs/puppetlabs-vcsrepo', + :ref => 'v3.1.0' + +mod 'git_resource', + :git => 'https://github.com/voxpupuli/puppet-git_resource', + :ref => 'v1.0.2' + +mod 'xinetd', + :git => 'https://github.com/puppetlabs/puppetlabs-xinetd', + :ref => 'v3.3.0' + +mod 'memcached', + :git => 'https://github.com/saz/puppet-memcached', + :ref => 'v3.4.0' + +mod 'python', + :git => 'https://github.com/voxpupuli/puppet-python', + :ref => 'v4.0.0' + +mod 'dns', + :git => 'https://github.com/theforeman/puppet-dns', + :ref => '6.2.0' + +mod 'archive', + :git => 'https://github.com/voxpupuli/puppet-archive', + :ref => 'v4.4.0' + +mod 'corosync', + :git => 'https://github.com/voxpupuli/puppet-corosync', + :ref => 'v5.0.0' + +mod 'ssh_keygen', + :git => 'https://github.com/voxpupuli/puppet-ssh_keygen', + :ref => 'v2.0.1' + diff --git a/configure_facts.sh b/configure_facts.sh index 0ac683864..7f3252935 100644 --- a/configure_facts.sh +++ b/configure_facts.sh @@ -13,10 +13,13 @@ # License for the specific language governing permissions and limitations # under the License. +source /etc/os-release +OS_NAME_VERS=${REDHAT_SUPPORT_PRODUCT}${REDHAT_SUPPORT_PRODUCT_VERSION} + # Write out facts to the facter folder when we generate them. export WRITE_FACTS=${WRITE_FACTS:-true} -export DLRN_BASE=${DLRN_BASE:-centos7-master/puppet-passed-ci} -export DLRN_DEPS_BASE=${DLRN_DEPS_BASE:-centos7-master/deps/latest/} +export DLRN_BASE=${DLRN_BASE:-${OS_NAME_VERS}-master/puppet-passed-ci} +export DLRN_DEPS_BASE=${DLRN_DEPS_BASE:-${OS_NAME_VERS}-master/deps/latest/} export CEPH_VERSION=${CEPH_VERSION:-nautilus} export SCRIPT_DIR=$(cd `dirname $0` && pwd -P) @@ -63,7 +66,7 @@ else fi rdo_dlrn=`curl --silent ${NODEPOOL_RDO_PROXY}/${DLRN_BASE}/delorean.repo | grep baseurl | cut -d= -f2` -if [[ -z "$rdo_dlrn" ]]; then +if [[ -z "$rdo_dlrn" ]] && [ -f /etc/redhat-release ] ; then echo "Failed to parse dlrn hash" exit 1 fi diff --git a/external_modules.txt b/external_modules.txt index 414760065..10638f584 100644 --- a/external_modules.txt +++ b/external_modules.txt @@ -8,7 +8,7 @@ puppetlabs/puppetlabs-apt,2.4.0 puppetlabs/puppetlabs-concat,2.2.1 puppetlabs/puppetlabs-firewall puppetlabs/puppetlabs-inifile,2.2.0 -puppetlabs/puppetlabs-mysql,6.2.0 +puppetlabs/puppetlabs-mysql puppetlabs/puppetlabs-postgresql voxpupuli/puppet-rabbitmq puppetlabs/puppetlabs-rsync diff --git a/fixtures/scenario001.pp b/fixtures/scenario001.pp index 914940991..26b1706cf 100644 --- a/fixtures/scenario001.pp +++ b/fixtures/scenario001.pp @@ -44,6 +44,15 @@ if ($::os['name'] == 'Ubuntu') or ($::os['name'] == 'Fedora') or $ssl = true } +# FIXME(ykarel) Enable ceph in CentOS8 once Ceph repos are available +if ($::os['family'] == 'RedHat' and Integer.new($::os['release']['major']) > 7) { + $backend = undef + $ceph = false +} else { + $backend = 'rbd' + $ceph = true +} + case $::osfamily { 'Debian': { $ipv6 = false @@ -87,18 +96,18 @@ class { 'openstack_integration::keystone': token_expiration => '2400', } class { 'openstack_integration::glance': - backend => 'rbd', + backend => $backend, } class { 'openstack_integration::neutron': notification_topics => $notification_topics, } include openstack_integration::placement class { 'openstack_integration::nova': - libvirt_rbd => true, + libvirt_rbd => $ceph, notification_topics => $notification_topics, } class { 'openstack_integration::cinder': - backend => 'rbd', + backend => $backend, } include openstack_integration::ceilometer class { 'openstack_integration::aodh': @@ -107,13 +116,17 @@ class { 'openstack_integration::aodh': if $enable_vitrage { include openstack_integration::vitrage } -include openstack_integration::ceph +if $ceph { + include openstack_integration::ceph +} class { 'openstack_integration::heat': notification_topics => $notification_topics, } include openstack_integration::provision include openstack_integration::redis -include openstack_integration::gnocchi +class { 'openstack_integration::gnocchi': + integration_enable => $ceph, +} include openstack_integration::panko class { 'openstack_integration::tempest': diff --git a/fixtures/scenario003.pp b/fixtures/scenario003.pp index 8ba5fd1f2..5744ce621 100644 --- a/fixtures/scenario003.pp +++ b/fixtures/scenario003.pp @@ -97,8 +97,15 @@ include openstack_integration::rabbitmq include openstack_integration::mysql include openstack_integration::keystone include openstack_integration::glance +# RHEL8 has an issue with linuxbridge driver https://bugzilla.redhat.com/show_bug.cgi?id=1720637 +if ($::os['family'] == 'RedHat' and Integer.new($::os['release']['major']) > 7) { + $neutron_driver = undef +} else { + $neutron_driver = 'linuxbridge' +} + class { 'openstack_integration::neutron': - driver => 'linuxbridge', + driver => $neutron_driver, } include openstack_integration::placement include openstack_integration::nova diff --git a/fixtures/scenario004.pp b/fixtures/scenario004.pp index c29ee1853..ed6c2d586 100644 --- a/fixtures/scenario004.pp +++ b/fixtures/scenario004.pp @@ -44,6 +44,15 @@ if ($::os['name'] == 'Ubuntu') or ($::os['name'] == 'Fedora') or $ssl = true } +# FIXME(ykarel) Enable ceph in CentOS8 once Ceph repos are available +if ($::os['family'] == 'RedHat' and Integer.new($::os['release']['major']) > 7) { + $backend = undef + $ceph = false +} else { + $backend = 'swift' + $ceph = true +} + if $::operatingsystem == 'Ubuntu' { $ipv6 = false # Watcher packages are not available in Ubuntu repository. @@ -76,7 +85,7 @@ include openstack_integration::rabbitmq include openstack_integration::mysql include openstack_integration::keystone class { 'openstack_integration::glance': - backend => 'swift', + backend => $backend, } class { 'openstack_integration::neutron': bgpvpn_enabled => $bgpvpn_enabled, @@ -85,12 +94,14 @@ class { 'openstack_integration::neutron': } include openstack_integration::placement class { 'openstack_integration::nova': - libvirt_rbd => true, + libvirt_rbd => $ceph, } -class { 'openstack_integration::ceph': - deploy_rgw => true, - swift_dropin => true, +if $ceph { + class { 'openstack_integration::ceph': + deploy_rgw => true, + swift_dropin => true, + } } if $watcher_enabled { include openstack_integration::watcher diff --git a/functions b/functions index 28c6f7428..5c19d463b 100644 --- a/functions +++ b/functions @@ -8,6 +8,11 @@ # # - ``SCRIPT_DIR`` must be set to script path # - ``GEM_BIN_DIR`` must be set to Gem bin directory +BASE_PUPPETFILE=Puppetfile +source /etc/os-release +if [[ "${REDHAT_SUPPORT_PRODUCT,,}" = "centos" && ${REDHAT_SUPPORT_PRODUCT_VERSION} = "7" ]]; then + BASE_PUPPETFILE=Puppetfile_centos7 +fi install_external() { PUPPETFILE=${SCRIPT_DIR}/Puppetfile1 r10k -v DEBUG puppetfile install } @@ -58,7 +63,7 @@ install_openstack() { install_all() { # When installing from local source, we want to install the current source # we're working from. - PUPPETFILE=${SCRIPT_DIR}/Puppetfile r10k -v DEBUG puppetfile install + PUPPETFILE=${SCRIPT_DIR}/${BASE_PUPPETFILE} r10k -v DEBUG puppetfile install cp -a ${SCRIPT_DIR} ${PUPPETFILE_DIR}/openstack_integration } @@ -71,7 +76,7 @@ install_all() { # - ``ZUUL_BRANCH`` must be set to Zuul branch install_modules() { if [ -d /home/zuul/src/opendev.org ] ; then - csplit ${SCRIPT_DIR}/Puppetfile /'External modules'/ \ + csplit ${SCRIPT_DIR}/${BASE_PUPPETFILE} /'External modules'/ \ --prefix ${SCRIPT_DIR}/Puppetfile \ --suffix '%d' install_external @@ -90,7 +95,7 @@ install_modules() { # - ``ZUUL_BRANCH`` must be set to Zuul branch install_modules_unit() { if [ -d /home/zuul/src/opendev.org ] ; then - csplit ${SCRIPT_DIR}/Puppetfile /'External modules'/ \ + csplit ${SCRIPT_DIR}/${BASE_PUPPETFILE} /'External modules'/ \ --prefix ${SCRIPT_DIR}/Puppetfile \ --suffix '%d' cat ${SCRIPT_DIR}/Puppetfile_unit >> ${SCRIPT_DIR}/Puppetfile1 @@ -220,17 +225,18 @@ function catch_selinux_alerts() { $SUDO sealert -a /var/log/audit/audit.log if $SUDO grep -iq 'type=AVC' /var/log/audit/audit.log; then echo "AVC detected in /var/log/audit/audit.log" + source /etc/os-release # TODO: figure why latest rabbitmq deployed with SSL tries to write in SSL pem file. # https://bugzilla.redhat.com/show_bug.cgi?id=1341738 if $SUDO grep -iqE 'denied.*system_r:rabbitmq_t' /var/log/audit/audit.log; then echo "non-critical RabbitMQ AVC, ignoring it now." # FIXME(ykarel) catch_selinux_alerts not work with non ssl scenarios(no rabbitmq alert), - # currently running all scenarios without ssl in Fedora, and scenario-py3 in CentOS/Fedora, + # currently running all scenarios without ssl in Fedora and CentOS8, # because glance,nova,mistral py3 has issues when running with eventlet + ssl: # glance https://bugs.launchpad.net/glance/+bug/1769006 # nova https://bugs.launchpad.net/nova/+bug/1808975 # mistral https://bugs.launchpad.net/mistral/+bug/1808953 - elif [ -f /etc/fedora-release -o "$SCENARIO" = "scenario-py3" ]; then + elif [ -f /etc/fedora-release ] || [[ "${REDHAT_SUPPORT_PRODUCT,,}" = "centos" && ${REDHAT_SUPPORT_PRODUCT_VERSION} = "8" ]]; then echo "non ssl scenario, ignoring it now." else echo "Please file a bug on https://bugzilla.redhat.com/enter_bug.cgi?product=Red%20Hat%20OpenStack&component=openstack-selinux showing sealert output." diff --git a/manifests/config.pp b/manifests/config.pp index f63f31571..bcf812b08 100644 --- a/manifests/config.pp +++ b/manifests/config.pp @@ -53,6 +53,7 @@ class openstack_integration::config ( $rabbit_env = { 'RABBITMQ_NODE_IP_ADDRESS' => $host, 'RABBITMQ_SERVER_START_ARGS' => '"-proto_dist inet6_tcp"', + 'LC_ALL' => 'en_US.UTF-8', } } $ip_version = '6' @@ -63,7 +64,9 @@ class openstack_integration::config ( $tooz_url = "redis://[${host}]:6379" } else { $host = '127.0.0.1' - $rabbit_env = {} + $rabbit_env = { + 'LC_ALL' => 'en_US.UTF-8', + } $ip_version = '4' $memcached_servers = ["${host}:11211"] $swift_memcached_servers = $memcached_servers diff --git a/manifests/mysql.pp b/manifests/mysql.pp index 4fabb8c50..d3a7e085e 100644 --- a/manifests/mysql.pp +++ b/manifests/mysql.pp @@ -3,8 +3,8 @@ class openstack_integration::mysql { class { 'mysql::server': } # FIXME (amoralej) Required until a new release of mariadb-libs is released by RDO - # Fedora do not have mariadb-libs, so changing only for CentOS - if $::operatingsystem == 'CentOS' { + # Fedora and CentOS8 do not have mariadb-libs, so changing only for CentOS + if ($::operatingsystem == 'CentOS') and (versioncmp($::operatingsystemmajrelease, '7') == 0) { package { 'mariadb-libs': ensure => 'latest' } diff --git a/manifests/neutron.pp b/manifests/neutron.pp index ce5230e8b..3cd891513 100644 --- a/manifests/neutron.pp +++ b/manifests/neutron.pp @@ -49,6 +49,12 @@ class openstack_integration::neutron ( case $driver { 'openvswitch': { include vswitch::ovs + # In CentOS8 puppet-vswitch requires network-scripts package until it's ported to NM. + if ($::operatingsystem == 'CentOS') and (versioncmp($::operatingsystemmajrelease, '8') == 0) { + package { 'network-scripts-openvswitch': + ensure => 'latest' + } + } # Functional test for Open-vSwitch: # create dummy loopback interface to exercise adding a port to a bridge vs_bridge { 'br-ex': diff --git a/manifests/rabbitmq.pp b/manifests/rabbitmq.pp index c18b592a1..b61b16850 100644 --- a/manifests/rabbitmq.pp +++ b/manifests/rabbitmq.pp @@ -26,6 +26,7 @@ class openstack_integration::rabbitmq { ssl_key => "/etc/rabbitmq/ssl/private/${::fqdn}.pem", environment_variables => $::openstack_integration::config::rabbit_env, repos_ensure => false, + manage_python => false, } } else { class { 'rabbitmq': @@ -33,6 +34,7 @@ class openstack_integration::rabbitmq { delete_guest_user => true, environment_variables => $::openstack_integration::config::rabbit_env, repos_ensure => false, + manage_python => false, } } rabbitmq_vhost { '/': diff --git a/zuul.d/integration.yaml b/zuul.d/integration.yaml index c5019fb34..16310d265 100644 --- a/zuul.d/integration.yaml +++ b/zuul.d/integration.yaml @@ -114,6 +114,14 @@ parent: puppet-openstack-integration-5-scenario001 nodeset: centos-7 +- job: + name: puppet-openstack-integration-5-scenario001-tempest-centos-8 + parent: puppet-openstack-integration-5-scenario001 + nodeset: + nodes: + - name: centos8 + label: centos-8 + - job: name: puppet-openstack-integration-5-scenario001-tempest-debian-stable-luminous parent: puppet-openstack-integration-5-scenario001 @@ -140,6 +148,14 @@ parent: puppet-openstack-integration-5-scenario002 nodeset: centos-7 +- job: + name: puppet-openstack-integration-5-scenario002-tempest-centos-8 + parent: puppet-openstack-integration-5-scenario002 + nodeset: + nodes: + - name: centos8 + label: centos-8 + - job: name: puppet-openstack-integration-5-scenario002-tempest-debian-stable parent: puppet-openstack-integration-5-scenario002 @@ -166,6 +182,14 @@ parent: puppet-openstack-integration-5-scenario003 nodeset: centos-7 +- job: + name: puppet-openstack-integration-5-scenario003-tempest-centos-8 + parent: puppet-openstack-integration-5-scenario003 + nodeset: + nodes: + - name: centos8 + label: centos-8 + - job: name: puppet-openstack-integration-5-scenario003-tempest-debian-stable parent: puppet-openstack-integration-5-scenario003 @@ -196,6 +220,16 @@ vars: ceph: nautilus +- job: + name: puppet-openstack-integration-5-scenario004-tempest-centos-8 + parent: puppet-openstack-integration-5-scenario004 + nodeset: + nodes: + - name: centos8 + label: centos-8 + vars: + ceph: nautilus + - job: name: puppet-openstack-integration-5-scenario004-tempest-debian-stable-luminous parent: puppet-openstack-integration-5-scenario004