Implement a script to generate Puppetfile

* Implement generate_puppetfile.sh
This script will read from openstack_modules.txt and
external_modules.txt that contain the list of Puppet modules we rely to
run our CI. It will build a Puppetfile by using master for Puppet
OpenStack modules and the latest tag for external modules.

* Remove Puppetfile, that is not useful anymore since it's generated.

* Run the script everytime our CI needs to install modules.
Which means our CI will always test the latests versions of our external
modules.

* Allow to pin modules.
Example:
puppetlabs/puppetlabs-apache,1.8.1

Change-Id: I2d529cc16b335d112f996c9f1f0266f440eab065

Puppetfile

@@ -1,5 +1,6 @@
-# OpenStack modules
+# Auto-generated Puppetfile for Puppet OpenStack project
 
+## OpenStack modules
 mod 'aodh',
   :git => 'https://git.openstack.org/openstack/puppet-aodh',
   :ref => 'master'
@@ -108,10 +109,6 @@ mod 'trove',
   :git => 'https://git.openstack.org/openstack/puppet-trove',
   :ref => 'master'
 
-mod 'tuskar',
-  :git => 'https://git.openstack.org/openstack/puppet-tuskar',
-  :ref => 'master'
-
 mod 'vswitch',
   :git => 'https://git.openstack.org/openstack/puppet-vswitch',
   :ref => 'master'
@@ -120,88 +117,84 @@ mod 'zaqar',
   :git => 'https://git.openstack.org/openstack/puppet-zaqar',
   :ref => 'master'
 
-# External modules
+## External modules
-
-mod 'apache',
-  :git => 'https://github.com/puppetlabs/puppetlabs-apache',
-  :branch => '1.8.x'
-
-mod 'apt',
-  :git => 'https://github.com/puppetlabs/puppetlabs-apt',
-  :branch => '2.2.x'
-
-mod 'concat',
-  :git => 'https://github.com/puppetlabs/puppetlabs-concat',
-  :branch => '1.2.x'
-
-mod 'corosync',
-  :git => 'https://github.com/puppet-community/puppet-corosync',
-  :tag => '0.8.0'
-
-mod 'dns',
-  :git => 'https://github.com/theforeman/puppet-dns',
-  :tag => '3.1.0'
-
-mod 'firewall',
-  :git => 'https://github.com/puppetlabs/puppetlabs-firewall',
-  :branch => '1.7.x'
-
-mod 'inifile',
-  :git => 'https://github.com/puppetlabs/puppetlabs-inifile',
-  :branch => '1.4.x'
-
-mod 'memcached',
-  :git => 'https://github.com/saz/puppet-memcached',
-  :tag => 'v2.8.1'
-
-mod 'mongodb',
-  :git => 'https://github.com/puppetlabs/puppetlabs-mongodb',
-  :branch => '0.12.0'
-
-mod 'mysql',
-  :git => 'https://github.com/puppetlabs/puppetlabs-mysql',
-  :branch => '3.6.x'
-
-mod 'postgresql',
-  :git => 'https://github.com/puppetlabs/puppetlabs-postgresql',
-  :branch => '4.6.x'
-
 mod 'powerdns',
   :git => 'https://github.com/antonlindstrom/puppet-powerdns',
-  :tag => '0.0.5'
+  :ref => '0.0.5'
-
-mod 'python',
-  :git => 'https://github.com/stankevich/puppet-python',
-  :tag => '1.10.0'
-
-mod 'qpid',
-  :git => 'https://github.com/dprince/puppet-qpid',
-  :tag => '1.0.3'
-
-mod 'rabbitmq',
-  :git => 'https://github.com/puppetlabs/puppetlabs-rabbitmq',
-  :tag => '5.3.1'
-
-mod 'rsync',
-  :git => 'https://github.com/puppetlabs/puppetlabs-rsync',
-  :tag => '0.4.0'
-
-mod 'staging',
-  :git => 'https://github.com/nanliu/puppet-staging',
-  :tag => '1.0.4'
-
-mod 'stdlib',
-  :git => 'https://github.com/puppetlabs/puppetlabs-stdlib',
-  :branch => '4.9.x'
 
 mod 'sysctl',
   :git => 'https://github.com/duritong/puppet-sysctl',
-  :tag => 'v0.0.11'
+  :ref => 'v0.0.11'
+
+mod 'staging',
+  :git => 'https://github.com/nanliu/puppet-staging',
+  :ref => '1.0.4'
+
+mod 'corosync',
+  :git => 'https://github.com/puppet-community/puppet-corosync',
+  :ref => '0.8.0'
+
+mod 'apache',
+  :git => 'https://github.com/puppetlabs/puppetlabs-apache',
+  :ref => '1.8.1'
+
+mod 'apt',
+  :git => 'https://github.com/puppetlabs/puppetlabs-apt',
+  :ref => '2.2.2'
+
+mod 'concat',
+  :git => 'https://github.com/puppetlabs/puppetlabs-concat',
+  :ref => '1.2.5'
+
+mod 'firewall',
+  :git => 'https://github.com/puppetlabs/puppetlabs-firewall',
+  :ref => '1.8.0'
+
+mod 'inifile',
+  :git => 'https://github.com/puppetlabs/puppetlabs-inifile',
+  :ref => '1.5.0'
+
+mod 'mongodb',
+  :git => 'https://github.com/puppetlabs/puppetlabs-mongodb',
+  :ref => '0.13.0'
+
+mod 'mysql',
+  :git => 'https://github.com/puppetlabs/puppetlabs-mysql',
+  :ref => '3.7.0'
+
+mod 'postgresql',
+  :git => 'https://github.com/puppetlabs/puppetlabs-postgresql',
+  :ref => '4.7.1'
+
+mod 'rabbitmq',
+  :git => 'https://github.com/puppetlabs/puppetlabs-rabbitmq',
+  :ref => '5.3.1'
+
+mod 'rsync',
+  :git => 'https://github.com/puppetlabs/puppetlabs-rsync',
+  :ref => '0.4.0'
+
+mod 'stdlib',
+  :git => 'https://github.com/puppetlabs/puppetlabs-stdlib',
+  :ref => '4.11.0'
 
 mod 'vcsrepo',
   :git => 'https://github.com/puppetlabs/puppetlabs-vcsrepo',
-  :branch => '1.3.x'
+  :ref => '1.3.2'
 
 mod 'xinetd',
   :git => 'https://github.com/puppetlabs/puppetlabs-xinetd',
-  :tag => '1.5.0'
+  :ref => '1.5.0'
+
+mod 'memcached',
+  :git => 'https://github.com/saz/puppet-memcached',
+  :ref => 'v2.8.1'
+
+mod 'python',
+  :git => 'https://github.com/stankevich/puppet-python',
+  :ref => '1.12.0'
+
+mod 'dns',
+  :git => 'https://github.com/theforeman/puppet-dns',
+  :ref => '3.2.0'
+

external_modules.txt

@@ -0,0 +1,20 @@
+antonlindstrom/puppet-powerdns
+duritong/puppet-sysctl
+nanliu/puppet-staging
+puppet-community/puppet-corosync
+puppetlabs/puppetlabs-apache
+puppetlabs/puppetlabs-apt
+puppetlabs/puppetlabs-concat,1.2.5
+puppetlabs/puppetlabs-firewall
+puppetlabs/puppetlabs-inifile
+puppetlabs/puppetlabs-mongodb
+puppetlabs/puppetlabs-mysql
+puppetlabs/puppetlabs-postgresql
+puppetlabs/puppetlabs-rabbitmq
+puppetlabs/puppetlabs-rsync
+puppetlabs/puppetlabs-stdlib
+puppetlabs/puppetlabs-vcsrepo
+puppetlabs/puppetlabs-xinetd
+saz/puppet-memcached
+stankevich/puppet-python
+theforeman/puppet-dns

generate_puppetfile.sh

@@ -0,0 +1,57 @@
+#!/bin/bash
+#
+# Build a Puppetfile with latest dependencies
+#
+
+# cleanup
+rm -rf modules Puppetfile
+mkdir modules
+
+# header
+echo -e "# Auto-generated Puppetfile for Puppet OpenStack project\n" > Puppetfile
+
+# OpenStack Modules
+echo "## OpenStack modules" >> Puppetfile
+for p in $(cat openstack_modules.txt); do
+    # hack for puppet-openstack-integration
+    # where namespace is openstack_integration
+    title=$(echo $p | sed 's/-/_/g')
+    # TODO(emilien) we need to add support for stable branches
+    cat >> Puppetfile <<EOF
+mod '$title',
+  :git => 'https://git.openstack.org/openstack/puppet-$p',
+  :ref => 'master'
+
+EOF
+done
+
+# External Modules
+echo -e "## External modules" >> Puppetfile
+for e in $(cat external_modules.txt); do
+    namespace=$(echo $e | awk -F'/' '{print $1}' | cut -d "," -f 1)
+    module=$(echo $e | awk -F'/' '{print $2}' | cut -d "," -f 1)
+    title=$(echo $module | awk -F'/' '{print $1}' | cut -d "-" -f 2)
+    pin=$(echo $e | grep "," | cut -d "," -f 2)
+    if [ ! -z "$pin" ]; then
+        git ls-remote --exit-code https://github.com/$namespace/$module $pin
+        if (($? == 2)); then
+            echo "Wrong pin: $pin does not exist in $module module."
+            exit 1
+        else
+            tag=$pin
+        fi
+    else
+      git clone https://github.com/$namespace/$module modules/$module
+      tag=$(cd modules/$module; git describe --tags $(git rev-list --tags --max-count=1))
+      rm -rf modules/$module
+    fi
+    cat >> Puppetfile <<EOF
+mod '$title',
+  :git => 'https://github.com/$namespace/$module',
+  :ref => '$tag'
+
+EOF
+done
+
+# for debug
+cat Puppetfile

openstack_modules.txt

@@ -0,0 +1,29 @@
+aodh
+barbican
+ceilometer
+ceph
+cinder
+designate
+glance
+gnocchi
+heat
+horizon
+ironic
+keystone
+manila
+mistral
+monasca
+murano
+neutron
+nova
+octavia
+openstack-integration
+openstack_extras
+openstacklib
+oslo
+sahara
+swift
+tempest
+trove
+vswitch
+zaqar