From b55ed05a274e5da40b567ad127a3d1c5808e48c6 Mon Sep 17 00:00:00 2001
From: Monty Taylor <mordred@inaugust.com>
Date: Mon, 17 Mar 2014 04:01:33 -0400
Subject: [PATCH] Drive puppet from the master over ssh
We'd like to be able to control sequencing of how and when puppet
runs across our machines. Currently, it's just a set of agents
that run kinda whenever they run. At times they hang and we don't
know about it. Also, cross-server sequencing is impossible to
achieve.
Change the operation away from agents running on the machine as
daemons, and instead ssh from the master to each machine.
Change-Id: I76e41e63c6d0825e8735c484ba4580d545515e43
---
doc/source/puppet.rst | 47 ++++++++++++++-----
doc/source/running-your-own.rst | 3 --
launch/README | 20 --------
.../openstack_project/files/puppet.default | 7 +++
modules/openstack_project/manifests/base.pp | 12 +++++
.../manifests/puppetmaster.pp | 2 +-
6 files changed, 55 insertions(+), 36 deletions(-)
create mode 100644 modules/openstack_project/files/puppet.default
diff --git a/doc/source/puppet.rst b/doc/source/puppet.rst
index 75616b956a..2520939239 100644
--- a/doc/source/puppet.rst
+++ b/doc/source/puppet.rst
@@ -82,13 +82,6 @@ On the new server connecting (for example, review.openstack.org) to the puppet m
sudo apt-get install puppet
-Then edit the ``/etc/default/puppet`` file to change the start variable:
-
-.. code-block:: ini
-
- # Start puppet on boot?
- START=yes
-
The node then needs to be configured to set a fixed hostname and the hostname
of the puppet master with the following additions to ``/etc/puppet/puppet.conf``:
@@ -121,15 +114,45 @@ If you see the new node there you can sign its cert on the puppet master with:
sudo puppet cert sign review.openstack.org
-Finally on the puppet agent you need to start the agent daemon:
+Once the cert is signed, the puppet running orchestration will pick up
+the node and run puppet on it as needed.
+
+Running Puppet on Nodes
+-----------------------
+
+In OpenStack's Infrastructure, puppet runs are triggered from a cronjob
+running on the puppetmaster which in turn runs a single run of puppet on
+each host we know about. We do not use the daemon mode of puppet agent
+because it experiences random hangs, and also does not allow us to control
+sequencing in any meaningful way.
+
+The entry point for this process is ``/opt/config/production/run_all.sh``
+
+There are a set of nodes, which are configured in puppet as "override" nodes,
+which are run in sequence before the rest of the nodes are run in parellel.
+At the moment, this allows creation of git repos on the git slaves before
+creation of the master repos on the gerrit server.
+
+Disabling Puppet on Nodes
+-------------------------
+
+In the case of needing to disable the running of puppet on a node, it's a
+simple matter of disabling the agent:
.. code-block:: bash
- sudo service puppet start
+ sudo puppet agent --disable
+
+This will prevent any subsequent runs of the agent, including ones triggered
+globally by the run_all script. If, as an admin, you need to run puppet on
+a node where it has been disabled, you need to specify an alternate disable
+lock file which will allow your local run of puppet without allowing the
+globally orchestrated runs to occur:
+
+.. code-block:: bash
+
+ sudo puppet agent --test --puppetdlockfile=/tmp/alt-lock-file
-Now that it is signed the puppet agent will execute any instructions for its
-node on the next run (default is every 30 minutes). You can trigger this
-earlier by restarting the puppet service on the agent node.
Important Notes
---------------
diff --git a/doc/source/running-your-own.rst b/doc/source/running-your-own.rst
index 7a308a5393..a182610ea7 100644
--- a/doc/source/running-your-own.rst
+++ b/doc/source/running-your-own.rst
@@ -129,9 +129,6 @@ details)::
* Run the DNS update commands [nb: install your DNS API by hand at the moment]
-* ssh into the new node and update its ``/etc/default/puppet`` to autostart
- per the launch README.
-
Stage 3 - gerrit
~~~~~~~~~~~~~~~~
diff --git a/launch/README b/launch/README
index 0b339edcd2..2105a3eac1 100644
--- a/launch/README
+++ b/launch/README
@@ -64,23 +64,3 @@ run to configure DNS for a newly launched server. To see the commands
for an existing server, run:
./dns.py $FQDN
-
-Activate Puppet Agent
-=====================
-
-If this is a Jenkins slave, Puppet configuration is applied through
-an already installed cron job, so you can ignore this section. If
-this is ''not'' a Jenkins slave, you'll want to log into it via SSH
-and turn on the Puppet agent so it will start checking into the
-master on its own. on Debian/Ubuntu::
-
- sudo sed -i 's/^START=.*/START=yes/' /etc/default/puppet
- sudo su -c 'invoke-rc.d puppet start'
-
-...or on CentOS/Fedora/RHEL::
-
- sudo chkconfig puppet on
- sudo su -c 'service puppet start'
-
-You should be able to tell from the Puppet Dashboard when it begins
-to check in, which normally happens at 10-minute intervals.
diff --git a/modules/openstack_project/files/puppet.default b/modules/openstack_project/files/puppet.default
new file mode 100644
index 0000000000..4bc02a0c29
--- /dev/null
+++ b/modules/openstack_project/files/puppet.default
@@ -0,0 +1,7 @@
+# Defaults for puppet - sourced by /etc/init.d/puppet
+
+# Start puppet on boot?
+START=no
+
+# Startup options
+DAEMON_OPTS=""
diff --git a/modules/openstack_project/manifests/base.pp b/modules/openstack_project/manifests/base.pp
index e504abd85a..3f4ca78df8 100644
--- a/modules/openstack_project/manifests/base.pp
+++ b/modules/openstack_project/manifests/base.pp
@@ -100,6 +100,15 @@ class openstack_project::base(
replace => true,
}
+ file { '/etc/default/puppet':
+ ensure => present,
+ owner => 'root',
+ group => 'root',
+ mode => '0444',
+ source => 'puppet:///modules/openstack_project/puppet.default',
+ replace => true,
+ }
+
}
file { '/etc/puppet/puppet.conf':
@@ -111,6 +120,9 @@ class openstack_project::base(
replace => true,
}
+ service { 'puppet':
+ ensure => stopped,
+ }
}
# vim:sw=2:ts=2:expandtab:textwidth=79
diff --git a/modules/openstack_project/manifests/puppetmaster.pp b/modules/openstack_project/manifests/puppetmaster.pp
index 12c803e75c..d6bc2d96bb 100644
--- a/modules/openstack_project/manifests/puppetmaster.pp
+++ b/modules/openstack_project/manifests/puppetmaster.pp
@@ -20,7 +20,7 @@ class openstack_project::puppetmaster (
cron { 'updatepuppetmaster':
user => 'root',
minute => '*/15',
- command => 'sleep $((RANDOM\%600)) && cd /opt/config/production && git fetch -q && git reset -q --hard @{u} && ./install_modules.sh && touch manifests/site.pp',
+ command => 'bash /opt/config/production/run_all.sh',
environment => 'PATH=/var/lib/gems/1.8/bin:/usr/bin:/bin:/usr/sbin:/sbin',
}