Add pypi mirror building slaves.

Change-Id: I9c5ea870d365f65ff19d91ca579ee127f67d9ea8
Reviewed-on: https://review.openstack.org/25314
Reviewed-by: Jeremy Stanley <fungi@yuggoth.org>
Approved: Monty Taylor <mordred@inaugust.com>
Reviewed-by: Monty Taylor <mordred@inaugust.com>
Tested-by: Jenkins

manifests/site.pp

@@ -239,6 +239,18 @@ node /^ci-backup-.*\.openstack\.org$/ {
 # Jenkins slaves:
 #
 
+node 'mirror26.slave.openstack.org' {
+  class { 'openstack_project::mirror26_slave':
+    jenkins_ssh_private_key => hiera('jenkins_ssh_private_key_contents')
+  }
+}
+
+node 'mirror27.slave.openstack.org' {
+  class { 'openstack_project::mirror27_slave':
+    jenkins_ssh_private_key => hiera('jenkins_ssh_private_key_contents')
+  }
+}
+
 node 'devstack-launch.slave.openstack.org' {
   class { 'openstack_project::devstack_launch_slave':
     jenkins_api_user        => hiera('jenkins_api_user'),

modules/openstack_project/files/jenkins-pypi-mirror.yaml

@@ -0,0 +1,15 @@
+# This file is managed by puppet.
+# https://github.com/openstack-infra/config
+
+cache-root: /home/jenkins/pypimirror/cache
+
+mirrors:
+  - name: openstack
+    projects:
+      - https://github.com/openstack/requirements
+    output: /home/jenkins/pypimirror/mirror/openstack
+
+  - name: openstack-infra
+    projects:
+      - https://github.com/openstack-infra/config
+    output: /home/jenkins/pypimirror/mirror/openstack-infra

modules/openstack_project/manifests/mirror26_slave.pp

@@ -0,0 +1,24 @@
+# Copyright 2013  OpenStack Foundation
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+#
+# Install a python26 mirror building slave.
+
+class openstack_project::mirror26_slave (
+  $jenkins_ssh_private_key,
+) {
+
+  class { 'openstack_project::mirror_slave':
+    jenkins_ssh_private_key => $jenkins_ssh_private_key,
+  }
+}

modules/openstack_project/manifests/mirror27_slave.pp

@@ -0,0 +1,24 @@
+# Copyright 2013  OpenStack Foundation
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+#
+# Install a python27 mirror building slave.
+
+class openstack_project::mirror27_slave (
+  $jenkins_ssh_private_key,
+) {
+
+  class { 'openstack_project::mirror_slave':
+    jenkins_ssh_private_key => $jenkins_ssh_private_key,
+  }
+}

modules/openstack_project/manifests/mirror_slave.pp

@@ -0,0 +1,76 @@
+# Copyright 2013  OpenStack Foundation
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+#
+# Install a python mirror building slave.
+
+class openstack_project::mirror_slave (
+  $jenkins_ssh_private_key,
+) {
+
+  include openstack_project::slave
+  include jeepyb
+
+  if ! defined(Package['python-yaml']) {
+    package { 'python-yaml':
+      ensure => present,
+    }
+  }
+
+  file { '/home/jenkins/.ssh/id_rsa':
+    owner   => 'jenkins',
+    group   => 'jenkins',
+    mode    => '0400',
+    require => File['/home/jenkins/.ssh'],
+    content => $jenkins_ssh_private_key,
+  }
+
+  file { '/home/jenkins/pypimirror':
+    ensure  => directory,
+    mode    => '0755',
+    owner   => 'jenkins',
+    group   => 'jenkins',
+    require => User['jenkins'],
+  }
+
+  file { '/home/jenkins/pypimirror/etc':
+    ensure  => directory,
+    mode    => '0755',
+    owner   => 'jenkins',
+    group   => 'jenkins',
+    require => File['/home/jenkins/pypimirror'],
+  }
+
+  file { '/home/jenkins/pypimirror/cache':
+    ensure  => directory,
+    mode    => '0755',
+    owner   => 'jenkins',
+    group   => 'jenkins',
+    require => File['/home/jenkins/pypimirror'],
+  }
+
+  file { '/home/jenkins/pypimirror/mirror':
+    ensure  => directory,
+    mode    => '0755',
+    owner   => 'jenkins',
+    group   => 'jenkins',
+    require => File['/home/jenkins/pypimirror'],
+  }
+
+  file { '/home/jenkins/pypimirror/etc/pypi-mirror.yaml':
+    ensure  => present,
+    source  => 'puppet:///modules/openstack_project/jenkins-pypi-mirror.yaml',
+    require => File['/home/jenkins/pypimirror/etc'],
+  }
+
+}

modules/openstack_project/manifests/static.pp

@@ -51,6 +51,13 @@ class openstack_project::static (
     template => 'openstack_project/logs.vhost.erb',
   }
 
+  apache::vhost { 'pypi.openstack.org':
+    port     => 80,
+    priority => '50',
+    docroot  => '/srv/static/pypi',
+    require  => File['/srv/static/pypi'],
+  }
+
   apache::vhost { 'docs-draft.openstack.org':
     port     => 80,
     priority => '50',
@@ -91,6 +98,13 @@ class openstack_project::static (
     require => User['jenkins'],
   }
 
+  file { '/srv/static/pypi':
+    ensure  => directory,
+    owner   => 'jenkins',
+    group   => 'jenkins',
+    require => User['jenkins'],
+  }
+
   file { '/srv/static/logs/robots.txt':
     ensure  => present,
     owner   => 'root',
@@ -100,6 +114,15 @@ class openstack_project::static (
     require => File['/srv/static/logs'],
   }
 
+  file { '/srv/static/pypi/robots.txt':
+    ensure  => present,
+    owner   => 'root',
+    group   => 'root',
+    mode    => '0444',
+    source  => 'puppet:///modules/openstack_project/disallow_robots.txt',
+    require => File['/srv/static/pypi'],
+  }
+
   file { '/srv/static/docs-draft':
     ensure => directory,
   }