From 9d37eba442fb1ed7d0a81c566be49d3c5ba65434 Mon Sep 17 00:00:00 2001 From: Sergey Kolekonov Date: Mon, 25 May 2015 16:06:07 +0300 Subject: [PATCH] Add deployment scripts for App Catalog CI - Added Puppet module and additional shell scripts - Deployment documentation can be found at deployment/README.md file Change-Id: Ic81bb539a23a155f4c48eb69c916d953bb12c918 --- deployment/README.md | 44 +++++++ .../catalog-ci-jenkins/manifests/site.pp | 1 + .../modules/catalog_ci/files/config.xml | 45 +++++++ .../catalog_ci/files/gerrit-trigger.xml | 67 ++++++++++ .../modules/catalog_ci/files/jobs/job.yaml | 67 ++++++++++ .../catalog_ci/files/scripts/app-catalog.sh | 96 ++++++++++++++ .../files/scripts/generate_names.py | 50 +++++++ .../modules/catalog_ci/manifests/init.pp | 123 ++++++++++++++++++ .../catalog_ci/templates/jenkins_jobs.ini.erb | 8 ++ deployment/deploy.sh | 19 +++ 10 files changed, 520 insertions(+) create mode 100644 deployment/README.md create mode 100644 deployment/catalog-ci-jenkins/manifests/site.pp create mode 100644 deployment/catalog-ci-jenkins/modules/catalog_ci/files/config.xml create mode 100644 deployment/catalog-ci-jenkins/modules/catalog_ci/files/gerrit-trigger.xml create mode 100644 deployment/catalog-ci-jenkins/modules/catalog_ci/files/jobs/job.yaml create mode 100755 deployment/catalog-ci-jenkins/modules/catalog_ci/files/scripts/app-catalog.sh create mode 100755 deployment/catalog-ci-jenkins/modules/catalog_ci/files/scripts/generate_names.py create mode 100644 deployment/catalog-ci-jenkins/modules/catalog_ci/manifests/init.pp create mode 100644 deployment/catalog-ci-jenkins/modules/catalog_ci/templates/jenkins_jobs.ini.erb create mode 100755 deployment/deploy.sh diff --git a/deployment/README.md b/deployment/README.md new file mode 100644 index 0000000..2074b1b --- /dev/null +++ b/deployment/README.md @@ -0,0 +1,44 @@ +apps-catalog-ci +======== +Description +---------------------- +This is a collection of deployment scripts for apps-catalog CI project. +It consists of Puppet module catalog_ci and an additional shell script. +The scripts allow to setup Jenkins with access to Gerrit to trigger +appropriate jobs on a commit to the apps-catalog project. + +Deployment +---------------------- +Execute deploy.sh script to begin deployment: +(some operations require superuser access rights) + +./deploy.sh + +The script will install necessary packages and deploy/configure Jenkins. +You will be able to access it at http://server_ip:8080 +The following steps are required after deployment: +- Setup access rights. By default Jenkins uses Launchpad OpenID and all users + have full access to Jenkins. + Proceed to Manage Jenkins -> Configure Global Security and setup security. +- Finish Gerrit auth setup: + Add a private key file (id_rsa) to Jenkins ssh directory: + * sudo mkdir -p /var/lib/jenkins/.ssh + * sudo cp id_rsa /var/lib/jenkins/.ssh + * sudo chown -R jenkins:jenkins /var/lib/jenkins/.ssh + * sudo chmod 600 /var/lib/jenkins/.ssh/id_rsa + Then proceed to Manage Jenkins -> Gerrit Trigger and press the button + in 'Status' column. If button will change its color to green, your connection + to Gerrit works OK and Jenkins is receiving Gerrit events. Otherwise please + check Gerrit server parameters. +- rclone (http://rclone.org/) is used to upload images to CDN. + Please install and configure it manually, if it's required. + 'jenkins' user should be able to access default rclone configuration file + in order to use it. + +Jenkins Jobs +---------------------- +Jenkins Job Builder is used to configure Jenkins jobs. It will be automatically +installed by deployment scripts. Jobs configuration files will be placed to +/etc/jenkins_jobs/jobs. You can use the following command to apply your changes + +jenkins-jobs update /etc/jenkins_jobs/jobs diff --git a/deployment/catalog-ci-jenkins/manifests/site.pp b/deployment/catalog-ci-jenkins/manifests/site.pp new file mode 100644 index 0000000..ed619c1 --- /dev/null +++ b/deployment/catalog-ci-jenkins/manifests/site.pp @@ -0,0 +1 @@ +include catalog_ci diff --git a/deployment/catalog-ci-jenkins/modules/catalog_ci/files/config.xml b/deployment/catalog-ci-jenkins/modules/catalog_ci/files/config.xml new file mode 100644 index 0000000..c5d8377 --- /dev/null +++ b/deployment/catalog-ci-jenkins/modules/catalog_ci/files/config.xml @@ -0,0 +1,45 @@ + + + + 1.596.2 + 3 + NORMAL + true + + hudson.model.Hudson.Administer:anonymous + hudson.model.Hudson.Read:anonymous + hudson.model.Item.Read:anonymous + + + https://login.launchpad.net/+openid + + false + + ${JENKINS_HOME}/workspace/${ITEM_FULLNAME} + ${ITEM_ROOTDIR}/builds + + false + + + + + + + 5 + 0 + + + + All + false + false + + + + All + 0 + + + + true + diff --git a/deployment/catalog-ci-jenkins/modules/catalog_ci/files/gerrit-trigger.xml b/deployment/catalog-ci-jenkins/modules/catalog_ci/files/gerrit-trigger.xml new file mode 100644 index 0000000..c5bf289 --- /dev/null +++ b/deployment/catalog-ci-jenkins/modules/catalog_ci/files/gerrit-trigger.xml @@ -0,0 +1,67 @@ + + + + + review.openstack.org + false + + review.openstack.org + 29418 + + catalog-ci + catalog-ci@mirantis.com + /var/lib/jenkins/.ssh/id_rsa + false + false + false + false + gerrit review <CHANGE>,<PATCHSET> --message 'Build Successful <BUILDS_STATS>' --verified <VERIFIED> --code-review <CODE_REVIEW> + gerrit review <CHANGE>,<PATCHSET> --message 'Build Unstable <BUILDS_STATS>' --verified <VERIFIED> --code-review <CODE_REVIEW> + gerrit review <CHANGE>,<PATCHSET> --message 'Build Failed <BUILDS_STATS>' --verified <VERIFIED> --code-review <CODE_REVIEW> + + gerrit review <CHANGE>,<PATCHSET> --message 'No Builds Executed <BUILDS_STATS>' --verified <VERIFIED> --code-review <CODE_REVIEW> + https://review.openstack.org/ + 0 + 0 + 1 + 0 + -1 + 0 + 0 + 0 + 0 + 0 + true + true + 3 + 30 + + + CRVW + Code Review + + + VRIF + Verified + + + + false + + false + + 0 + + + + + ALL + + + + + 3 + 1 + 360 + + diff --git a/deployment/catalog-ci-jenkins/modules/catalog_ci/files/jobs/job.yaml b/deployment/catalog-ci-jenkins/modules/catalog_ci/files/jobs/job.yaml new file mode 100644 index 0000000..7ec076e --- /dev/null +++ b/deployment/catalog-ci-jenkins/modules/catalog_ci/files/jobs/job.yaml @@ -0,0 +1,67 @@ +- job: + name: check-image + node: master + project-type: freestyle + description: "This job checks glance images" + defaults: global + disabled: false + concurrent: true + + scm: + - git: + url: https://git.openstack.org/stackforge/apps-catalog + refspec: $GERRIT_REFSPEC + name: + choosing-strategy: gerrit + skip-tag: false + wipe-workspace: true + branches: + - "$GERRIT_BRANCH" + + triggers: + - gerrit: + server-name: "review.openstack.org" + trigger-on-patchset-uploaded-event: true + projects: + - project-compare-type: 'PLAIN' + project-pattern: 'stackforge/apps-catalog' + branch-compare-type: 'PLAIN' + branch-pattern: master + + builders: + - shell: "git checkout $FETCH_HEAD\ + \n$JENKINS_HOME/scripts/app-catalog.sh" + +- job: + name: merge-image + node: master + project-type: freestyle + description: "This job checks and uploads glance images to CDN" + defaults: global + disabled: false + concurrent: true + + scm: + - git: + url: https://git.openstack.org/stackforge/apps-catalog + refspec: $GERRIT_REFSPEC + name: + choosing-strategy: gerrit + skip-tag: false + wipe-workspace: true + branches: + - "$GERRIT_BRANCH" + + triggers: + - gerrit: + server-name: "review.openstack.org" + trigger-on-change-merged-event: true + projects: + - project-compare-type: 'PLAIN' + project-pattern: 'stackforge/apps-catalog' + branch-compare-type: 'PLAIN' + branch-pattern: master + + builders: + - shell: "git checkout $FETCH_HEAD\ + \n$JENKINS_HOME/scripts/app-catalog.sh" diff --git a/deployment/catalog-ci-jenkins/modules/catalog_ci/files/scripts/app-catalog.sh b/deployment/catalog-ci-jenkins/modules/catalog_ci/files/scripts/app-catalog.sh new file mode 100755 index 0000000..46b912c --- /dev/null +++ b/deployment/catalog-ci-jenkins/modules/catalog_ci/files/scripts/app-catalog.sh @@ -0,0 +1,96 @@ +#!/bin/bash -x + +WDIR=$JENKINS_HOME/scripts +PATCH=$GERRIT_CHANGE_NUMBER +CODES=(100 200 302) +TMP_DIR=$(mktemp -d) +TMP_FILE=$(mktemp) +EVENT=$GERRIT_EVENT_TYPE +FILE_LIST=$(git diff HEAD~1 --name-only) +IMAGE_CONFIG="openstack_catalog/web/static/glance_images.yaml" +IMAGE_CDN_PATH="catalog_ci:catalog/images" + +clean() { + rm -rf $TMP_DIR + rm -f $TMP_FILE +} + +upload_image () { + local CONFIG + local OLD_CONFIG + local IMAGE_NAME + local IMAGE_PATH + + IMAGE_PATH=$1 + CONFIG=$TMP_DIR/$(basename $IMAGE_CONFIG) + OLD_CONFIG=$TMP_DIR/$(basename $IMAGE_CONFIG).old + IMAGE_NAME=$(python $WDIR/generate_names.py glance $OLD_CONFIG $CONFIG) + if [ -z "$IMAGE_NAME" ]; then + echo "Image file can't be generated" + exit 1 + fi + + echo "Uploading image $IMAGE_NAME from $IMAGE_PATH" + mv $IMAGE_PATH $(dirname $IMAGE_PATH)/$IMAGE_NAME + rclone copy $(dirname $IMAGE_PATH)/$IMAGE_NAME $IMAGE_CDN_PATH + clean +} + +main() { + local URL + local HASH + local REAL_HASH + local HTTP_CODE + local IMAGE + + ssh -p 29418 catalog-ci@review.openstack.org gerrit query $PATCH > $TMP_FILE + URL=$(cat $TMP_FILE | egrep "^\s*Image-URL:\s(https?|ftp)://.*" | egrep -o "(https?|ftp)://.*$") + HASH=$(cat $TMP_FILE | egrep "^\s*Image-hash:\s[A-Za-z0-9]*$" | egrep -o "[A-Za-z0-9]*$") + + cat $TMP_FILE | grep Image-URL | grep -q Unknown && exit 0 +# if [ $(cat $TMP_FILE | grep Image-URL | grep Unknown) ]; then +# echo "Image URL is unknown, no checks and changes will be performed" +# exit 0 +# fi + + if [ -z "$URL" -o -z "$HASH" ]; then + echo "Image URL or hash wasn't found" + clean + exit 0 + else + HTTP_CODE=$(curl -o /dev/null --silent --head --write-out '%{http_code}\n' $URL) + if ! [[ " ${CODES[*]} " == *" $HTTP_CODE "* ]]; then + echo "File wasn't found" + clean + exit 1 + fi + fi + + wget $URL -P $TMP_DIR + if [ "$HASH" == "Unknown" ]; then + echo "Image hash is unknown, skipping checks..." + else + REAL_HASH=$(md5sum $TMP_DIR/* | awk '{print $1}') + if [ "$REAL_HASH" != "$HASH" ]; then + echo "Hash mismatch" + clean + exit 1 + else + echo "Image hash is correct" + fi + fi + + if [ "$EVENT" == "change-merged" ]; then + IMAGE=$(ls $TMP_DIR/*) + cp $IMAGE_CONFIG $TMP_DIR + git checkout HEAD~1 + cp $IMAGE_CONFIG $TMP_DIR/$(basename $IMAGE_CONFIG).old + upload_image $IMAGE + else + clean + fi +} + +if [[ ${FILE_LIST[*]} =~ "$IMAGE_CONFIG" ]]; then + main "$@" +fi diff --git a/deployment/catalog-ci-jenkins/modules/catalog_ci/files/scripts/generate_names.py b/deployment/catalog-ci-jenkins/modules/catalog_ci/files/scripts/generate_names.py new file mode 100755 index 0000000..0d82412 --- /dev/null +++ b/deployment/catalog-ci-jenkins/modules/catalog_ci/files/scripts/generate_names.py @@ -0,0 +1,50 @@ +#!/usr/bin/python +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import itertools +import re +from sys import argv +import yaml + + +def yaml_to_dict(infile, k): + stream = open(infile, 'r') + rdict = yaml.load(stream)[k] + return rdict + + +def diff_images_config(images1, images2): + if images1 == images2: + return '' + intersec = [item for item in images1 if item in images2] + sym_diff = [item for item in itertools.chain( + images1, images2) if item not in intersec] + name = '' + d_size = len(sym_diff) + if d_size <= 2: + i = d_size - 1 + else: + return '' + + if 'name' in sym_diff[i].keys() and 'format' in sym_diff[i].keys(): + i_name = re.sub('[(){}<>]', '', sym_diff[i]['name']) + i_type = sym_diff[i]['format'] + name = i_name + '.' + i_type + name = name.lower().replace(" ", "_") + return name + +if __name__ == '__main__': + if argv[1] == 'glance': + images1 = yaml_to_dict(argv[2], 'images') + images2 = yaml_to_dict(argv[3], 'images') + print(diff_images_config(images1, images2)) diff --git a/deployment/catalog-ci-jenkins/modules/catalog_ci/manifests/init.pp b/deployment/catalog-ci-jenkins/modules/catalog_ci/manifests/init.pp new file mode 100644 index 0000000..88b0010 --- /dev/null +++ b/deployment/catalog-ci-jenkins/modules/catalog_ci/manifests/init.pp @@ -0,0 +1,123 @@ +class catalog_ci { + + $user = 'admin' + $password = 'adminpassword' + $jenkins_config = '/var/lib/jenkins/config.xml' + $gerrit_config = '/var/lib/jenkins/gerrit-trigger.xml' + + class{ 'jenkins': + lts => true, + install_java => true, + plugin_hash => { + 'git' => {}, + 'parameterized-trigger' => {}, + 'token-macro' => {}, + 'mailer' => {}, + 'scm-api' => {}, + 'promoted-builds' => {}, + 'matrix-project' => {}, + 'git-client' => {}, + 'ssh-credentials' => {}, + 'credentials' => {}, + 'gerrit-trigger' => {}, + 'rebuild' => {}, + 'git-client' => {}, + 'rabbitmq-consumer' => {}, + 'openid' => {}, + 'openid4java' => {}, + } + } + + jenkins::user { "$user": + email => 'admin@example.com', + password => "$password", + } + + class{ 'jenkins::security': + security_model => 'full_control', + } + + Class['jenkins'] -> Jenkins::User["$user"] -> Class['jenkins::security'] + -> Exec['create_jobs'] -> File["$jenkins_config"] + + package {'python-pip': + ensure => present, + } + + package {'deepdiff': + ensure => present, + provider => 'pip', + } + + package {'pyyaml': + ensure => present, + provider => 'pip', + } + + package {'jenkins-job-builder': + ensure => present, + provider => 'pip', + } + + file {'/var/lib/jenkins/scripts': + owner => 'root', + group => 'root', + ensure => directory, + source => 'puppet:///modules/catalog_ci/scripts', + recurse => true, + require => File['/etc/jenkins_jobs'], + } + + file {'/etc/jenkins_jobs': + owner => 'root', + group => 'root', + ensure => directory, + require => Package['jenkins-job-builder'], + } + + file {'/etc/jenkins_jobs/jenkins_jobs.ini': + owner => 'root', + group => 'root', + content => template('catalog_ci/jenkins_jobs.ini.erb'), + require => File['/etc/jenkins_jobs'], + } + + file {'/etc/jenkins_jobs/jobs': + owner => 'root', + group => 'root', + ensure => directory, + source => 'puppet:///modules/catalog_ci/jobs', + recurse => true, + require => File['/etc/jenkins_jobs'], + } + + file {"$jenkins_config": + owner => 'jenkins', + group => 'jenkins', + ensure => present, + source => 'puppet:///modules/catalog_ci/config.xml', + } + + file {"$gerrit_config": + owner => 'jenkins', + group => 'jenkins', + ensure => present, + source => 'puppet:///modules/catalog_ci/gerrit-trigger.xml', + require => File["$jenkins_config"], + } + + exec {'create_jobs': + command => 'jenkins-jobs update /etc/jenkins_jobs/jobs', + path => '/bin:/sbin:/usr/bin:/usr/sbin:/usr/local/bin', + require => File['/etc/jenkins_jobs/jobs'], + } + + exec{'restart_jenkins': + command => 'service jenkins restart', + path => '/bin:/sbin:/usr/bin:/usr/sbin', + subscribe => [ File["$jenkins_config"], File["$gerrit_config"] ], + } + + Package['python-pip'] -> Package['deepdiff'] -> + Package['pyyaml'] -> Package['jenkins-job-builder'] +} diff --git a/deployment/catalog-ci-jenkins/modules/catalog_ci/templates/jenkins_jobs.ini.erb b/deployment/catalog-ci-jenkins/modules/catalog_ci/templates/jenkins_jobs.ini.erb new file mode 100644 index 0000000..ff07d41 --- /dev/null +++ b/deployment/catalog-ci-jenkins/modules/catalog_ci/templates/jenkins_jobs.ini.erb @@ -0,0 +1,8 @@ +[job_builder] +ignore_cache=True + +[jenkins] +user=<%= @user %> +password=<%= @password %> +url=http://127.0.0.1:8080 +query_plugins_info=False diff --git a/deployment/deploy.sh b/deployment/deploy.sh new file mode 100755 index 0000000..5a1717b --- /dev/null +++ b/deployment/deploy.sh @@ -0,0 +1,19 @@ +#!/bin/bash -e + +# Generate a password for service account +ADMIN_PASS=$(< /dev/urandom tr -dc _A-Z-a-z-0-9 | head -c10) +sed -i "s/adminpassword/$ADMIN_PASS/g" catalog-ci-jenkins/modules/catalog_ci/manifests/init.pp + +sudo apt-get update +sudo apt-get install git puppet -y + +# Using custom (forked) puppet-jenkins module +# due to critical problem in the upstream one +git clone https://github.com/skolekonov/puppet-jenkins.git +tar czf rtyler-jenkins-1.3.0.tar.gz puppet-jenkins/* +sudo puppet module install rtyler-jenkins-1.3.0.tar.gz + +sudo puppet apply -vd --modulepath catalog-ci-jenkins/modules:/etc/puppet/modules catalog-ci-jenkins/manifests/site.pp + +echo "Deployment completed" +echo "WARNING. Please open Jenkins WebUI and setup user access matrix"