jjb/jenkins-job-builder
Code Issues Proposed changes
jenkins-job-builder/jenkins_jobs/builder.py
James E. Blair 8f09841693 Refactor modules and templating. 
Switch to using entry points for loading modules as well as
individual buliders, triggers, publishers, etc.

Remove most openstack-specific python code.

Change templating so it's less repetitive -- a single project
definition will suffice for multiple jobs or job-groups.

This outputs XML that is identical to the current production XML,
warts and all.  There are significant improvements that can be made
to the YAML in a separate change, as they will cause minor changes
to existing jobs (adding timestamps, logrotate, etc.).  These are
mostly marked with TODO in this change.

Change-Id: Idcfddb3b43b6cfef4b20919a84540706d7a0a0b1
Reviewed-on: https://review.openstack.org/11000
Approved: James E. Blair <corvus@inaugust.com>
Reviewed-by: James E. Blair <corvus@inaugust.com>
Tested-by: Jenkins
2012-08-10 16:00:42 +00:00

275 lines
9.2 KiB
Python
Raw Blame History

#!/usr/bin/env python
# Copyright (C) 2012 OpenStack, LLC.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
# Manage jobs in Jenkins server
import os
import hashlib
import yaml
import xml.etree.ElementTree as XML
from xml.dom import minidom
import jenkins
import ConfigParser
from StringIO import StringIO
import re
import pkgutil
import pkg_resources
import pprint
import sys
class JenkinsJobsException(Exception): pass
class YamlParser(object):
def __init__(self):
self.registry = ModuleRegistry()
self.data = {}
self.jobs = []
def parse(self, fn):
data = yaml.load(open(fn))
for item in data:
cls, dfn = item.items()[0]
group = self.data.get(cls, {})
name = dfn['name']
group[name] = dfn
self.data[cls] = group
def getJob(self, name):
return self.data.get('job', {}).get(name, None)
def getJobGroup(self, name):
return self.data.get('job-group', {}).get(name, None)
def getJobTemplate(self, name):
return self.data.get('job-template', {}).get(name, None)
def generateXML(self):
changed = True
while changed:
changed = False
for module in self.registry.modules:
if hasattr(module, 'handle_data'):
if module.handle_data(self):
changed = True
for job in self.data.get('job', {}).values():
self.getXMLForJob(job)
for project in self.data.get('project', {}).values():
for jobname in project.get('jobs', []):
job = self.getJob(jobname)
if job:
# Just naming an existing defined job
continue
# see if it's a job group
group = self.getJobGroup(jobname)
if group:
for group_jobname in group['jobs']:
job = self.getJob(group_jobname)
if job:
continue
template = self.getJobTemplate(group_jobname)
# Allow a group to override parameters set by a project
d = {}
d.update(project)
d.update(group)
# Except name, since the group's name is not useful
d['name'] = project['name']
if template:
self.getXMLForTemplateJob(d, template)
continue
# see if it's a template
template = self.getJobTemplate(jobname)
if template:
self.getXMLForTemplateJob(project, template)
def getXMLForTemplateJob(self, project, template):
s = yaml.dump(template, default_flow_style=False)
s = s.format(**project)
data = yaml.load(s)
self.getXMLForJob(data)
def getXMLForJob(self, data):
kind = data.get('project-type', 'freestyle')
for ep in pkg_resources.iter_entry_points(
group='jenkins_jobs.projects', name=kind):
Mod = ep.load()
mod = Mod(self.registry)
xml = mod.root_xml(data)
self.gen_xml(xml, data)
job = XmlJob(xml, data['name'])
self.jobs.append(job)
break
def gen_xml(self, xml, data):
XML.SubElement(xml, 'actions')
description = XML.SubElement(xml, 'description')
description.text = "THIS JOB IS MANAGED BY PUPPET AND WILL BE OVERWRITTEN.\n\n\
DON'T EDIT THIS JOB THROUGH THE WEB\n\n\
If you would like to make changes to this job, please see:\n\n\
https://github.com/openstack/openstack-ci-puppet\n\n\
In modules/jenkins_jobs"
XML.SubElement(xml, 'keepDependencies').text = 'false'
if data.get('disabled'):
XML.SubElement(xml, 'disabled').text = 'true'
else:
XML.SubElement(xml, 'disabled').text = 'false'
XML.SubElement(xml, 'blockBuildWhenDownstreamBuilding').text = 'false'
XML.SubElement(xml, 'blockBuildWhenUpstreamBuilding').text = 'false'
if data.get('concurrent'):
XML.SubElement(xml, 'concurrentBuild').text = 'true'
else:
XML.SubElement(xml, 'concurrentBuild').text = 'false'
for module in self.registry.modules:
if hasattr(module, 'gen_xml'):
module.gen_xml(self, xml, data)
class ModuleRegistry(object):
# TODO: make this extensible
def __init__(self):
self.modules = []
self.handlers = {}
for entrypoint in pkg_resources.iter_entry_points(
group='jenkins_jobs.modules'):
Mod = entrypoint.load()
mod = Mod(self)
self.modules.append(mod)
self.modules.sort(lambda a, b: cmp(a.sequence, b.sequence))
def registerHandler(self, category, name, method):
cat_dict = self.handlers.get(category, {})
if not cat_dict:
self.handlers[category] = cat_dict
cat_dict[name] = method
def getHandler(self, category, name):
return self.handlers[category][name]
class XmlJob(object):
def __init__(self, xml, name):
self.xml = xml
self.name = name
def md5(self):
return hashlib.md5(self.output()).hexdigest()
# Pretty printing ideas from http://stackoverflow.com/questions/749796/pretty-printing-xml-in-python
pretty_text_re = re.compile('>\n\s+([^<>\s].*?)\n\s+</', re.DOTALL)
def output(self):
out = minidom.parseString(XML.tostring(self.xml)).toprettyxml(indent=' ')
return self.pretty_text_re.sub('>\g<1></', out)
class CacheStorage(object):
def __init__(self):
self.cachefilename = os.path.expanduser('~/.jenkins_jobs_cache.yml')
try:
yfile = file(self.cachefilename, 'r')
except IOError:
self.data = {}
return
self.data = yaml.load(yfile)
yfile.close()
def set(self, job, md5):
self.data[job] = md5
yfile = file(self.cachefilename, 'w')
yaml.dump(self.data, yfile)
yfile.close()
def is_cached(self, job):
if self.data.has_key(job):
return True
return False
def has_changed(self, job, md5):
if self.data.has_key(job) and self.data[job] == md5:
return False
return True
class Jenkins(object):
def __init__(self, url, user, password):
self.jenkins = jenkins.Jenkins(url, user, password)
def update_job(self, job_name, xml):
if self.is_job(job_name):
self.jenkins.reconfig_job(job_name, xml)
else:
self.jenkins.create_job(job_name, xml)
def is_job(self, job_name):
return self.jenkins.job_exists(job_name)
def get_job_md5(self, job_name):
xml = self.jenkins.get_job_config(job_name)
return hashlib.md5(xml).hexdigest()
def delete_job(self, job_name):
if self.is_job(job_name):
self.jenkins.delete_job(job_name)
class Builder(object):
def __init__(self, jenkins_url, jenkins_user, jenkins_password):
self.jenkins = Jenkins(jenkins_url, jenkins_user, jenkins_password)
self.cache = CacheStorage()
def delete_job(self):
self.jenkins.delete_job(options.name)
def update_job(self, fn, name=None, output_dir=None):
if os.path.isdir(fn):
files_to_process = [os.path.join(fn, f)
for f in os.listdir(fn)
if (f.endswith('.yml') or f.endswith('.yaml'))]
else:
files_to_process = [fn]
parser = YamlParser()
for in_file in files_to_process:
parser.parse(in_file)
parser.generateXML()
parser.jobs.sort(lambda a,b: cmp(a.name, b.name))
for job in parser.jobs:
if name and job.name != name:
continue
if output_dir:
#print '='*70
#print job.name
#print '-'*70
if name:
print job.output()
continue
fn = os.path.join(output_dir, job.name)
f = open(fn, 'w')
f.write(job.output())
f.close()
continue
md5 = job.md5()
if (remote_jenkins.is_job(job.nam)
and not self.cache.is_cached(job.name)):
old_md5 = remote_jenkins.get_job_md5(job.name)
self.cache.set(job.name, old_md5)
if self.cache.has_changed(job.name, md5):
remote_jenkins.update_job(job.name, xml.output())
self.cache.set(job.name, md5)
View Git Blame Copy Permalink