Switch Zuul/JJB to use LOG_PATH

This calculates the full LOG_PATH in the Zuul config and pases it
to Jenkins.  The new path is similar to the old but with the
substitution of a short ZUUL_UUID at the end instead of the Jenkins
build number in order to avoid collisions from multiple Jenkins
masters.

Periodic jobs add a node name to their log path to avoid collisions
from multiple masters.  Unfortunately, that value is not accessible
to the logstash worker.  This can be solved by having Zuul trigger
periodic jobs.

Add the ZUUL_REF to logstash as "build_ref".

Requires https://review.openstack.org/#/c/39130/

Change-Id: I40bad59e3ad8ed6b4706762ed8b833fd15c13b0d
This commit is contained in:
James E. Blair 2013-07-29 10:46:53 -07:00
parent 75e2eec220
commit 0ccf564937
8 changed files with 33 additions and 33 deletions

View File

@ -311,7 +311,7 @@ node 'zuul.openstack.org' {
gerrit_server => 'review.openstack.org', gerrit_server => 'review.openstack.org',
gerrit_user => 'jenkins', gerrit_user => 'jenkins',
zuul_ssh_private_key => hiera('jenkins_ssh_private_key_contents'), zuul_ssh_private_key => hiera('jenkins_ssh_private_key_contents'),
url_pattern => 'http://logs.openstack.org/{build.parameters[BASE_LOG_PATH]}/{job.name}/{build.number}', url_pattern => 'http://logs.openstack.org/{build.parameters[LOG_PATH]}',
sysadmins => hiera('sysadmins'), sysadmins => hiera('sysadmins'),
statsd_host => 'graphite.openstack.org', statsd_host => 'graphite.openstack.org',
gearman_workers => [ gearman_workers => [
@ -326,7 +326,7 @@ node 'zuul-dev.openstack.org' {
gerrit_server => 'review.openstack.org', gerrit_server => 'review.openstack.org',
gerrit_user => 'zuul-dev', gerrit_user => 'zuul-dev',
zuul_ssh_private_key => hiera('zuul_dev_ssh_private_key_contents'), zuul_ssh_private_key => hiera('zuul_dev_ssh_private_key_contents'),
url_pattern => 'http://logs.openstack.org/{build.parameters[BASE_LOG_PATH]}/{job.name}/{build.number}', url_pattern => 'http://logs.openstack.org/{build.parameters[LOG_PATH]}',
sysadmins => hiera('sysadmins'), sysadmins => hiera('sysadmins'),
statsd_host => 'graphite.openstack.org', statsd_host => 'graphite.openstack.org',
gearman_workers => [ gearman_workers => [

View File

@ -193,10 +193,10 @@
builders: builders:
- shell: | - shell: |
#!/bin/sh #!/bin/sh
if test "$BASE_LOG_PATH" ; then if test "$LOG_PATH" ; then
echo "Detailed logs: http://logs.openstack.org/$BASE_LOG_PATH/$JOB_NAME/$BUILD_NUMBER/" echo "Detailed logs: http://logs.openstack.org/$LOG_PATH/"
else else
echo "Detailed logs: http://logs.openstack.org/periodic/$JOB_NAME/$BUILD_NUMBER/" echo "Detailed logs: http://logs.openstack.org/periodic/$JOB_NAME/$NODE_NAME/$BUILD_NUMBER/"
fi fi
- builder: - builder:
@ -239,7 +239,7 @@
- scp: - scp:
site: 'static.openstack.org' site: 'static.openstack.org'
files: files:
- target: 'logs/$BASE_LOG_PATH/$JOB_NAME/$BUILD_NUMBER' - target: 'logs/$LOG_PATH'
copy-console: true copy-console: true
copy-after-failure: true copy-after-failure: true
@ -249,7 +249,7 @@
- scp: - scp:
site: 'static.openstack.org' site: 'static.openstack.org'
files: files:
- target: 'logs/periodic/$JOB_NAME/$BUILD_NUMBER' - target: 'logs/periodic/$JOB_NAME/$NODE_NAME/$BUILD_NUMBER'
copy-console: true copy-console: true
copy-after-failure: true copy-after-failure: true
@ -259,15 +259,15 @@
- scp: - scp:
site: 'static.openstack.org' site: 'static.openstack.org'
files: files:
- target: 'logs/$BASE_LOG_PATH/$JOB_NAME/$BUILD_NUMBER' - target: 'logs/$LOG_PATH'
source: 'logs/**' source: 'logs/**'
keep-hierarchy: true keep-hierarchy: true
copy-after-failure: true copy-after-failure: true
- target: 'logs/$BASE_LOG_PATH/$JOB_NAME/$BUILD_NUMBER' - target: 'logs/$LOG_PATH'
source: '**/testr_results.html.gz' source: '**/testr_results.html.gz'
keep-hierarchy: false keep-hierarchy: false
copy-after-failure: true copy-after-failure: true
- target: 'logs/$BASE_LOG_PATH/$JOB_NAME/$BUILD_NUMBER' - target: 'logs/$LOG_PATH'
source: '**/subunit_log.txt.gz' source: '**/subunit_log.txt.gz'
keep-hierarchy: false keep-hierarchy: false
copy-after-failure: true copy-after-failure: true
@ -278,15 +278,15 @@
- scp: - scp:
site: 'static.openstack.org' site: 'static.openstack.org'
files: files:
- target: 'logs/periodic/$JOB_NAME/$BUILD_NUMBER' - target: 'logs/periodic/$JOB_NAME/$NODE_NAME/$BUILD_NUMBER'
source: 'logs/**' source: 'logs/**'
keep-hierarchy: true keep-hierarchy: true
copy-after-failure: true copy-after-failure: true
- target: 'logs/periodic/$JOB_NAME/$BUILD_NUMBER' - target: 'logs/periodic/$JOB_NAME/$NODE_NAME/$BUILD_NUMBER'
source: '**/testr_results.html.gz' source: '**/testr_results.html.gz'
keep-hierarchy: false keep-hierarchy: false
copy-after-failure: true copy-after-failure: true
- target: 'logs/periodic/$JOB_NAME/$BUILD_NUMBER' - target: 'logs/periodic/$JOB_NAME/$NODE_NAME/$BUILD_NUMBER'
source: '**/subunit_log.txt.gz' source: '**/subunit_log.txt.gz'
keep-hierarchy: false keep-hierarchy: false
copy-after-failure: true copy-after-failure: true
@ -309,7 +309,7 @@
- scp: - scp:
site: 'static.openstack.org' site: 'static.openstack.org'
files: files:
- target: 'logs/$BASE_LOG_PATH/$JOB_NAME/$BUILD_NUMBER' - target: 'logs/$LOG_PATH'
source: 'cover/**' source: 'cover/**'
keep-hierarchy: true keep-hierarchy: true
copy-after-failure: true copy-after-failure: true
@ -320,19 +320,19 @@
- scp: - scp:
site: 'static.openstack.org' site: 'static.openstack.org'
files: files:
- target: 'logs/$BASE_LOG_PATH/$JOB_NAME/$BUILD_NUMBER' - target: 'logs/$LOG_PATH'
source: '**/*nose_results.html' source: '**/*nose_results.html'
keep-hierarchy: false keep-hierarchy: false
copy-after-failure: true copy-after-failure: true
- target: 'logs/$BASE_LOG_PATH/$JOB_NAME/$BUILD_NUMBER' - target: 'logs/$LOG_PATH'
source: '**/*testr_results.html.gz' source: '**/*testr_results.html.gz'
keep-hierarchy: false keep-hierarchy: false
copy-after-failure: true copy-after-failure: true
- target: 'logs/$BASE_LOG_PATH/$JOB_NAME/$BUILD_NUMBER' - target: 'logs/$LOG_PATH'
source: '.testrepository/tmp*' source: '.testrepository/tmp*'
keep-hierarchy: false keep-hierarchy: false
copy-after-failure: true copy-after-failure: true
- target: 'logs/$BASE_LOG_PATH/$JOB_NAME/$BUILD_NUMBER' - target: 'logs/$LOG_PATH'
source: '**/*subunit_log.txt.gz' source: '**/*subunit_log.txt.gz'
keep-hierarchy: false keep-hierarchy: false
copy-after-failure: true copy-after-failure: true
@ -343,7 +343,7 @@
- scp: - scp:
site: 'static.openstack.org' site: 'static.openstack.org'
files: files:
- target: 'docs-draft/$BASE_LOG_PATH/$JOB_NAME/$BUILD_NUMBER/' - target: 'docs-draft/$LOG_PATH'
source: 'doc/build/html/**' source: 'doc/build/html/**'
keep-hierarchy: true keep-hierarchy: true
copy-after-failure: true copy-after-failure: true

View File

@ -19,7 +19,7 @@
- scp: - scp:
site: 'static.openstack.org' site: 'static.openstack.org'
files: files:
- target: 'docs-draft/$BASE_LOG_PATH/$JOB_NAME/$BUILD_NUMBER/ops/' - target: 'docs-draft/$LOG_PATH/ops/'
source: 'doc/src/docbkx/openstack-ops/target/docbkx/webhelp/**' source: 'doc/src/docbkx/openstack-ops/target/docbkx/webhelp/**'
keep-hierarchy: true keep-hierarchy: true
copy-after-failure: true copy-after-failure: true

View File

@ -35,7 +35,7 @@
- scp: - scp:
site: 'static.openstack.org' site: 'static.openstack.org'
files: files:
- target: 'logs/$BASE_LOG_PATH/$JOB_NAME/$BUILD_NUMBER' - target: 'logs/$LOG_PATH'
source: '**/pip.log' source: '**/pip.log'
keep-hierarchy: false keep-hierarchy: false
copy-after-failure: true copy-after-failure: true

View File

@ -76,10 +76,7 @@ class EventProcessor(threading.Thread):
def _get_log_dir(self, event): def _get_log_dir(self, event):
parameters = event["build"].get("parameters", {}) parameters = event["build"].get("parameters", {})
base = parameters.get('BASE_LOG_PATH', 'periodic') base = parameters.get('LOG_PATH', 'UNKNOWN')
base += '/{name}/{number}/'.format(
name=event.get("name", "UNKNOWN"),
number=event["build"].get("number", "UNKNOWN"))
return base return base
def _parse_fields(self, event, filename): def _parse_fields(self, event, filename):
@ -90,13 +87,14 @@ class EventProcessor(threading.Thread):
fields["build_number"] = event["build"].get("number", "UNKNOWN") fields["build_number"] = event["build"].get("number", "UNKNOWN")
parameters = event["build"].get("parameters", {}) parameters = event["build"].get("parameters", {})
fields["build_queue"] = parameters.get("ZUUL_PIPELINE", "UNKNOWN") fields["build_queue"] = parameters.get("ZUUL_PIPELINE", "UNKNOWN")
if fields["build_queue"] in ["check", "gate"]: fields["build_ref"] = parameters.get("ZUUL_REF", "UNKNOWN")
if parameters.get("ZUUL_CHANGE"):
fields["build_change"] = parameters.get("ZUUL_CHANGE", "UNKNOWN") fields["build_change"] = parameters.get("ZUUL_CHANGE", "UNKNOWN")
fields["build_patchset"] = parameters.get("ZUUL_PATCHSET", fields["build_patchset"] = parameters.get("ZUUL_PATCHSET",
"UNKNOWN") "UNKNOWN")
elif fields["build_queue"] in ["post", "pre-release", "release"]: elif parameters.get("ZUUL_NEWREV"):
fields["build_shortref"] = parameters.get("ZUUL_SHORT_NEWREV", fields["build_newrev"] = parameters.get("ZUUL_NEWREV",
"UNKNOWN") "UNKNOWN")
return fields return fields
def _parse_event(self, event, fileopts): def _parse_event(self, event, fileopts):

View File

@ -58,9 +58,9 @@ jobs:
- name: openstack-ha-guide - name: openstack-ha-guide
branch: ^master$ branch: ^master$
- name: gate-openstack-cli-guide - name: gate-openstack-cli-guide
success-pattern: http://docs-draft.openstack.org/{build.parameters[BASE_LOG_PATH]}/{build.number}/cli/quick-start success-pattern: http://docs-draft.openstack.org/{build.parameters[LOG_PATH]}/cli/quick-start
- name: ^gate-.*-docs$ - name: ^gate-.*-docs$
success-pattern: http://docs-draft.openstack.org/{build.parameters[BASE_LOG_PATH]}/{job.name}/{build.number}/doc/build/html/ success-pattern: http://docs-draft.openstack.org/{build.parameters[LOG_PATH]}/doc/build/html/
- name: ^gate-.*-requirements$ - name: ^gate-.*-requirements$
failure-message: Incompatible requirement found; see https://wiki.openstack.org/wiki/Requirements failure-message: Incompatible requirement found; see https://wiki.openstack.org/wiki/Requirements
branch: ^(?!stable/(folsom|grizzly)).*$ branch: ^(?!stable/(folsom|grizzly)).*$

View File

@ -141,7 +141,7 @@ jobs:
- name: openstack-security-guide - name: openstack-security-guide
branch: ^master$ branch: ^master$
- name: ^gate-.*-docs$ - name: ^gate-.*-docs$
success-pattern: http://docs-draft.openstack.org/{build.parameters[BASE_LOG_PATH]}/{job.name}/{build.number}/doc/build/html/ success-pattern: http://docs-draft.openstack.org/{build.parameters[LOG_PATH]}/doc/build/html/
- name: infra-publications-publish - name: infra-publications-publish
branch: ^(?!master).*$ branch: ^(?!master).*$
- name: ^gate-.*-requirements$ - name: ^gate-.*-requirements$

View File

@ -1,4 +1,4 @@
def set_log_url(item, params): def set_log_url(item, job, params):
if hasattr(item.change, 'refspec'): if hasattr(item.change, 'refspec'):
path = "%s/%s/%s/%s" % ( path = "%s/%s/%s/%s" % (
params['ZUUL_CHANGE'][-2:], params['ZUUL_CHANGE'], params['ZUUL_CHANGE'][-2:], params['ZUUL_CHANGE'],
@ -8,3 +8,5 @@ def set_log_url(item, params):
params['ZUUL_NEWREV'][:2], params['ZUUL_NEWREV'], params['ZUUL_NEWREV'][:2], params['ZUUL_NEWREV'],
params['ZUUL_PIPELINE']) params['ZUUL_PIPELINE'])
params['BASE_LOG_PATH'] = path params['BASE_LOG_PATH'] = path
params['LOG_PATH'] = path + '/%s/%s' % (job.name,
params['ZUUL_UUID'][:7])