jjb/jenkins-job-builder
Code Issues Proposed changes

Macro now use parameters from defaults

Browse Source 
Also fix error message for missing defaults.

Change-Id: I5d912c2bfb607c8388010ac6e39780be93073f5a
This commit is contained in:
Vsevolod Fedorov 2024-02-22 11:26:32 +03:00
parent 06366b02e7
commit fb913725e2
19 changed files with 205 additions and 102 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
doc/source/changelog.rst
View File

@ -9,6 +9,11 @@ Features added
* Update TAP publisher plugin (by Kienan Stewart).
* Add tags: ``!include-raw-verbatim:`` and ``!include-raw-expand:``. Tags ``!include-raw:`` and ``!include-raw-escape:`` are now deprecated.
* Macros can now use parameters specified in defaults, the same way as job-templates.
See this examples:
* `macro-uses-global-defaults.yaml <https://review.opendev.org/c/jjb/jenkins-job-builder/+/910877/4/tests/yamlparser/job_fixtures/macro-uses-global-defaults.yaml>`_
* `macro-uses-custom-defaults.yaml <https://review.opendev.org/c/jjb/jenkins-job-builder/+/910877/4/tests/yamlparser/job_fixtures/macro-uses-custom-defaults.yaml>`_
.. note::
After moving to 6.1.0 release, to remove deprecation warnings make these adjustments to your JJB sources:

57
jenkins_jobs/macro.py
View File

@ -13,6 +13,10 @@
from dataclasses import dataclass
from functools import partial
from .root_base import ElementBase
from .expander import Expander, StringsOnlyExpander
from .yaml_objects import BaseYamlObject
from .loc_loader import LocDict
from .errors import JenkinsJobsException
from .position import Pos
@ -33,8 +37,12 @@ macro_specs = [
@dataclass
class Macro:
class Macro(ElementBase):
_expander: Expander
_str_expander: StringsOnlyExpander
_type_name: str
name: str
defaults_name: str
pos: Pos
elements: list
@ -50,16 +58,61 @@ class Macro:
):
d = data.copy()
name = d.pop_required_loc_string("name")
defaults = d.pop_loc_string("defaults", "global")
elements = d.pop_required_element(elements_name)
expander = Expander(config)
str_expander = StringsOnlyExpander(config)
if d:
example_key = next(iter(d.keys()))
raise JenkinsJobsException(
f"In {type_name} macro {name!r}: unexpected elements: {','.join(d.keys())}",
pos=data.key_pos.get(example_key),
)
macro = cls(name, pos, elements or [])
macro = cls(
roots.defaults,
expander,
str_expander,
type_name,
name,
defaults,
pos,
elements or [],
)
roots.assign(roots.macros[type_name], name, macro, "macro")
def __str__(self):
return f"macro {self.name!r}"
def dispatch_elements(self, registry, xml_parent, component_data, job_data, params):
defaults = self._pick_defaults(self.defaults_name)
full_params = LocDict.merge(
defaults.params,
params,
)
element_list = self.elements
if isinstance(element_list, BaseYamlObject):
# Expand !j2-yaml tag if it is right below macro body.
# But do not expand yaml tags inside it - they will be expanded later.
element_list = element_list.expand(self._str_expander, full_params)
for element in element_list:
try:
expanded_element = self._expander.expand(element, full_params)
except JenkinsJobsException as x:
raise x.with_context(
f"While expanding {self}",
pos=self.pos,
)
# Pass component_data in as template data to this function
# so that if the macro is invoked with arguments,
# the arguments are interpolated into the real defn.
registry.dispatch(
self._type_name,
xml_parent,
expanded_element,
component_data,
job_data=job_data,
)
macro_adders = {
macro_type: partial(Macro.add, macro_type, elements_name)

30
jenkins_jobs/registry.py
View File

@ -27,8 +27,6 @@ from six import PY2
from jenkins.plugins import PluginVersion
from jenkins_jobs.errors import JenkinsJobsException
from jenkins_jobs.expander import Expander, StringsOnlyExpander
from jenkins_jobs.yaml_objects import BaseYamlObject
__all__ = ["ModuleRegistry"]
@ -48,8 +46,6 @@ class ModuleRegistry(object):
self.jjb_config = jjb_config
self.masked_warned = {}
self._macros = {}
self._expander = Expander(jjb_config)
self._str_expander = StringsOnlyExpander(jjb_config)
if plugins_list is None:
self._plugin_version = {}
@ -290,30 +286,8 @@ class ModuleRegistry(object):
)
if component_data is None:
component_data = {}
expander_params = {**component_data, **(job_data or {})}
elements = macro.elements
if isinstance(elements, BaseYamlObject):
# Expand !j2-yaml tag if it is right below macro body.
# But do not expand yaml tags inside it - they will be expanded later.
elements = elements.expand(self._str_expander, expander_params)
for b in elements:
try:
element = self._expander.expand(b, expander_params)
except JenkinsJobsException as x:
raise x.with_context(
f"While expanding macro {name!r}",
pos=macro.pos,
)
# Pass component_data in as template data to this function
# so that if the macro is invoked with arguments,
# the arguments are interpolated into the real defn.
self.dispatch(
component_type,
xml_parent,
element,
component_data,
job_data=job_data,
)
params = {**component_data, **(job_data or {})}
macro.dispatch_elements(self, xml_parent, component_data, job_data, params)
def _load_eps(self, component_list_type, component_type, entry_point, name):
logging.debug("Caching entrypoints for %s" % component_list_type)

47
jenkins_jobs/root_base.py
View File

@ -40,10 +40,34 @@ class JobViewData:
@dataclass
class RootBase:
"""Base class for YAML root elements - job, view or template"""
class ElementBase:
"""Base class for YAML elements - job, view, template, or macro"""
_defaults: dict
@property
def title(self):
return str(self).capitalize()
def _pick_defaults(self, name):
try:
defaults = self._defaults[name]
except KeyError:
if name == "global":
return Defaults.empty()
raise JenkinsJobsException(
f"{self.title} wants defaults {name!r}, but it was never defined",
pos=name.pos,
)
if name == "global":
return defaults
return defaults.merged_with_global(self._pick_defaults("global"))
@dataclass
class RootBase(ElementBase):
"""Base class for YAML root elements - job, view or template"""
_expander: Expander
_keep_descriptions: bool
_id: str
@ -61,10 +85,6 @@ class RootBase:
else:
return self.name
@property
def title(self):
return str(self).capitalize()
@property
def contents(self):
contents = self._contents.copy()
@ -80,21 +100,6 @@ class RootBase:
expanded_contents["description"] = amended_description
return expanded_contents
def _pick_defaults(self, name):
try:
defaults = self._defaults[name]
except KeyError:
if name == "global":
return Defaults.empty()
raise JenkinsJobsException(
f"{self.title} wants defaults {self.defaults_name!r}"
" but it was never defined",
pos=name.pos,
)
if name == "global":
return defaults
return defaults.merged_with_global(self._pick_defaults("global"))
class NonTemplateRootMixin:
def top_level_generate_items(self):

18
tests/yamlparser/error_fixtures/macro-uses-custom-defaults.error
View File

@ -1,18 +0,0 @@
macro-uses-custom-defaults.yaml:34:3: In project 'sample-project'
- project:
^
macro-uses-custom-defaults.yaml:37:7: Defined here
- sample-job-template
^
macro-uses-custom-defaults.yaml:26:3: In job template 'sample-job-template'
- job-template:
^
macro-uses-custom-defaults.yaml:30:7: While expanding builder macro call 'builder-without-params'
- builder-without-params
^
macro-uses-custom-defaults.yaml:7:3: While expanding macro 'builder-without-params'
- builder:
^
macro-uses-custom-defaults.yaml:12:9: While formatting string 'echo "builder-without-params: Fails when trying to expand: {global_param}"\n...': Missing parameter: 'global_param'
echo "builder-without-params: Fails when tryi ...
^

1
tests/yamlparser/error_fixtures/macro-uses-custom-defaults.yaml.inc
View File

@ -1 +0,0 @@
echo "include-raw-expand: Fails when trying to expand: {global_param}"

18
tests/yamlparser/error_fixtures/macro-uses-global-defaults.error
View File

@ -1,18 +0,0 @@
macro-uses-global-defaults.yaml:32:3: In project 'sample-project'
- project:
^
macro-uses-global-defaults.yaml:35:7: Defined here
- sample-job-template
^
macro-uses-global-defaults.yaml:24:3: In job template 'sample-job-template'
- job-template:
^
macro-uses-global-defaults.yaml:28:7: While expanding builder macro call 'builder-without-params'
- builder-without-params
^
macro-uses-global-defaults.yaml:7:3: While expanding macro 'builder-without-params'
- builder:
^
macro-uses-global-defaults.yaml:11:9: While formatting string 'echo "builder-without-params: Fails when trying to expand: {global_param}"\n...': Missing parameter: 'global_param'
echo "builder-without-params: Fails when tryi ...
^

1
tests/yamlparser/error_fixtures/macro-uses-global-defaults.yaml.inc
View File

@ -1 +0,0 @@
echo "include-raw-expand: Fails when trying to expand: {global_param}"

2
tests/yamlparser/error_fixtures/missing_defaults_at_job.error
View File

@ -1,6 +1,6 @@
missing_defaults_at_job.yaml:1:3: In job 'sample-job'
- job:
^
missing_defaults_at_job.yaml:3:15: Job 'sample-job' wants defaults 'missing-defaults' but it was never defined
missing_defaults_at_job.yaml:3:15: Job 'sample-job' wants defaults 'missing-defaults', but it was never defined
defaults: missing-defaults
^

2
tests/yamlparser/error_fixtures/missing_defaults_at_job_template.error
View File

@ -4,6 +4,6 @@ missing_defaults_at_job_template.yaml:5:3: In project 'sample-project'
missing_defaults_at_job_template.yaml:1:3: In job template 'sample-job'
- job-template:
^
missing_defaults_at_job_template.yaml:3:15: Job template 'sample-job' wants defaults 'missing-defaults' but it was never defined
missing_defaults_at_job_template.yaml:3:15: Job template 'sample-job' wants defaults 'missing-defaults', but it was never defined
defaults: missing-defaults
^

15
tests/yamlparser/error_fixtures/missing_defaults_at_macro.error Normal file
View File

@ -0,0 +1,15 @@
missing_defaults_at_macro.yaml:12:3: In project 'sample-project'
- project:
^
missing_defaults_at_macro.yaml:15:7: Defined here
- sample-job
^
missing_defaults_at_macro.yaml:7:3: In job template 'sample-job'
- job-template:
^
missing_defaults_at_macro.yaml:10:7: While expanding builder macro call 'sample-builder'
- sample-builder
^
missing_defaults_at_macro.yaml:3:15: Macro 'sample-builder' wants defaults 'missing-defaults', but it was never defined
defaults: missing-defaults
^

15
tests/yamlparser/error_fixtures/missing_defaults_at_macro.yaml Normal file
View File

@ -0,0 +1,15 @@
- builder:
name: sample-builder
defaults: missing-defaults
builders:
- shell: hello
- job-template:
name: sample-job
builders:
- sample-builder
- project:
name: sample-project
jobs:
- sample-job

2
tests/yamlparser/error_fixtures/missing_defaults_at_project.error
View File

@ -4,6 +4,6 @@ missing_defaults_at_project.yaml:4:3: In project 'sample-project'
missing_defaults_at_project.yaml:1:3: In job template 'sample-job'
- job-template:
^
missing_defaults_at_project.yaml:6:15: Job template 'sample-job' wants defaults 'global' but it was never defined
missing_defaults_at_project.yaml:6:15: Job template 'sample-job' wants defaults 'missing-defaults', but it was never defined
defaults: missing-defaults
^

37
tests/yamlparser/job_fixtures/macro-uses-custom-defaults.xml Normal file
View File

@ -0,0 +1,37 @@
<?xml version="1.0" encoding="utf-8"?>
<project>
<actions/>
<description>&lt;!-- Managed by Jenkins Job Builder --&gt;</description>
<keepDependencies>false</keepDependencies>
<displayName>sample-job-template</displayName>
<blockBuildWhenDownstreamBuilding>false</blockBuildWhenDownstreamBuilding>
<blockBuildWhenUpstreamBuilding>false</blockBuildWhenUpstreamBuilding>
<concurrentBuild>false</concurrentBuild>
<canRoam>true</canRoam>
<properties/>
<scm class="hudson.scm.NullSCM"/>
<builders>
<hudson.tasks.Shell>
<command>echo &quot;builder-without-params: Should be expanded: sample global param value&quot;
</command>
</hudson.tasks.Shell>
<hudson.tasks.Shell>
<command>echo &quot;include-raw-expand: Should be expanded: sample global param value&quot;
</command>
</hudson.tasks.Shell>
<hudson.tasks.Shell>
<command>echo &quot;builder-with-params: Should be expanded: sample macro param value&quot;
</command>
</hudson.tasks.Shell>
<hudson.tasks.Shell>
<command>echo &quot;builder-with-params: Should be expanded: sample global param value&quot;
</command>
</hudson.tasks.Shell>
<hudson.tasks.Shell>
<command>echo &quot;include-raw-expand: Should be expanded: sample global param value&quot;
</command>
</hudson.tasks.Shell>
</builders>
<publishers/>
<buildWrappers/>
</project>

11
tests/yamlparser/error_fixtures/macro-uses-custom-defaults.yaml → tests/yamlparser/job_fixtures/macro-uses-custom-defaults.yaml
View File

@ -1,4 +1,4 @@
# Macros do not use custom defaults.
# Macros use custom defaults.
- defaults:
name: custom
@ -6,21 +6,20 @@
- builder:
name: builder-without-params
# defaults: custom
defaults: custom
builders:
- shell: |
echo "builder-without-params: Fails when trying to expand: {global_param}"
# Also fails when trying to expand:
echo "builder-without-params: Should be expanded: {global_param}"
- shell: !include-raw-expand: macro-uses-custom-defaults.yaml.inc
- builder:
name: builder-with-params
# defaults: custom
defaults: custom
builders:
- shell: |
echo "builder-with-params: Should be expanded: {param}"
- shell: |
echo "builder-with-params: Fails when trying to expand: {global_param}"
echo "builder-with-params: Should be expanded: {global_param}"
- shell: !include-raw-expand: macro-uses-custom-defaults.yaml.inc
- job-template:

1
tests/yamlparser/job_fixtures/macro-uses-custom-defaults.yaml.inc Normal file
View File

@ -0,0 +1 @@
echo "include-raw-expand: Should be expanded: {global_param}"

37
tests/yamlparser/job_fixtures/macro-uses-global-defaults.xml Normal file
View File

@ -0,0 +1,37 @@
<?xml version="1.0" encoding="utf-8"?>
<project>
<actions/>
<description>&lt;!-- Managed by Jenkins Job Builder --&gt;</description>
<keepDependencies>false</keepDependencies>
<displayName>sample-job-template</displayName>
<blockBuildWhenDownstreamBuilding>false</blockBuildWhenDownstreamBuilding>
<blockBuildWhenUpstreamBuilding>false</blockBuildWhenUpstreamBuilding>
<concurrentBuild>false</concurrentBuild>
<canRoam>true</canRoam>
<properties/>
<scm class="hudson.scm.NullSCM"/>
<builders>
<hudson.tasks.Shell>
<command>echo &quot;builder-without-params: Should be expanded: sample global param value&quot;
</command>
</hudson.tasks.Shell>
<hudson.tasks.Shell>
<command>echo &quot;include-raw-expand: Should be expanded: sample global param value&quot;
</command>
</hudson.tasks.Shell>
<hudson.tasks.Shell>
<command>echo &quot;builder-with-params: Should be expanded: sample macro param value&quot;
</command>
</hudson.tasks.Shell>
<hudson.tasks.Shell>
<command>echo &quot;builder-with-params: Should be expanded: sample global param value&quot;
</command>
</hudson.tasks.Shell>
<hudson.tasks.Shell>
<command>echo &quot;include-raw-expand: Should be expanded: sample global param value&quot;
</command>
</hudson.tasks.Shell>
</builders>
<publishers/>
<buildWrappers/>
</project>

7
tests/yamlparser/error_fixtures/macro-uses-global-defaults.yaml → tests/yamlparser/job_fixtures/macro-uses-global-defaults.yaml
View File

@ -1,4 +1,4 @@
# Macros do not use global defaults.
# Macros use global defaults.
- defaults:
name: global
@ -8,8 +8,7 @@
name: builder-without-params
builders:
- shell: |
echo "builder-without-params: Fails when trying to expand: {global_param}"
# Also fails when trying to expand:
echo "builder-without-params: Should be expanded: {global_param}"
- shell: !include-raw-expand: macro-uses-global-defaults.yaml.inc
- builder:
@ -18,7 +17,7 @@
- shell: |
echo "builder-with-params: Should be expanded: {param}"
- shell: |
echo "builder-with-params: Fails when trying to expand: {global_param}"
echo "builder-with-params: Should be expanded: {global_param}"
- shell: !include-raw-expand: macro-uses-global-defaults.yaml.inc
- job-template:

1
tests/yamlparser/job_fixtures/macro-uses-global-defaults.yaml.inc Normal file
View File

@ -0,0 +1 @@
echo "include-raw-expand: Should be expanded: {global_param}"