Macro now use parameters from defaults

Also fix error message for missing defaults.

Change-Id: I5d912c2bfb607c8388010ac6e39780be93073f5a

doc/source/changelog.rst

@@ -9,6 +9,11 @@ Features added
 
 * Update TAP publisher plugin (by Kienan Stewart).
 * Add tags: ``!include-raw-verbatim:`` and ``!include-raw-expand:``. Tags ``!include-raw:`` and ``!include-raw-escape:`` are now deprecated.
+* Macros can now use parameters specified in defaults, the same way as job-templates.
+  See this examples:
+
+  * `macro-uses-global-defaults.yaml <https://review.opendev.org/c/jjb/jenkins-job-builder/+/910877/4/tests/yamlparser/job_fixtures/macro-uses-global-defaults.yaml>`_
+  * `macro-uses-custom-defaults.yaml <https://review.opendev.org/c/jjb/jenkins-job-builder/+/910877/4/tests/yamlparser/job_fixtures/macro-uses-custom-defaults.yaml>`_
 
 .. note::
    After moving to 6.1.0 release, to remove deprecation warnings make these adjustments to your JJB sources:

jenkins_jobs/macro.py

@@ -13,6 +13,10 @@
 from dataclasses import dataclass
 from functools import partial
 
+from .root_base import ElementBase
+from .expander import Expander, StringsOnlyExpander
+from .yaml_objects import BaseYamlObject
+from .loc_loader import LocDict
 from .errors import JenkinsJobsException
 from .position import Pos
 
@@ -33,8 +37,12 @@ macro_specs = [
 
 
 @dataclass
-class Macro:
+class Macro(ElementBase):
+    _expander: Expander
+    _str_expander: StringsOnlyExpander
+    _type_name: str
     name: str
+    defaults_name: str
     pos: Pos
     elements: list
 
@@ -50,16 +58,61 @@ class Macro:
     ):
         d = data.copy()
         name = d.pop_required_loc_string("name")
+        defaults = d.pop_loc_string("defaults", "global")
         elements = d.pop_required_element(elements_name)
+        expander = Expander(config)
+        str_expander = StringsOnlyExpander(config)
         if d:
             example_key = next(iter(d.keys()))
             raise JenkinsJobsException(
                 f"In {type_name} macro {name!r}: unexpected elements: {','.join(d.keys())}",
                 pos=data.key_pos.get(example_key),
             )
-        macro = cls(name, pos, elements or [])
+        macro = cls(
+            roots.defaults,
+            expander,
+            str_expander,
+            type_name,
+            name,
+            defaults,
+            pos,
+            elements or [],
+        )
         roots.assign(roots.macros[type_name], name, macro, "macro")
 
+    def __str__(self):
+        return f"macro {self.name!r}"
+
+    def dispatch_elements(self, registry, xml_parent, component_data, job_data, params):
+        defaults = self._pick_defaults(self.defaults_name)
+        full_params = LocDict.merge(
+            defaults.params,
+            params,
+        )
+        element_list = self.elements
+        if isinstance(element_list, BaseYamlObject):
+            # Expand !j2-yaml tag if it is right below macro body.
+            # But do not expand yaml tags inside it - they will be expanded later.
+            element_list = element_list.expand(self._str_expander, full_params)
+        for element in element_list:
+            try:
+                expanded_element = self._expander.expand(element, full_params)
+            except JenkinsJobsException as x:
+                raise x.with_context(
+                    f"While expanding {self}",
+                    pos=self.pos,
+                )
+            # Pass component_data in as template data to this function
+            # so that if the macro is invoked with arguments,
+            # the arguments are interpolated into the real defn.
+            registry.dispatch(
+                self._type_name,
+                xml_parent,
+                expanded_element,
+                component_data,
+                job_data=job_data,
+            )
+
 
 macro_adders = {
     macro_type: partial(Macro.add, macro_type, elements_name)

jenkins_jobs/registry.py

@@ -27,8 +27,6 @@ from six import PY2
 
 from jenkins.plugins import PluginVersion
 from jenkins_jobs.errors import JenkinsJobsException
-from jenkins_jobs.expander import Expander, StringsOnlyExpander
-from jenkins_jobs.yaml_objects import BaseYamlObject
 
 __all__ = ["ModuleRegistry"]
 
@@ -48,8 +46,6 @@ class ModuleRegistry(object):
         self.jjb_config = jjb_config
         self.masked_warned = {}
         self._macros = {}
-        self._expander = Expander(jjb_config)
-        self._str_expander = StringsOnlyExpander(jjb_config)
 
         if plugins_list is None:
             self._plugin_version = {}
@@ -290,30 +286,8 @@ class ModuleRegistry(object):
             )
         if component_data is None:
             component_data = {}
-        expander_params = {**component_data, **(job_data or {})}
+        params = {**component_data, **(job_data or {})}
-        elements = macro.elements
+        macro.dispatch_elements(self, xml_parent, component_data, job_data, params)
-        if isinstance(elements, BaseYamlObject):
-            # Expand !j2-yaml tag if it is right below macro body.
-            # But do not expand yaml tags inside it - they will be expanded later.
-            elements = elements.expand(self._str_expander, expander_params)
-        for b in elements:
-            try:
-                element = self._expander.expand(b, expander_params)
-            except JenkinsJobsException as x:
-                raise x.with_context(
-                    f"While expanding macro {name!r}",
-                    pos=macro.pos,
-                )
-            # Pass component_data in as template data to this function
-            # so that if the macro is invoked with arguments,
-            # the arguments are interpolated into the real defn.
-            self.dispatch(
-                component_type,
-                xml_parent,
-                element,
-                component_data,
-                job_data=job_data,
-            )
 
     def _load_eps(self, component_list_type, component_type, entry_point, name):
         logging.debug("Caching entrypoints for %s" % component_list_type)

jenkins_jobs/root_base.py

@@ -40,10 +40,34 @@ class JobViewData:
 
 
 @dataclass
-class RootBase:
+class ElementBase:
-    """Base class for YAML root elements - job, view or template"""
+    """Base class for YAML elements - job, view, template, or macro"""
 
     _defaults: dict
+
+    @property
+    def title(self):
+        return str(self).capitalize()
+
+    def _pick_defaults(self, name):
+        try:
+            defaults = self._defaults[name]
+        except KeyError:
+            if name == "global":
+                return Defaults.empty()
+            raise JenkinsJobsException(
+                f"{self.title} wants defaults {name!r}, but it was never defined",
+                pos=name.pos,
+            )
+        if name == "global":
+            return defaults
+        return defaults.merged_with_global(self._pick_defaults("global"))
+
+
+@dataclass
+class RootBase(ElementBase):
+    """Base class for YAML root elements - job, view or template"""
+
     _expander: Expander
     _keep_descriptions: bool
     _id: str
@@ -61,10 +85,6 @@ class RootBase:
         else:
             return self.name
 
-    @property
-    def title(self):
-        return str(self).capitalize()
-
     @property
     def contents(self):
         contents = self._contents.copy()
@@ -80,21 +100,6 @@ class RootBase:
             expanded_contents["description"] = amended_description
         return expanded_contents
 
-    def _pick_defaults(self, name):
-        try:
-            defaults = self._defaults[name]
-        except KeyError:
-            if name == "global":
-                return Defaults.empty()
-            raise JenkinsJobsException(
-                f"{self.title} wants defaults {self.defaults_name!r}"
-                " but it was never defined",
-                pos=name.pos,
-            )
-        if name == "global":
-            return defaults
-        return defaults.merged_with_global(self._pick_defaults("global"))
-
 
 class NonTemplateRootMixin:
     def top_level_generate_items(self):

tests/yamlparser/error_fixtures/macro-uses-custom-defaults.error

@@ -1,18 +0,0 @@
-macro-uses-custom-defaults.yaml:34:3: In project 'sample-project'
-    - project:
-      ^
-macro-uses-custom-defaults.yaml:37:7: Defined here
-        - sample-job-template
-          ^
-macro-uses-custom-defaults.yaml:26:3: In job template 'sample-job-template'
-    - job-template:
-      ^
-macro-uses-custom-defaults.yaml:30:7: While expanding builder macro call 'builder-without-params'
-        - builder-without-params
-          ^
-macro-uses-custom-defaults.yaml:7:3: While expanding macro 'builder-without-params'
-    - builder:
-      ^
-macro-uses-custom-defaults.yaml:12:9: While formatting string 'echo "builder-without-params: Fails when trying to expand: {global_param}"\n...': Missing parameter: 'global_param'
-            echo "builder-without-params: Fails when tryi ...
-            ^

tests/yamlparser/error_fixtures/macro-uses-custom-defaults.yaml.inc

@@ -1 +0,0 @@
-echo "include-raw-expand: Fails when trying to expand: {global_param}"

tests/yamlparser/error_fixtures/macro-uses-global-defaults.error

@@ -1,18 +0,0 @@
-macro-uses-global-defaults.yaml:32:3: In project 'sample-project'
-    - project:
-      ^
-macro-uses-global-defaults.yaml:35:7: Defined here
-        - sample-job-template
-          ^
-macro-uses-global-defaults.yaml:24:3: In job template 'sample-job-template'
-    - job-template:
-      ^
-macro-uses-global-defaults.yaml:28:7: While expanding builder macro call 'builder-without-params'
-        - builder-without-params
-          ^
-macro-uses-global-defaults.yaml:7:3: While expanding macro 'builder-without-params'
-    - builder:
-      ^
-macro-uses-global-defaults.yaml:11:9: While formatting string 'echo "builder-without-params: Fails when trying to expand: {global_param}"\n...': Missing parameter: 'global_param'
-            echo "builder-without-params: Fails when tryi ...
-            ^

tests/yamlparser/error_fixtures/macro-uses-global-defaults.yaml.inc

@@ -1 +0,0 @@
-echo "include-raw-expand: Fails when trying to expand: {global_param}"

tests/yamlparser/error_fixtures/missing_defaults_at_job.error

@@ -1,6 +1,6 @@
 missing_defaults_at_job.yaml:1:3: In job 'sample-job'
     - job:
       ^
-missing_defaults_at_job.yaml:3:15: Job 'sample-job' wants defaults 'missing-defaults' but it was never defined
+missing_defaults_at_job.yaml:3:15: Job 'sample-job' wants defaults 'missing-defaults', but it was never defined
         defaults: missing-defaults
                   ^

tests/yamlparser/error_fixtures/missing_defaults_at_job_template.error

@@ -4,6 +4,6 @@ missing_defaults_at_job_template.yaml:5:3: In project 'sample-project'
 missing_defaults_at_job_template.yaml:1:3: In job template 'sample-job'
     - job-template:
       ^
-missing_defaults_at_job_template.yaml:3:15: Job template 'sample-job' wants defaults 'missing-defaults' but it was never defined
+missing_defaults_at_job_template.yaml:3:15: Job template 'sample-job' wants defaults 'missing-defaults', but it was never defined
         defaults: missing-defaults
                   ^

tests/yamlparser/error_fixtures/missing_defaults_at_macro.error

@@ -0,0 +1,15 @@
+missing_defaults_at_macro.yaml:12:3: In project 'sample-project'
+    - project:
+      ^
+missing_defaults_at_macro.yaml:15:7: Defined here
+        - sample-job
+          ^
+missing_defaults_at_macro.yaml:7:3: In job template 'sample-job'
+    - job-template:
+      ^
+missing_defaults_at_macro.yaml:10:7: While expanding builder macro call 'sample-builder'
+        - sample-builder
+          ^
+missing_defaults_at_macro.yaml:3:15: Macro 'sample-builder' wants defaults 'missing-defaults', but it was never defined
+        defaults: missing-defaults
+                  ^

tests/yamlparser/error_fixtures/missing_defaults_at_macro.yaml

@@ -0,0 +1,15 @@
+- builder:
+    name: sample-builder
+    defaults: missing-defaults
+    builders:
+    - shell: hello
+
+- job-template:
+    name: sample-job
+    builders:
+    - sample-builder
+
+- project:
+    name: sample-project
+    jobs:
+    - sample-job

tests/yamlparser/error_fixtures/missing_defaults_at_project.error

@@ -4,6 +4,6 @@ missing_defaults_at_project.yaml:4:3: In project 'sample-project'
 missing_defaults_at_project.yaml:1:3: In job template 'sample-job'
     - job-template:
       ^
-missing_defaults_at_project.yaml:6:15: Job template 'sample-job' wants defaults 'global' but it was never defined
+missing_defaults_at_project.yaml:6:15: Job template 'sample-job' wants defaults 'missing-defaults', but it was never defined
         defaults: missing-defaults
                   ^

tests/yamlparser/job_fixtures/macro-uses-custom-defaults.xml

@@ -0,0 +1,37 @@
+<?xml version="1.0" encoding="utf-8"?>
+<project>
+  <actions/>
+  <description>&lt;!-- Managed by Jenkins Job Builder --&gt;</description>
+  <keepDependencies>false</keepDependencies>
+  <displayName>sample-job-template</displayName>
+  <blockBuildWhenDownstreamBuilding>false</blockBuildWhenDownstreamBuilding>
+  <blockBuildWhenUpstreamBuilding>false</blockBuildWhenUpstreamBuilding>
+  <concurrentBuild>false</concurrentBuild>
+  <canRoam>true</canRoam>
+  <properties/>
+  <scm class="hudson.scm.NullSCM"/>
+  <builders>
+    <hudson.tasks.Shell>
+      <command>echo &quot;builder-without-params: Should be expanded: sample global param value&quot;
+</command>
+    </hudson.tasks.Shell>
+    <hudson.tasks.Shell>
+      <command>echo &quot;include-raw-expand: Should be expanded: sample global param value&quot;
+</command>
+    </hudson.tasks.Shell>
+    <hudson.tasks.Shell>
+      <command>echo &quot;builder-with-params: Should be expanded: sample macro param value&quot;
+</command>
+    </hudson.tasks.Shell>
+    <hudson.tasks.Shell>
+      <command>echo &quot;builder-with-params: Should be expanded: sample global param value&quot;
+</command>
+    </hudson.tasks.Shell>
+    <hudson.tasks.Shell>
+      <command>echo &quot;include-raw-expand: Should be expanded: sample global param value&quot;
+</command>
+    </hudson.tasks.Shell>
+  </builders>
+  <publishers/>
+  <buildWrappers/>
+</project>

tests/yamlparser/job_fixtures/macro-uses-custom-defaults.yaml

@@ -1,4 +1,4 @@
-# Macros do not use custom defaults.
+# Macros use custom defaults.
 
 - defaults:
     name: custom
@@ -6,21 +6,20 @@
 
 - builder:
     name: builder-without-params
-    # defaults: custom
+    defaults: custom
     builders:
     - shell: |
-        echo "builder-without-params: Fails when trying to expand: {global_param}"
+        echo "builder-without-params: Should be expanded: {global_param}"
-    # Also fails when trying to expand:
     - shell: !include-raw-expand: macro-uses-custom-defaults.yaml.inc
 
 - builder:
     name: builder-with-params
-    # defaults: custom
+    defaults: custom
     builders:
     - shell: |
         echo "builder-with-params: Should be expanded: {param}"
     - shell: |
-        echo "builder-with-params: Fails when trying to expand: {global_param}"
+        echo "builder-with-params: Should be expanded: {global_param}"
     - shell: !include-raw-expand: macro-uses-custom-defaults.yaml.inc
 
 - job-template:

tests/yamlparser/job_fixtures/macro-uses-custom-defaults.yaml.inc

@@ -0,0 +1 @@
+echo "include-raw-expand: Should be expanded: {global_param}"

tests/yamlparser/job_fixtures/macro-uses-global-defaults.xml

@@ -0,0 +1,37 @@
+<?xml version="1.0" encoding="utf-8"?>
+<project>
+  <actions/>
+  <description>&lt;!-- Managed by Jenkins Job Builder --&gt;</description>
+  <keepDependencies>false</keepDependencies>
+  <displayName>sample-job-template</displayName>
+  <blockBuildWhenDownstreamBuilding>false</blockBuildWhenDownstreamBuilding>
+  <blockBuildWhenUpstreamBuilding>false</blockBuildWhenUpstreamBuilding>
+  <concurrentBuild>false</concurrentBuild>
+  <canRoam>true</canRoam>
+  <properties/>
+  <scm class="hudson.scm.NullSCM"/>
+  <builders>
+    <hudson.tasks.Shell>
+      <command>echo &quot;builder-without-params: Should be expanded: sample global param value&quot;
+</command>
+    </hudson.tasks.Shell>
+    <hudson.tasks.Shell>
+      <command>echo &quot;include-raw-expand: Should be expanded: sample global param value&quot;
+</command>
+    </hudson.tasks.Shell>
+    <hudson.tasks.Shell>
+      <command>echo &quot;builder-with-params: Should be expanded: sample macro param value&quot;
+</command>
+    </hudson.tasks.Shell>
+    <hudson.tasks.Shell>
+      <command>echo &quot;builder-with-params: Should be expanded: sample global param value&quot;
+</command>
+    </hudson.tasks.Shell>
+    <hudson.tasks.Shell>
+      <command>echo &quot;include-raw-expand: Should be expanded: sample global param value&quot;
+</command>
+    </hudson.tasks.Shell>
+  </builders>
+  <publishers/>
+  <buildWrappers/>
+</project>

tests/yamlparser/job_fixtures/macro-uses-global-defaults.yaml

@@ -1,4 +1,4 @@
-# Macros do not use global defaults.
+# Macros use global defaults.
 
 - defaults:
     name: global
@@ -8,8 +8,7 @@
     name: builder-without-params
     builders:
     - shell: |
-        echo "builder-without-params: Fails when trying to expand: {global_param}"
+        echo "builder-without-params: Should be expanded: {global_param}"
-    # Also fails when trying to expand:
     - shell: !include-raw-expand: macro-uses-global-defaults.yaml.inc
 
 - builder:
@@ -18,7 +17,7 @@
     - shell: |
         echo "builder-with-params: Should be expanded: {param}"
     - shell: |
-        echo "builder-with-params: Fails when trying to expand: {global_param}"
+        echo "builder-with-params: Should be expanded: {global_param}"
     - shell: !include-raw-expand: macro-uses-global-defaults.yaml.inc
 
 - job-template:

tests/yamlparser/job_fixtures/macro-uses-global-defaults.yaml.inc

@@ -0,0 +1 @@
+echo "include-raw-expand: Should be expanded: {global_param}"