Add context for macro calls

Change-Id: I674b153770297c3a0abbfcee26d840e0f2be490b

jenkins_jobs/modules/base.py

@@ -14,6 +14,8 @@
 
 # Base class for a jenkins_jobs module
 
+from jenkins_jobs.loc_loader import LocList
+
 
 class Base(object):
     """
@@ -73,3 +75,21 @@ class Base(object):
         """
 
         pass
+
+    def dispatch_component_list(
+        self, component_type, component_list, xml_parent, job_data=None
+    ):
+        if not component_list:
+            return
+        for idx, component in enumerate(component_list):
+            if isinstance(component_list, LocList):
+                pos = component_list.value_pos[idx]
+            else:
+                pos = None
+            self.registry.dispatch(
+                component_type,
+                xml_parent,
+                component,
+                job_data=job_data,
+                component_pos=pos,
+            )

jenkins_jobs/modules/builders.py

@@ -2576,9 +2576,8 @@ class Builders(jenkins_jobs.modules.base.Base):
 
         for alias in ["prebuilders", "builders", "postbuilders"]:
             if alias in data:
-                builders = XML.SubElement(xml_parent, alias)
+                xml_builders = XML.SubElement(xml_parent, alias)
-                for builder in data[alias]:
+                self.dispatch_component_list("builder", data[alias], xml_builders)
-                    self.registry.dispatch("builder", builders, builder)
 
         # Make sure freestyle projects always have a <builders> entry
         # or Jenkins v1.472 (at least) will NPE.

jenkins_jobs/modules/notifications.py

@@ -95,5 +95,6 @@ class Notifications(jenkins_jobs.modules.base.Base):
             )
             endpoints_element = XML.SubElement(notify_element, "endpoints")
 
-            for endpoint in notifications:
+            self.dispatch_component_list(
-                self.registry.dispatch("notification", endpoints_element, endpoint)
+                "notification", notifications, endpoints_element
+            )

jenkins_jobs/modules/parameters.py

@@ -34,6 +34,7 @@ Example::
 
 import xml.etree.ElementTree as XML
 
+from jenkins_jobs.loc_loader import LocList
 from jenkins_jobs.errors import JenkinsJobsException
 from jenkins_jobs.errors import MissingAttributeError
 from jenkins_jobs.errors import InvalidAttributeError
@@ -1624,7 +1625,11 @@ class Parameters(jenkins_jobs.modules.base.Base):
             pdefs = pdefp.find("parameterDefinitions")
             if pdefs is None:
                 pdefs = XML.SubElement(pdefp, "parameterDefinitions")
-            for param in parameters:
+            for idx, param in enumerate(parameters):
+                if isinstance(parameters, LocList):
+                    param_pos = parameters.value_pos[idx]
+                else:
+                    param_pos = None
                 if not isinstance(param, dict):
                     # Macro parameter without arguments
                     param = {param: {}}
@@ -1640,4 +1645,6 @@ class Parameters(jenkins_jobs.modules.base.Base):
                             self._extend_uno_choice_param_data(
                                 macro_param, macro_param_type, data
                             )
-                self.registry.dispatch("parameter", pdefs, param)
+                self.registry.dispatch(
+                    "parameter", pdefs, param, component_pos=param_pos
+                )

jenkins_jobs/modules/properties.py

@@ -1513,9 +1513,10 @@ class Properties(jenkins_jobs.modules.base.Base):
     component_list_type = "properties"
 
     def gen_xml(self, xml_parent, data):
-        properties = xml_parent.find("properties")
+        xml_properties = xml_parent.find("properties")
-        if properties is None:
+        if xml_properties is None:
-            properties = XML.SubElement(xml_parent, "properties")
+            xml_properties = XML.SubElement(xml_parent, "properties")
 
-        for prop in data.get("properties", []):
+        self.dispatch_component_list(
-            self.registry.dispatch("property", properties, prop, job_data=data)
+            "property", data.get("properties", []), xml_properties, job_data=data
+        )

jenkins_jobs/modules/publishers.py

@@ -8400,7 +8400,7 @@ class Publishers(jenkins_jobs.modules.base.Base):
             logger.debug("Publishers skipped for Pipeline job")
             return
 
-        publishers = XML.SubElement(xml_parent, "publishers")
+        xml_publishers = XML.SubElement(xml_parent, "publishers")
 
-        for action in data.get("publishers", []):
+        component_list = data.get("publishers", [])
-            self.registry.dispatch("publisher", publishers, action)
+        self.dispatch_component_list("publisher", component_list, xml_publishers)

jenkins_jobs/modules/reporters.py

@@ -151,7 +151,7 @@ class Reporters(jenkins_jobs.modules.base.Base):
                 "Reporters may only be used for Maven " "modules."
             )
 
-        reporters = XML.SubElement(xml_parent, "reporters")
+        xml_reporters = XML.SubElement(xml_parent, "reporters")
 
-        for action in data.get("reporters", []):
+        component_list = data.get("reporters", [])
-            self.registry.dispatch("reporter", reporters, action)
+        self.dispatch_component_list("reporter", component_list, xml_reporters)

jenkins_jobs/modules/scm.py

@@ -39,6 +39,7 @@ Example of an empty ``scm``:
 import logging
 import xml.etree.ElementTree as XML
 
+from jenkins_jobs.loc_loader import LocList
 from jenkins_jobs.errors import InvalidAttributeError
 from jenkins_jobs.errors import JenkinsJobsException
 import jenkins_jobs.modules.base
@@ -1769,8 +1770,8 @@ class SCM(jenkins_jobs.modules.base.Base):
             return
 
         scms_parent = XML.Element("scms")
-        for scm in data.get("scm", []):
+        component_list = data.get("scm", [])
-            self.registry.dispatch("scm", scms_parent, scm)
+        self.dispatch_component_list("scm", component_list, scms_parent)
         scms_count = len(scms_parent)
         if scms_count == 0:
             XML.SubElement(xml_parent, "scm", {"class": "hudson.scm.NullSCM"})
@@ -1806,7 +1807,14 @@ class PipelineSCM(jenkins_jobs.modules.base.Base):
             if scms_count == 0:
                 raise JenkinsJobsException("'scm' missing or empty")
             elif scms_count == 1:
-                self.registry.dispatch("scm", definition_parent, scms[0])
+                component = scms[0]
+                if isinstance(scms, LocList):
+                    component_pos = scms.value_pos[0]
+                else:
+                    component_pos = None
+                self.registry.dispatch(
+                    "scm", definition_parent, component, component_pos=component_pos
+                )
                 mapping = [
                     ("script-path", "scriptPath", "Jenkinsfile"),
                     ("lightweight-checkout", "lightweight", None, [True, False]),

jenkins_jobs/modules/triggers.py

@@ -2784,7 +2784,7 @@ class Triggers(jenkins_jobs.modules.base.Base):
             return
 
         if data.get("project-type", "freestyle") != "pipeline":
-            trig_e = XML.SubElement(xml_parent, "triggers", {"class": "vector"})
+            xml_triggers = XML.SubElement(xml_parent, "triggers", {"class": "vector"})
         else:
             properties = xml_parent.find("properties")
             if properties is None:
@@ -2793,7 +2793,6 @@ class Triggers(jenkins_jobs.modules.base.Base):
                 properties,
                 "org.jenkinsci.plugins.workflow.job.properties.PipelineTriggersJobProperty",
             )
-            trig_e = XML.SubElement(pipeline_trig_prop, "triggers")
+            xml_triggers = XML.SubElement(pipeline_trig_prop, "triggers")
 
-        for trigger in triggers:
+        self.dispatch_component_list("trigger", triggers, xml_triggers)
-            self.registry.dispatch("trigger", trig_e, trigger)

jenkins_jobs/modules/wrappers.py

@@ -3005,7 +3005,6 @@ class Wrappers(jenkins_jobs.modules.base.Base):
             logger.debug("Build wrappers skipped for Pipeline job")
             return
 
-        wrappers = XML.SubElement(xml_parent, "buildWrappers")
+        xml_wrappers = XML.SubElement(xml_parent, "buildWrappers")
 
-        for wrap in data.get("wrappers", []):
+        self.dispatch_component_list("wrapper", data.get("wrappers", []), xml_wrappers)
-            self.registry.dispatch("wrapper", wrappers, wrap)

jenkins_jobs/registry.py

@@ -191,7 +191,13 @@ class ModuleRegistry(object):
         return component_list_type
 
     def dispatch(
-        self, component_type, xml_parent, component, template_data={}, job_data=None
+        self,
+        component_type,
+        xml_parent,
+        component,
+        template_data={},
+        job_data=None,
+        component_pos=None,
     ):
         """This is a method that you can call from your implementation of
         Base.gen_xml or component.  It allows modules to define a type
@@ -243,9 +249,24 @@ class ModuleRegistry(object):
         macro_dict = self.macros.get(component_type, {})
         macro = macro_dict.get(name)
         if macro:
-            self._dispatch_macro(
+            try:
-                component_data, component_type, eps, job_data, macro, name, xml_parent
+                self._dispatch_macro(
-            )
+                    component_data,
+                    component_type,
+                    eps,
+                    job_data,
+                    macro,
+                    name,
+                    xml_parent,
+                )
+            except JenkinsJobsException as x:
+                if component_pos is not None:
+                    raise x.with_context(
+                        f"While expanding {component_type} macro call {name!r}",
+                        pos=component_pos,
+                    )
+                else:
+                    raise
         elif name in eps:
             try:
                 func = eps[name]

tests/yamlparser/error_fixtures/macro_error_context_for_builder.error

@@ -0,0 +1,18 @@
+macro_error_context_for_builder.yaml:14:3: In project 'sample-project'
+    - project:
+      ^
+macro_error_context_for_builder.yaml:17:9: Defined here
+          - sample-job
+            ^
+macro_error_context_for_builder.yaml:8:3: In job template 'sample-job'
+    - job-template:
+      ^
+macro_error_context_for_builder.yaml:11:7: While expanding builder macro call 'sample-builder'
+        - sample-builder:
+          ^
+macro_error_context_for_builder.yaml:3:3: While expanding macro 'sample-builder'
+    - builder:
+      ^
+macro_error_context_for_builder.yaml:6:15: While formatting string 'echo {missing_parameter}': Missing parameter: 'missing_parameter'
+        - shell: 'echo {missing_parameter}'
+                  ^

tests/yamlparser/error_fixtures/macro_error_context_for_builder.yaml

@@ -0,0 +1,17 @@
+# Check proper context is shown for error in builder macro expansion.
+
+- builder:
+    name: sample-builder
+    builders:
+    - shell: 'echo {missing_parameter}'
+
+- job-template:
+    name: sample-job
+    builders:
+    - sample-builder:
+        unused_param: abc
+
+- project:
+    name: sample-project
+    jobs:
+      - sample-job

tests/yamlparser/error_fixtures/macro_error_context_for_notification.error

@@ -0,0 +1,18 @@
+macro_error_context_for_notification.yaml:16:3: In project 'sample-project'
+    - project:
+      ^
+macro_error_context_for_notification.yaml:19:9: Defined here
+          - sample-job
+            ^
+macro_error_context_for_notification.yaml:9:3: In job template 'sample-job'
+    - job-template:
+      ^
+macro_error_context_for_notification.yaml:13:7: While expanding notification macro call 'sample-notification'
+        - sample-notification:
+          ^
+macro_error_context_for_notification.yaml:3:3: While expanding macro 'sample-notification'
+    - notification:
+      ^
+macro_error_context_for_notification.yaml:7:15: While formatting string '{missing_parameter}': Missing parameter: 'missing_parameter'
+            url: '{missing_parameter}'
+                  ^

tests/yamlparser/error_fixtures/macro_error_context_for_notification.yaml

@@ -0,0 +1,19 @@
+# Check proper context is shown for error in builder macro expansion.
+
+- notification:
+    name: sample-notification
+    notifications:
+    - http:
+        url: '{missing_parameter}'
+
+- job-template:
+    name: sample-job
+    project-type: maven
+    notifications:
+    - sample-notification:
+        unused_param: abc
+
+- project:
+    name: sample-project
+    jobs:
+      - sample-job

tests/yamlparser/error_fixtures/macro_error_context_for_parameter.error

@@ -0,0 +1,18 @@
+macro_error_context_for_parameter.yaml:16:3: In project 'sample-project'
+    - project:
+      ^
+macro_error_context_for_parameter.yaml:19:9: Defined here
+          - sample-job
+            ^
+macro_error_context_for_parameter.yaml:10:3: In job template 'sample-job'
+    - job-template:
+      ^
+macro_error_context_for_parameter.yaml:13:7: While expanding parameter macro call 'sample-parameter'
+        - sample-parameter:
+          ^
+macro_error_context_for_parameter.yaml:3:3: While expanding macro 'sample-parameter'
+    - parameter:
+      ^
+macro_error_context_for_parameter.yaml:8:19: While formatting string '{missing_parameter}': Missing parameter: 'missing_parameter'
+            default: '{missing_parameter}'
+                      ^

tests/yamlparser/error_fixtures/macro_error_context_for_parameter.yaml

@@ -0,0 +1,19 @@
+# Check proper context is shown for error in parameter macro expansion.
+
+- parameter:
+    name: sample-parameter
+    parameters:
+    - string:
+        name: FOO
+        default: '{missing_parameter}'
+
+- job-template:
+    name: sample-job
+    parameters:
+    - sample-parameter:
+        unused_param: abc
+
+- project:
+    name: sample-project
+    jobs:
+      - sample-job

tests/yamlparser/error_fixtures/macro_error_context_for_pipeline_scm.error

@@ -0,0 +1,18 @@
+macro_error_context_for_pipeline_scm.yaml:17:3: In project 'sample-project'
+    - project:
+      ^
+macro_error_context_for_pipeline_scm.yaml:20:9: Defined here
+          - sample-job
+            ^
+macro_error_context_for_pipeline_scm.yaml:9:3: In job template 'sample-job'
+    - job-template:
+      ^
+macro_error_context_for_pipeline_scm.yaml:14:9: While expanding scm macro call 'sample-scm'
+          - sample-scm:
+            ^
+macro_error_context_for_pipeline_scm.yaml:3:3: While expanding macro 'sample-scm'
+    - scm:
+      ^
+macro_error_context_for_pipeline_scm.yaml:7:15: While formatting string '{missing_parameter}': Missing parameter: 'missing_parameter'
+            url: '{missing_parameter}'
+                  ^

tests/yamlparser/error_fixtures/macro_error_context_for_pipeline_scm.yaml

@@ -0,0 +1,20 @@
+# Check proper context is shown for error in builder macro expansion.
+
+- scm:
+    name: sample-scm
+    scm:
+    - git:
+        url: '{missing_parameter}'
+
+- job-template:
+    name: sample-job
+    project-type: pipeline
+    pipeline-scm:
+      scm:
+      - sample-scm:
+          unused_param: abc
+
+- project:
+    name: sample-project
+    jobs:
+      - sample-job

tests/yamlparser/error_fixtures/macro_error_context_for_property.error

@@ -0,0 +1,18 @@
+macro_error_context_for_property.yaml:15:3: In project 'sample-project'
+    - project:
+      ^
+macro_error_context_for_property.yaml:18:9: Defined here
+          - sample-job
+            ^
+macro_error_context_for_property.yaml:9:3: In job template 'sample-job'
+    - job-template:
+      ^
+macro_error_context_for_property.yaml:12:7: While expanding property macro call 'sample-property'
+        - sample-property:
+          ^
+macro_error_context_for_property.yaml:3:3: While expanding macro 'sample-property'
+    - property:
+      ^
+macro_error_context_for_property.yaml:7:28: While formatting string '{missing_parameter}': Missing parameter: 'missing_parameter'
+            number-of-builds: '{missing_parameter}'
+                               ^

tests/yamlparser/error_fixtures/macro_error_context_for_property.yaml

@@ -0,0 +1,18 @@
+# Check proper context is shown for error in builder macro expansion.
+
+- property:
+    name: sample-property
+    properties:
+    - branch-api:
+        number-of-builds: '{missing_parameter}'
+
+- job-template:
+    name: sample-job
+    properties:
+    - sample-property:
+        unused_param: abc
+
+- project:
+    name: sample-project
+    jobs:
+      - sample-job

tests/yamlparser/error_fixtures/macro_error_context_for_publisher.error

@@ -0,0 +1,18 @@
+macro_error_context_for_publisher.yaml:15:3: In project 'sample-project'
+    - project:
+      ^
+macro_error_context_for_publisher.yaml:18:9: Defined here
+          - sample-job
+            ^
+macro_error_context_for_publisher.yaml:9:3: In job template 'sample-job'
+    - job-template:
+      ^
+macro_error_context_for_publisher.yaml:12:7: While expanding publisher macro call 'sample-publisher'
+        - sample-publisher:
+          ^
+macro_error_context_for_publisher.yaml:3:3: While expanding macro 'sample-publisher'
+    - publisher:
+      ^
+macro_error_context_for_publisher.yaml:7:21: While formatting string '{missing_parameter}': Missing parameter: 'missing_parameter'
+            artifacts: '{missing_parameter}'
+                        ^

tests/yamlparser/error_fixtures/macro_error_context_for_publisher.yaml

@@ -0,0 +1,18 @@
+# Check proper context is shown for error in builder macro expansion.
+
+- publisher:
+    name: sample-publisher
+    publishers:
+    - archive:
+        artifacts: '{missing_parameter}'
+
+- job-template:
+    name: sample-job
+    publishers:
+    - sample-publisher:
+        unused_param: abc
+
+- project:
+    name: sample-project
+    jobs:
+      - sample-job

tests/yamlparser/error_fixtures/macro_error_context_for_reporter.error

@@ -0,0 +1,18 @@
+macro_error_context_for_reporter.yaml:16:3: In project 'sample-project'
+    - project:
+      ^
+macro_error_context_for_reporter.yaml:19:9: Defined here
+          - sample-job
+            ^
+macro_error_context_for_reporter.yaml:9:3: In job template 'sample-job'
+    - job-template:
+      ^
+macro_error_context_for_reporter.yaml:13:7: While expanding reporter macro call 'sample-reporter'
+        - sample-reporter:
+          ^
+macro_error_context_for_reporter.yaml:3:3: While expanding macro 'sample-reporter'
+    - reporter:
+      ^
+macro_error_context_for_reporter.yaml:7:21: While formatting string '{missing_parameter}': Missing parameter: 'missing_parameter'
+            recipient: '{missing_parameter}'
+                        ^

tests/yamlparser/error_fixtures/macro_error_context_for_reporter.yaml

@@ -0,0 +1,19 @@
+# Check proper context is shown for error in builder macro expansion.
+
+- reporter:
+    name: sample-reporter
+    reporters:
+    - email:
+        recipient: '{missing_parameter}'
+
+- job-template:
+    name: sample-job
+    project-type: maven
+    reporters:
+    - sample-reporter:
+        unused_param: abc
+
+- project:
+    name: sample-project
+    jobs:
+      - sample-job

tests/yamlparser/error_fixtures/macro_error_context_for_scm.error

@@ -0,0 +1,18 @@
+macro_error_context_for_scm.yaml:15:3: In project 'sample-project'
+    - project:
+      ^
+macro_error_context_for_scm.yaml:18:9: Defined here
+          - sample-job
+            ^
+macro_error_context_for_scm.yaml:9:3: In job template 'sample-job'
+    - job-template:
+      ^
+macro_error_context_for_scm.yaml:12:7: While expanding scm macro call 'sample-scm'
+        - sample-scm:
+          ^
+macro_error_context_for_scm.yaml:3:3: While expanding macro 'sample-scm'
+    - scm:
+      ^
+macro_error_context_for_scm.yaml:7:15: While formatting string '{missing_parameter}': Missing parameter: 'missing_parameter'
+            url: '{missing_parameter}'
+                  ^

tests/yamlparser/error_fixtures/macro_error_context_for_scm.yaml

@@ -0,0 +1,18 @@
+# Check proper context is shown for error in builder macro expansion.
+
+- scm:
+    name: sample-scm
+    scm:
+    - git:
+        url: '{missing_parameter}'
+
+- job-template:
+    name: sample-job
+    scm:
+    - sample-scm:
+        unused_param: abc
+
+- project:
+    name: sample-project
+    jobs:
+      - sample-job

tests/yamlparser/error_fixtures/macro_error_context_for_trigger.error

@@ -0,0 +1,18 @@
+macro_error_context_for_trigger.yaml:15:3: In project 'sample-project'
+    - project:
+      ^
+macro_error_context_for_trigger.yaml:18:9: Defined here
+          - sample-job
+            ^
+macro_error_context_for_trigger.yaml:9:3: In job template 'sample-job'
+    - job-template:
+      ^
+macro_error_context_for_trigger.yaml:12:7: While expanding trigger macro call 'sample-trigger'
+        - sample-trigger:
+          ^
+macro_error_context_for_trigger.yaml:3:3: While expanding macro 'sample-trigger'
+    - trigger:
+      ^
+macro_error_context_for_trigger.yaml:7:16: While formatting string '{missing_parameter}': Missing parameter: 'missing_parameter'
+            cron: '{missing_parameter}'
+                   ^

tests/yamlparser/error_fixtures/macro_error_context_for_trigger.yaml

@@ -0,0 +1,18 @@
+# Check proper context is shown for error in builder macro expansion.
+
+- trigger:
+    name: sample-trigger
+    triggers:
+    - pollscm:
+        cron: '{missing_parameter}'
+
+- job-template:
+    name: sample-job
+    triggers:
+    - sample-trigger:
+        unused_param: abc
+
+- project:
+    name: sample-project
+    jobs:
+      - sample-job

tests/yamlparser/error_fixtures/macro_error_context_for_wrapper.error

@@ -0,0 +1,18 @@
+macro_error_context_for_wrapper.yaml:15:3: In project 'sample-project'
+    - project:
+      ^
+macro_error_context_for_wrapper.yaml:18:9: Defined here
+          - sample-job
+            ^
+macro_error_context_for_wrapper.yaml:9:3: In job template 'sample-job'
+    - job-template:
+      ^
+macro_error_context_for_wrapper.yaml:12:7: While expanding wrapper macro call 'sample-wrapper'
+        - sample-wrapper:
+          ^
+macro_error_context_for_wrapper.yaml:3:3: While expanding macro 'sample-wrapper'
+    - wrapper:
+      ^
+macro_error_context_for_wrapper.yaml:7:19: While formatting string '{missing_parameter}': Missing parameter: 'missing_parameter'
+            timeout: '{missing_parameter}'
+                      ^

tests/yamlparser/error_fixtures/macro_error_context_for_wrapper.yaml

@@ -0,0 +1,18 @@
+# Check proper context is shown for error in builder macro expansion.
+
+- wrapper:
+    name: sample-wrapper
+    wrappers:
+    - timeout:
+        timeout: '{missing_parameter}'
+
+- job-template:
+    name: sample-job
+    wrappers:
+    - sample-wrapper:
+        unused_param: abc
+
+- project:
+    name: sample-project
+    jobs:
+      - sample-job

tests/yamlparser/error_fixtures/missing_param_jinja2_macro_direct.error

@@ -7,6 +7,9 @@ missing_param_jinja2_macro_direct.yaml:4:9: Defined here
 missing_param_jinja2_macro_direct.yaml:12:3: In job template 'sample-job'
     - job-template:
       ^
+missing_param_jinja2_macro_direct.yaml:15:9: While expanding builder macro call 'sample-builder'
+          - sample-builder:
+            ^
 missing_param_jinja2_macro_direct.yaml:6:3: While expanding macro 'sample-builder'
     - builder:
       ^

tests/yamlparser/error_fixtures/missing_param_simple_macro_direct.error

@@ -7,6 +7,9 @@ missing_param_simple_macro_direct.yaml:4:9: Defined here
 missing_param_simple_macro_direct.yaml:12:3: In job template 'sample-job'
     - job-template:
       ^
+missing_param_simple_macro_direct.yaml:15:9: While expanding builder macro call 'sample-builder'
+          - sample-builder:
+            ^
 missing_param_simple_macro_direct.yaml:6:3: While expanding macro 'sample-builder'
     - builder:
       ^