jenkins-job-builder/jenkins_jobs/yaml_objects.py

# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.

# Provides local yaml parsing classes and extends yaml module.

"""Custom application specific yamls tags are supported to provide
enhancements when reading yaml configuration.

Action Tags
^^^^^^^^^^^

These allow manipulation of data being stored in one layout in the source
yaml for convenience and/or clarity, to another format to be processed by
the targeted module instead of requiring all modules in JJB being capable
of supporting multiple input formats.

The tag ``!join:`` will treat the first element of the following list as
the delimiter to use, when joining the remaining elements into a string
and returning a single string to be consumed by the specified module option.

This allows users to maintain elements of data in a list structure for ease
of review/maintenance, and have the yaml parser convert it to a string for
consumption as any argument for modules. The main expected use case is to
allow for generic plugin data such as shell properties to be populated from
a list construct which the yaml parser converts to a single string, instead
of trying to support this within the module code which would require a
templating engine similar to Jinja.

Generic Example:

    .. literalinclude:: /../../tests/loader/fixtures/joinlists.yaml


Environment Inject:

    .. literalinclude:: /../../tests/yamlparser/job_fixtures/string_join.yaml


While this mechanism can also be used items where delimiters are supported by
the module, that should be considered a bug that the existing code doesn't
handle being provided a list and delimiter to perform the correct conversion
for you. Should you discover a module that takes arguments with delimiters and
the existing JJB codebase does not handle accepting lists, then this can be
used as a temporary solution in place of using very long strings:

Extended Params Example:

    .. literalinclude::
        /../../tests/parameters/fixtures/extended-choice-param-full.yaml


Inclusion Tags
^^^^^^^^^^^^^^

These allow inclusion of arbitrary files as a method of having blocks of data
managed separately to the yaml job configurations. A specific usage of this is
inlining scripts contained in separate files, although such tags may also be
used to simplify usage of macros or job templates.

The tag ``!include:`` will treat the following string as file which should be
parsed as yaml configuration data.

Example:

    .. literalinclude:: /../../tests/loader/fixtures/include001.yaml

    contents of include001.yaml.inc:

    .. literalinclude:: /../../tests/yamlparser/job_fixtures/include001.yaml.inc


The tag ``!include-raw:`` will treat the given string or list of strings as
filenames to be opened as one or more data blob, which should be read into
the calling yaml construct without any further parsing. Any data in a file
included through this tag, will be treated as string data.

Examples:

    .. literalinclude:: /../../tests/loader/fixtures/include-raw001-job.yaml

    contents of include-raw001-hello-world.sh:

        .. literalinclude::
            /../../tests/loader/fixtures/include-raw001-hello-world.sh

    contents of include-raw001-vars.sh:

        .. literalinclude::
            /../../tests/loader/fixtures/include-raw001-vars.sh

    using a list of files:

    .. literalinclude::
        /../../tests/loader/fixtures/include-raw-multi001.yaml

The tag ``!include-raw-escape:`` treats the given string or list of strings as
filenames to be opened as one or more data blobs, which should be escaped
before being read in as string data. This allows job-templates to use this tag
to include scripts from files without needing to escape braces in the original
file.

.. warning::

    When used as a macro ``!include-raw-escape:`` should only be used if
    parameters are passed into the escaped file and you would like to escape
    those parameters. If the file does not have any jjb parameters passed into
    it then ``!include-raw:`` should be used instead otherwise you will run
    into an interesting issue where ``include-raw-escape:`` actually adds
    additional curly braces around existing curly braces. For example
    ${PROJECT} becomes ${{PROJECT}} which may break bash scripts.

Examples:

    .. literalinclude::
        /../../tests/loader/fixtures/include-raw-escaped001-template.yaml

    contents of include-raw001-hello-world.sh:

        .. literalinclude::
            /../../tests/loader/fixtures/include-raw001-hello-world.sh

    contents of include-raw001-vars.sh:

        .. literalinclude::
            /../../tests/loader/fixtures/include-raw001-vars.sh

    using a list of files:

    .. literalinclude::
        /../../tests/loader/fixtures/include-raw-escaped-multi001.yaml


For all the multi file includes, the files are simply appended using a newline
character.


To allow for job templates to perform substitution on the path names, when a
filename containing a python format placeholder is encountered, lazy loading
support is enabled, where instead of returning the contents back during yaml
parsing, it is delayed until the variable substitution is performed.

Example:

    .. literalinclude:: /../../tests/yamlparser/job_fixtures/lazy-load-jobs001.yaml

    using a list of files:

    .. literalinclude::
        /../../tests/yamlparser/job_fixtures/lazy-load-jobs-multi001.yaml

.. note::

    Because lazy-loading involves performing the substitution on the file
    name, it means that jenkins-job-builder can not call the variable
    substitution on the contents of the file. This means that the
    ``!include-raw:`` tag will behave as though ``!include-raw-escape:`` tag
    was used instead whenever name substitution on the filename is to be
    performed.

    Given the behaviour described above, when substitution is to be performed
    on any filename passed via ``!include-raw-escape:`` the tag will be
    automatically converted to ``!include-raw:`` and no escaping will be
    performed.


The tag ``!include-jinja2:`` will treat the given string or list of strings as
filenames to be opened as Jinja2 templates, which should be rendered to a
string and included in the calling YAML construct.  (This is analogous to the
templating that will happen with ``!include-raw``.)

Examples:

    .. literalinclude:: /../../tests/yamlparser/job_fixtures/jinja01.yaml

    contents of jinja01.yaml.inc:

        .. literalinclude:: /../../tests/yamlparser/job_fixtures/jinja01.yaml.inc


The tag ``!j2:`` takes a string and treats it as a Jinja2 template.  It will be
rendered (with the variables in that context) and included in the calling YAML
construct.

Examples:

    .. literalinclude:: /../../tests/yamlparser/job_fixtures/jinja-string01.yaml

The tag ``!j2-yaml:`` is similar to the ``!j2:`` tag, just that it loads the
Jinja-rendered string as YAML and embeds it in the calling YAML construct. This
provides a very flexible and convenient way of generating pieces of YAML
structures. One of use cases is defining complex YAML structures with much
simpler configuration, without any duplication.

Examples:

    .. literalinclude:: /../../tests/yamlparser/job_fixtures/jinja-yaml01.yaml

Another use case is controlling lists dynamically, like conditionally adding
list elements based on project configuration.

Examples:

    .. literalinclude:: /../../tests/yamlparser/job_fixtures/jinja-yaml02.yaml

"""

import abc
import os.path
import logging
import sys
from pathlib import Path

import jinja2
import jinja2.meta
import yaml

from .errors import JenkinsJobsException
from .formatter import CustomFormatter, enum_str_format_required_params

logger = logging.getLogger(__name__)


if sys.version_info >= (3, 8):
    from functools import cached_property
else:
    from functools import lru_cache

    # cached_property was introduced in python 3.8.
    # Recipe from https://stackoverflow.com/a/19979379
    def cached_property(fn):
        return property(lru_cache()(fn))


class BaseYamlObject(metaclass=abc.ABCMeta):
    @staticmethod
    def path_list_from_node(loader, node):
        if isinstance(node, yaml.ScalarNode):
            return [loader.construct_yaml_str(node)]
        elif isinstance(node, yaml.SequenceNode):
            return loader.construct_sequence(node)
        else:
            raise yaml.constructor.ConstructorError(
                None,
                None,
                f"expected either a sequence or scalar node, but found {node.id}",
                node.start_mark,
            )

    @classmethod
    def from_yaml(cls, loader, node):
        value = loader.construct_yaml_str(node)
        return cls(loader.jjb_config, loader, value)

    def __init__(self, jjb_config, loader):
        self._search_path = jjb_config.yamlparser["include_path"]
        if loader.source_path:
            # Loaded from a file, find includes beside it too.
            self._search_path.append(os.path.dirname(loader.source_path))
        self._loader = loader
        allow_empty = jjb_config.yamlparser["allow_empty_variables"]
        self._formatter = CustomFormatter(allow_empty)

    @abc.abstractmethod
    def expand(self, expander, params):
        """Expand object but do not substitute template parameters"""
        pass

    def subst(self, expander, params):
        """Expand object and substitute template parameters"""
        return self.expand(expander, params)

    def _find_file(self, rel_path):
        search_path = self._search_path
        if "." not in search_path:
            search_path.append(".")
        dir_list = [Path(d).expanduser() for d in self._search_path]
        for dir in dir_list:
            candidate = dir.joinpath(rel_path)
            if candidate.is_file():
                logger.debug("Including file %r from path %r", str(rel_path), str(dir))
                return candidate
        raise JenkinsJobsException(
            f"File {rel_path} does not exist on any of include directories:"
            f" {','.join([str(d) for d in dir_list])}"
        )


class J2BaseYamlObject(BaseYamlObject):
    def __init__(self, jjb_config, loader):
        super().__init__(jjb_config, loader)
        self._jinja2_env = jinja2.Environment(
            loader=jinja2.FileSystemLoader(self._search_path),
            undefined=jinja2.StrictUndefined,
        )

    @staticmethod
    def _render_template(template_text, template, params):
        try:
            return template.render(params)
        except jinja2.UndefinedError as x:
            if len(template_text) > 40:
                text = template_text[:40] + "..."
            else:
                text = template_text
            raise JenkinsJobsException(
                f"While formatting jinja2 template {text!r}: {x}"
            )


class J2Template(J2BaseYamlObject):
    def __init__(self, jjb_config, loader, template_text):
        super().__init__(jjb_config, loader)
        self._template_text = template_text
        self._template = self._jinja2_env.from_string(template_text)

    @cached_property
    def required_params(self):
        ast = self._jinja2_env.parse(self._template_text)
        return jinja2.meta.find_undeclared_variables(ast)

    def _render(self, params):
        return self._render_template(self._template_text, self._template, params)


class J2String(J2Template):
    yaml_tag = "!j2:"

    def expand(self, expander, params):
        return self._render(params)


class J2Yaml(J2Template):
    yaml_tag = "!j2-yaml:"

    def expand(self, expander, params):
        text = self._render(params)
        data = self._loader.load(text)
        return expander.expand(data, params)


class IncludeJinja2(J2BaseYamlObject):
    yaml_tag = "!include-jinja2:"

    @classmethod
    def from_yaml(cls, loader, node):
        path_list = cls.path_list_from_node(loader, node)
        return cls(loader.jjb_config, loader, path_list)

    def __init__(self, jjb_config, loader, path_list):
        super().__init__(jjb_config, loader)
        self._path_list = path_list

    @property
    def required_params(self):
        return []

    def expand(self, expander, params):
        return "\n".join(
            self._expand_path(expander, params, path) for path in self._path_list
        )

    def _expand_path(self, expander, params, path_template):
        rel_path = self._formatter.format(path_template, **params)
        full_path = self._find_file(rel_path)
        template_text = full_path.read_text()
        template = self._jinja2_env.from_string(template_text)
        return self._render_template(template_text, template, params)


class IncludeBaseObject(BaseYamlObject):
    @classmethod
    def from_yaml(cls, loader, node):
        path_list = cls.path_list_from_node(loader, node)
        return cls(loader.jjb_config, loader, path_list)

    def __init__(self, jjb_config, loader, path_list):
        super().__init__(jjb_config, loader)
        self._path_list = path_list

    @property
    def required_params(self):
        for path in self._path_list:
            yield from enum_str_format_required_params(path)


class YamlInclude(IncludeBaseObject):
    yaml_tag = "!include:"

    def expand(self, expander, params):
        yaml_list = [
            self._expand_path(expander, params, path) for path in self._path_list
        ]
        if len(yaml_list) == 1:
            return yaml_list[0]
        else:
            return "\n".join(yaml_list)

    def _expand_path(self, expander, params, path_template):
        rel_path = self._formatter.format(path_template, **params)
        full_path = self._find_file(rel_path)
        text = full_path.read_text()
        data = self._loader.load(text)
        return expander.expand(data, params)


class IncludeRawBase(IncludeBaseObject):
    def expand(self, expander, params):
        return "\n".join(self._expand_path(path, params) for path in self._path_list)

    def subst(self, expander, params):
        return "\n".join(self._subst_path(path, params) for path in self._path_list)


class IncludeRaw(IncludeRawBase):
    yaml_tag = "!include-raw:"

    def _expand_path(self, rel_path_template, params):
        rel_path = self._formatter.format(rel_path_template, **params)
        full_path = self._find_file(rel_path)
        return full_path.read_text()

    def _subst_path(self, rel_path_template, params):
        rel_path = self._formatter.format(rel_path_template, **params)
        full_path = self._find_file(rel_path)
        template = full_path.read_text()
        return self._formatter.format(template, **params)


class IncludeRawEscape(IncludeRawBase):
    yaml_tag = "!include-raw-escape:"

    def _expand_path(self, rel_path_template, params):
        rel_path = self._formatter.format(rel_path_template, **params)
        full_path = self._find_file(rel_path)
        text = full_path.read_text()
        # Backward compatibility:
        # if used inside job or macro without parameters, curly braces are duplicated.
        return text.replace("{", "{{").replace("}", "}}")

    def _subst_path(self, rel_path_template, params):
        rel_path = self._formatter.format(rel_path_template, **params)
        full_path = self._find_file(rel_path)
        return full_path.read_text()


class YamlListJoin:
    yaml_tag = "!join:"

    @classmethod
    def from_yaml(cls, loader, node):
        value = loader.construct_sequence(node, deep=True)
        if len(value) != 2:
            raise yaml.constructor.ConstructorError(
                None,
                None,
                "Join value should contain 2 elements: delimiter and string list,"
                f" but contains {len(value)} elements: {value!r}",
                node.start_mark,
            )
        delimiter, seq = value
        return delimiter.join(seq)