Bring over the 'templater' from bzr

Change-Id: Iefcceb1c0d47ae455985f1d027465b0db0a0e467

cloudinit/templater.py

@@ -0,0 +1,105 @@
+# Copyright 2015 Canonical Ltd.
+# This file is part of cloud-init.  See LICENCE file for license information.
+#
+# vi: ts=4 expandtab
+
+import collections
+import logging
+import os
+import re
+
+try:
+    import jinja2
+    from jinja2 import Template as JTemplate
+    JINJA_AVAILABLE = True
+except (ImportError, AttributeError):
+    JINJA_AVAILABLE = False  # noqa
+
+LOG = logging.getLogger(__name__)
+TYPE_MATCHER = re.compile(r"##\s*template:(.*)", re.I)
+BASIC_MATCHER = re.compile(r'\$\{([A-Za-z0-9_.]+)\}|\$([A-Za-z0-9_.]+)')
+
+
+def basic_render(content, params):
+    """This does simple replacement of bash variable like templates.
+
+    It identifies patterns like ${a} or $a and can also identify patterns like
+    ${a.b} or $a.b which will look for a key 'b' in the dictionary rooted
+    by key 'a'.
+    """
+
+    def replacer(match):
+        # Only 1 of the 2 groups will actually have a valid entry.
+        name = match.group(1) or match.group(2)
+        if name is None:
+            # not sure how this can possibly occur
+            raise RuntimeError("Match encountered but no valid group present")
+        path = collections.deque(name.split("."))
+        selected_params = params
+        while len(path) > 1:
+            key = path.popleft()
+            if not isinstance(selected_params, dict):
+                raise TypeError(
+                    "Can not traverse into non-dictionary '%s' of type %s "
+                    "while looking for subkey '%s'" %
+                    (selected_params, type(selected_params), key))
+            selected_params = selected_params[key]
+        key = path.popleft()
+        if not isinstance(selected_params, dict):
+            raise TypeError("Can not extract key '%s' from non-dictionary"
+                            " '%s' of type %s"
+                            % (key, selected_params, type(selected_params)))
+        return str(selected_params[key])
+
+    return BASIC_MATCHER.sub(replacer, content)
+
+
+def detect_template(text):
+
+    def jinja_render(content, params):
+        # keep_trailing_newline is in jinja2 2.7+, not 2.6
+        add = "\n" if content.endswith("\n") else ""
+        return JTemplate(content,
+                         undefined=jinja2.StrictUndefined,
+                         trim_blocks=True).render(**params) + add
+
+    if "\n" in text:
+        ident, rest = text.split("\n", 1)
+    else:
+        ident = text
+        rest = ''
+    type_match = TYPE_MATCHER.match(ident)
+    if not type_match:
+        return ('basic', basic_render, text)
+    else:
+        template_type = type_match.group(1).lower().strip()
+        if template_type not in ('jinja', 'basic'):
+            raise ValueError("Unknown template rendering type '%s' requested"
+                             % template_type)
+        if template_type == 'jinja' and not JINJA_AVAILABLE:
+            raise ValueError("Template requested jinja as renderer, but Jinja "
+                             "is not available.")
+        elif template_type == 'jinja' and JINJA_AVAILABLE:
+            return ('jinja', jinja_render, rest)
+        # Only thing left over is the basic renderer (it is always available).
+        return ('basic', basic_render, rest)
+
+
+def render_from_file(fn, params, encoding='utf-8'):
+    with open(fn, 'rb') as fh:
+        content = fh.read()
+    content = content.decode(encoding)
+    _, renderer, content = detect_template(content)
+    return renderer(content, params)
+
+
+def render_to_file(fn, outfn, params, mode=0o644, encoding='utf-8'):
+    contents = render_from_file(fn, params, encoding=encoding)
+    with open(outfn, 'wb') as fh:
+        fh.write(contents.encode(encoding))
+    os.chmod(outfn, mode)
+
+
+def render_string(content, params):
+    _, renderer, content = detect_template(content)
+    return renderer(content, params)

cloudinit/tests/test_templating.py

@@ -0,0 +1,137 @@
+# Copyright 2015 Canonical Ltd.
+# This file is part of cloud-init.  See LICENCE file for license information.
+#
+# vi: ts=4 expandtab
+
+import fixtures
+import mock
+import os
+import textwrap
+
+from cloudinit import templater
+from cloudinit import test
+
+
+class TestTemplates(test.TestCase):
+    jinja_tmpl = '\n'.join((
+        "## template:jinja",
+        "{{a}},{{b}}",
+        ""
+    ))
+    jinja_params = {'a': '1', 'b': '2'}
+    jinja_expected = '1,2\n'
+
+    def test_render_basic(self):
+        in_data = textwrap.dedent("""
+            ${b}
+
+            c = d
+            """)
+        in_data = in_data.strip()
+        expected_data = textwrap.dedent("""
+            2
+
+            c = d
+            """)
+        out_data = templater.basic_render(in_data, {'b': 2})
+        self.assertEqual(expected_data.strip(), out_data)
+
+    def test_render_jinja(self):
+        c = templater.render_string(self.jinja_tmpl, self.jinja_params)
+        self.assertEqual(self.jinja_expected, c)
+
+    def test_render_jinja_crlf(self):
+        blob = '\r\n'.join((
+            "## template:jinja",
+            "{{a}},{{b}}"))
+        c = templater.render_string(blob, {"a": 1, "b": 2})
+        self.assertEqual("1,2", c)
+
+    def test_render_default(self):
+        blob = '''$a,$b'''
+        c = templater.render_string(blob, {"a": 1, "b": 2})
+        self.assertEqual("1,2", c)
+
+    def test_render_explict_default(self):
+        blob = '\n'.join(('## template: basic', '$a,$b',))
+        c = templater.render_string(blob, {"a": 1, "b": 2})
+        self.assertEqual("1,2", c)
+
+    def test_render_basic_deeper(self):
+        hn = 'myfoohost.yahoo.com'
+        expected_data = "h=%s\nc=d\n" % hn
+        in_data = "h=$hostname.canonical_name\nc=d\n"
+        params = {
+            "hostname": {
+                "canonical_name": hn,
+            },
+        }
+        out_data = templater.render_string(in_data, params)
+        self.assertEqual(expected_data, out_data)
+
+    def test_render_basic_no_parens(self):
+        hn = "myfoohost"
+        in_data = "h=$hostname\nc=d\n"
+        expected_data = "h=%s\nc=d\n" % hn
+        out_data = templater.basic_render(in_data, {'hostname': hn})
+        self.assertEqual(expected_data, out_data)
+
+    def test_render_basic_parens(self):
+        hn = "myfoohost"
+        in_data = "h = ${hostname}\nc=d\n"
+        expected_data = "h = %s\nc=d\n" % hn
+        out_data = templater.basic_render(in_data, {'hostname': hn})
+        self.assertEqual(expected_data, out_data)
+
+    def test_render_basic2(self):
+        mirror = "mymirror"
+        codename = "zany"
+        in_data = "deb $mirror $codename-updates main contrib non-free"
+        ex_data = "deb %s %s-updates main contrib non-free" % (mirror,
+                                                               codename)
+
+        out_data = templater.basic_render(
+            in_data, {'mirror': mirror, 'codename': codename})
+        self.assertEqual(ex_data, out_data)
+
+    def test_render_basic_exception_1(self):
+        in_data = "h=${foo.bar}"
+        self.assertRaises(
+            TypeError, templater.basic_render, in_data, {'foo': [1, 2]})
+
+    def test_unknown_renderer_raises_exception(self):
+        blob = '\n'.join((
+            "## template:bigfastcat",
+            "Hellow $name"
+            ""))
+        self.assertRaises(
+            ValueError, templater.render_string, blob, {'name': 'foo'})
+
+    @mock.patch.object(templater, 'JINJA_AVAILABLE', False)
+    def test_jinja_without_jinja_raises_exception(self):
+        blob = '\n'.join((
+            "## template:jinja",
+            "Hellow {{name}}"
+            ""))
+        templater.JINJA_AVAILABLE = False
+        self.assertRaises(
+            ValueError, templater.render_string, blob, {'name': 'foo'})
+
+    def test_render_from_file(self):
+        td = self.useFixture(fixtures.TempDir()).path
+        fname = os.path.join(td, "myfile")
+        with open(fname, "w") as fp:
+            fp.write(self.jinja_tmpl)
+        rendered = templater.render_from_file(fname, self.jinja_params)
+        self.assertEqual(rendered, self.jinja_expected)
+
+    def test_render_to_file(self):
+        td = self.useFixture(fixtures.TempDir()).path
+        src = os.path.join(td, "src")
+        target = os.path.join(td, "target")
+        with open(src, "w") as fp:
+            fp.write(self.jinja_tmpl)
+        templater.render_to_file(src, target, self.jinja_params)
+        with open(target, "r") as fp:
+            rendered = fp.read()
+        self.assertEqual(rendered, self.jinja_expected)

requirements.txt

@@ -10,3 +10,4 @@ pyyaml
 jsonpatch
 requests>=1.0
 requests-oauthlib
+jinja2

test-requirements.txt

@@ -6,6 +6,7 @@ httpretty>=0.7.1
 mock
 nose
 testtools
+fixtures
 
 # For doc building
 sphinx>=1.1.2,!=1.2.0,!=1.3b1,<1.3