From 92405e6712267589a245bedee422739cdd81255e Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Thu, 2 Apr 2015 12:44:48 -0700 Subject: [PATCH] Bring over the 'templater' from bzr Change-Id: Iefcceb1c0d47ae455985f1d027465b0db0a0e467 --- cloudinit/templater.py | 105 ++++++++++++++++++++++ cloudinit/tests/test_templating.py | 137 +++++++++++++++++++++++++++++ requirements.txt | 1 + test-requirements.txt | 1 + 4 files changed, 244 insertions(+) create mode 100644 cloudinit/templater.py create mode 100644 cloudinit/tests/test_templating.py diff --git a/cloudinit/templater.py b/cloudinit/templater.py new file mode 100644 index 00000000..b38764b5 --- /dev/null +++ b/cloudinit/templater.py @@ -0,0 +1,105 @@ +# Copyright 2015 Canonical Ltd. +# This file is part of cloud-init. See LICENCE file for license information. +# +# vi: ts=4 expandtab + +import collections +import logging +import os +import re + +try: + import jinja2 + from jinja2 import Template as JTemplate + JINJA_AVAILABLE = True +except (ImportError, AttributeError): + JINJA_AVAILABLE = False # noqa + +LOG = logging.getLogger(__name__) +TYPE_MATCHER = re.compile(r"##\s*template:(.*)", re.I) +BASIC_MATCHER = re.compile(r'\$\{([A-Za-z0-9_.]+)\}|\$([A-Za-z0-9_.]+)') + + +def basic_render(content, params): + """This does simple replacement of bash variable like templates. + + It identifies patterns like ${a} or $a and can also identify patterns like + ${a.b} or $a.b which will look for a key 'b' in the dictionary rooted + by key 'a'. + """ + + def replacer(match): + # Only 1 of the 2 groups will actually have a valid entry. + name = match.group(1) or match.group(2) + if name is None: + # not sure how this can possibly occur + raise RuntimeError("Match encountered but no valid group present") + path = collections.deque(name.split(".")) + selected_params = params + while len(path) > 1: + key = path.popleft() + if not isinstance(selected_params, dict): + raise TypeError( + "Can not traverse into non-dictionary '%s' of type %s " + "while looking for subkey '%s'" % + (selected_params, type(selected_params), key)) + selected_params = selected_params[key] + key = path.popleft() + if not isinstance(selected_params, dict): + raise TypeError("Can not extract key '%s' from non-dictionary" + " '%s' of type %s" + % (key, selected_params, type(selected_params))) + return str(selected_params[key]) + + return BASIC_MATCHER.sub(replacer, content) + + +def detect_template(text): + + def jinja_render(content, params): + # keep_trailing_newline is in jinja2 2.7+, not 2.6 + add = "\n" if content.endswith("\n") else "" + return JTemplate(content, + undefined=jinja2.StrictUndefined, + trim_blocks=True).render(**params) + add + + if "\n" in text: + ident, rest = text.split("\n", 1) + else: + ident = text + rest = '' + type_match = TYPE_MATCHER.match(ident) + if not type_match: + return ('basic', basic_render, text) + else: + template_type = type_match.group(1).lower().strip() + if template_type not in ('jinja', 'basic'): + raise ValueError("Unknown template rendering type '%s' requested" + % template_type) + if template_type == 'jinja' and not JINJA_AVAILABLE: + raise ValueError("Template requested jinja as renderer, but Jinja " + "is not available.") + elif template_type == 'jinja' and JINJA_AVAILABLE: + return ('jinja', jinja_render, rest) + # Only thing left over is the basic renderer (it is always available). + return ('basic', basic_render, rest) + + +def render_from_file(fn, params, encoding='utf-8'): + with open(fn, 'rb') as fh: + content = fh.read() + content = content.decode(encoding) + _, renderer, content = detect_template(content) + return renderer(content, params) + + +def render_to_file(fn, outfn, params, mode=0o644, encoding='utf-8'): + contents = render_from_file(fn, params, encoding=encoding) + with open(outfn, 'wb') as fh: + fh.write(contents.encode(encoding)) + os.chmod(outfn, mode) + + +def render_string(content, params): + _, renderer, content = detect_template(content) + return renderer(content, params) diff --git a/cloudinit/tests/test_templating.py b/cloudinit/tests/test_templating.py new file mode 100644 index 00000000..9bd8d241 --- /dev/null +++ b/cloudinit/tests/test_templating.py @@ -0,0 +1,137 @@ +# Copyright 2015 Canonical Ltd. +# This file is part of cloud-init. See LICENCE file for license information. +# +# vi: ts=4 expandtab + +import fixtures +import mock +import os +import textwrap + +from cloudinit import templater +from cloudinit import test + + +class TestTemplates(test.TestCase): + jinja_tmpl = '\n'.join(( + "## template:jinja", + "{{a}},{{b}}", + "" + )) + jinja_params = {'a': '1', 'b': '2'} + jinja_expected = '1,2\n' + + def test_render_basic(self): + in_data = textwrap.dedent(""" + ${b} + + c = d + """) + in_data = in_data.strip() + expected_data = textwrap.dedent(""" + 2 + + c = d + """) + out_data = templater.basic_render(in_data, {'b': 2}) + self.assertEqual(expected_data.strip(), out_data) + + def test_render_jinja(self): + c = templater.render_string(self.jinja_tmpl, self.jinja_params) + self.assertEqual(self.jinja_expected, c) + + def test_render_jinja_crlf(self): + blob = '\r\n'.join(( + "## template:jinja", + "{{a}},{{b}}")) + c = templater.render_string(blob, {"a": 1, "b": 2}) + self.assertEqual("1,2", c) + + def test_render_default(self): + blob = '''$a,$b''' + c = templater.render_string(blob, {"a": 1, "b": 2}) + self.assertEqual("1,2", c) + + def test_render_explict_default(self): + blob = '\n'.join(('## template: basic', '$a,$b',)) + c = templater.render_string(blob, {"a": 1, "b": 2}) + self.assertEqual("1,2", c) + + def test_render_basic_deeper(self): + hn = 'myfoohost.yahoo.com' + expected_data = "h=%s\nc=d\n" % hn + in_data = "h=$hostname.canonical_name\nc=d\n" + params = { + "hostname": { + "canonical_name": hn, + }, + } + out_data = templater.render_string(in_data, params) + self.assertEqual(expected_data, out_data) + + def test_render_basic_no_parens(self): + hn = "myfoohost" + in_data = "h=$hostname\nc=d\n" + expected_data = "h=%s\nc=d\n" % hn + out_data = templater.basic_render(in_data, {'hostname': hn}) + self.assertEqual(expected_data, out_data) + + def test_render_basic_parens(self): + hn = "myfoohost" + in_data = "h = ${hostname}\nc=d\n" + expected_data = "h = %s\nc=d\n" % hn + out_data = templater.basic_render(in_data, {'hostname': hn}) + self.assertEqual(expected_data, out_data) + + def test_render_basic2(self): + mirror = "mymirror" + codename = "zany" + in_data = "deb $mirror $codename-updates main contrib non-free" + ex_data = "deb %s %s-updates main contrib non-free" % (mirror, + codename) + + out_data = templater.basic_render( + in_data, {'mirror': mirror, 'codename': codename}) + self.assertEqual(ex_data, out_data) + + def test_render_basic_exception_1(self): + in_data = "h=${foo.bar}" + self.assertRaises( + TypeError, templater.basic_render, in_data, {'foo': [1, 2]}) + + def test_unknown_renderer_raises_exception(self): + blob = '\n'.join(( + "## template:bigfastcat", + "Hellow $name" + "")) + self.assertRaises( + ValueError, templater.render_string, blob, {'name': 'foo'}) + + @mock.patch.object(templater, 'JINJA_AVAILABLE', False) + def test_jinja_without_jinja_raises_exception(self): + blob = '\n'.join(( + "## template:jinja", + "Hellow {{name}}" + "")) + templater.JINJA_AVAILABLE = False + self.assertRaises( + ValueError, templater.render_string, blob, {'name': 'foo'}) + + def test_render_from_file(self): + td = self.useFixture(fixtures.TempDir()).path + fname = os.path.join(td, "myfile") + with open(fname, "w") as fp: + fp.write(self.jinja_tmpl) + rendered = templater.render_from_file(fname, self.jinja_params) + self.assertEqual(rendered, self.jinja_expected) + + def test_render_to_file(self): + td = self.useFixture(fixtures.TempDir()).path + src = os.path.join(td, "src") + target = os.path.join(td, "target") + with open(src, "w") as fp: + fp.write(self.jinja_tmpl) + templater.render_to_file(src, target, self.jinja_params) + with open(target, "r") as fp: + rendered = fp.read() + self.assertEqual(rendered, self.jinja_expected) diff --git a/requirements.txt b/requirements.txt index 44f21809..0219db9f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -10,3 +10,4 @@ pyyaml jsonpatch requests>=1.0 requests-oauthlib +jinja2 diff --git a/test-requirements.txt b/test-requirements.txt index 2c2b0846..33f413c7 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -6,6 +6,7 @@ httpretty>=0.7.1 mock nose testtools +fixtures # For doc building sphinx>=1.1.2,!=1.2.0,!=1.3b1,<1.3