make bddeb work with python3 or python2

painful, and not perfect, but at this point the output builds
on a vivid system python2 (bddeb --python2) or python3.

 * remove use of cheetah by bddeb in favor of builtin renderer
 * add '--python2' flag to bddeb and knowledge of python 2 and python3
   package names.
 * read-dependencies can now read test-requirements also.
 * differenciate from build-requirements and runtime requirements.

packages/bddeb

@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/env python3
 
 import os
 import shutil
@@ -27,23 +27,35 @@ import argparse
 # Package names that will showup in requires to what we can actually
 # use in our debian 'control' file, this is a translation of the 'requires'
 # file pypi package name to a debian/ubuntu package name.
-PKG_MP = {
+STD_NAMED_PACKAGES = [
-    'argparse': 'python-argparse',
+    'configobj',
-    'cheetah': 'python-cheetah',
+    'jinja2',
-    'configobj': 'python-configobj',
+    'jsonpatch',
-    'jinja2': 'python-jinja2',
+    'oauthlib',
-    'jsonpatch': 'python-jsonpatch | python-json-patch',
+    'prettytable',
-    'oauth': 'python-oauth',
+    'requests',
-    'prettytable': 'python-prettytable',
+    'six',
-    'pyserial': 'python-serial',
+    'httpretty',
-    'pyyaml': 'python-yaml',
+    'mock',
-    'requests': 'python-requests',
+    'nose',
-    'six': 'python-six',
+    'setuptools',
+]
+NONSTD_NAMED_PACKAGES = {
+    'argparse': ('python-argparse', None),
+    'contextlib2': ('python-contextlib2', None),
+    'cheetah': ('python-cheetah', None),
+    'pyserial': ('python-serial', 'python3-serial'),
+    'pyyaml': ('python-yaml', 'python3-yaml'),
+    'six': ('python-six', 'python3-six'),
+    'pep8': ('pep8', 'python3-pep8'),
+    'pyflakes': ('pyflakes', 'pyflakes'),
 }
+
 DEBUILD_ARGS = ["-S", "-d"]
 
 
-def write_debian_folder(root, version, revno, append_requires=[]):
+def write_debian_folder(root, version, revno, pkgmap,
+                        pyver="3", append_requires=[]):
     deb_dir = util.abs_join(root, 'debian')
     os.makedirs(deb_dir)
 
@@ -59,25 +71,42 @@ def write_debian_folder(root, version, revno, append_requires=[]):
     # Write out the control file template
     cmd = [util.abs_join(find_root(), 'tools', 'read-dependencies')]
     (stdout, _stderr) = util.subp(cmd)
-    pkgs = [p.lower().strip() for p in stdout.splitlines()]
+    pypi_pkgs = [p.lower().strip() for p in stdout.splitlines()]
+
+    (stdout, _stderr) = util.subp(cmd + ['test-requirements.txt'])
+    pypi_test_pkgs = [p.lower().strip() for p in stdout.splitlines()]
 
     # Map to known packages
     requires = append_requires
-    for p in pkgs:
+    test_requires = []
-        tgt_pkg = PKG_MP.get(p)
+    lists = ((pypi_pkgs, requires), (pypi_test_pkgs, test_requires))
-        if not tgt_pkg:
+    for pypilist, target in lists:
-            raise RuntimeError(("Do not know how to translate pypi dependency"
+        for p in pypilist:
-                                " %r to a known package") % (p))
+            if p not in pkgmap:
+                raise RuntimeError(("Do not know how to translate pypi "
+                                    "dependency %r to a known package") % (p))
+            elif pkgmap[p]:
+                target.append(pkgmap[p])
+
+    if pyver == "3":
+        python = "python3"
     else:
-            requires.append(tgt_pkg)
+        python = "python"
 
     templater.render_to_file(util.abs_join(find_root(),
                                            'packages', 'debian', 'control.in'),
                              util.abs_join(deb_dir, 'control'),
-                             params={'requires': requires})
+                             params={'requires': ','.join(requires),
+                                     'test_requires': ','.join(test_requires),
+                                     'python': python})
+
+    templater.render_to_file(util.abs_join(find_root(),
+                                           'packages', 'debian', 'rules.in'),
+                             util.abs_join(deb_dir, 'rules'),
+                             params={'python': python, 'pyver': pyver})
 
     # Just copy the following directly
-    for base_fn in ['dirs', 'copyright', 'compat', 'rules']:
+    for base_fn in ['dirs', 'copyright', 'compat']:
         shutil.copy(util.abs_join(find_root(),
                                   'packages', 'debian', base_fn),
                     util.abs_join(deb_dir, base_fn))
@@ -91,12 +120,16 @@ def main():
                               " (default: %(default)s)"),
                         default=False,
                         action='store_true')
-    parser.add_argument("--no-cloud-utils", dest="no_cloud_utils",
+    parser.add_argument("--cloud-utils", dest="cloud_utils",
-                        help=("don't depend on cloud-utils package"
+                        help=("depend on cloud-utils package"
                               " (default: %(default)s)"),
                         default=False,
                         action='store_true')
 
+    parser.add_argument("--python2", dest="python2",
+                        help=("build debs for python2 rather than python3"),
+                        default=False, action='store_true')
+
     parser.add_argument("--init-system", dest="init_system",
                         help=("build deb with INIT_SYSTEM=xxx"
                               " (default: %(default)s"),
@@ -123,6 +156,18 @@ def main():
     if args.verbose:
         capture = False
 
+    pkgmap = {}
+    for p in NONSTD_NAMED_PACKAGES:
+        pkgmap[p] = NONSTD_NAMED_PACKAGES[p][int(not args.python2)]
+
+    for p in STD_NAMED_PACKAGES:
+        if args.python2:
+            pkgmap[p] = "python-" + p
+            pyver = "2"
+        else:
+            pkgmap[p] = "python3-" + p
+            pyver = "3"
+
     with util.tempdir() as tdir:
 
         cmd = [util.abs_join(find_root(), 'tools', 'read-version')]
@@ -153,11 +198,12 @@ def main():
         shutil.move(extracted_name, xdir)
 
         print("Creating a debian/ folder in %r" % (xdir))
-        if not args.no_cloud_utils:
+        if args.cloud_utils:
             append_requires=['cloud-utils | cloud-guest-utils']
         else:
             append_requires=[]
-        write_debian_folder(xdir, version, revno, append_requires)
+        write_debian_folder(xdir, version, revno, pkgmap,
+                            pyver=pyver, append_requires=append_requires)
 
         # The naming here seems to follow some debian standard
         # so it will whine if it is changed...

packages/debian/changelog.in

@@ -1,4 +1,4 @@
-## This is a cheetah template
+## template:basic
 cloud-init (${version}~bzr${revision}-1) UNRELEASED; urgency=low
 
   * build

packages/debian/control.in

@@ -1,4 +1,4 @@
-## This is a cheetah template
+## template:basic
 Source: cloud-init
 Section: admin
 Priority: optional
@@ -6,31 +6,22 @@ Maintainer: Scott Moser <smoser@ubuntu.com>
 Build-Depends: debhelper (>= 9),
                dh-python,
                dh-systemd,
-               python (>= 2.6.6-3~),
-               python-nose,
                pyflakes,
-               python-setuptools,
+               ${python},
-               python-selinux,
+               ${test_requires},
-               python-cheetah,
+               ${requires}
-               python-mocker,
-               python-httpretty,
-#for $r in $requires
-               ${r},
-#end for
 XS-Python-Version: all
-Standards-Version: 3.9.3
+Standards-Version: 3.9.6
 
 Package: cloud-init
 Architecture: all
 Depends: procps,
-         python,
+         ${python},
-#for $r in $requires
+         ${requires},
-         ${r},
+         software-properties-common,
-#end for
+         ${misc:Depends},
-         python-software-properties | software-properties-common,
-         \${misc:Depends},
 Recommends: sudo
-XB-Python-Version: \${python:Versions}
+XB-Python-Version: ${python:Versions}
 Description: Init scripts for cloud instances
  Cloud instances need special scripts to run during initialisation
  to retrieve and install ssh keys and to let the user run various scripts.

packages/debian/rules.in

@@ -1,10 +1,12 @@
+## template:basic
 #!/usr/bin/make -f
 
 INIT_SYSTEM ?= upstart,systemd
+PYVER ?= python${pyver}
 export PYBUILD_INSTALL_ARGS=--init-system=$(INIT_SYSTEM)
 
 %:
-	dh $@ --with python2,systemd --buildsystem pybuild
+	dh $@ --with $(PYVER),systemd --buildsystem pybuild
 
 override_dh_install:
 	dh_install
@@ -12,6 +14,6 @@ override_dh_install:
 	cp tools/21-cloudinit.conf debian/cloud-init/etc/rsyslog.d/21-cloudinit.conf
 
 override_dh_auto_test:
-	# Becuase setup tools didn't copy data...
+	# Because setup tools didn't copy data...
-	cp -r tests/data .pybuild/pythonX.Y_2.7/build/tests
+	[ ! -d .pybuild/pythonX.Y_?.?/build/tests ] || cp -r tests/data .pybuild/pythonX.Y_?.?/build/tests
 	http_proxy= dh_auto_test -- --test-nose

test-requirements.txt

@@ -1,6 +1,7 @@
 httpretty>=0.7.1
 mock
-mocker
 nose
 pep8==1.5.7
 pyflakes
+contextlib2
+setuptools

tests/unittests/helpers.py

@@ -1,3 +1,5 @@
+from __future__ import print_function
+
 import os
 import sys
 import shutil
@@ -275,3 +277,17 @@ def populate_dir(path, files):
         with open(os.path.join(path, name), "w") as fp:
             fp.write(content)
             fp.close()
+
+try:
+    skipIf = unittest.skipIf
+except AttributeError:
+    # Python 2.6.  Doesn't have to be high fidelity.
+    def skipIf(condition, reason):
+        def decorator(func):
+            def wrapper(*args, **kws):
+                if condition:
+                    return func(*args, **kws)
+                else:
+                    print(reason, file=sys.stderr)
+            return wrapper
+        return decorator

tests/unittests/test_handler/test_handler_chef.py

@@ -18,6 +18,8 @@ import tempfile
 
 LOG = logging.getLogger(__name__)
 
+CLIENT_TEMPL = os.path.sep.join(["templates", "chef_client.rb.tmpl"])
+
 
 class TestChef(t_help.FilesystemMockingTestCase):
     def setUp(self):
@@ -41,9 +43,13 @@ class TestChef(t_help.FilesystemMockingTestCase):
         for d in cc_chef.CHEF_DIRS:
             self.assertFalse(os.path.isdir(d))
 
+    @t_help.skipIf(not os.path.isfile(CLIENT_TEMPL),
+                   CLIENT_TEMPL + " is not available")
     def test_basic_config(self):
-        # This should create a file of the format...
         """
+        test basic config looks sane
+
+        # This should create a file of the format...
         # Created by cloud-init v. 0.7.6 on Sat, 11 Oct 2014 23:57:21 +0000
         log_level              :info
         ssl_verify_mode        :verify_none
@@ -105,6 +111,8 @@ class TestChef(t_help.FilesystemMockingTestCase):
                 'c': 'd',
             }, json.loads(c))
 
+    @t_help.skipIf(not os.path.isfile(CLIENT_TEMPL),
+                   CLIENT_TEMPL + " is not available")
     def test_template_deletes(self):
         tpl_file = util.load_file('templates/chef_client.rb.tmpl')
         self.patchUtils(self.tmp)

tests/unittests/test_templating.py

@@ -27,20 +27,6 @@ import textwrap
 
 from cloudinit import templater
 
-try:
-    skipIf = unittest.skipIf
-except AttributeError:
-    # Python 2.6.  Doesn't have to be high fidelity.
-    def skipIf(condition, reason):
-        def decorator(func):
-            def wrapper(*args, **kws):
-                if condition:
-                    return func(*args, **kws)
-                else:
-                    print(reason, file=sys.stderr)
-            return wrapper
-        return decorator
-
 
 class TestTemplates(test_helpers.TestCase):
     def test_render_basic(self):
@@ -58,7 +44,7 @@ class TestTemplates(test_helpers.TestCase):
         out_data = templater.basic_render(in_data, {'b': 2})
         self.assertEqual(expected_data.strip(), out_data)
 
-    @skipIf(six.PY3, 'Cheetah is not compatible with Python 3')
+    @test_helpers.skipIf(six.PY3, 'Cheetah is not compatible with Python 3')
     def test_detection(self):
         blob = "## template:cheetah"
 

tools/read-dependencies

@@ -1,6 +1,7 @@
 #!/usr/bin/env python
 
 import os
+import re
 import sys
 
 if 'CLOUD_INIT_TOP_D' in os.environ:
@@ -14,10 +15,15 @@ for fname in ("setup.py", "requirements.txt"):
                          "exist in cloud-init root directory." % fname)
         sys.exit(1)
 
-with open(os.path.join(topd, "requirements.txt"), "r") as fp:
+if len(sys.argv) > 1:
+   reqfile = sys.argv[1]
+else:
+   reqfile = "requirements.txt"
+   
+with open(os.path.join(topd, reqfile), "r") as fp:
     for line in fp:
         if not line.strip() or line.startswith("#"):
             continue
-        sys.stdout.write(line)
+        sys.stdout.write(re.split("[>=.<]*", line)[0].strip() + "\n")
 
 sys.exit(0)