x/cloud-init
Code Issues Proposed changes

make bddeb work with python3 or python2

Browse Source 
painful, and not perfect, but at this point the output builds
on a vivid system python2 (bddeb --python2) or python3.

 * remove use of cheetah by bddeb in favor of builtin renderer
 * add '--python2' flag to bddeb and knowledge of python 2 and python3
   package names.
 * read-dependencies can now read test-requirements also.
 * differenciate from build-requirements and runtime requirements.
This commit is contained in:
Scott Moser 2015-02-10 20:32:32 +00:00
parent 11efc148bf
commit 028beaac09
9 changed files with 126 additions and 70 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

102
packages/bddeb
View File

@ -1,4 +1,4 @@
#!/usr/bin/python
#!/usr/bin/env python3
import os
import shutil
@ -27,23 +27,35 @@ import argparse
# Package names that will showup in requires to what we can actually
# use in our debian 'control' file, this is a translation of the 'requires'
# file pypi package name to a debian/ubuntu package name.
PKG_MP = {
'argparse': 'python-argparse',
'cheetah': 'python-cheetah',
'configobj': 'python-configobj',
'jinja2': 'python-jinja2',
'jsonpatch': 'python-jsonpatch | python-json-patch',
'oauth': 'python-oauth',
'prettytable': 'python-prettytable',
'pyserial': 'python-serial',
'pyyaml': 'python-yaml',
'requests': 'python-requests',
'six': 'python-six',
STD_NAMED_PACKAGES = [
'configobj',
'jinja2',
'jsonpatch',
'oauthlib',
'prettytable',
'requests',
'six',
'httpretty',
'mock',
'nose',
'setuptools',
]
NONSTD_NAMED_PACKAGES = {
'argparse': ('python-argparse', None),
'contextlib2': ('python-contextlib2', None),
'cheetah': ('python-cheetah', None),
'pyserial': ('python-serial', 'python3-serial'),
'pyyaml': ('python-yaml', 'python3-yaml'),
'six': ('python-six', 'python3-six'),
'pep8': ('pep8', 'python3-pep8'),
'pyflakes': ('pyflakes', 'pyflakes'),
}
DEBUILD_ARGS = ["-S", "-d"]
def write_debian_folder(root, version, revno, append_requires=[]):
def write_debian_folder(root, version, revno, pkgmap,
pyver="3", append_requires=[]):
deb_dir = util.abs_join(root, 'debian')
os.makedirs(deb_dir)
@ -59,25 +71,42 @@ def write_debian_folder(root, version, revno, append_requires=[]):
# Write out the control file template
cmd = [util.abs_join(find_root(), 'tools', 'read-dependencies')]
(stdout, _stderr) = util.subp(cmd)
pkgs = [p.lower().strip() for p in stdout.splitlines()]
pypi_pkgs = [p.lower().strip() for p in stdout.splitlines()]
(stdout, _stderr) = util.subp(cmd + ['test-requirements.txt'])
pypi_test_pkgs = [p.lower().strip() for p in stdout.splitlines()]
# Map to known packages
requires = append_requires
for p in pkgs:
tgt_pkg = PKG_MP.get(p)
if not tgt_pkg:
raise RuntimeError(("Do not know how to translate pypi dependency"
" %r to a known package") % (p))
else:
requires.append(tgt_pkg)
test_requires = []
lists = ((pypi_pkgs, requires), (pypi_test_pkgs, test_requires))
for pypilist, target in lists:
for p in pypilist:
if p not in pkgmap:
raise RuntimeError(("Do not know how to translate pypi "
"dependency %r to a known package") % (p))
elif pkgmap[p]:
target.append(pkgmap[p])
if pyver == "3":
python = "python3"
else:
python = "python"
templater.render_to_file(util.abs_join(find_root(),
'packages', 'debian', 'control.in'),
util.abs_join(deb_dir, 'control'),
params={'requires': requires})
params={'requires': ','.join(requires),
'test_requires': ','.join(test_requires),
'python': python})
templater.render_to_file(util.abs_join(find_root(),
'packages', 'debian', 'rules.in'),
util.abs_join(deb_dir, 'rules'),
params={'python': python, 'pyver': pyver})
# Just copy the following directly
for base_fn in ['dirs', 'copyright', 'compat', 'rules']:
for base_fn in ['dirs', 'copyright', 'compat']:
shutil.copy(util.abs_join(find_root(),
'packages', 'debian', base_fn),
util.abs_join(deb_dir, base_fn))
@ -91,12 +120,16 @@ def main():
" (default: %(default)s)"),
default=False,
action='store_true')
parser.add_argument("--no-cloud-utils", dest="no_cloud_utils",
help=("don't depend on cloud-utils package"
parser.add_argument("--cloud-utils", dest="cloud_utils",
help=("depend on cloud-utils package"
" (default: %(default)s)"),
default=False,
action='store_true')
parser.add_argument("--python2", dest="python2",
help=("build debs for python2 rather than python3"),
default=False, action='store_true')
parser.add_argument("--init-system", dest="init_system",
help=("build deb with INIT_SYSTEM=xxx"
" (default: %(default)s"),
@ -123,6 +156,18 @@ def main():
if args.verbose:
capture = False
pkgmap = {}
for p in NONSTD_NAMED_PACKAGES:
pkgmap[p] = NONSTD_NAMED_PACKAGES[p][int(not args.python2)]
for p in STD_NAMED_PACKAGES:
if args.python2:
pkgmap[p] = "python-" + p
pyver = "2"
else:
pkgmap[p] = "python3-" + p
pyver = "3"
with util.tempdir() as tdir:
cmd = [util.abs_join(find_root(), 'tools', 'read-version')]
@ -153,11 +198,12 @@ def main():
shutil.move(extracted_name, xdir)
print("Creating a debian/ folder in %r" % (xdir))
if not args.no_cloud_utils:
if args.cloud_utils:
append_requires=['cloud-utils | cloud-guest-utils']
else:
append_requires=[]
write_debian_folder(xdir, version, revno, append_requires)
write_debian_folder(xdir, version, revno, pkgmap,
pyver=pyver, append_requires=append_requires)
# The naming here seems to follow some debian standard
# so it will whine if it is changed...

2
packages/debian/changelog.in
View File

@ -1,4 +1,4 @@
## This is a cheetah template
## template:basic
cloud-init (${version}~bzr${revision}-1) UNRELEASED; urgency=low
* build

29
packages/debian/control.in
View File

@ -1,4 +1,4 @@
## This is a cheetah template
## template:basic
Source: cloud-init
Section: admin
Priority: optional
@ -6,31 +6,22 @@ Maintainer: Scott Moser <smoser@ubuntu.com>
Build-Depends: debhelper (>= 9),
dh-python,
dh-systemd,
python (>= 2.6.6-3~),
python-nose,
pyflakes,
python-setuptools,
python-selinux,
python-cheetah,
python-mocker,
python-httpretty,
#for $r in $requires
${r},
#end for
${python},
${test_requires},
${requires}
XS-Python-Version: all
Standards-Version: 3.9.3
Standards-Version: 3.9.6
Package: cloud-init
Architecture: all
Depends: procps,
python,
#for $r in $requires
${r},
#end for
python-software-properties | software-properties-common,
\${misc:Depends},
${python},
${requires},
software-properties-common,
${misc:Depends},
Recommends: sudo
XB-Python-Version: \${python:Versions}
XB-Python-Version: ${python:Versions}
Description: Init scripts for cloud instances
Cloud instances need special scripts to run during initialisation
to retrieve and install ssh keys and to let the user run various scripts.

8
packages/debian/rules → packages/debian/rules.in
View File

@ -1,10 +1,12 @@
## template:basic
#!/usr/bin/make -f
INIT_SYSTEM ?= upstart,systemd
PYVER ?= python${pyver}
export PYBUILD_INSTALL_ARGS=--init-system=$(INIT_SYSTEM)
%:
dh $@ --with python2,systemd --buildsystem pybuild
dh $@ --with $(PYVER),systemd --buildsystem pybuild
override_dh_install:
dh_install
@ -12,6 +14,6 @@ override_dh_install:
cp tools/21-cloudinit.conf debian/cloud-init/etc/rsyslog.d/21-cloudinit.conf
override_dh_auto_test:
# Becuase setup tools didn't copy data...
cp -r tests/data .pybuild/pythonX.Y_2.7/build/tests
# Because setup tools didn't copy data...
[ ! -d .pybuild/pythonX.Y_?.?/build/tests ] || cp -r tests/data .pybuild/pythonX.Y_?.?/build/tests
http_proxy= dh_auto_test -- --test-nose

3
test-requirements.txt
View File

@ -1,6 +1,7 @@
httpretty>=0.7.1
mock
mocker
nose
pep8==1.5.7
pyflakes
contextlib2
setuptools

16
tests/unittests/helpers.py
View File

@ -1,3 +1,5 @@
from __future__ import print_function
import os
import sys
import shutil
@ -275,3 +277,17 @@ def populate_dir(path, files):
with open(os.path.join(path, name), "w") as fp:
fp.write(content)
fp.close()
try:
skipIf = unittest.skipIf
except AttributeError:
# Python 2.6. Doesn't have to be high fidelity.
def skipIf(condition, reason):
def decorator(func):
def wrapper(*args, **kws):
if condition:
return func(*args, **kws)
else:
print(reason, file=sys.stderr)
return wrapper
return decorator

10
tests/unittests/test_handler/test_handler_chef.py
View File

@ -18,6 +18,8 @@ import tempfile
LOG = logging.getLogger(__name__)
CLIENT_TEMPL = os.path.sep.join(["templates", "chef_client.rb.tmpl"])
class TestChef(t_help.FilesystemMockingTestCase):
def setUp(self):
@ -41,9 +43,13 @@ class TestChef(t_help.FilesystemMockingTestCase):
for d in cc_chef.CHEF_DIRS:
self.assertFalse(os.path.isdir(d))
@t_help.skipIf(not os.path.isfile(CLIENT_TEMPL),
CLIENT_TEMPL + " is not available")
def test_basic_config(self):
# This should create a file of the format...
"""
test basic config looks sane
# This should create a file of the format...
# Created by cloud-init v. 0.7.6 on Sat, 11 Oct 2014 23:57:21 +0000
log_level :info
ssl_verify_mode :verify_none
@ -105,6 +111,8 @@ class TestChef(t_help.FilesystemMockingTestCase):
'c': 'd',
}, json.loads(c))
@t_help.skipIf(not os.path.isfile(CLIENT_TEMPL),
CLIENT_TEMPL + " is not available")
def test_template_deletes(self):
tpl_file = util.load_file('templates/chef_client.rb.tmpl')
self.patchUtils(self.tmp)

16
tests/unittests/test_templating.py
View File

@ -27,20 +27,6 @@ import textwrap
from cloudinit import templater
try:
skipIf = unittest.skipIf
except AttributeError:
# Python 2.6. Doesn't have to be high fidelity.
def skipIf(condition, reason):
def decorator(func):
def wrapper(*args, **kws):
if condition:
return func(*args, **kws)
else:
print(reason, file=sys.stderr)
return wrapper
return decorator
class TestTemplates(test_helpers.TestCase):
def test_render_basic(self):
@ -58,7 +44,7 @@ class TestTemplates(test_helpers.TestCase):
out_data = templater.basic_render(in_data, {'b': 2})
self.assertEqual(expected_data.strip(), out_data)
@skipIf(six.PY3, 'Cheetah is not compatible with Python 3')
@test_helpers.skipIf(six.PY3, 'Cheetah is not compatible with Python 3')
def test_detection(self):
blob = "## template:cheetah"

10
tools/read-dependencies
View File

@ -1,6 +1,7 @@
#!/usr/bin/env python
import os
import re
import sys
if 'CLOUD_INIT_TOP_D' in os.environ:
@ -14,10 +15,15 @@ for fname in ("setup.py", "requirements.txt"):
"exist in cloud-init root directory." % fname)
sys.exit(1)
with open(os.path.join(topd, "requirements.txt"), "r") as fp:
if len(sys.argv) > 1:
reqfile = sys.argv[1]
else:
reqfile = "requirements.txt"
with open(os.path.join(topd, reqfile), "r") as fp:
for line in fp:
if not line.strip() or line.startswith("#"):
continue
sys.stdout.write(line)
sys.stdout.write(re.split("[>=.<]*", line)[0].strip() + "\n")
sys.exit(0)