fix snappy package installation.

Some tests added here to make sure this works.
The install '--config' will only start to work with the next
version of snappy-go.
This commit is contained in:
Scott Moser 2015-03-27 17:04:59 -04:00
commit d0982b0755
3 changed files with 523 additions and 43 deletions

View File

@ -29,6 +29,7 @@
- readurl, read_file_or_url returns bytes, user must convert as necessary
- SmartOS: use v2 metadata service (LP: #1436417) [Daniel Watkins]
- NoCloud: fix local datasource claiming found without explicit dsmode
- Snappy: add support for installing snappy packages and configuring.
0.7.6:
- open 0.7.6
- Enable vendordata on CloudSigma datasource (LP: #1303986)

View File

@ -1,5 +1,46 @@
# vi: ts=4 expandtab
#
"""
snappy modules allows configuration of snappy.
Example config:
#cloud-config
snappy:
system_snappy: auto
ssh_enabled: False
packages: [etcd, pkg2.smoser]
config:
pkgname:
key2: value2
pkg2:
key1: value1
packages_dir: '/writable/user-data/cloud-init/snaps'
- ssh_enabled:
This defaults to 'False'. Set to a non-false value to enable ssh service
- snap installation and config
The above would install 'etcd', and then install 'pkg2.smoser' with a
'--config=<file>' argument where 'file' as 'config-blob' inside it.
If 'pkgname' is installed already, then 'snappy config pkgname <file>'
will be called where 'file' has 'pkgname-config-blob' as its content.
Entries in 'config' can be namespaced or non-namespaced for a package.
In either case, the config provided to snappy command is non-namespaced.
The package name is provided as it appears.
If 'packages_dir' has files in it that end in '.snap', then they are
installed. Given 3 files:
<packages_dir>/foo.snap
<packages_dir>/foo.config
<packages_dir>/bar.snap
cloud-init will invoke:
snappy install "--config=<packages_dir>/foo.config" \
<packages_dir>/foo.snap
snappy install <packages_dir>/bar.snap
Note, that if provided a 'config' entry for 'ubuntu-core', then
cloud-init will invoke: snappy config ubuntu-core <config>
Allowing you to configure ubuntu-core in this way.
"""
from cloudinit import log as logging
from cloudinit import templater
@ -7,63 +48,168 @@ from cloudinit import util
from cloudinit.settings import PER_INSTANCE
import glob
import six
import tempfile
import os
LOG = logging.getLogger(__name__)
frequency = PER_INSTANCE
SNAPPY_ENV_PATH = "/writable/system-data/etc/snappy.env"
SNAPPY_CMD = "snappy"
NAMESPACE_DELIM = '.'
BUILTIN_CFG = {
'packages': [],
'packages_dir': '/writable/user-data/cloud-init/click_packages',
'packages_dir': '/writable/user-data/cloud-init/snaps',
'ssh_enabled': False,
'system_snappy': "auto"
'system_snappy': "auto",
'config': {},
}
"""
snappy:
system_snappy: auto
ssh_enabled: True
packages:
- etcd
- {'name': 'pkg1', 'config': "wark"}
"""
def parse_filename(fname):
fname = os.path.basename(fname)
fname_noext = fname.rpartition(".")[0]
name = fname_noext.partition("_")[0]
shortname = name.partition(".")[0]
return(name, shortname, fname_noext)
def install_package(pkg_name, config=None):
cmd = ["snappy", "install"]
if config:
if os.path.isfile(config):
cmd.append("--config-file=" + config)
def get_fs_package_ops(fspath):
if not fspath:
return []
ops = []
for snapfile in sorted(glob.glob(os.path.sep.join([fspath, '*.snap']))):
(name, shortname, fname_noext) = parse_filename(snapfile)
cfg = None
for cand in (fname_noext, name, shortname):
fpcand = os.path.sep.join([fspath, cand]) + ".config"
if os.path.isfile(fpcand):
cfg = fpcand
break
ops.append(makeop('install', name, config=None,
path=snapfile, cfgfile=cfg))
return ops
def makeop(op, name, config=None, path=None, cfgfile=None):
return({'op': op, 'name': name, 'config': config, 'path': path,
'cfgfile': cfgfile})
def get_package_config(configs, name):
# load the package's config from the configs dict.
# prefer full-name entry (config-example.canonical)
# over short name entry (config-example)
if name in configs:
return configs[name]
return configs.get(name.partition(NAMESPACE_DELIM)[0])
def get_package_ops(packages, configs, installed=None, fspath=None):
# get the install an config operations that should be done
if installed is None:
installed = read_installed_packages()
short_installed = [p.partition(NAMESPACE_DELIM)[0] for p in installed]
if not packages:
packages = []
if not configs:
configs = {}
ops = []
ops += get_fs_package_ops(fspath)
for name in packages:
ops.append(makeop('install', name, get_package_config(configs, name)))
to_install = [f['name'] for f in ops]
short_to_install = [f['name'].partition(NAMESPACE_DELIM)[0] for f in ops]
for name in configs:
if name in to_install:
continue
shortname = name.partition(NAMESPACE_DELIM)[0]
if shortname in short_to_install:
continue
if name in installed or shortname in short_installed:
ops.append(makeop('config', name,
config=get_package_config(configs, name)))
# prefer config entries to filepath entries
for op in ops:
if op['op'] != 'install' or not op['cfgfile']:
continue
name = op['name']
fromcfg = get_package_config(configs, op['name'])
if fromcfg:
LOG.debug("preferring configs[%(name)s] over '%(cfgfile)s'", op)
op['cfgfile'] = None
op['config'] = fromcfg
return ops
def render_snap_op(op, name, path=None, cfgfile=None, config=None):
if op not in ('install', 'config'):
raise ValueError("cannot render op '%s'" % op)
shortname = name.partition(NAMESPACE_DELIM)[0]
try:
cfg_tmpf = None
if config is not None:
# input to 'snappy config packagename' must have nested data. odd.
# config:
# packagename:
# config
# Note, however, we do not touch config files on disk.
nested_cfg = {'config': {shortname: config}}
(fd, cfg_tmpf) = tempfile.mkstemp()
os.write(fd, util.yaml_dumps(nested_cfg).encode())
os.close(fd)
cfgfile = cfg_tmpf
cmd = [SNAPPY_CMD, op]
if op == 'install':
if cfgfile:
cmd.append('--config=' + cfgfile)
if path:
cmd.append("--allow-unauthenticated")
cmd.append(path)
else:
cmd.append("--config=" + config)
cmd.append(pkg_name)
cmd.append(name)
elif op == 'config':
cmd += [name, cfgfile]
util.subp(cmd)
finally:
if cfg_tmpf:
os.unlink(cfg_tmpf)
def install_packages(package_dir, packages):
local_pkgs = glob.glob(os.path.sep.join([package_dir, '*.click']))
LOG.debug("installing local packages %s" % local_pkgs)
if local_pkgs:
for pkg in local_pkgs:
cfg = pkg.replace(".click", ".config")
if not os.path.isfile(cfg):
cfg = None
install_package(pkg, config=cfg)
LOG.debug("installing click packages")
if packages:
for pkg in packages:
if not pkg:
continue
if isinstance(pkg, str):
name = pkg
config = None
elif pkg:
name = pkg.get('name', pkg)
config = pkg.get('config')
install_package(pkg_name=name, config=config)
def read_installed_packages():
ret = []
for (name, date, version, dev) in read_pkg_data():
if dev:
ret.append(NAMESPACE_DELIM.join([name, dev]))
else:
ret.append(name)
return ret
def read_pkg_data():
out, err = util.subp([SNAPPY_CMD, "list"])
pkg_data = []
for line in out.splitlines()[1:]:
toks = line.split(sep=None, maxsplit=3)
if len(toks) == 3:
(name, date, version) = toks
dev = None
else:
(name, date, version, dev) = toks
pkg_data.append((name, date, version, dev,))
return pkg_data
def disable_enable_ssh(enabled):
@ -92,6 +238,15 @@ def system_is_snappy():
return False
def set_snappy_command():
global SNAPPY_CMD
if util.which("snappy-go"):
SNAPPY_CMD = "snappy-go"
else:
SNAPPY_CMD = "snappy"
LOG.debug("snappy command is '%s'", SNAPPY_CMD)
def handle(name, cfg, cloud, log, args):
cfgin = cfg.get('snappy')
if not cfgin:
@ -107,7 +262,22 @@ def handle(name, cfg, cloud, log, args):
LOG.debug("%s: 'auto' mode, and system not snappy", name)
return
install_packages(mycfg['packages_dir'],
mycfg['packages'])
set_snappy_command()
pkg_ops = get_package_ops(packages=mycfg['packages'],
configs=mycfg['config'],
fspath=mycfg['packages_dir'])
fails = []
for pkg_op in pkg_ops:
try:
render_snap_op(**pkg_op)
except Exception as e:
fails.append((pkg_op, e,))
LOG.warn("'%s' failed for '%s': %s",
pkg_op['op'], pkg_op['name'], e)
disable_enable_ssh(mycfg.get('ssh_enabled', False))
if fails:
raise Exception("failed to install/configure snaps")

View File

@ -0,0 +1,309 @@
from cloudinit.config.cc_snappy import (
makeop, get_package_ops, render_snap_op)
from cloudinit import util
from .. import helpers as t_help
import os
import shutil
import tempfile
import yaml
ALLOWED = (dict, list, int, str)
class TestInstallPackages(t_help.TestCase):
def setUp(self):
super(TestInstallPackages, self).setUp()
self.unapply = []
# by default 'which' has nothing in its path
self.apply_patches([(util, 'subp', self._subp)])
self.subp_called = []
self.snapcmds = []
self.tmp = tempfile.mkdtemp(prefix="TestInstallPackages")
def tearDown(self):
apply_patches([i for i in reversed(self.unapply)])
shutil.rmtree(self.tmp)
def apply_patches(self, patches):
ret = apply_patches(patches)
self.unapply += ret
def populate_tmp(self, files):
return t_help.populate_dir(self.tmp, files)
def _subp(self, *args, **kwargs):
# supports subp calling with cmd as args or kwargs
if 'args' not in kwargs:
kwargs['args'] = args[0]
self.subp_called.append(kwargs)
snap_cmds = []
args = kwargs['args']
# here we basically parse the snappy command invoked
# and append to snapcmds a list of (mode, pkg, config)
if args[0:2] == ['snappy', 'config']:
if args[3] == "-":
config = kwargs.get('data', '')
else:
with open(args[3], "rb") as fp:
config = yaml.safe_load(fp.read())
self.snapcmds.append(['config', args[2], config])
elif args[0:2] == ['snappy', 'install']:
config = None
pkg = None
for arg in args[2:]:
if arg.startswith("--config="):
cfgfile = arg.partition("=")[2]
if cfgfile == "-":
config = kwargs.get('data', '')
elif cfgfile:
with open(cfgfile, "rb") as fp:
config = yaml.safe_load(fp.read())
elif not pkg and not arg.startswith("-"):
pkg = arg
self.snapcmds.append(['install', pkg, config])
def test_package_ops_1(self):
ret = get_package_ops(
packages=['pkg1', 'pkg2', 'pkg3'],
configs={'pkg2': b'mycfg2'}, installed=[])
self.assertEqual(
ret, [makeop('install', 'pkg1', None, None),
makeop('install', 'pkg2', b'mycfg2', None),
makeop('install', 'pkg3', None, None)])
def test_package_ops_config_only(self):
ret = get_package_ops(
packages=None,
configs={'pkg2': b'mycfg2'}, installed=['pkg1', 'pkg2'])
self.assertEqual(
ret, [makeop('config', 'pkg2', b'mycfg2')])
def test_package_ops_install_and_config(self):
ret = get_package_ops(
packages=['pkg3', 'pkg2'],
configs={'pkg2': b'mycfg2', 'xinstalled': b'xcfg'},
installed=['xinstalled'])
self.assertEqual(
ret, [makeop('install', 'pkg3'),
makeop('install', 'pkg2', b'mycfg2'),
makeop('config', 'xinstalled', b'xcfg')])
def test_package_ops_install_long_config_short(self):
# a package can be installed by full name, but have config by short
cfg = {'k1': 'k2'}
ret = get_package_ops(
packages=['config-example.canonical'],
configs={'config-example': cfg}, installed=[])
self.assertEqual(
ret, [makeop('install', 'config-example.canonical', cfg)])
def test_package_ops_with_file(self):
self.populate_tmp(
{"snapf1.snap": b"foo1", "snapf1.config": b"snapf1cfg",
"snapf2.snap": b"foo2", "foo.bar": "ignored"})
ret = get_package_ops(
packages=['pkg1'], configs={}, installed=[], fspath=self.tmp)
self.assertEqual(
ret,
[makeop_tmpd(self.tmp, 'install', 'snapf1', path="snapf1.snap",
cfgfile="snapf1.config"),
makeop_tmpd(self.tmp, 'install', 'snapf2', path="snapf2.snap"),
makeop('install', 'pkg1')])
def test_package_ops_common_filename(self):
# fish package name from filename
# package names likely look like: pkgname.namespace_version_arch.snap
fname = "xkcd-webserver.canonical_0.3.4_all.snap"
name = "xkcd-webserver.canonical"
shortname = "xkcd-webserver"
# find filenames
self.populate_tmp(
{"pkg-ws.smoser_0.3.4_all.snap": "pkg-ws-snapdata",
"pkg-ws.config": "pkg-ws-config",
"pkg1.smoser_1.2.3_all.snap": "pkg1.snapdata",
"pkg1.smoser.config": "pkg1.smoser.config-data",
"pkg1.config": "pkg1.config-data",
"pkg2.smoser_0.0_amd64.snap": "pkg2-snapdata",
"pkg2.smoser_0.0_amd64.config": "pkg2.config",
})
ret = get_package_ops(
packages=[], configs={}, installed=[], fspath=self.tmp)
self.assertEqual(
ret,
[makeop_tmpd(self.tmp, 'install', 'pkg-ws.smoser',
path="pkg-ws.smoser_0.3.4_all.snap",
cfgfile="pkg-ws.config"),
makeop_tmpd(self.tmp, 'install', 'pkg1.smoser',
path="pkg1.smoser_1.2.3_all.snap",
cfgfile="pkg1.smoser.config"),
makeop_tmpd(self.tmp, 'install', 'pkg2.smoser',
path="pkg2.smoser_0.0_amd64.snap",
cfgfile="pkg2.smoser_0.0_amd64.config"),
])
def test_package_ops_config_overrides_file(self):
# config data overrides local file .config
self.populate_tmp(
{"snapf1.snap": b"foo1", "snapf1.config": b"snapf1cfg"})
ret = get_package_ops(
packages=[], configs={'snapf1': 'snapf1cfg-config'},
installed=[], fspath=self.tmp)
self.assertEqual(
ret, [makeop_tmpd(self.tmp, 'install', 'snapf1',
path="snapf1.snap", config="snapf1cfg-config")])
def test_package_ops_namespacing(self):
cfgs = {
'config-example': {'k1': 'v1'},
'pkg1': {'p1': 'p2'},
'ubuntu-core': {'c1': 'c2'},
'notinstalled.smoser': {'s1': 's2'},
}
cfg = {'config-example-k1': 'config-example-k2'}
ret = get_package_ops(
packages=['config-example.canonical'], configs=cfgs,
installed=['config-example.smoser', 'pkg1.canonical',
'ubuntu-core'])
expected_configs = [
makeop('config', 'pkg1', config=cfgs['pkg1']),
makeop('config', 'ubuntu-core', config=cfgs['ubuntu-core'])]
expected_installs = [
makeop('install', 'config-example.canonical',
config=cfgs['config-example'])]
installs = [i for i in ret if i['op'] == 'install']
configs = [c for c in ret if c['op'] == 'config']
self.assertEqual(installs, expected_installs)
# configs are not ordered
self.assertEqual(len(configs), len(expected_configs))
self.assertTrue(all(found in expected_configs for found in configs))
def test_render_op_localsnap(self):
self.populate_tmp({"snapf1.snap": b"foo1"})
op = makeop_tmpd(self.tmp, 'install', 'snapf1',
path='snapf1.snap')
render_snap_op(**op)
self.assertEqual(
self.snapcmds, [['install', op['path'], None]])
def test_render_op_localsnap_localconfig(self):
self.populate_tmp(
{"snapf1.snap": b"foo1", 'snapf1.config': b'snapf1cfg'})
op = makeop_tmpd(self.tmp, 'install', 'snapf1',
path='snapf1.snap', cfgfile='snapf1.config')
render_snap_op(**op)
self.assertEqual(
self.snapcmds, [['install', op['path'], 'snapf1cfg']])
def test_render_op_snap(self):
op = makeop('install', 'snapf1')
render_snap_op(**op)
self.assertEqual(
self.snapcmds, [['install', 'snapf1', None]])
def test_render_op_snap_config(self):
mycfg = {'key1': 'value1'}
name = "snapf1"
op = makeop('install', name, config=mycfg)
render_snap_op(**op)
self.assertEqual(
self.snapcmds, [['install', name, {'config': {name: mycfg}}]])
def test_render_op_config_bytes(self):
name = "snapf1"
mycfg = b'myconfig'
op = makeop('config', name, config=mycfg)
render_snap_op(**op)
self.assertEqual(
self.snapcmds, [['config', 'snapf1', {'config': {name: mycfg}}]])
def test_render_op_config_string(self):
name = 'snapf1'
mycfg = 'myconfig: foo\nhisconfig: bar\n'
op = makeop('config', name, config=mycfg)
render_snap_op(**op)
self.assertEqual(
self.snapcmds, [['config', 'snapf1', {'config': {name: mycfg}}]])
def test_render_op_config_dict(self):
# config entry for package can be a dict, not a string blob
mycfg = {'foo': 'bar'}
name = 'snapf1'
op = makeop('config', name, config=mycfg)
render_snap_op(**op)
# snapcmds is a list of 3-entry lists. data_found will be the
# blob of data in the file in 'snappy install --config=<file>'
data_found = self.snapcmds[0][2]
self.assertEqual(mycfg, data_found['config'][name])
def test_render_op_config_list(self):
# config entry for package can be a list, not a string blob
mycfg = ['foo', 'bar', 'wark', {'f1': 'b1'}]
name = "snapf1"
op = makeop('config', name, config=mycfg)
render_snap_op(**op)
data_found = self.snapcmds[0][2]
self.assertEqual(mycfg, data_found['config'][name])
def test_render_op_config_int(self):
# config entry for package can be a list, not a string blob
mycfg = 1
name = 'snapf1'
op = makeop('config', name, config=mycfg)
render_snap_op(**op)
data_found = self.snapcmds[0][2]
self.assertEqual(mycfg, data_found['config'][name])
def test_render_long_configs_short(self):
# install a namespaced package should have un-namespaced config
mycfg = {'k1': 'k2'}
name = 'snapf1'
op = makeop('install', name + ".smoser", config=mycfg)
render_snap_op(**op)
data_found = self.snapcmds[0][2]
self.assertEqual(mycfg, data_found['config'][name])
def test_render_does_not_pad_cfgfile(self):
# package_ops with cfgfile should not modify --file= content.
mydata = "foo1: bar1\nk: [l1, l2, l3]\n"
self.populate_tmp(
{"snapf1.snap": b"foo1", "snapf1.config": mydata.encode()})
ret = get_package_ops(
packages=[], configs={}, installed=[], fspath=self.tmp)
self.assertEqual(
ret,
[makeop_tmpd(self.tmp, 'install', 'snapf1', path="snapf1.snap",
cfgfile="snapf1.config")])
# now the op was ok, but test that render didn't mess it up.
render_snap_op(**ret[0])
data_found = self.snapcmds[0][2]
# the data found gets loaded in the snapcmd interpretation
# so this comparison is a bit lossy, but input to snappy config
# is expected to be yaml loadable, so it should be OK.
self.assertEqual(yaml.safe_load(mydata), data_found)
def makeop_tmpd(tmpd, op, name, config=None, path=None, cfgfile=None):
if cfgfile:
cfgfile = os.path.sep.join([tmpd, cfgfile])
if path:
path = os.path.sep.join([tmpd, path])
return(makeop(op=op, name=name, config=config, path=path, cfgfile=cfgfile))
def apply_patches(patches):
ret = []
for (ref, name, replace) in patches:
if replace is None:
continue
orig = getattr(ref, name)
setattr(ref, name, replace)
ret.append((ref, name, orig))
return ret