x/cloud-init
Code Issues Proposed changes

Merge with upstream sources

Browse Source
This commit is contained in:
Ben Howard 2014-01-09 08:17:41 -07:00
commit 0d9f85fdec
10 changed files with 338 additions and 221 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
ChangeLog
View File

@ -1,5 +1,12 @@
0.7.5:
- open 0.7.5
- Add a debug log message around import failures
- add a 'debug' module for easily printing out some information about
datasource and cloud-init [Shraddha Pandhe]
- support running apt with 'eatmydata' via configuration token
apt_get_wrapper (LP: #1236531).
- convert paths provided in config-drive 'files' to string before writing
(LP: #1260072).
0.7.4:
- fix issue mounting 'ephemeral0' if ephemeral0 was an alias for a
partitioned block device with target filesystem on ephemeral0.1.

79
cloudinit/config/cc_debug.py Normal file
View File

@ -0,0 +1,79 @@
# vi: ts=4 expandtab
#
# Copyright (C) 2013 Yahoo! Inc.
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License version 3, as
# published by the Free Software Foundation.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
from StringIO import StringIO
from cloudinit import util
from cloudinit import type_utils
import copy
def _make_header(text):
header = StringIO()
header.write("-" * 80)
header.write("\n")
header.write(text.center(80, ' '))
header.write("\n")
header.write("-" * 80)
header.write("\n")
return header.getvalue()
def handle(name, cfg, cloud, log, args):
verbose = util.get_cfg_by_path(cfg, ('debug', 'verbose'), default=True)
if args:
# if args are provided (from cmdline) then explicitly set verbose
out_file = args[0]
verbose = True
else:
out_file = util.get_cfg_by_path(cfg, ('debug', 'output'))
if not verbose:
log.debug(("Skipping module named %s,"
" verbose printing disabled"), name)
return
# Clean out some keys that we just don't care about showing...
dump_cfg = copy.deepcopy(cfg)
for k in ['log_cfgs']:
dump_cfg.pop(k, None)
all_keys = list(dump_cfg.keys())
for k in all_keys:
if k.startswith("_"):
dump_cfg.pop(k, None)
# Now dump it...
to_print = StringIO()
to_print.write(_make_header("Config"))
to_print.write(util.yaml_dumps(dump_cfg))
to_print.write("\n")
to_print.write(_make_header("MetaData"))
to_print.write(util.yaml_dumps(cloud.datasource.metadata))
to_print.write("\n")
to_print.write(_make_header("Misc"))
to_print.write("Datasource: %s\n" %
(type_utils.obj_name(cloud.datasource)))
to_print.write("Distro: %s\n" % (type_utils.obj_name(cloud.distro)))
to_print.write("Hostname: %s\n" % (cloud.get_hostname(True)))
to_print.write("Instance ID: %s\n" % (cloud.get_instance_id()))
to_print.write("Locale: %s\n" % (cloud.get_locale()))
to_print.write("Launch IDX: %s\n" % (cloud.launch_index))
contents = to_print.getvalue()
content_to_file = []
for line in contents.splitlines():
line = "ci-info: %s\n" % (line)
content_to_file.append(line)
if out_file:
util.write_file(out_file, "".join(content_to_file), 0644, "w")
else:
util.multi_log("".join(content_to_file), console=True, stderr=False)

28
cloudinit/distros/debian.py
View File

@ -36,6 +36,10 @@ LOG = logging.getLogger(__name__)
APT_GET_COMMAND = ('apt-get', '--option=Dpkg::Options::=--force-confold',
'--option=Dpkg::options::=--force-unsafe-io',
'--assume-yes', '--quiet')
APT_GET_WRAPPER = {
'command': 'eatmydata',
'enabled': 'auto',
}
class Distro(distros.Distro):
@ -148,7 +152,13 @@ class Distro(distros.Distro):
# See: http://tiny.cc/kg91fw
# Or: http://tiny.cc/mh91fw
e['DEBIAN_FRONTEND'] = 'noninteractive'
cmd = list(self.get_option("apt_get_command", APT_GET_COMMAND))
wcfg = self.get_option("apt_get_wrapper", APT_GET_WRAPPER)
cmd = _get_wrapper_prefix(
wcfg.get('command', APT_GET_WRAPPER['command']),
wcfg.get('enabled', APT_GET_WRAPPER['enabled']))
cmd.extend(list(self.get_option("apt_get_command", APT_GET_COMMAND)))
if args and isinstance(args, str):
cmd.append(args)
@ -166,7 +176,9 @@ class Distro(distros.Distro):
cmd.extend(pkglist)
# Allow the output of this to flow outwards (ie not be captured)
util.subp(cmd, env=e, capture=False)
util.log_time(logfunc=LOG.debug,
msg="apt-%s [%s]" % (command, ' '.join(cmd)), func=util.subp,
args=(cmd,), kwargs={'env': e, 'capture': False})
def update_package_sources(self):
self._runner.run("update-sources", self.package_command,
@ -175,3 +187,15 @@ class Distro(distros.Distro):
def get_primary_arch(self):
(arch, _err) = util.subp(['dpkg', '--print-architecture'])
return str(arch).strip()
def _get_wrapper_prefix(cmd, mode):
if isinstance(cmd, str):
cmd = [str(cmd)]
if (util.is_true(mode) or
(str(mode).lower() == "auto" and cmd[0] and
util.which(cmd[0]))):
return cmd
else:
return []

6
cloudinit/importer.py
View File

@ -36,6 +36,7 @@ def find_module(base_name, search_paths, required_attrs=None):
found_places = []
if not required_attrs:
required_attrs = []
# NOTE(harlowja): translate the search paths to include the base name.
real_paths = []
for path in search_paths:
real_path = []
@ -50,8 +51,9 @@ def find_module(base_name, search_paths, required_attrs=None):
mod = None
try:
mod = import_module(full_path)
except ImportError:
pass
except ImportError as e:
LOG.debug("Failed at attempted import of '%s' due to: %s",
full_path, e)
if not mod:
continue
found_attrs = 0

2
cloudinit/util.py
View File

@ -170,6 +170,8 @@ class SeLinuxGuard(object):
def __exit__(self, excp_type, excp_value, excp_traceback):
if self.selinux and self.selinux.is_selinux_enabled():
path = os.path.realpath(os.path.expanduser(self.path))
# path should be a string, not unicode
path = str(path)
do_restore = False
try:
# See if even worth restoring??

1
doc/examples/cloud-config-datasources.txt
View File

@ -49,7 +49,6 @@ datasource:
hostname_bounce:
interface: eth0
policy: on # [can be 'on', 'off' or 'force']
}
SmartOS:
# Smart OS datasource works over a serial console interacting with

422
doc/examples/cloud-config-disk-setup.txt
View File

@ -1,24 +1,24 @@
Cloud-init supports the creation of simple partition tables and file systems
on devices.
# Cloud-init supports the creation of simple partition tables and file systems
# on devices.
Default disk definitions for AWS
--------------------------------
(Not implemented yet, but provided for future documentation)
# Default disk definitions for AWS
# --------------------------------
# (Not implemented yet, but provided for future documentation)
disk_setup:
ephmeral0:
type: 'mbr'
layout: True
overwrite: False
disk_setup:
ephmeral0:
type: 'mbr'
layout: True
overwrite: False
fs_setup:
- label: None,
filesystem: ext3
device: ephemeral0
partition: auto
fs_setup:
- label: None,
filesystem: ext3
device: ephemeral0
partition: auto
Default disk definitions for Windows Azure
------------------------------------------
# Default disk definitions for Windows Azure
# ------------------------------------------
device_aliases: {'ephemeral0': '/dev/sdb'}
disk_setup:
@ -34,8 +34,8 @@ fs_setup:
replace_fs: ntfs
Default disk definitions for SmartOS
------------------------------------
# Default disk definitions for SmartOS
# ------------------------------------
device_aliases: {'ephemeral0': '/dev/sdb'}
disk_setup:
@ -49,203 +49,203 @@ fs_setup:
filesystem: ext3
device: ephemeral0.0
Cavaut for SmartOS: if ephemeral disk is not defined, then the disk will
not be automatically added to the mounts.
# Cavaut for SmartOS: if ephemeral disk is not defined, then the disk will
# not be automatically added to the mounts.
The default definition is used to make sure that the ephemeral storage is
setup properly.
# The default definition is used to make sure that the ephemeral storage is
# setup properly.
"disk_setup": disk partitioning
--------------------------------
# "disk_setup": disk partitioning
# --------------------------------
The disk_setup directive instructs Cloud-init to partition a disk. The format is:
# The disk_setup directive instructs Cloud-init to partition a disk. The format is:
disk_setup:
ephmeral0:
type: 'mbr'
layout: 'auto'
/dev/xvdh:
type: 'mbr'
layout:
- 33
- [33, 82]
- 33
overwrite: True
disk_setup:
ephmeral0:
type: 'mbr'
layout: 'auto'
/dev/xvdh:
type: 'mbr'
layout:
- 33
- [33, 82]
- 33
overwrite: True
The format is a list of dicts of dicts. The first value is the name of the
device and the subsequent values define how to create and layout the partition.
# The format is a list of dicts of dicts. The first value is the name of the
# device and the subsequent values define how to create and layout the
# partition.
# The general format is:
# disk_setup:
# <DEVICE>:
# type: 'mbr'
# layout: <LAYOUT|BOOL>
# overwrite: <BOOL>
#
# Where:
# <DEVICE>: The name of the device. 'ephemeralX' and 'swap' are special
# values which are specific to the cloud. For these devices
# Cloud-init will look up what the real devices is and then
# use it.
#
# For other devices, the kernel device name is used. At this
# time only simply kernel devices are supported, meaning
# that device mapper and other targets may not work.
#
# Note: At this time, there is no handling or setup of
# device mapper targets.
#
# type=<TYPE>: Currently the following are supported:
# 'mbr': default and setups a MS-DOS partition table
#
# Note: At this time only 'mbr' partition tables are allowed.
# It is anticipated in the future that we'll have GPT as
# option in the future, or even "RAID" to create a mdadm
# RAID.
#
# layout={...}: The device layout. This is a list of values, with the
# percentage of disk that partition will take.
# Valid options are:
# [<SIZE>, [<SIZE>, <PART_TYPE]]
#
# Where <SIZE> is the _percentage_ of the disk to use, while
# <PART_TYPE> is the numerical value of the partition type.
#
# The following setups two partitions, with the first
# partition having a swap label, taking 1/3 of the disk space
# and the remainder being used as the second partition.
# /dev/xvdh':
# type: 'mbr'
# layout:
# - [33,82]
# - 66
# overwrite: True
#
# When layout is "true" it means single partition the entire
# device.
#
# When layout is "false" it means don't partition or ignore
# existing partitioning.
#
# If layout is set to "true" and overwrite is set to "false",
# it will skip partitioning the device without a failure.
#
# overwrite=<BOOL>: This describes whether to ride with saftey's on and
# everything holstered.
#
# 'false' is the default, which means that:
# 1. The device will be checked for a partition table
# 2. The device will be checked for a file system
# 3. If either a partition of file system is found, then
# the operation will be _skipped_.
#
# 'true' is cowboy mode. There are no checks and things are
# done blindly. USE with caution, you can do things you
# really, really don't want to do.
#
#
# fs_setup: Setup the file system
# -------------------------------
#
# fs_setup describes the how the file systems are supposed to look.
The general format is:
disk_setup:
<DEVICE>:
type: 'mbr'
layout: <LAYOUT|BOOL>
overwrite: <BOOL>
fs_setup:
- label: ephemeral0
filesystem: 'ext3'
device: 'ephemeral0'
partition: 'auto'
- label: mylabl2
filesystem: 'ext4'
device: '/dev/xvda1'
- special:
cmd: mkfs -t %(FILESYSTEM)s -L %(LABEL)s %(DEVICE)s
filesystem: 'btrfs'
device: '/dev/xvdh'
Where:
<DEVICE>: The name of the device. 'ephemeralX' and 'swap' are special
values which are specific to the cloud. For these devices
Cloud-init will look up what the real devices is and then
use it.
For other devices, the kernel device name is used. At this
time only simply kernel devices are supported, meaning
that device mapper and other targets may not work.
Note: At this time, there is no handling or setup of
device mapper targets.
type=<TYPE>: Currently the following are supported:
'mbr': default and setups a MS-DOS partition table
Note: At this time only 'mbr' partition tables are allowed.
It is anticipated in the future that we'll have GPT as
option in the future, or even "RAID" to create a mdadm
RAID.
layout={...}: The device layout. This is a list of values, with the
percentage of disk that partition will take.
Valid options are:
[<SIZE>, [<SIZE>, <PART_TYPE]]
Where <SIZE> is the _percentage_ of the disk to use, while
<PART_TYPE> is the numerical value of the partition type.
The following setups two partitions, with the first
partition having a swap label, taking 1/3 of the disk space
and the remainder being used as the second partition.
/dev/xvdh':
type: 'mbr'
layout:
- [33,82]
- 66
overwrite: True
When layout is "true" it means single partition the entire
device.
When layout is "false" it means don't partition or ignore
existing partitioning.
If layout is set to "true" and overwrite is set to "false",
it will skip partitioning the device without a failure.
overwrite=<BOOL>: This describes whether to ride with saftey's on and
everything holstered.
'false' is the default, which means that:
1. The device will be checked for a partition table
2. The device will be checked for a file system
3. If either a partition of file system is found, then
the operation will be _skipped_.
'true' is cowboy mode. There are no checks and things are
done blindly. USE with caution, you can do things you
really, really don't want to do.
fs_setup: Setup the file system
-------------------------------
fs_setup describes the how the file systems are supposed to look.
fs_setup:
- label: ephemeral0
filesystem: 'ext3'
device: 'ephemeral0'
partition: 'auto'
- label: mylabl2
filesystem: 'ext4'
device: '/dev/xvda1'
- special:
cmd: mkfs -t %(FILESYSTEM)s -L %(LABEL)s %(DEVICE)s
filesystem: 'btrfs'
device: '/dev/xvdh'
The general format is:
fs_setup:
- label: <LABEL>
filesystem: <FS_TYPE>
device: <DEVICE>
partition: <PART_VALUE>
overwrite: <OVERWRITE>
replace_fs: <FS_TYPE>
Where:
<LABEL>: The file system label to be used. If set to None, no label is
used.
<FS_TYPE>: The file system type. It is assumed that the there
will be a "mkfs.<FS_TYPE>" that behaves likes "mkfs". On a standard
Ubuntu Cloud Image, this means that you have the option of ext{2,3,4},
and vfat by default.
<DEVICE>: The device name. Special names of 'ephemeralX' or 'swap'
are allowed and the actual device is acquired from the cloud datasource.
When using 'ephemeralX' (i.e. ephemeral0), make sure to leave the
label as 'ephemeralX' otherwise there may be issues with the mounting
of the ephemeral storage layer.
If you define the device as 'ephemeralX.Y' then Y will be interpetted
as a partition value. However, ephermalX.0 is the _same_ as ephemeralX.
<PART_VALUE>:
Partition definitions are overwriten if you use the '<DEVICE>.Y' notation.
The valid options are:
"auto|any": tell cloud-init not to care whether there is a partition
or not. Auto will use the first partition that does not contain a
file system already. In the absence of a partition table, it will
put it directly on the disk.
"auto": If a file system that matches the specification in terms of
label, type and device, then cloud-init will skip the creation of
the file system.
"any": If a file system that matches the file system type and device,
then cloud-init will skip the creation of the file system.
Devices are selected based on first-detected, starting with partitions
and then the raw disk. Consider the following:
NAME FSTYPE LABEL
xvdb
|-xvdb1 ext4
|-xvdb2
|-xvdb3 btrfs test
\-xvdb4 ext4 test
If you ask for 'auto', label of 'test, and file system of 'ext4'
then cloud-init will select the 2nd partition, even though there
is a partition match at the 4th partition.
If you ask for 'any' and a label of 'test', then cloud-init will
select the 1st partition.
If you ask for 'auto' and don't define label, then cloud-init will
select the 1st partition.
In general, if you have a specific partition configuration in mind,
you should define either the device or the partition number. 'auto'
and 'any' are specifically intended for formating ephemeral storage or
for simple schemes.
"none": Put the file system directly on the device.
<NUM>: where NUM is the actual partition number.
<OVERWRITE>: Defines whether or not to overwrite any existing
filesystem.
"true": Indiscriminately destroy any pre-existing file system. Use at
your own peril.
"false": If an existing file system exists, skip the creation.
<REPLACE_FS>: This is a special directive, used for Windows Azure that
instructs cloud-init to replace a file system of <FS_TYPE>. NOTE:
unless you define a label, this requires the use of the 'any' partition
directive.
Behavior Caveat: The default behavior is to _check_ if the file system exists.
If a file system matches the specification, then the operation is a no-op.
# The general format is:
# fs_setup:
# - label: <LABEL>
# filesystem: <FS_TYPE>
# device: <DEVICE>
# partition: <PART_VALUE>
# overwrite: <OVERWRITE>
# replace_fs: <FS_TYPE>
#
# Where:
# <LABEL>: The file system label to be used. If set to None, no label is
# used.
#
# <FS_TYPE>: The file system type. It is assumed that the there
# will be a "mkfs.<FS_TYPE>" that behaves likes "mkfs". On a standard
# Ubuntu Cloud Image, this means that you have the option of ext{2,3,4},
# and vfat by default.
#
# <DEVICE>: The device name. Special names of 'ephemeralX' or 'swap'
# are allowed and the actual device is acquired from the cloud datasource.
# When using 'ephemeralX' (i.e. ephemeral0), make sure to leave the
# label as 'ephemeralX' otherwise there may be issues with the mounting
# of the ephemeral storage layer.
#
# If you define the device as 'ephemeralX.Y' then Y will be interpetted
# as a partition value. However, ephermalX.0 is the _same_ as ephemeralX.
#
# <PART_VALUE>:
# Partition definitions are overwriten if you use the '<DEVICE>.Y' notation.
#
# The valid options are:
# "auto|any": tell cloud-init not to care whether there is a partition
# or not. Auto will use the first partition that does not contain a
# file system already. In the absence of a partition table, it will
# put it directly on the disk.
#
# "auto": If a file system that matches the specification in terms of
# label, type and device, then cloud-init will skip the creation of
# the file system.
#
# "any": If a file system that matches the file system type and device,
# then cloud-init will skip the creation of the file system.
#
# Devices are selected based on first-detected, starting with partitions
# and then the raw disk. Consider the following:
# NAME FSTYPE LABEL
# xvdb
# |-xvdb1 ext4
# |-xvdb2
# |-xvdb3 btrfs test
# \-xvdb4 ext4 test
#
# If you ask for 'auto', label of 'test, and file system of 'ext4'
# then cloud-init will select the 2nd partition, even though there
# is a partition match at the 4th partition.
#
# If you ask for 'any' and a label of 'test', then cloud-init will
# select the 1st partition.
#
# If you ask for 'auto' and don't define label, then cloud-init will
# select the 1st partition.
#
# In general, if you have a specific partition configuration in mind,
# you should define either the device or the partition number. 'auto'
# and 'any' are specifically intended for formating ephemeral storage or
# for simple schemes.
#
# "none": Put the file system directly on the device.
#
# <NUM>: where NUM is the actual partition number.
#
# <OVERWRITE>: Defines whether or not to overwrite any existing
# filesystem.
#
# "true": Indiscriminately destroy any pre-existing file system. Use at
# your own peril.
#
# "false": If an existing file system exists, skip the creation.
#
# <REPLACE_FS>: This is a special directive, used for Windows Azure that
# instructs cloud-init to replace a file system of <FS_TYPE>. NOTE:
# unless you define a label, this requires the use of the 'any' partition
# directive.
#
# Behavior Caveat: The default behavior is to _check_ if the file system exists.
# If a file system matches the specification, then the operation is a no-op.

11
doc/examples/cloud-config.txt
View File

@ -74,7 +74,7 @@ apt_preserve_sources_list: true
# 'source' entries in apt-sources that match this python regex
# expression will be passed to add-apt-repository
add_apt_repo_match = "^[\w-]+:\w"
add_apt_repo_match: '^[\w-]+:\w'
apt_sources:
- source: "deb http://ppa.launchpad.net/byobu/ppa/ubuntu karmic main"
@ -147,8 +147,13 @@ apt_sources:
# '--option=Dpkg::options::=--force-unsafe-io', '--assume-yes', '--quiet']
#
# apt_get_upgrade_subcommand:
# Specify a different subcommand for 'upgrade. The default is 'dist-upgrade'.
# This is the subcommand that is invoked if package_upgrade is set to true above.
# Specify a different subcommand for 'upgrade. The default is 'dist-upgrade'.
# This is the subcommand that is invoked if package_upgrade is set to true above.
#
# apt_get_wrapper:
# command: eatmydata
# enabled: [True, False, "auto"]
#
# Install additional packages on first boot
#

2
packages/bddeb
View File

@ -32,7 +32,7 @@ PKG_MP = {
'boto': 'python-boto',
'cheetah': 'python-cheetah',
'configobj': 'python-configobj',
'jsonpatch': 'python-json-patch',
'jsonpatch': 'python-jsonpatch | python-json-patch',
'oauth': 'python-oauth',
'prettytable': 'python-prettytable',
'pyserial': 'python-serial',

1
packages/debian/control.in
View File

@ -25,7 +25,6 @@ Depends: procps,
#end for
python-software-properties | software-properties-common,
\${misc:Depends},
\${python:Depends}
XB-Python-Version: \${python:Versions}
Description: Init scripts for cloud instances
Cloud instances need special scripts to run during initialisation