x/cloud-init
Code Issues Proposed changes

Merge with upstream sources

Browse Source
This commit is contained in:
Ben Howard 2014-01-09 08:17:41 -07:00
commit 0d9f85fdec
10 changed files with 338 additions and 221 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
ChangeLog
View File

@ -1,5 +1,12 @@
0.7.5: 0.7.5:
- open 0.7.5 - open 0.7.5
- Add a debug log message around import failures
- add a 'debug' module for easily printing out some information about
datasource and cloud-init [Shraddha Pandhe]
- support running apt with 'eatmydata' via configuration token
apt_get_wrapper (LP: #1236531).
- convert paths provided in config-drive 'files' to string before writing
(LP: #1260072).
0.7.4: 0.7.4:
- fix issue mounting 'ephemeral0' if ephemeral0 was an alias for a - fix issue mounting 'ephemeral0' if ephemeral0 was an alias for a
partitioned block device with target filesystem on ephemeral0.1. partitioned block device with target filesystem on ephemeral0.1.

79
cloudinit/config/cc_debug.py Normal file
View File

@ -0,0 +1,79 @@
# vi: ts=4 expandtab
#
# Copyright (C) 2013 Yahoo! Inc.
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License version 3, as
# published by the Free Software Foundation.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
from StringIO import StringIO
from cloudinit import util
from cloudinit import type_utils
import copy
def _make_header(text):
header = StringIO()
header.write("-" * 80)
header.write("\n")
header.write(text.center(80, ' '))
header.write("\n")
header.write("-" * 80)
header.write("\n")
return header.getvalue()
def handle(name, cfg, cloud, log, args):
verbose = util.get_cfg_by_path(cfg, ('debug', 'verbose'), default=True)
if args:
# if args are provided (from cmdline) then explicitly set verbose
out_file = args[0]
verbose = True
else:
out_file = util.get_cfg_by_path(cfg, ('debug', 'output'))
if not verbose:
log.debug(("Skipping module named %s,"
" verbose printing disabled"), name)
return
# Clean out some keys that we just don't care about showing...
dump_cfg = copy.deepcopy(cfg)
for k in ['log_cfgs']:
dump_cfg.pop(k, None)
all_keys = list(dump_cfg.keys())
for k in all_keys:
if k.startswith("_"):
dump_cfg.pop(k, None)
# Now dump it...
to_print = StringIO()
to_print.write(_make_header("Config"))
to_print.write(util.yaml_dumps(dump_cfg))
to_print.write("\n")
to_print.write(_make_header("MetaData"))
to_print.write(util.yaml_dumps(cloud.datasource.metadata))
to_print.write("\n")
to_print.write(_make_header("Misc"))
to_print.write("Datasource: %s\n" %
(type_utils.obj_name(cloud.datasource)))
to_print.write("Distro: %s\n" % (type_utils.obj_name(cloud.distro)))
to_print.write("Hostname: %s\n" % (cloud.get_hostname(True)))
to_print.write("Instance ID: %s\n" % (cloud.get_instance_id()))
to_print.write("Locale: %s\n" % (cloud.get_locale()))
to_print.write("Launch IDX: %s\n" % (cloud.launch_index))
contents = to_print.getvalue()
content_to_file = []
for line in contents.splitlines():
line = "ci-info: %s\n" % (line)
content_to_file.append(line)
if out_file:
util.write_file(out_file, "".join(content_to_file), 0644, "w")
else:
util.multi_log("".join(content_to_file), console=True, stderr=False)

28
cloudinit/distros/debian.py
View File

@ -36,6 +36,10 @@ LOG = logging.getLogger(__name__)
APT_GET_COMMAND = ('apt-get', '--option=Dpkg::Options::=--force-confold', APT_GET_COMMAND = ('apt-get', '--option=Dpkg::Options::=--force-confold',
'--option=Dpkg::options::=--force-unsafe-io', '--option=Dpkg::options::=--force-unsafe-io',
'--assume-yes', '--quiet') '--assume-yes', '--quiet')
APT_GET_WRAPPER = {
'command': 'eatmydata',
'enabled': 'auto',
}
class Distro(distros.Distro): class Distro(distros.Distro):
@ -148,7 +152,13 @@ class Distro(distros.Distro):
# See: http://tiny.cc/kg91fw # See: http://tiny.cc/kg91fw
# Or: http://tiny.cc/mh91fw # Or: http://tiny.cc/mh91fw
e['DEBIAN_FRONTEND'] = 'noninteractive' e['DEBIAN_FRONTEND'] = 'noninteractive'
cmd = list(self.get_option("apt_get_command", APT_GET_COMMAND))
wcfg = self.get_option("apt_get_wrapper", APT_GET_WRAPPER)
cmd = _get_wrapper_prefix(
wcfg.get('command', APT_GET_WRAPPER['command']),
wcfg.get('enabled', APT_GET_WRAPPER['enabled']))
cmd.extend(list(self.get_option("apt_get_command", APT_GET_COMMAND)))
if args and isinstance(args, str): if args and isinstance(args, str):
cmd.append(args) cmd.append(args)
@ -166,7 +176,9 @@ class Distro(distros.Distro):
cmd.extend(pkglist) cmd.extend(pkglist)
# Allow the output of this to flow outwards (ie not be captured) # Allow the output of this to flow outwards (ie not be captured)
util.subp(cmd, env=e, capture=False) util.log_time(logfunc=LOG.debug,
msg="apt-%s [%s]" % (command, ' '.join(cmd)), func=util.subp,
args=(cmd,), kwargs={'env': e, 'capture': False})
def update_package_sources(self): def update_package_sources(self):
self._runner.run("update-sources", self.package_command, self._runner.run("update-sources", self.package_command,
@ -175,3 +187,15 @@ class Distro(distros.Distro):
def get_primary_arch(self): def get_primary_arch(self):
(arch, _err) = util.subp(['dpkg', '--print-architecture']) (arch, _err) = util.subp(['dpkg', '--print-architecture'])
return str(arch).strip() return str(arch).strip()
def _get_wrapper_prefix(cmd, mode):
if isinstance(cmd, str):
cmd = [str(cmd)]
if (util.is_true(mode) or
(str(mode).lower() == "auto" and cmd[0] and
util.which(cmd[0]))):
return cmd
else:
return []

6
cloudinit/importer.py
View File

@ -36,6 +36,7 @@ def find_module(base_name, search_paths, required_attrs=None):
found_places = [] found_places = []
if not required_attrs: if not required_attrs:
required_attrs = [] required_attrs = []
# NOTE(harlowja): translate the search paths to include the base name.
real_paths = [] real_paths = []
for path in search_paths: for path in search_paths:
real_path = [] real_path = []
@ -50,8 +51,9 @@ def find_module(base_name, search_paths, required_attrs=None):
mod = None mod = None
try: try:
mod = import_module(full_path) mod = import_module(full_path)
except ImportError: except ImportError as e:
pass LOG.debug("Failed at attempted import of '%s' due to: %s",
full_path, e)
if not mod: if not mod:
continue continue
found_attrs = 0 found_attrs = 0

2
cloudinit/util.py
View File

@ -170,6 +170,8 @@ class SeLinuxGuard(object):
def __exit__(self, excp_type, excp_value, excp_traceback): def __exit__(self, excp_type, excp_value, excp_traceback):
if self.selinux and self.selinux.is_selinux_enabled(): if self.selinux and self.selinux.is_selinux_enabled():
path = os.path.realpath(os.path.expanduser(self.path)) path = os.path.realpath(os.path.expanduser(self.path))
# path should be a string, not unicode
path = str(path)
do_restore = False do_restore = False
try: try:
# See if even worth restoring?? # See if even worth restoring??

1
doc/examples/cloud-config-datasources.txt
View File

@ -49,7 +49,6 @@ datasource:
hostname_bounce: hostname_bounce:
interface: eth0 interface: eth0
policy: on # [can be 'on', 'off' or 'force'] policy: on # [can be 'on', 'off' or 'force']
}
SmartOS: SmartOS:
# Smart OS datasource works over a serial console interacting with # Smart OS datasource works over a serial console interacting with

422
doc/examples/cloud-config-disk-setup.txt
View File

@ -1,24 +1,24 @@
Cloud-init supports the creation of simple partition tables and file systems # Cloud-init supports the creation of simple partition tables and file systems
on devices. # on devices.
Default disk definitions for AWS # Default disk definitions for AWS
-------------------------------- # --------------------------------
(Not implemented yet, but provided for future documentation) # (Not implemented yet, but provided for future documentation)
disk_setup: disk_setup:
ephmeral0: ephmeral0:
type: 'mbr' type: 'mbr'
layout: True layout: True
overwrite: False overwrite: False
fs_setup: fs_setup:
- label: None, - label: None,
filesystem: ext3 filesystem: ext3
device: ephemeral0 device: ephemeral0
partition: auto partition: auto
Default disk definitions for Windows Azure # Default disk definitions for Windows Azure
------------------------------------------ # ------------------------------------------
device_aliases: {'ephemeral0': '/dev/sdb'} device_aliases: {'ephemeral0': '/dev/sdb'}
disk_setup: disk_setup:
@ -34,8 +34,8 @@ fs_setup:
replace_fs: ntfs replace_fs: ntfs
Default disk definitions for SmartOS # Default disk definitions for SmartOS
------------------------------------ # ------------------------------------
device_aliases: {'ephemeral0': '/dev/sdb'} device_aliases: {'ephemeral0': '/dev/sdb'}
disk_setup: disk_setup:
@ -49,203 +49,203 @@ fs_setup:
filesystem: ext3 filesystem: ext3
device: ephemeral0.0 device: ephemeral0.0
Cavaut for SmartOS: if ephemeral disk is not defined, then the disk will # Cavaut for SmartOS: if ephemeral disk is not defined, then the disk will
not be automatically added to the mounts. # not be automatically added to the mounts.
The default definition is used to make sure that the ephemeral storage is # The default definition is used to make sure that the ephemeral storage is
setup properly. # setup properly.
"disk_setup": disk partitioning # "disk_setup": disk partitioning
-------------------------------- # --------------------------------
The disk_setup directive instructs Cloud-init to partition a disk. The format is: # The disk_setup directive instructs Cloud-init to partition a disk. The format is:
disk_setup: disk_setup:
ephmeral0: ephmeral0:
type: 'mbr' type: 'mbr'
layout: 'auto' layout: 'auto'
/dev/xvdh: /dev/xvdh:
type: 'mbr' type: 'mbr'
layout: layout:
- 33 - 33
- [33, 82] - [33, 82]
- 33 - 33
overwrite: True overwrite: True
The format is a list of dicts of dicts. The first value is the name of the # The format is a list of dicts of dicts. The first value is the name of the
device and the subsequent values define how to create and layout the partition. # device and the subsequent values define how to create and layout the
# partition.
# The general format is:
# disk_setup:
# <DEVICE>:
# type: 'mbr'
# layout: <LAYOUT|BOOL>
# overwrite: <BOOL>
#
# Where:
# <DEVICE>: The name of the device. 'ephemeralX' and 'swap' are special
# values which are specific to the cloud. For these devices
# Cloud-init will look up what the real devices is and then
# use it.
#
# For other devices, the kernel device name is used. At this
# time only simply kernel devices are supported, meaning
# that device mapper and other targets may not work.
#
# Note: At this time, there is no handling or setup of
# device mapper targets.
#
# type=<TYPE>: Currently the following are supported:
# 'mbr': default and setups a MS-DOS partition table
#
# Note: At this time only 'mbr' partition tables are allowed.
# It is anticipated in the future that we'll have GPT as
# option in the future, or even "RAID" to create a mdadm
# RAID.
#
# layout={...}: The device layout. This is a list of values, with the
# percentage of disk that partition will take.
# Valid options are:
# [<SIZE>, [<SIZE>, <PART_TYPE]]
#
# Where <SIZE> is the _percentage_ of the disk to use, while
# <PART_TYPE> is the numerical value of the partition type.
#
# The following setups two partitions, with the first
# partition having a swap label, taking 1/3 of the disk space
# and the remainder being used as the second partition.
# /dev/xvdh':
# type: 'mbr'
# layout:
# - [33,82]
# - 66
# overwrite: True
#
# When layout is "true" it means single partition the entire
# device.
#
# When layout is "false" it means don't partition or ignore
# existing partitioning.
#
# If layout is set to "true" and overwrite is set to "false",
# it will skip partitioning the device without a failure.
#
# overwrite=<BOOL>: This describes whether to ride with saftey's on and
# everything holstered.
#
# 'false' is the default, which means that:
# 1. The device will be checked for a partition table
# 2. The device will be checked for a file system
# 3. If either a partition of file system is found, then
# the operation will be _skipped_.
#
# 'true' is cowboy mode. There are no checks and things are
# done blindly. USE with caution, you can do things you
# really, really don't want to do.
#
#
# fs_setup: Setup the file system
# -------------------------------
#
# fs_setup describes the how the file systems are supposed to look.
The general format is: fs_setup:
disk_setup: - label: ephemeral0
<DEVICE>: filesystem: 'ext3'
type: 'mbr' device: 'ephemeral0'
layout: <LAYOUT|BOOL> partition: 'auto'
overwrite: <BOOL> - label: mylabl2
filesystem: 'ext4'
device: '/dev/xvda1'
- special:
cmd: mkfs -t %(FILESYSTEM)s -L %(LABEL)s %(DEVICE)s
filesystem: 'btrfs'
device: '/dev/xvdh'
Where: # The general format is:
<DEVICE>: The name of the device. 'ephemeralX' and 'swap' are special # fs_setup:
values which are specific to the cloud. For these devices # - label: <LABEL>
Cloud-init will look up what the real devices is and then # filesystem: <FS_TYPE>
use it. # device: <DEVICE>
# partition: <PART_VALUE>
For other devices, the kernel device name is used. At this # overwrite: <OVERWRITE>
time only simply kernel devices are supported, meaning # replace_fs: <FS_TYPE>
that device mapper and other targets may not work. #
# Where:
Note: At this time, there is no handling or setup of # <LABEL>: The file system label to be used. If set to None, no label is
device mapper targets. # used.
#
type=<TYPE>: Currently the following are supported: # <FS_TYPE>: The file system type. It is assumed that the there
'mbr': default and setups a MS-DOS partition table # will be a "mkfs.<FS_TYPE>" that behaves likes "mkfs". On a standard
# Ubuntu Cloud Image, this means that you have the option of ext{2,3,4},
Note: At this time only 'mbr' partition tables are allowed. # and vfat by default.
It is anticipated in the future that we'll have GPT as #
option in the future, or even "RAID" to create a mdadm # <DEVICE>: The device name. Special names of 'ephemeralX' or 'swap'
RAID. # are allowed and the actual device is acquired from the cloud datasource.
# When using 'ephemeralX' (i.e. ephemeral0), make sure to leave the
layout={...}: The device layout. This is a list of values, with the # label as 'ephemeralX' otherwise there may be issues with the mounting
percentage of disk that partition will take. # of the ephemeral storage layer.
Valid options are: #
[<SIZE>, [<SIZE>, <PART_TYPE]] # If you define the device as 'ephemeralX.Y' then Y will be interpetted
# as a partition value. However, ephermalX.0 is the _same_ as ephemeralX.
Where <SIZE> is the _percentage_ of the disk to use, while #
<PART_TYPE> is the numerical value of the partition type. # <PART_VALUE>:
# Partition definitions are overwriten if you use the '<DEVICE>.Y' notation.
The following setups two partitions, with the first #
partition having a swap label, taking 1/3 of the disk space # The valid options are:
and the remainder being used as the second partition. # "auto|any": tell cloud-init not to care whether there is a partition
/dev/xvdh': # or not. Auto will use the first partition that does not contain a
type: 'mbr' # file system already. In the absence of a partition table, it will
layout: # put it directly on the disk.
- [33,82] #
- 66 # "auto": If a file system that matches the specification in terms of
overwrite: True # label, type and device, then cloud-init will skip the creation of
# the file system.
When layout is "true" it means single partition the entire #
device. # "any": If a file system that matches the file system type and device,
# then cloud-init will skip the creation of the file system.
When layout is "false" it means don't partition or ignore #
existing partitioning. # Devices are selected based on first-detected, starting with partitions
# and then the raw disk. Consider the following:
If layout is set to "true" and overwrite is set to "false", # NAME FSTYPE LABEL
it will skip partitioning the device without a failure. # xvdb
# |-xvdb1 ext4
overwrite=<BOOL>: This describes whether to ride with saftey's on and # |-xvdb2
everything holstered. # |-xvdb3 btrfs test
# \-xvdb4 ext4 test
'false' is the default, which means that: #
1. The device will be checked for a partition table # If you ask for 'auto', label of 'test, and file system of 'ext4'
2. The device will be checked for a file system # then cloud-init will select the 2nd partition, even though there
3. If either a partition of file system is found, then # is a partition match at the 4th partition.
the operation will be _skipped_. #
# If you ask for 'any' and a label of 'test', then cloud-init will
'true' is cowboy mode. There are no checks and things are # select the 1st partition.
done blindly. USE with caution, you can do things you #
really, really don't want to do. # If you ask for 'auto' and don't define label, then cloud-init will
# select the 1st partition.
#
fs_setup: Setup the file system # In general, if you have a specific partition configuration in mind,
------------------------------- # you should define either the device or the partition number. 'auto'
# and 'any' are specifically intended for formating ephemeral storage or
fs_setup describes the how the file systems are supposed to look. # for simple schemes.
#
fs_setup: # "none": Put the file system directly on the device.
- label: ephemeral0 #
filesystem: 'ext3' # <NUM>: where NUM is the actual partition number.
device: 'ephemeral0' #
partition: 'auto' # <OVERWRITE>: Defines whether or not to overwrite any existing
- label: mylabl2 # filesystem.
filesystem: 'ext4' #
device: '/dev/xvda1' # "true": Indiscriminately destroy any pre-existing file system. Use at
- special: # your own peril.
cmd: mkfs -t %(FILESYSTEM)s -L %(LABEL)s %(DEVICE)s #
filesystem: 'btrfs' # "false": If an existing file system exists, skip the creation.
device: '/dev/xvdh' #
# <REPLACE_FS>: This is a special directive, used for Windows Azure that
The general format is: # instructs cloud-init to replace a file system of <FS_TYPE>. NOTE:
fs_setup: # unless you define a label, this requires the use of the 'any' partition
- label: <LABEL> # directive.
filesystem: <FS_TYPE> #
device: <DEVICE> # Behavior Caveat: The default behavior is to _check_ if the file system exists.
partition: <PART_VALUE> # If a file system matches the specification, then the operation is a no-op.
overwrite: <OVERWRITE>
replace_fs: <FS_TYPE>
Where:
<LABEL>: The file system label to be used. If set to None, no label is
used.
<FS_TYPE>: The file system type. It is assumed that the there
will be a "mkfs.<FS_TYPE>" that behaves likes "mkfs". On a standard
Ubuntu Cloud Image, this means that you have the option of ext{2,3,4},
and vfat by default.
<DEVICE>: The device name. Special names of 'ephemeralX' or 'swap'
are allowed and the actual device is acquired from the cloud datasource.
When using 'ephemeralX' (i.e. ephemeral0), make sure to leave the
label as 'ephemeralX' otherwise there may be issues with the mounting
of the ephemeral storage layer.
If you define the device as 'ephemeralX.Y' then Y will be interpetted
as a partition value. However, ephermalX.0 is the _same_ as ephemeralX.
<PART_VALUE>:
Partition definitions are overwriten if you use the '<DEVICE>.Y' notation.
The valid options are:
"auto|any": tell cloud-init not to care whether there is a partition
or not. Auto will use the first partition that does not contain a
file system already. In the absence of a partition table, it will
put it directly on the disk.
"auto": If a file system that matches the specification in terms of
label, type and device, then cloud-init will skip the creation of
the file system.
"any": If a file system that matches the file system type and device,
then cloud-init will skip the creation of the file system.
Devices are selected based on first-detected, starting with partitions
and then the raw disk. Consider the following:
NAME FSTYPE LABEL
xvdb
|-xvdb1 ext4
|-xvdb2
|-xvdb3 btrfs test
\-xvdb4 ext4 test
If you ask for 'auto', label of 'test, and file system of 'ext4'
then cloud-init will select the 2nd partition, even though there
is a partition match at the 4th partition.
If you ask for 'any' and a label of 'test', then cloud-init will
select the 1st partition.
If you ask for 'auto' and don't define label, then cloud-init will
select the 1st partition.
In general, if you have a specific partition configuration in mind,
you should define either the device or the partition number. 'auto'
and 'any' are specifically intended for formating ephemeral storage or
for simple schemes.
"none": Put the file system directly on the device.
<NUM>: where NUM is the actual partition number.
<OVERWRITE>: Defines whether or not to overwrite any existing
filesystem.
"true": Indiscriminately destroy any pre-existing file system. Use at
your own peril.
"false": If an existing file system exists, skip the creation.
<REPLACE_FS>: This is a special directive, used for Windows Azure that
instructs cloud-init to replace a file system of <FS_TYPE>. NOTE:
unless you define a label, this requires the use of the 'any' partition
directive.
Behavior Caveat: The default behavior is to _check_ if the file system exists.
If a file system matches the specification, then the operation is a no-op.

11
doc/examples/cloud-config.txt
View File

@ -74,7 +74,7 @@ apt_preserve_sources_list: true
# 'source' entries in apt-sources that match this python regex # 'source' entries in apt-sources that match this python regex
# expression will be passed to add-apt-repository # expression will be passed to add-apt-repository
add_apt_repo_match = "^[\w-]+:\w" add_apt_repo_match: '^[\w-]+:\w'
apt_sources: apt_sources:
- source: "deb http://ppa.launchpad.net/byobu/ppa/ubuntu karmic main" - source: "deb http://ppa.launchpad.net/byobu/ppa/ubuntu karmic main"
@ -147,8 +147,13 @@ apt_sources:
# '--option=Dpkg::options::=--force-unsafe-io', '--assume-yes', '--quiet'] # '--option=Dpkg::options::=--force-unsafe-io', '--assume-yes', '--quiet']
# #
# apt_get_upgrade_subcommand: # apt_get_upgrade_subcommand:
# Specify a different subcommand for 'upgrade. The default is 'dist-upgrade'. # Specify a different subcommand for 'upgrade. The default is 'dist-upgrade'.
# This is the subcommand that is invoked if package_upgrade is set to true above. # This is the subcommand that is invoked if package_upgrade is set to true above.
#
# apt_get_wrapper:
# command: eatmydata
# enabled: [True, False, "auto"]
#
# Install additional packages on first boot # Install additional packages on first boot
# #

2
packages/bddeb
View File

@ -32,7 +32,7 @@ PKG_MP = {
'boto': 'python-boto', 'boto': 'python-boto',
'cheetah': 'python-cheetah', 'cheetah': 'python-cheetah',
'configobj': 'python-configobj', 'configobj': 'python-configobj',
'jsonpatch': 'python-json-patch', 'jsonpatch': 'python-jsonpatch | python-json-patch',
'oauth': 'python-oauth', 'oauth': 'python-oauth',
'prettytable': 'python-prettytable', 'prettytable': 'python-prettytable',
'pyserial': 'python-serial', 'pyserial': 'python-serial',

1
packages/debian/control.in
View File

@ -25,7 +25,6 @@ Depends: procps,
#end for #end for
python-software-properties | software-properties-common, python-software-properties | software-properties-common,
\${misc:Depends}, \${misc:Depends},
\${python:Depends}
XB-Python-Version: \${python:Versions} XB-Python-Version: \${python:Versions}
Description: Init scripts for cloud instances Description: Init scripts for cloud instances
Cloud instances need special scripts to run during initialisation Cloud instances need special scripts to run during initialisation