x/cloud-init
Code Issues Proposed changes

Merge with upstream sources

Browse Source
This commit is contained in:
Ben Howard 2014-01-09 08:17:41 -07:00
commit 0d9f85fdec
10 changed files with 338 additions and 221 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
ChangeLog
View File

@ -1,5 +1,12 @@
0.7.5: 0.7.5:
- open 0.7.5 - open 0.7.5
- Add a debug log message around import failures
- add a 'debug' module for easily printing out some information about
datasource and cloud-init [Shraddha Pandhe]
- support running apt with 'eatmydata' via configuration token
apt_get_wrapper (LP: #1236531).
- convert paths provided in config-drive 'files' to string before writing
(LP: #1260072).
0.7.4: 0.7.4:
- fix issue mounting 'ephemeral0' if ephemeral0 was an alias for a - fix issue mounting 'ephemeral0' if ephemeral0 was an alias for a
partitioned block device with target filesystem on ephemeral0.1. partitioned block device with target filesystem on ephemeral0.1.

79
cloudinit/config/cc_debug.py Normal file
View File

@ -0,0 +1,79 @@
# vi: ts=4 expandtab
#
# Copyright (C) 2013 Yahoo! Inc.
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License version 3, as
# published by the Free Software Foundation.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
from StringIO import StringIO
from cloudinit import util
from cloudinit import type_utils
import copy
def _make_header(text):
header = StringIO()
header.write("-" * 80)
header.write("\n")
header.write(text.center(80, ' '))
header.write("\n")
header.write("-" * 80)
header.write("\n")
return header.getvalue()
def handle(name, cfg, cloud, log, args):
verbose = util.get_cfg_by_path(cfg, ('debug', 'verbose'), default=True)
if args:
# if args are provided (from cmdline) then explicitly set verbose
out_file = args[0]
verbose = True
else:
out_file = util.get_cfg_by_path(cfg, ('debug', 'output'))
if not verbose:
log.debug(("Skipping module named %s,"
" verbose printing disabled"), name)
return
# Clean out some keys that we just don't care about showing...
dump_cfg = copy.deepcopy(cfg)
for k in ['log_cfgs']:
dump_cfg.pop(k, None)
all_keys = list(dump_cfg.keys())
for k in all_keys:
if k.startswith("_"):
dump_cfg.pop(k, None)
# Now dump it...
to_print = StringIO()
to_print.write(_make_header("Config"))
to_print.write(util.yaml_dumps(dump_cfg))
to_print.write("\n")
to_print.write(_make_header("MetaData"))
to_print.write(util.yaml_dumps(cloud.datasource.metadata))
to_print.write("\n")
to_print.write(_make_header("Misc"))
to_print.write("Datasource: %s\n" %
(type_utils.obj_name(cloud.datasource)))
to_print.write("Distro: %s\n" % (type_utils.obj_name(cloud.distro)))
to_print.write("Hostname: %s\n" % (cloud.get_hostname(True)))
to_print.write("Instance ID: %s\n" % (cloud.get_instance_id()))
to_print.write("Locale: %s\n" % (cloud.get_locale()))
to_print.write("Launch IDX: %s\n" % (cloud.launch_index))
contents = to_print.getvalue()
content_to_file = []
for line in contents.splitlines():
line = "ci-info: %s\n" % (line)
content_to_file.append(line)
if out_file:
util.write_file(out_file, "".join(content_to_file), 0644, "w")
else:
util.multi_log("".join(content_to_file), console=True, stderr=False)

28
cloudinit/distros/debian.py
View File

@ -36,6 +36,10 @@ LOG = logging.getLogger(__name__)
APT_GET_COMMAND = ('apt-get', '--option=Dpkg::Options::=--force-confold', APT_GET_COMMAND = ('apt-get', '--option=Dpkg::Options::=--force-confold',
'--option=Dpkg::options::=--force-unsafe-io', '--option=Dpkg::options::=--force-unsafe-io',
'--assume-yes', '--quiet') '--assume-yes', '--quiet')
APT_GET_WRAPPER = {
'command': 'eatmydata',
'enabled': 'auto',
}
class Distro(distros.Distro): class Distro(distros.Distro):
@ -148,7 +152,13 @@ class Distro(distros.Distro):
# See: http://tiny.cc/kg91fw # See: http://tiny.cc/kg91fw
# Or: http://tiny.cc/mh91fw # Or: http://tiny.cc/mh91fw
e['DEBIAN_FRONTEND'] = 'noninteractive' e['DEBIAN_FRONTEND'] = 'noninteractive'
cmd = list(self.get_option("apt_get_command", APT_GET_COMMAND))
wcfg = self.get_option("apt_get_wrapper", APT_GET_WRAPPER)
cmd = _get_wrapper_prefix(
wcfg.get('command', APT_GET_WRAPPER['command']),
wcfg.get('enabled', APT_GET_WRAPPER['enabled']))
cmd.extend(list(self.get_option("apt_get_command", APT_GET_COMMAND)))
if args and isinstance(args, str): if args and isinstance(args, str):
cmd.append(args) cmd.append(args)
@ -166,7 +176,9 @@ class Distro(distros.Distro):
cmd.extend(pkglist) cmd.extend(pkglist)
# Allow the output of this to flow outwards (ie not be captured) # Allow the output of this to flow outwards (ie not be captured)
util.subp(cmd, env=e, capture=False) util.log_time(logfunc=LOG.debug,
msg="apt-%s [%s]" % (command, ' '.join(cmd)), func=util.subp,
args=(cmd,), kwargs={'env': e, 'capture': False})
def update_package_sources(self): def update_package_sources(self):
self._runner.run("update-sources", self.package_command, self._runner.run("update-sources", self.package_command,
@ -175,3 +187,15 @@ class Distro(distros.Distro):
def get_primary_arch(self): def get_primary_arch(self):
(arch, _err) = util.subp(['dpkg', '--print-architecture']) (arch, _err) = util.subp(['dpkg', '--print-architecture'])
return str(arch).strip() return str(arch).strip()
def _get_wrapper_prefix(cmd, mode):
if isinstance(cmd, str):
cmd = [str(cmd)]
if (util.is_true(mode) or
(str(mode).lower() == "auto" and cmd[0] and
util.which(cmd[0]))):
return cmd
else:
return []

6
cloudinit/importer.py
View File

@ -36,6 +36,7 @@ def find_module(base_name, search_paths, required_attrs=None):
found_places = [] found_places = []
if not required_attrs: if not required_attrs:
required_attrs = [] required_attrs = []
# NOTE(harlowja): translate the search paths to include the base name.
real_paths = [] real_paths = []
for path in search_paths: for path in search_paths:
real_path = [] real_path = []
@ -50,8 +51,9 @@ def find_module(base_name, search_paths, required_attrs=None):
mod = None mod = None
try: try:
mod = import_module(full_path) mod = import_module(full_path)
except ImportError: except ImportError as e:
pass LOG.debug("Failed at attempted import of '%s' due to: %s",
full_path, e)
if not mod: if not mod:
continue continue
found_attrs = 0 found_attrs = 0

2
cloudinit/util.py
View File

@ -170,6 +170,8 @@ class SeLinuxGuard(object):
def __exit__(self, excp_type, excp_value, excp_traceback): def __exit__(self, excp_type, excp_value, excp_traceback):
if self.selinux and self.selinux.is_selinux_enabled(): if self.selinux and self.selinux.is_selinux_enabled():
path = os.path.realpath(os.path.expanduser(self.path)) path = os.path.realpath(os.path.expanduser(self.path))
# path should be a string, not unicode
path = str(path)
do_restore = False do_restore = False
try: try:
# See if even worth restoring?? # See if even worth restoring??

1
doc/examples/cloud-config-datasources.txt
View File

@ -49,7 +49,6 @@ datasource:
hostname_bounce: hostname_bounce:
interface: eth0 interface: eth0
policy: on # [can be 'on', 'off' or 'force'] policy: on # [can be 'on', 'off' or 'force']
}
SmartOS: SmartOS:
# Smart OS datasource works over a serial console interacting with # Smart OS datasource works over a serial console interacting with

356
doc/examples/cloud-config-disk-setup.txt
View File

@ -1,9 +1,9 @@
Cloud-init supports the creation of simple partition tables and file systems # Cloud-init supports the creation of simple partition tables and file systems
on devices. # on devices.
Default disk definitions for AWS # Default disk definitions for AWS
-------------------------------- # --------------------------------
(Not implemented yet, but provided for future documentation) # (Not implemented yet, but provided for future documentation)
disk_setup: disk_setup:
ephmeral0: ephmeral0:
@ -17,8 +17,8 @@ Default disk definitions for AWS
device: ephemeral0 device: ephemeral0
partition: auto partition: auto
Default disk definitions for Windows Azure # Default disk definitions for Windows Azure
------------------------------------------ # ------------------------------------------
device_aliases: {'ephemeral0': '/dev/sdb'} device_aliases: {'ephemeral0': '/dev/sdb'}
disk_setup: disk_setup:
@ -34,8 +34,8 @@ fs_setup:
replace_fs: ntfs replace_fs: ntfs
Default disk definitions for SmartOS # Default disk definitions for SmartOS
------------------------------------ # ------------------------------------
device_aliases: {'ephemeral0': '/dev/sdb'} device_aliases: {'ephemeral0': '/dev/sdb'}
disk_setup: disk_setup:
@ -49,17 +49,17 @@ fs_setup:
filesystem: ext3 filesystem: ext3
device: ephemeral0.0 device: ephemeral0.0
Cavaut for SmartOS: if ephemeral disk is not defined, then the disk will # Cavaut for SmartOS: if ephemeral disk is not defined, then the disk will
not be automatically added to the mounts. # not be automatically added to the mounts.
The default definition is used to make sure that the ephemeral storage is # The default definition is used to make sure that the ephemeral storage is
setup properly. # setup properly.
"disk_setup": disk partitioning # "disk_setup": disk partitioning
-------------------------------- # --------------------------------
The disk_setup directive instructs Cloud-init to partition a disk. The format is: # The disk_setup directive instructs Cloud-init to partition a disk. The format is:
disk_setup: disk_setup:
ephmeral0: ephmeral0:
@ -73,82 +73,82 @@ The disk_setup directive instructs Cloud-init to partition a disk. The format is
- 33 - 33
overwrite: True overwrite: True
The format is a list of dicts of dicts. The first value is the name of the # The format is a list of dicts of dicts. The first value is the name of the
device and the subsequent values define how to create and layout the partition. # device and the subsequent values define how to create and layout the
# partition.
The general format is: # The general format is:
disk_setup: # disk_setup:
<DEVICE>: # <DEVICE>:
type: 'mbr' # type: 'mbr'
layout: <LAYOUT|BOOL> # layout: <LAYOUT|BOOL>
overwrite: <BOOL> # overwrite: <BOOL>
#
Where: # Where:
<DEVICE>: The name of the device. 'ephemeralX' and 'swap' are special # <DEVICE>: The name of the device. 'ephemeralX' and 'swap' are special
values which are specific to the cloud. For these devices # values which are specific to the cloud. For these devices
Cloud-init will look up what the real devices is and then # Cloud-init will look up what the real devices is and then
use it. # use it.
#
For other devices, the kernel device name is used. At this # For other devices, the kernel device name is used. At this
time only simply kernel devices are supported, meaning # time only simply kernel devices are supported, meaning
that device mapper and other targets may not work. # that device mapper and other targets may not work.
#
Note: At this time, there is no handling or setup of # Note: At this time, there is no handling or setup of
device mapper targets. # device mapper targets.
#
type=<TYPE>: Currently the following are supported: # type=<TYPE>: Currently the following are supported:
'mbr': default and setups a MS-DOS partition table # 'mbr': default and setups a MS-DOS partition table
#
Note: At this time only 'mbr' partition tables are allowed. # Note: At this time only 'mbr' partition tables are allowed.
It is anticipated in the future that we'll have GPT as # It is anticipated in the future that we'll have GPT as
option in the future, or even "RAID" to create a mdadm # option in the future, or even "RAID" to create a mdadm
RAID. # RAID.
#
layout={...}: The device layout. This is a list of values, with the # layout={...}: The device layout. This is a list of values, with the
percentage of disk that partition will take. # percentage of disk that partition will take.
Valid options are: # Valid options are:
[<SIZE>, [<SIZE>, <PART_TYPE]] # [<SIZE>, [<SIZE>, <PART_TYPE]]
#
Where <SIZE> is the _percentage_ of the disk to use, while # Where <SIZE> is the _percentage_ of the disk to use, while
<PART_TYPE> is the numerical value of the partition type. # <PART_TYPE> is the numerical value of the partition type.
#
The following setups two partitions, with the first # The following setups two partitions, with the first
partition having a swap label, taking 1/3 of the disk space # partition having a swap label, taking 1/3 of the disk space
and the remainder being used as the second partition. # and the remainder being used as the second partition.
/dev/xvdh': # /dev/xvdh':
type: 'mbr' # type: 'mbr'
layout: # layout:
- [33,82] # - [33,82]
- 66 # - 66
overwrite: True # overwrite: True
#
When layout is "true" it means single partition the entire # When layout is "true" it means single partition the entire
device. # device.
#
When layout is "false" it means don't partition or ignore # When layout is "false" it means don't partition or ignore
existing partitioning. # existing partitioning.
#
If layout is set to "true" and overwrite is set to "false", # If layout is set to "true" and overwrite is set to "false",
it will skip partitioning the device without a failure. # it will skip partitioning the device without a failure.
#
overwrite=<BOOL>: This describes whether to ride with saftey's on and # overwrite=<BOOL>: This describes whether to ride with saftey's on and
everything holstered. # everything holstered.
#
'false' is the default, which means that: # 'false' is the default, which means that:
1. The device will be checked for a partition table # 1. The device will be checked for a partition table
2. The device will be checked for a file system # 2. The device will be checked for a file system
3. If either a partition of file system is found, then # 3. If either a partition of file system is found, then
the operation will be _skipped_. # the operation will be _skipped_.
#
'true' is cowboy mode. There are no checks and things are # 'true' is cowboy mode. There are no checks and things are
done blindly. USE with caution, you can do things you # done blindly. USE with caution, you can do things you
really, really don't want to do. # really, really don't want to do.
#
#
fs_setup: Setup the file system # fs_setup: Setup the file system
------------------------------- # -------------------------------
#
fs_setup describes the how the file systems are supposed to look. # fs_setup describes the how the file systems are supposed to look.
fs_setup: fs_setup:
- label: ephemeral0 - label: ephemeral0
@ -163,89 +163,89 @@ fs_setup describes the how the file systems are supposed to look.
filesystem: 'btrfs' filesystem: 'btrfs'
device: '/dev/xvdh' device: '/dev/xvdh'
The general format is: # The general format is:
fs_setup: # fs_setup:
- label: <LABEL> # - label: <LABEL>
filesystem: <FS_TYPE> # filesystem: <FS_TYPE>
device: <DEVICE> # device: <DEVICE>
partition: <PART_VALUE> # partition: <PART_VALUE>
overwrite: <OVERWRITE> # overwrite: <OVERWRITE>
replace_fs: <FS_TYPE> # replace_fs: <FS_TYPE>
#
Where: # Where:
<LABEL>: The file system label to be used. If set to None, no label is # <LABEL>: The file system label to be used. If set to None, no label is
used. # used.
#
<FS_TYPE>: The file system type. It is assumed that the there # <FS_TYPE>: The file system type. It is assumed that the there
will be a "mkfs.<FS_TYPE>" that behaves likes "mkfs". On a standard # will be a "mkfs.<FS_TYPE>" that behaves likes "mkfs". On a standard
Ubuntu Cloud Image, this means that you have the option of ext{2,3,4}, # Ubuntu Cloud Image, this means that you have the option of ext{2,3,4},
and vfat by default. # and vfat by default.
#
<DEVICE>: The device name. Special names of 'ephemeralX' or 'swap' # <DEVICE>: The device name. Special names of 'ephemeralX' or 'swap'
are allowed and the actual device is acquired from the cloud datasource. # are allowed and the actual device is acquired from the cloud datasource.
When using 'ephemeralX' (i.e. ephemeral0), make sure to leave the # When using 'ephemeralX' (i.e. ephemeral0), make sure to leave the
label as 'ephemeralX' otherwise there may be issues with the mounting # label as 'ephemeralX' otherwise there may be issues with the mounting
of the ephemeral storage layer. # of the ephemeral storage layer.
#
If you define the device as 'ephemeralX.Y' then Y will be interpetted # If you define the device as 'ephemeralX.Y' then Y will be interpetted
as a partition value. However, ephermalX.0 is the _same_ as ephemeralX. # as a partition value. However, ephermalX.0 is the _same_ as ephemeralX.
#
<PART_VALUE>: # <PART_VALUE>:
Partition definitions are overwriten if you use the '<DEVICE>.Y' notation. # Partition definitions are overwriten if you use the '<DEVICE>.Y' notation.
#
The valid options are: # The valid options are:
"auto|any": tell cloud-init not to care whether there is a partition # "auto|any": tell cloud-init not to care whether there is a partition
or not. Auto will use the first partition that does not contain a # or not. Auto will use the first partition that does not contain a
file system already. In the absence of a partition table, it will # file system already. In the absence of a partition table, it will
put it directly on the disk. # put it directly on the disk.
#
"auto": If a file system that matches the specification in terms of # "auto": If a file system that matches the specification in terms of
label, type and device, then cloud-init will skip the creation of # label, type and device, then cloud-init will skip the creation of
the file system. # the file system.
#
"any": If a file system that matches the file system type and device, # "any": If a file system that matches the file system type and device,
then cloud-init will skip the creation of the file system. # then cloud-init will skip the creation of the file system.
#
Devices are selected based on first-detected, starting with partitions # Devices are selected based on first-detected, starting with partitions
and then the raw disk. Consider the following: # and then the raw disk. Consider the following:
NAME FSTYPE LABEL # NAME FSTYPE LABEL
xvdb # xvdb
|-xvdb1 ext4 # |-xvdb1 ext4
|-xvdb2 # |-xvdb2
|-xvdb3 btrfs test # |-xvdb3 btrfs test
\-xvdb4 ext4 test # \-xvdb4 ext4 test
#
If you ask for 'auto', label of 'test, and file system of 'ext4' # If you ask for 'auto', label of 'test, and file system of 'ext4'
then cloud-init will select the 2nd partition, even though there # then cloud-init will select the 2nd partition, even though there
is a partition match at the 4th partition. # is a partition match at the 4th partition.
#
If you ask for 'any' and a label of 'test', then cloud-init will # If you ask for 'any' and a label of 'test', then cloud-init will
select the 1st partition. # select the 1st partition.
#
If you ask for 'auto' and don't define label, then cloud-init will # If you ask for 'auto' and don't define label, then cloud-init will
select the 1st partition. # select the 1st partition.
#
In general, if you have a specific partition configuration in mind, # In general, if you have a specific partition configuration in mind,
you should define either the device or the partition number. 'auto' # you should define either the device or the partition number. 'auto'
and 'any' are specifically intended for formating ephemeral storage or # and 'any' are specifically intended for formating ephemeral storage or
for simple schemes. # for simple schemes.
#
"none": Put the file system directly on the device. # "none": Put the file system directly on the device.
#
<NUM>: where NUM is the actual partition number. # <NUM>: where NUM is the actual partition number.
#
<OVERWRITE>: Defines whether or not to overwrite any existing # <OVERWRITE>: Defines whether or not to overwrite any existing
filesystem. # filesystem.
#
"true": Indiscriminately destroy any pre-existing file system. Use at # "true": Indiscriminately destroy any pre-existing file system. Use at
your own peril. # your own peril.
#
"false": If an existing file system exists, skip the creation. # "false": If an existing file system exists, skip the creation.
#
<REPLACE_FS>: This is a special directive, used for Windows Azure that # <REPLACE_FS>: This is a special directive, used for Windows Azure that
instructs cloud-init to replace a file system of <FS_TYPE>. NOTE: # instructs cloud-init to replace a file system of <FS_TYPE>. NOTE:
unless you define a label, this requires the use of the 'any' partition # unless you define a label, this requires the use of the 'any' partition
directive. # directive.
#
Behavior Caveat: The default behavior is to _check_ if the file system exists. # Behavior Caveat: The default behavior is to _check_ if the file system exists.
If a file system matches the specification, then the operation is a no-op. # If a file system matches the specification, then the operation is a no-op.

7
doc/examples/cloud-config.txt
View File

@ -74,7 +74,7 @@ apt_preserve_sources_list: true
# 'source' entries in apt-sources that match this python regex # 'source' entries in apt-sources that match this python regex
# expression will be passed to add-apt-repository # expression will be passed to add-apt-repository
add_apt_repo_match = "^[\w-]+:\w" add_apt_repo_match: '^[\w-]+:\w'
apt_sources: apt_sources:
- source: "deb http://ppa.launchpad.net/byobu/ppa/ubuntu karmic main" - source: "deb http://ppa.launchpad.net/byobu/ppa/ubuntu karmic main"
@ -149,6 +149,11 @@ apt_sources:
# apt_get_upgrade_subcommand: # apt_get_upgrade_subcommand:
# Specify a different subcommand for 'upgrade. The default is 'dist-upgrade'. # Specify a different subcommand for 'upgrade. The default is 'dist-upgrade'.
# This is the subcommand that is invoked if package_upgrade is set to true above. # This is the subcommand that is invoked if package_upgrade is set to true above.
#
# apt_get_wrapper:
# command: eatmydata
# enabled: [True, False, "auto"]
#
# Install additional packages on first boot # Install additional packages on first boot
# #

2
packages/bddeb
View File

@ -32,7 +32,7 @@ PKG_MP = {
'boto': 'python-boto', 'boto': 'python-boto',
'cheetah': 'python-cheetah', 'cheetah': 'python-cheetah',
'configobj': 'python-configobj', 'configobj': 'python-configobj',
'jsonpatch': 'python-json-patch', 'jsonpatch': 'python-jsonpatch | python-json-patch',
'oauth': 'python-oauth', 'oauth': 'python-oauth',
'prettytable': 'python-prettytable', 'prettytable': 'python-prettytable',
'pyserial': 'python-serial', 'pyserial': 'python-serial',

1
packages/debian/control.in
View File

@ -25,7 +25,6 @@ Depends: procps,
#end for #end for
python-software-properties | software-properties-common, python-software-properties | software-properties-common,
\${misc:Depends}, \${misc:Depends},
\${python:Depends}
XB-Python-Version: \${python:Versions} XB-Python-Version: \${python:Versions}
Description: Init scripts for cloud instances Description: Init scripts for cloud instances
Cloud instances need special scripts to run during initialisation Cloud instances need special scripts to run during initialisation