x/cloud-init
Code Issues Proposed changes

Added ability to define disks via 'ephemeralX.Y'.

Browse Source 
Modified cc_mounts to identify whether ephermalX is partitioned.
Changed datasources for Azure and SmartOS to use 'ephemeralX.Y' format.
Added disk remove functionally
This commit is contained in:
Ben Howard 2013-10-02 15:05:15 -06:00
parent db0fc22e12
commit 738a3472f1
7 changed files with 241 additions and 37 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

135
cloudinit/config/cc_disk_setup.py
View File

@ -16,8 +16,8 @@
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
from cloudinit.settings import PER_INSTANCE
from cloudinit import util
from cloudinit.settings import PER_INSTANCE
import logging
import shlex
@ -29,13 +29,13 @@ SFDISK_CMD = util.which("sfdisk")
LSBLK_CMD = util.which("lsblk")
BLKID_CMD = util.which("blkid")
BLKDEV_CMD = util.which("blockdev")
WIPEFS_CMD = util.which("wipefs")
LOG = logging.getLogger(__name__)
def handle(_name, cfg, cloud, log, _args):
"""
Call util.prep_disk for disk_setup cloud-config.
See doc/examples/cloud-config_disk-setup.txt for documentation on the
format.
"""
@ -203,8 +203,38 @@ def is_filesystem(device):
return fs_type
def enumerate_disk(device):
"""
Enumerate the elements of a child device. Return a dict of name,
type, fstype, and label
"""
lsblk_cmd = [LSBLK_CMD, '--pairs', '--out', 'NAME,TYPE,FSTYPE,LABEL',
device]
info = None
try:
info, _err = util.subp(lsblk_cmd)
except Exception as e:
raise Exception("Failed during disk check for %s\n%s" % (device, e))
parts = [x for x in (info.strip()).splitlines() if len(x.split()) > 0]
for part in parts:
d = {'name': None,
'type': None,
'fstype': None,
'label': None,
}
for key, value in value_splitter(part):
d[key.lower()] = value
LOG.info(d)
yield d
def find_device_node(device, fs_type=None, label=None, valid_targets=None,
label_match=True):
label_match=True, replace_fs=None):
"""
Find a device that is either matches the spec, or the first
@ -221,26 +251,12 @@ def find_device_node(device, fs_type=None, label=None, valid_targets=None,
if not valid_targets:
valid_targets = ['disk', 'part']
lsblk_cmd = [LSBLK_CMD, '--pairs', '--out', 'NAME,TYPE,FSTYPE,LABEL',
device]
info = None
try:
info, _err = util.subp(lsblk_cmd)
except Exception as e:
raise Exception("Failed during disk check for %s\n%s" % (device, e))
raw_device_used = False
parts = [x for x in (info.strip()).splitlines() if len(x.split()) > 0]
for d in enumerate_disk(device):
for part in parts:
d = {'name': None,
'type': None,
'fstype': None,
'label': None,
}
for key, value in value_splitter(part):
d[key.lower()] = value
if d['fstype'] == replace_fs and label_match == False:
# We found a device where we want to replace the FS
return ('/dev/%s' % d['name'], False)
if (d['fstype'] == fs_type and
((label_match and d['label'] == label) or not label_match)):
@ -454,6 +470,42 @@ def get_partition_mbr_layout(size, layout):
return sfdisk_definition
def purge_disk(device):
"""
Remove parition table entries
"""
# wipe any file systems first
for d in enumerate_disk(device):
LOG.info(d)
if d['type'] not in ["disk", "crypt"]:
wipefs_cmd = [WIPEFS_CMD, "--all", "/dev/%s" % d['name']]
try:
LOG.info("Purging filesystem on /dev/%s" % d['name'])
util.subp(wipefs_cmd)
LOG.info("Purged filesystem on /dev/%s" % d['name'])
except Exception as e:
raise Exception("Failed FS purge of /dev/%s" % d['name'])
dd_cmd = util.which("dd")
last_seek = int(get_hdd_size(device) / 1024) - 2
first_mb = [dd_cmd, "if=/dev/zero", "of=%s" % device, "bs=1M", "count=1"]
last_mb = [dd_cmd, "if=/dev/zero", "of=%s" % device, "bs=1M", "seek=%s" % last_seek]
try:
util.subp(first_mb)
LOG.info("Purged MBR/Partition table from %s" % device)
util.subp(last_mb, rcs=[0,1])
LOG.info("Purged any chance of GPT table from %s" % device)
# Wipe it for good measure
wipefs_cmd = [WIPEFS_CMD, "--all", device]
util.subp(wipefs_cmd)
except Exception as e:
LOG.critical(e)
raise Exception("Failed to remove MBR/Part from %s" % device)
read_parttbl(device)
def get_partition_layout(table_type, size, layout):
"""
@ -542,6 +594,12 @@ def mkpart(device, definition):
if not is_device_valid(device):
raise Exception("Device %s is not a disk device!", device)
# Remove the partition table entries
if isinstance(layout, str) and layout.lower() == "remove":
LOG.debug("Instructed to remove partition table entries")
purge_disk(device)
return
LOG.debug("Checking if device layout matches")
if check_partition_layout(table_type, device, layout):
LOG.debug("Device partitioning layout matches")
@ -565,6 +623,26 @@ def mkpart(device, definition):
LOG.debug("Partition table created for %s", device)
def lookup_force_flag(fs):
"""
A force flag might be -F or -F, this look it up
"""
flags = {'ext': '-F',
'btrfs': '-f',
'xfs': '-f',
'reiserfs': '-f',
}
if 'ext' in fs.lower():
fs = 'ext'
if fs.lower() in flags:
return flags[fs]
LOG.warn("Force flag for %s is unknown." % fs)
return ''
def mkfs(fs_cfg):
"""
Create a file system on the device.
@ -592,6 +670,7 @@ def mkfs(fs_cfg):
fs_type = fs_cfg.get('filesystem')
fs_cmd = fs_cfg.get('cmd', [])
fs_opts = fs_cfg.get('extra_opts', [])
fs_replace = fs_cfg.get('replace_fs', False)
overwrite = fs_cfg.get('overwrite', False)
# This allows you to define the default ephemeral or swap
@ -632,17 +711,23 @@ def mkfs(fs_cfg):
label_match = False
device, reuse = find_device_node(device, fs_type=fs_type, label=label,
label_match=label_match)
label_match=label_match,
replace_fs=fs_replace)
LOG.debug("Automatic device for %s identified as %s", odevice, device)
if reuse:
LOG.debug("Found filesystem match, skipping formating.")
return
if not reuse and fs_replace and device:
LOG.debug("Replacing file system on %s as instructed." % device)
if not device:
LOG.debug("No device aviable that matches request. "
"Skipping fs creation for %s", fs_cfg)
return
elif not partition or str(partition).lower() == 'none':
LOG.debug("Using the raw device to place filesystem %s on" % label)
else:
LOG.debug("Error in device identification handling.")
@ -682,12 +767,16 @@ def mkfs(fs_cfg):
if label:
fs_cmd.extend(["-L", label])
# File systems that support the -F flag
if not fs_cmd and (overwrite or device_type(device) == "disk"):
fs_cmd.append(lookup_force_flag(fs_type))
# Add the extends FS options
if fs_opts:
fs_cmd.extend(fs_opts)
LOG.debug("Creating file system %s on %s", label, device)
LOG.debug(" Using cmd: %s", "".join(fs_cmd))
LOG.debug(" Using cmd: %s", " ".join(fs_cmd))
try:
util.subp(fs_cmd)
except Exception as e:

37
cloudinit/config/cc_mounts.py
View File

@ -20,6 +20,7 @@
from string import whitespace # pylint: disable=W0402
import os.path
import re
from cloudinit import type_utils
@ -75,7 +76,9 @@ def handle(_name, cfg, cloud, log, _args):
"name from ephemeral to ephemeral0"), (i + 1))
if is_mdname(startname):
newname = cloud.device_name_to_device(startname)
candidate_name = cloud.device_name_to_device(startname)
newname = disk_or_part(candidate_name)
if not newname:
log.debug("Ignoring nonexistant named mount %s", startname)
cfgmnt[i][1] = None
@ -119,7 +122,8 @@ def handle(_name, cfg, cloud, log, _args):
# entry has the same device name
for defmnt in defmnts:
startname = defmnt[0]
devname = cloud.device_name_to_device(startname)
candidate_name = cloud.device_name_to_device(startname)
devname = disk_or_part(candidate_name)
if devname is None:
log.debug("Ignoring nonexistant named default mount %s", startname)
continue
@ -198,3 +202,32 @@ def handle(_name, cfg, cloud, log, _args):
util.subp(("mount", "-a"))
except:
util.logexc(log, "Activating mounts via 'mount -a' failed")
def disk_or_part(device):
"""
Find where the file system is on the disk, either on
the disk itself or on the first partition. We don't go
any deeper than partition 1 though.
"""
if not device:
return None
short_name = device.split('/')[-1]
sys_path = "/sys/block/%s" % short_name
if not os.path.exists(sys_path):
LOG.warn("Device %s does not exist in sysfs" % device)
return None
sys_long_path = sys_path + "/" + short_name + "%s"
valid_mappings = [ sys_long_path % "1",
sys_long_path % "p1",
sys_path ]
for cdisk in valid_mappings:
if not os.path.exists(cdisk):
continue
return "/dev/%s" % cdisk.split('/')[-1]
return None

9
cloudinit/sources/DataSourceAzure.py
View File

@ -54,8 +54,9 @@ BUILTIN_CLOUD_CONFIG = {
'layout': True,
'overwrite': False}
},
'fs_setup': [{'filesystem': 'ext4', 'device': 'ephemeral0',
'partition': 'auto'}],
'fs_setup': [{'filesystem': 'ext4',
'device': 'ephemeral0.1',
'replace_fs': 'ntfs'}]
}
DS_CFG_PATH = ['datasource', DS_NAME]
@ -176,7 +177,9 @@ class DataSourceAzureNet(sources.DataSource):
return True
def device_name_to_device(self, name):
return self.ds_cfg['disk_aliases'].get(name)
device = name.split('.')[0]
return util.map_device_alias(self.ds_cfg['disk_aliases'].get(device),
alias=name)
def get_config_obj(self):
return self.cfg

9
cloudinit/sources/DataSourceSmartOS.py
View File

@ -81,8 +81,9 @@ BUILTIN_CLOUD_CONFIG = {
'layout': False,
'overwrite': False}
},
'fs_setup': [{'label': 'ephemeral0', 'filesystem': 'ext3',
'device': 'ephemeral0', 'partition': 'auto'}],
'fs_setup': [{'label': 'ephemeral0',
'filesystem': 'ext3',
'device': 'ephemeral0'}],
}
@ -155,7 +156,9 @@ class DataSourceSmartOS(sources.DataSource):
return True
def device_name_to_device(self, name):
return self.ds_cfg['disk_aliases'].get(name)
device = name.split('.')[0]
return util.map_device_alias(self.ds_cfg['disk_aliases'].get(device),
alias=name)
def get_config_obj(self):
return self.cfg

76
cloudinit/util.py
View File

@ -32,6 +32,7 @@ import grp
import gzip
import hashlib
import os
import os.path
import platform
import pwd
import random
@ -1826,3 +1827,78 @@ def log_time(logfunc, msg, func, args=None, kwargs=None, get_uptime=False):
except:
pass
return ret
def map_partition(alias):
"""
Return partition number for devices like ephemeral0.0 or ephemeral0.1
Parameters:
alaias: the alias, i.e. ephemeral0 or swap0
device: the actual device to markup
Rules:
- anything after a . is a parittion
- device.0 is the same as device
"""
if len(alias.split('.')) == 1:
return None
suffix = alias.split('.')[-1]
try:
if int(suffix) == 0:
return None
return int(suffix)
except ValueError:
pass
return None
def map_device_alias(device, partition=None, alias=None):
"""
Find the name of the partition. While this might seem rather
straight forward, its not since some devices are '<device><partition>'
while others are '<device>p<partition>'. For example, /dev/xvda3 on EC2
will present as /dev/xvda3p1 for the first partition since /dev/xvda3 is
a block device.
The primary use is to map 'ephemeral0.1' in the datasource to a
real device name
"""
if not device:
return None
if not partition and not alias:
raise Exception("partition or alias is required")
if alias:
partition = map_partition(alias)
# if the partition doesn't map, return the device
if not partition:
return device
short_name = device.split('/')[-1]
sys_path = "/sys/block/%s" % short_name
if not os.path.exists(sys_path):
return None
sys_long_path = sys_path + "/" + short_name
valid_mappings = [sys_long_path + "%s" % partition,
sys_long_path + "p%s" % partition]
for cdisk in valid_mappings:
if not os.path.exists(cdisk):
continue
dev_path = "/dev/%s" % cdisk.split('/')[-1]
if os.path.exists(dev_path):
return dev_path
return None

10
doc/examples/cloud-config-disk-setup.txt
View File

@ -30,8 +30,8 @@ disk_setup:
fs_setup:
- label: ephemeral0
filesystem: ext4
device: ephemeral0
partition: auto
device: ephemeral0.1
replace_fs: ntfs
Default disk definitions for SmartOS
@ -47,8 +47,7 @@ disk_setup:
fs_setup:
- label: ephemeral0
filesystem: ext3
device: ephemeral0
partition: auto
device: ephemeral0.0
Cavaut for SmartOS: if ephemeral disk is not defined, then the disk will
not be automatically added to the mounts.
@ -187,6 +186,9 @@ Where:
label as 'ephemeralX' otherwise there may be issues with the mounting
of the ephemeral storage layer.
If you define the device as 'ephemeralX.Y' then Y will be interpetted
as a partition value. However, ephermalX.0 is the _same_ as ephemeralX.
<PART_VALUE>: The valid options are:
"auto|any": tell cloud-init not to care whether there is a partition
or not. Auto will use the first partition that does not contain a

2
tests/unittests/test_datasource/test_azure.py
View File

@ -328,8 +328,6 @@ class TestAzureDataSource(MockerTestCase):
self.assertTrue(ret)
cfg = dsrc.get_config_obj()
self.assertTrue(cfg)
self.assertEquals(dsrc.device_name_to_device("ephemeral0"),
"/dev/sdc")
def test_userdata_arrives(self):
userdata = "This is my user-data"