x/cloud-init
Code Issues Proposed changes

add support for redirecting output of cloud-init, cloud-config and cloud-final

Browse Source
This commit is contained in:
Scott Moser 2011-01-25 02:26:22 +00:00
parent ba6d7276c4
commit 6bccbfb555
8 changed files with 181 additions and 22 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
ChangeLog
View File

@ -27,3 +27,5 @@
this is to ensure they don't get run off the 'get-console-ouptut' buffer
- user_scripts run via cloud-final and thus semaphore renamed from
user_scripts to config_user_scripts
- add support for redirecting output of cloud-init, cloud-config, cloud-final
via the config file, or user data config file

26
cloud-init-cfg.py
View File

@ -35,14 +35,15 @@ def main():
# read cloud config jobs from config (builtin -> system)
# and run all in order
modlist = "cloud_config"
modename = "config"
if len(sys.argv) < 2:
Usage(sys.stderr)
sys.exit(1)
if sys.argv[1] == "all":
name = "all"
if len(sys.argv) > 2:
modlist = sys.argv[2]
modename = sys.argv[2]
else:
freq = None
run_args = []
@ -54,10 +55,6 @@ def main():
if len(sys.argv) > 3:
run_args=sys.argv[3:]
cloudinit.logging_set_from_cfg_file()
log = logging.getLogger()
log.info("cloud-init-cfg %s" % sys.argv[1:])
cfg_path = cloudinit.get_ipath_cur("cloud_config")
cfg_env_name = cloudinit.cfg_env_name
if os.environ.has_key(cfg_env_name):
@ -65,19 +62,30 @@ def main():
cc = CC.CloudConfig(cfg_path)
try:
(outfmt, errfmt) = CC.get_output_cfg(cc.cfg,modename)
CC.redirect_output(outfmt, errfmt)
except Exception, e:
err("Failed to get and set output config: %s\n" % e)
cloudinit.logging_set_from_cfg(cc.cfg)
log = logging.getLogger()
log.info("cloud-init-cfg %s" % sys.argv[1:])
module_list = [ ]
if name == "all":
modlist_cfg_name = "%s_modules" % modlist
modlist_cfg_name = "cloud_%s_modules" % modename
print modlist_cfg_name
module_list = CC.read_cc_modules(cc.cfg,modlist_cfg_name)
if not len(module_list):
err("no modules to run in cloud_config [%s]" % modlist,log)
err("no modules to run in cloud_config [%s]" % modename,log)
sys.exit(0)
else:
module_list.append( [ name, freq ] + run_args )
failures = CC.run_cc_modules(cc,module_list,log)
if len(failures):
err("errors running cloud_config [%s]: %s" % (modlist,failures), log)
err("errors running cloud_config [%s]: %s" % (modename,failures), log)
sys.exit(len(failures))
def err(msg,log=None):

39
cloud-init.py
View File

@ -27,8 +27,8 @@ import time
import logging
import errno
def warn(str):
sys.stderr.write(str)
def warn(wstr):
sys.stderr.write(wstr)
def main():
cmds = ( "start", "start-local" )
@ -40,7 +40,7 @@ def main():
sys.stderr.write("bad command %s. use one of %s\n" % (cmd, cmds))
sys.exit(1)
now = time.strftime("%a, %d %b %Y %H:%M:%S %z")
now = time.strftime("%a, %d %b %Y %H:%M:%S %z",time.gmtime())
try:
uptimef=open("/proc/uptime")
uptime=uptimef.read().split(" ")[0]
@ -49,13 +49,24 @@ def main():
warn("unable to open /proc/uptime\n")
uptime = "na"
source_type = "all"
if cmd == "start-local":
source_type = "local"
try:
cfg = cloudinit.get_base_cfg()
(outfmt, errfmt) = CC.get_output_cfg(cfg,"init")
CC.redirect_output(outfmt, errfmt)
except Exception, e:
warn("Failed to get and set output config: %s\n" % e)
msg = "cloud-init %s running: %s. up %s seconds" % (cmd, now, uptime)
sys.stderr.write(msg + "\n")
sys.stderr.flush()
source_type = "all"
if cmd == "start-local":
source_type = "local"
cloudinit.logging_set_from_cfg(cfg)
log = logging.getLogger()
log.info(msg)
try:
cloudinit.initfs()
@ -63,10 +74,6 @@ def main():
warn("failed to initfs, likely bad things to come: %s\n" % str(e))
cloudinit.logging_set_from_cfg_file()
log = logging.getLogger()
log.info(msg)
# cache is not instance specific, so it has to be purged
# but we want 'start' to benefit from a cache if
# a previous start-local populated one
@ -104,6 +111,18 @@ def main():
cfg_path = cloudinit.get_ipath_cur("cloud_config")
cc = CC.CloudConfig(cfg_path, cloud)
# if the output config changed, update output and err
try:
outfmt_orig = outfmt
errfmt_orig = errfmt
(outfmt, errfmt) = CC.get_output_cfg(cc.cfg,"init")
if outfmt_orig != outfmt or errfmt_orig != errfmt:
warn("stdout, stderr changing to (%s,%s)" % (outfmt,errfmt))
CC.redirect_output(outfmt, errfmt)
except Exception, e:
warn("Failed to get and set output config: %s\n" % e)
module_list = CC.read_cc_modules(cc.cfg,"cloud_init_modules")
failures = []

109
cloudinit/CloudConfig/__init__.py
View File

@ -21,6 +21,8 @@ import cloudinit
import cloudinit.util as util
import sys
import traceback
import os
import subprocess
per_instance="once-per-instance"
per_always="always"
@ -103,3 +105,110 @@ def run_cc_modules(cc,module_list,log):
failures.append(name)
return(failures)
# always returns well formated values
# cfg is expected to have an entry 'output' in it, which is a dictionary
# that includes entries for 'init', 'config', 'final' or 'all'
# init: /var/log/cloud.out
# config: [ ">> /var/log/cloud-config.out", /var/log/cloud-config.err ]
# final:
# output: "| logger -p"
# error: "> /dev/null"
# this returns the specific 'mode' entry, cleanly formatted, with value
# None if if none is given
def get_output_cfg(cfg, mode="init"):
ret = [ None, None ]
if not 'output' in cfg: return ret
outcfg = cfg['output']
if mode in outcfg:
modecfg = outcfg[mode]
else:
if 'all' not in outcfg: return ret
# if there is a 'all' item in the output list
# then it applies to all users of this (init, config, final)
modecfg = outcfg['all']
# if value is a string, it specifies stdout
if isinstance(modecfg,str):
ret = [ modecfg, None ]
# if its a list, then we expect (stdout, stderr)
if isinstance(modecfg,list):
if len(modecfg) > 0: ret[0] = modecfg[0]
if len(modecfg) > 1:
ret[1] = modecfg[1]
# if it is a dictionary, expect 'out' and 'error'
# items, which indicate out and error
if isinstance(modecfg, dict):
if 'output' in modecfg:
ret[0] = modecfg['output']
if 'error' in modecfg:
ret[1] = modecfg['error']
# if err's entry == "&1", then make it same as stdout
# as in shell syntax of "echo foo >/dev/null 2>&1"
if ret[1] == "&1": ret[1] = ret[0]
swlist = [ ">>", ">", "|" ]
for i in range(len(ret)):
if not ret[i]: continue
val = ret[i].lstrip()
found = False
for s in swlist:
if val.startswith(s):
val = "%s %s" % (s,val[len(s):].strip())
found = True
break
if not found:
# default behavior is append
val = "%s %s" % ( ">>", val.strip())
ret[i] = val
return(ret)
# redirect_output(outfmt, errfmt, orig_out, orig_err)
# replace orig_out and orig_err with filehandles specified in outfmt or errfmt
# fmt can be:
# > FILEPATH
# >> FILEPATH
# | program [ arg1 [ arg2 [ ... ] ] ]
#
# with a '|', arguments are passed to shell, so one level of
# shell escape is required.
def redirect_output(outfmt,errfmt, o_out=sys.stdout, o_err=sys.stderr):
if outfmt:
(mode, arg) = outfmt.split(" ",1)
if mode == ">" or mode == ">>":
owith = "ab"
if mode == ">": owith = "wb"
new_fp = open(arg, owith)
elif mode == "|":
proc = subprocess.Popen(arg, shell=True, stdin=subprocess.PIPE)
new_fp = proc.stdin
else:
raise TypeError("invalid type for outfmt: %s" % outfmt)
if o_out:
os.dup2(new_fp.fileno(), o_out.fileno())
if errfmt == outfmt:
os.dup2(new_fp.fileno(), o_err.fileno())
return
if errfmt:
(mode, arg) = errfmt.split(" ",1)
if mode == ">" or mode == ">>":
owith = "ab"
if mode == ">": owith = "wb"
new_fp = open(arg, owith)
elif mode == "|":
proc = subprocess.Popen(arg, shell=True, stdin=subprocess.PIPE)
new_fp = proc.stdin
else:
raise TypeError("invalid type for outfmt: %s" % outfmt)
if o_err:
os.dup2(new_fp.fileno(), o_err.fileno())
return

5
cloudinit/__init__.py
View File

@ -73,7 +73,7 @@ log.addHandler(NullHandler())
def logging_set_from_cfg_file(cfg_file=system_config):
logging_set_from_cfg(util.get_base_cfg(cfg_file,cfg_builtin,parsed_cfgs))
def logging_set_from_cfg(cfg, logfile=None):
def logging_set_from_cfg(cfg):
log_cfgs = []
logcfg=util.get_cfg_option_str(cfg, "log_cfg", False)
if logcfg:
@ -530,5 +530,8 @@ def get_ipath_cur(name=None):
def get_cpath(name=None):
return("%s%s" % (varlibdir, pathmap[name]))
def get_base_cfg():
return(util.get_base_cfg(system_config,cfg_builtin,parsed_cfgs))
class DataSourceNotFoundException(Exception):
pass

18
doc/examples/cloud-config.txt
View File

@ -292,3 +292,21 @@ resize_rootfs: True
# this message is written by cloud-final when the system is finished
# its first boot
final_message: "The system is finally up, after $UPTIME seconds"
# configure where output will go
# 'output' entry is a dict with 'init', 'config', 'final' or 'all'
# entries. Each one defines where
# cloud-init, cloud-config, cloud-config-final or all output will go
# each entry in the dict can be a string, list or dict.
# if it is a string, it refers to stdout
# if it is a list, entry 0 is stdout, entry 1 is stderr
# if it is a dict, it is expected to have 'output' and 'error' fields
# default is to write to console only
# the special entry "&1" for an error means "same location as stdout"
# (Note, that '&1' has meaning in yaml, so it must be quoted)
output:
init: "> /var/log/my-cloud-init.log"
config: [ ">> /tmp/foo.out", "> /tmp/foo.err" ]
final:
output: "| tee /tmp/final.stdout | tee /tmp/bar.stdout"
error: "&1"

2
upstart/cloud-config.conf
View File

@ -5,4 +5,4 @@ start on (filesystem and started rsyslog)
console output
task
exec cloud-init-cfg all cloud_config
exec cloud-init-cfg all config

2
upstart/cloud-final.conf
View File

@ -7,4 +7,4 @@ start on (stopped rc RUNLEVEL=[2345] and stopped cloud-config)
console output
task
exec cloud-init-cfg all cloud_final
exec cloud-init-cfg all final