x/timmy
Code Issues Proposed changes

Many minor changes

Browse Source 
+ add quiet mode
+ add warning loglevel argument
+ minor arg changes (-F -> -G)
+ fix put_files bugs
+ add automatic shell command escaping and fix awk commands after this change
+ small log message changes
+ remove use of temporary files for get_logs
+ implement reliable stdin in ssh_node
+ add exit PIPESTATUS into get_logs
+ add function to check if given node attrs are non-empty, used for cli prints
+ make args optional in RunIttem and SemaphoreProcess
+ fix stderr utf-8 decode issue (not a fix, more of a workaround, need better fix)
+ 'inputfile' now unnecessary in ssh_node (not used) but left in there for now
+ used 'xxd' to encode stdin to keep \0 bytes. Can use base64 instead, no real difference.
This commit is contained in:
f3flight 2016-05-17 04:27:41 +00:00
parent 0d1e00d785
commit 44ac4b893f
5 changed files with 127 additions and 82 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
rq/scripts/nova-service-list
View File

@ -1 +1,2 @@
source "$OPENRC"
nova help | grep 'service-list' &> /dev/null && nova service-list || nova-manage service list nova help | grep 'service-list' &> /dev/null && nova service-list || nova-manage service list

93
timmy/cli.py
View File

@ -24,10 +24,12 @@ from timmy.conf import load_conf
from timmy.tools import interrupt_wrapper from timmy.tools import interrupt_wrapper
def pretty_run(msg, f, args=[], kwargs={}): def pretty_run(quiet, msg, f, args=[], kwargs={}):
if not quiet:
sys.stdout.write('%s...\r' % msg) sys.stdout.write('%s...\r' % msg)
sys.stdout.flush() sys.stdout.flush()
result = f(*args, **kwargs) result = f(*args, **kwargs)
if not quiet:
print('%s: done' % msg) print('%s: done' % msg)
return result return result
@ -65,17 +67,20 @@ def main(argv=None):
parser.add_argument('--fake-logs', parser.add_argument('--fake-logs',
help='Do not collect logs, only calculate size.', help='Do not collect logs, only calculate size.',
action='store_true') action='store_true')
parser.add_argument('-d', '--debug', parser.add_argument('-w', '--warning',
help='Be extremely verbose.', help='Sets log level to warning (default).',
action='store_true') action='store_true')
parser.add_argument('-v', '--verbose', parser.add_argument('-v', '--verbose',
help='Be verbose.', help='Be verbose.',
action='store_true') action='store_true')
parser.add_argument('-d', '--debug',
help='Be extremely verbose.',
action='store_true')
parser.add_argument('-C', '--command', parser.add_argument('-C', '--command',
help=('Enables shell mode. Shell command to' help=('Enables shell mode. Shell command to'
' execute. For help on shell mode, read' ' execute. For help on shell mode, read'
' timmy/conf.py')) ' timmy/conf.py'))
parser.add_argument('-F', '--file', action='append', parser.add_argument('-G', '--get', action='append',
help=('Enables shell mode. Can be specified multiple' help=('Enables shell mode. Can be specified multiple'
' times. Filemask to collect via "scp -r".' ' times. Filemask to collect via "scp -r".'
' Result is placed into a folder specified' ' Result is placed into a folder specified'
@ -93,10 +98,19 @@ def main(argv=None):
' accumulating results across runs.'), ' accumulating results across runs.'),
action='store_true') action='store_true')
parser.add_argument('-P', '--put', nargs=2, action='append', parser.add_argument('-P', '--put', nargs=2, action='append',
help=('Upload filemask via "scp -r" to node(s).' help=('Enables shell mode. Upload filemask via'
' Each argument must contain two strings -' ' "scp -r" to node(s). Each argument must'
' source file/path/mask and destination.')) 'contain two strings - source file/path/mask'
' and destination.'))
parser.add_argument('-q', '--quiet',
help=('Print only command execution results and log'
' messages. Good for quick runs / "watch" wrap.'
' Also sets default loglevel to ERROR.'),
action='store_true')
args = parser.parse_args(argv[1:]) args = parser.parse_args(argv[1:])
if args.quiet and not args.warning:
loglevel = logging.ERROR
else:
loglevel = logging.WARNING loglevel = logging.WARNING
if args.verbose: if args.verbose:
loglevel = logging.INFO loglevel = logging.INFO
@ -106,7 +120,7 @@ def main(argv=None):
level=loglevel, level=loglevel,
format='%(asctime)s %(levelname)s %(message)s') format='%(asctime)s %(levelname)s %(message)s')
conf = load_conf(args.conf) conf = load_conf(args.conf)
if args.command or args.file: if args.command or args.get or args.put:
conf['shell_mode'] = True conf['shell_mode'] = True
if args.no_clean: if args.no_clean:
conf['clean'] = False conf['clean'] = False
@ -118,12 +132,12 @@ def main(argv=None):
for k in conf: for k in conf:
if k.startswith(Node.conf_match_prefix): if k.startswith(Node.conf_match_prefix):
conf.pop(k) conf.pop(k)
for src_dst in args.put: if args.put:
conf[Node.pkey].append(src_dst) conf[Node.pkey] = args.put
if args.command: if args.command:
conf[Node.ckey] = [{'stdout': args.command}] conf[Node.ckey] = [{'stdout': args.command}]
if args.file: if args.get:
conf[Node.fkey] = args.file conf[Node.fkey] = args.get
else: else:
filter = conf['soft_filter'] filter = conf['soft_filter']
if args.role: if args.role:
@ -133,50 +147,59 @@ def main(argv=None):
main_arc = os.path.join(conf['archives'], 'general.tar.gz') main_arc = os.path.join(conf['archives'], 'general.tar.gz')
if args.dest_file: if args.dest_file:
main_arc = args.dest_file main_arc = args.dest_file
nm = pretty_run('Initializing node data', nm = pretty_run(args.quiet, 'Initializing node data',
NodeManager, NodeManager,
kwargs={'conf': conf, 'extended': args.extended}) kwargs={'conf': conf, 'extended': args.extended})
if not args.only_logs: if not args.only_logs:
if conf[Node.pkey]: if nm.has(Node.pkey):
pretty_run('Uploading files', nm.put_files) pretty_run(args.quiet, 'Uploading files', nm.put_files)
if not (conf['shell_mode'] and not args.command): if nm.has(Node.ckey, Node.skey):
pretty_run('Executing commands and scripts', nm.run_commands, pretty_run(args.quiet, 'Executing commands and scripts',
args=(conf['outdir'], args.maxthreads)) nm.run_commands, args=(conf['outdir'],
if not (conf['shell_mode'] and not args.file): args.maxthreads))
pretty_run('Collecting files and filelists', nm.get_files, if nm.has(Node.fkey, Node.flkey):
args=(conf['outdir'], args.maxthreads)) pretty_run(args.quiet, 'Collecting files and filelists',
if not args.no_archive: nm.get_files, args=(conf['outdir'], args.maxthreads))
pretty_run('Creating outputs and files archive', if not args.no_archive and nm.has(*Node.conf_archive_general):
nm.create_archive_general, pretty_run(args.quiet, 'Creating outputs and files archive',
args=(conf['outdir'], main_arc, 60)) nm.create_archive_general, args=(conf['outdir'],
main_arc, 60))
if args.only_logs or args.getlogs: if args.only_logs or args.getlogs:
size = pretty_run('Calculating logs size', nm.calculate_log_size, size = pretty_run(args.quiet, 'Calculating logs size',
args=(args.maxthreads,)) nm.calculate_log_size, args=(args.maxthreads,))
if size == 0: if size == 0:
logging.warning('Size zero - no logs to collect.') logging.warning('Size zero - no logs to collect.')
return return
enough = pretty_run('Checking free space', nm.is_enough_space, enough = pretty_run(args.quiet, 'Checking free space',
args=(conf['archives'],)) nm.is_enough_space, args=(conf['archives'],))
if enough: if enough:
pretty_run('Collecting and packing logs', nm.get_logs, pretty_run(args.quiet, 'Collecting and packing logs', nm.get_logs,
args=(conf['archives'], conf['compress_timeout']), args=(conf['archives'], conf['compress_timeout']),
kwargs={'maxthreads': args.logs_maxthreads, kwargs={'maxthreads': args.logs_maxthreads,
'fake': args.fake_logs}) 'fake': args.fake_logs})
else:
logging.warning(('Not enough space for logs in "%s", skipping'
'log collection.') %
conf['archives'])
logging.info("Nodes:\n%s" % nm) logging.info("Nodes:\n%s" % nm)
if not args.quiet:
print('Run complete. Node information:') print('Run complete. Node information:')
print(nm) print(nm)
if conf['shell_mode']: if conf['shell_mode']:
if args.command: if args.command:
if not args.quiet:
print('Results:') print('Results:')
for node in nm.nodes.values(): for node in nm.nodes.values():
for cmd, path in node.mapcmds.items(): for cmd, path in node.mapcmds.items():
with open(path, 'r') as f: with open(path, 'r') as f:
for line in f.readlines(): for line in f.readlines():
print('node-%s: %s' % (node.id, line.rstrip('\n'))) print('node-%s:\t%s' %
if args.file: (node.id, line.rstrip('\n')))
print('Files collected into "%s".' % conf['outdir']) if nm.has(Node.fkey, Node.flkey) and not args.quiet:
if not args.no_archive: print('Outputs and files available in "%s".' % conf['outdir'])
print('Results packed and available in "%s".' % conf['archives']) if all([not args.no_archive, nm.has(*Node.conf_archive_general),
not args.quiet]):
print('Archives available in "%s".' % conf['archives'])
return 0 return 0
if __name__ == '__main__': if __name__ == '__main__':

4
timmy/conf.py
View File

@ -30,6 +30,10 @@ def load_conf(filename):
conf['archives'] = os.path.join(gettempdir(), 'timmy', 'archives') conf['archives'] = os.path.join(gettempdir(), 'timmy', 'archives')
conf['cmds_archive'] = '' conf['cmds_archive'] = ''
conf['put'] = [] conf['put'] = []
conf['cmds'] = []
conf['scripts'] = []
conf['files'] = []
conf['filelists'] = []
conf['logs'] = {'path': '/var/log', conf['logs'] = {'path': '/var/log',
'exclude': '[-_]\d{8}$|atop[-_]|\.gz$'} 'exclude': '[-_]\d{8}$|atop[-_]|\.gz$'}
'''Shell mode - only run what was specified via command line. '''Shell mode - only run what was specified via command line.

53
timmy/nodes.py
View File

@ -39,6 +39,7 @@ class Node(object):
pkey = 'put' pkey = 'put'
conf_actionable = [lkey, ckey, skey, fkey, flkey, pkey] conf_actionable = [lkey, ckey, skey, fkey, flkey, pkey]
conf_appendable = [lkey, ckey, skey, fkey, flkey, pkey] conf_appendable = [lkey, ckey, skey, fkey, flkey, pkey]
conf_archive_general = [ckey, skey, fkey, flkey]
conf_keep_default = [skey, ckey, fkey, flkey] conf_keep_default = [skey, ckey, fkey, flkey]
conf_once_prefix = 'once_' conf_once_prefix = 'once_'
conf_match_prefix = 'by_' conf_match_prefix = 'by_'
@ -189,8 +190,8 @@ class Node(object):
logging.error("exec_cmd: can't write to file %s" % dfile) logging.error("exec_cmd: can't write to file %s" % dfile)
return self return self
def exec_simple_cmd(self, cmd, infile, outfile, timeout=15, def exec_simple_cmd(self, cmd, timeout=15, infile=None, outfile=None,
fake=False, ok_codes=None): fake=False, ok_codes=None, input=None):
logging.info('node:%s(%s), exec: %s' % (self.id, self.ip, cmd)) logging.info('node:%s(%s), exec: %s' % (self.id, self.ip, cmd))
if not fake: if not fake:
outs, errs, code = tools.ssh_node(ip=self.ip, outs, errs, code = tools.ssh_node(ip=self.ip,
@ -199,8 +200,8 @@ class Node(object):
env_vars=self.env_vars, env_vars=self.env_vars,
timeout=timeout, timeout=timeout,
outputfile=outfile, outputfile=outfile,
inputfile=infile, ok_codes=ok_codes,
ok_codes=ok_codes) input=input)
self.check_code(code, 'exec_simple_cmd', cmd, ok_codes) self.check_code(code, 'exec_simple_cmd', cmd, ok_codes)
def get_files(self, odir='info', timeout=15): def get_files(self, odir='info', timeout=15):
@ -241,7 +242,7 @@ class Node(object):
def put_files(self): def put_files(self):
logging.info('put_files: node: %s, IP: %s' % (self.id, self.ip)) logging.info('put_files: node: %s, IP: %s' % (self.id, self.ip))
for f in self.put: for f in self.put:
outs, errs, code = tools.get_file_scp(ip=self.ip, outs, errs, code = tools.put_file_scp(ip=self.ip,
file=f[0], file=f[0],
dest=f[1], dest=f[1],
recursive=True) recursive=True)
@ -453,7 +454,7 @@ class NodeManager(object):
self.nodes[node.ip] = node self.nodes[node.ip] = node
def get_version(self): def get_version(self):
cmd = "awk -F ':' '/release/ {print \$2}' /etc/nailgun/version.yaml" cmd = "awk -F ':' '/release/ {print $2}' /etc/nailgun/version.yaml"
fuelnode = self.nodes[self.fuelip] fuelnode = self.nodes[self.fuelip]
release, err, code = tools.ssh_node(ip=fuelnode.ip, release, err, code = tools.ssh_node(ip=fuelnode.ip,
command=cmd, command=cmd,
@ -468,7 +469,7 @@ class NodeManager(object):
logging.info('release:%s' % (self.version)) logging.info('release:%s' % (self.version))
def nodes_get_release(self): def nodes_get_release(self):
cmd = "awk -F ':' '/fuel_version/ {print \$2}' /etc/astute.yaml" cmd = "awk -F ':' '/fuel_version/ {print $2}' /etc/astute.yaml"
for node in self.nodes.values(): for node in self.nodes.values():
if node.id == 0: if node.id == 0:
# skip master # skip master
@ -605,7 +606,7 @@ class NodeManager(object):
@run_with_lock @run_with_lock
def get_logs(self, outdir, timeout, fake=False, maxthreads=10, speed=100): def get_logs(self, outdir, timeout, fake=False, maxthreads=10, speed=100):
if fake: if fake:
logging.info('archive_logs:skip creating archives(fake:%s)' % fake) logging.info('get_logs: fake = True, skipping' % fake)
return return
txtfl = [] txtfl = []
speed = self.find_adm_interface_speed(speed) speed = self.find_adm_interface_speed(speed)
@ -614,31 +615,27 @@ class NodeManager(object):
run_items = [] run_items = []
for node in [n for n in self.nodes.values() if not n.filtered_out]: for node in [n for n in self.nodes.values() if not n.filtered_out]:
if not node.logs_dict(): if not node.logs_dict():
logging.info(("create_archive_logs: node %s - no logs " logging.info(("get_logs: node %s - no logs "
"to collect") % node.id) "to collect") % node.id)
continue continue
node.archivelogsfile = os.path.join(outdir, node.archivelogsfile = os.path.join(outdir,
'logs-node-%s.tar.gz' % 'logs-node-%s.tar.gz' %
str(node.id)) str(node.id))
tools.mdir(outdir) tools.mdir(outdir)
logslistfile = node.archivelogsfile + '.txt' input = ''
txtfl.append(logslistfile)
try:
with open(logslistfile, 'w') as llf:
for fn in node.logs_dict(): for fn in node.logs_dict():
llf.write(fn.lstrip(os.path.abspath(os.sep))+"\0") input += '%s\0' % fn.lstrip(os.path.abspath(os.sep))
except: with open('test-%s' % node.id, 'w') as fi:
logging.error("create_archive_logs: Can't write to file %s" % fi.write(input)
logslistfile)
continue
cmd = ("tar --gzip -C %s --create --warning=no-file-changed " cmd = ("tar --gzip -C %s --create --warning=no-file-changed "
" --file - --null --files-from -" % os.path.abspath(os.sep)) " --file - --null --files-from -" % os.path.abspath(os.sep))
if not (node.ip == 'localhost' or node.ip.startswith('127.')): if not (node.ip == 'localhost' or node.ip.startswith('127.')):
cmd = ' '.join([cmd, "| python -c '%s'" % pythonslowpipe]) cmd = ' '.join([cmd, "| python -c '%s'; exit ${PIPESTATUS}" %
pythonslowpipe])
args = {'cmd': cmd, args = {'cmd': cmd,
'infile': logslistfile,
'outfile': node.archivelogsfile,
'timeout': timeout, 'timeout': timeout,
'outfile': node.archivelogsfile,
'input': input,
'ok_codes': [0, 1]} 'ok_codes': [0, 1]}
run_items.append(tools.RunItem(target=node.exec_simple_cmd, run_items.append(tools.RunItem(target=node.exec_simple_cmd,
args=args)) args=args))
@ -647,7 +644,7 @@ class NodeManager(object):
try: try:
os.remove(tfile) os.remove(tfile)
except: except:
logging.error("archive_logs: can't delete file %s" % tfile) logging.error("get_logs: can't delete file %s" % tfile)
@run_with_lock @run_with_lock
def get_files(self, odir=Node.fkey, timeout=15): def get_files(self, odir=Node.fkey, timeout=15):
@ -664,6 +661,18 @@ class NodeManager(object):
run_items.append(tools.RunItem(target=n.put_files)) run_items.append(tools.RunItem(target=n.put_files))
tools.run_batch(run_items, 10) tools.run_batch(run_items, 10)
def has(self, *keys):
nodes = {}
for k in keys:
for n in self.nodes.values():
if hasattr(n, k):
attr = getattr(n, k)
if attr:
if k not in nodes:
nodes[k] = []
nodes[k].append(n)
return nodes
def main(argv=None): def main(argv=None):
return 0 return 0

42
timmy/tools.py
View File

@ -28,6 +28,7 @@ import subprocess
import yaml import yaml
from flock import FLock from flock import FLock
from tempfile import gettempdir from tempfile import gettempdir
from pipes import quote
slowpipe = ''' slowpipe = '''
@ -73,19 +74,21 @@ def run_with_lock(f):
class RunItem(): class RunItem():
def __init__(self, target, args, key=None): def __init__(self, target, args=None, key=None):
self.target = target self.target = target
self.args = args self.args = args
self.key = key
self.process = None self.process = None
self.queue = None self.queue = None
self.key = key
class SemaphoreProcess(Process): class SemaphoreProcess(Process):
def __init__(self, semaphore, target, args, queue=None): def __init__(self, semaphore, target, args=None, queue=None):
Process.__init__(self) Process.__init__(self)
self.semaphore = semaphore self.semaphore = semaphore
self.target = target self.target = target
if not args:
args = {}
self.args = args self.args = args
self.queue = queue self.queue = queue
@ -188,12 +191,13 @@ def mdir(directory):
def launch_cmd(cmd, timeout, input=None, ok_codes=None): def launch_cmd(cmd, timeout, input=None, ok_codes=None):
def _log_msg(cmd, stderr, code, debug=False, stdout=None): def _log_msg(cmd, stderr, code, debug=False, stdin=None, stdout=None):
message = ('launch_cmd:\n' message = (u'launch_cmd:\n'
'___command: %s\n' '___command: %s\n'
'______code: %s\n' '______code: %s\n'
'____stderr: %s\n' % (cmd, code, stderr)) '____stderr: %s' % (cmd, code, stderr.decode('utf-8')))
if debug: if debug:
message += '\n_____stdin: %s\n' % stdin
message += '____stdout: %s' % stdout message += '____stdout: %s' % stdout
return message return message
@ -228,7 +232,8 @@ def launch_cmd(cmd, timeout, input=None, ok_codes=None):
if timeout_killer: if timeout_killer:
timeout_killer.cancel() timeout_killer.cancel()
logging.info(_log_msg(cmd, errs, p.returncode)) logging.info(_log_msg(cmd, errs, p.returncode))
logging.debug(_log_msg(cmd, errs, p.returncode, debug=True, stdout=outs)) logging.debug(_log_msg(cmd, errs, p.returncode, debug=True,
stdin=input, stdout=outs))
if p.returncode: if p.returncode:
if not ok_codes or p.returncode not in ok_codes: if not ok_codes or p.returncode not in ok_codes:
logging.warning(_log_msg(cmd, errs, p.returncode)) logging.warning(_log_msg(cmd, errs, p.returncode))
@ -237,7 +242,7 @@ def launch_cmd(cmd, timeout, input=None, ok_codes=None):
def ssh_node(ip, command='', ssh_opts=None, env_vars=None, timeout=15, def ssh_node(ip, command='', ssh_opts=None, env_vars=None, timeout=15,
filename=None, inputfile=None, outputfile=None, filename=None, inputfile=None, outputfile=None,
ok_codes=None, prefix='nice -n 19 ionice -c 3'): ok_codes=None, input=None, prefix='nice -n 19 ionice -c 3'):
if not ssh_opts: if not ssh_opts:
ssh_opts = '' ssh_opts = ''
if not env_vars: if not env_vars:
@ -252,21 +257,24 @@ def ssh_node(ip, command='', ssh_opts=None, env_vars=None, timeout=15,
env_vars, timeout) env_vars, timeout)
else: else:
logging.info("exec ssh") logging.info("exec ssh")
# base cmd str
bstr = "timeout '%s' ssh -t -T %s '%s' '%s' " % ( bstr = "timeout '%s' ssh -t -T %s '%s' '%s' " % (
timeout, ssh_opts, ip, env_vars) timeout, ssh_opts, ip, env_vars)
if filename is None: if filename is None:
cmd = bstr + '"' + prefix + ' ' + command + '"' cmd = '%s %s' % (bstr, quote(prefix + ' ' + command))
else:
cmd = bstr + " '%s bash -s' < '%s'" % (prefix, filename)
if inputfile is not None: if inputfile is not None:
cmd = bstr + '"' + prefix + " " + command + '" < ' + inputfile '''inputfile and stdin will not work together,
give priority to inputfile'''
input = None
cmd = "%s < '%s'" % (cmd, inputfile)
else:
cmd = "%s'%s bash -s' < '%s'" % (bstr, prefix, filename)
logging.info("ssh_node: inputfile selected, cmd: %s" % cmd) logging.info("ssh_node: inputfile selected, cmd: %s" % cmd)
if outputfile is not None: if outputfile is not None:
cmd += ' > "' + outputfile + '"' cmd = "%s > '%s'" % (cmd, outputfile)
cmd = ("trap 'kill $pid' 15; " + cmd = ("input=\"$(cat | xxd -p)\"; trap 'kill $pid' 15; " +
"trap 'kill $pid' 2; " + cmd + '&:; pid=$!; wait $!') "trap 'kill $pid' 2; echo -n \"$input\" | xxd -r -p | " + cmd +
return launch_cmd(cmd, timeout, ok_codes=ok_codes) ' &:; pid=$!; wait $!')
return launch_cmd(cmd, timeout, input=input, ok_codes=ok_codes)
def get_files_rsync(ip, data, ssh_opts, dpath, timeout=15): def get_files_rsync(ip, data, ssh_opts, dpath, timeout=15):