Many minor changes
+ add quiet mode + add warning loglevel argument + minor arg changes (-F -> -G) + fix put_files bugs + add automatic shell command escaping and fix awk commands after this change + small log message changes + remove use of temporary files for get_logs + implement reliable stdin in ssh_node + add exit PIPESTATUS into get_logs + add function to check if given node attrs are non-empty, used for cli prints + make args optional in RunIttem and SemaphoreProcess + fix stderr utf-8 decode issue (not a fix, more of a workaround, need better fix) + 'inputfile' now unnecessary in ssh_node (not used) but left in there for now + used 'xxd' to encode stdin to keep \0 bytes. Can use base64 instead, no real difference.
This commit is contained in:
f3flight
parent
0d1e00d785
commit
44ac4b893f
@ -1 +1,2 @@
|
||||
source "$OPENRC"
|
||||
nova help | grep 'service-list' &> /dev/null && nova service-list || nova-manage service list
|
||||
|
||||
107
timmy/cli.py
107
timmy/cli.py
@ -24,11 +24,13 @@ from timmy.conf import load_conf
|
||||
from timmy.tools import interrupt_wrapper
|
||||
|
||||
|
||||
def pretty_run(msg, f, args=[], kwargs={}):
|
||||
sys.stdout.write('%s...\r' % msg)
|
||||
sys.stdout.flush()
|
||||
def pretty_run(quiet, msg, f, args=[], kwargs={}):
|
||||
if not quiet:
|
||||
sys.stdout.write('%s...\r' % msg)
|
||||
sys.stdout.flush()
|
||||
result = f(*args, **kwargs)
|
||||
print('%s: done' % msg)
|
||||
if not quiet:
|
||||
print('%s: done' % msg)
|
||||
return result
|
||||
|
||||
|
||||
@ -65,17 +67,20 @@ def main(argv=None):
|
||||
parser.add_argument('--fake-logs',
|
||||
help='Do not collect logs, only calculate size.',
|
||||
action='store_true')
|
||||
parser.add_argument('-d', '--debug',
|
||||
help='Be extremely verbose.',
|
||||
parser.add_argument('-w', '--warning',
|
||||
help='Sets log level to warning (default).',
|
||||
action='store_true')
|
||||
parser.add_argument('-v', '--verbose',
|
||||
help='Be verbose.',
|
||||
action='store_true')
|
||||
parser.add_argument('-d', '--debug',
|
||||
help='Be extremely verbose.',
|
||||
action='store_true')
|
||||
parser.add_argument('-C', '--command',
|
||||
help=('Enables shell mode. Shell command to'
|
||||
' execute. For help on shell mode, read'
|
||||
' timmy/conf.py'))
|
||||
parser.add_argument('-F', '--file', action='append',
|
||||
parser.add_argument('-G', '--get', action='append',
|
||||
help=('Enables shell mode. Can be specified multiple'
|
||||
' times. Filemask to collect via "scp -r".'
|
||||
' Result is placed into a folder specified'
|
||||
@ -93,11 +98,20 @@ def main(argv=None):
|
||||
' accumulating results across runs.'),
|
||||
action='store_true')
|
||||
parser.add_argument('-P', '--put', nargs=2, action='append',
|
||||
help=('Upload filemask via "scp -r" to node(s).'
|
||||
' Each argument must contain two strings -'
|
||||
' source file/path/mask and destination.'))
|
||||
help=('Enables shell mode. Upload filemask via'
|
||||
' "scp -r" to node(s). Each argument must'
|
||||
'contain two strings - source file/path/mask'
|
||||
' and destination.'))
|
||||
parser.add_argument('-q', '--quiet',
|
||||
help=('Print only command execution results and log'
|
||||
' messages. Good for quick runs / "watch" wrap.'
|
||||
' Also sets default loglevel to ERROR.'),
|
||||
action='store_true')
|
||||
args = parser.parse_args(argv[1:])
|
||||
loglevel = logging.WARNING
|
||||
if args.quiet and not args.warning:
|
||||
loglevel = logging.ERROR
|
||||
else:
|
||||
loglevel = logging.WARNING
|
||||
if args.verbose:
|
||||
loglevel = logging.INFO
|
||||
if args.debug:
|
||||
@ -106,7 +120,7 @@ def main(argv=None):
|
||||
level=loglevel,
|
||||
format='%(asctime)s %(levelname)s %(message)s')
|
||||
conf = load_conf(args.conf)
|
||||
if args.command or args.file:
|
||||
if args.command or args.get or args.put:
|
||||
conf['shell_mode'] = True
|
||||
if args.no_clean:
|
||||
conf['clean'] = False
|
||||
@ -118,12 +132,12 @@ def main(argv=None):
|
||||
for k in conf:
|
||||
if k.startswith(Node.conf_match_prefix):
|
||||
conf.pop(k)
|
||||
for src_dst in args.put:
|
||||
conf[Node.pkey].append(src_dst)
|
||||
if args.put:
|
||||
conf[Node.pkey] = args.put
|
||||
if args.command:
|
||||
conf[Node.ckey] = [{'stdout': args.command}]
|
||||
if args.file:
|
||||
conf[Node.fkey] = args.file
|
||||
if args.get:
|
||||
conf[Node.fkey] = args.get
|
||||
else:
|
||||
filter = conf['soft_filter']
|
||||
if args.role:
|
||||
@ -133,50 +147,59 @@ def main(argv=None):
|
||||
main_arc = os.path.join(conf['archives'], 'general.tar.gz')
|
||||
if args.dest_file:
|
||||
main_arc = args.dest_file
|
||||
nm = pretty_run('Initializing node data',
|
||||
nm = pretty_run(args.quiet, 'Initializing node data',
|
||||
NodeManager,
|
||||
kwargs={'conf': conf, 'extended': args.extended})
|
||||
if not args.only_logs:
|
||||
if conf[Node.pkey]:
|
||||
pretty_run('Uploading files', nm.put_files)
|
||||
if not (conf['shell_mode'] and not args.command):
|
||||
pretty_run('Executing commands and scripts', nm.run_commands,
|
||||
args=(conf['outdir'], args.maxthreads))
|
||||
if not (conf['shell_mode'] and not args.file):
|
||||
pretty_run('Collecting files and filelists', nm.get_files,
|
||||
args=(conf['outdir'], args.maxthreads))
|
||||
if not args.no_archive:
|
||||
pretty_run('Creating outputs and files archive',
|
||||
nm.create_archive_general,
|
||||
args=(conf['outdir'], main_arc, 60))
|
||||
if nm.has(Node.pkey):
|
||||
pretty_run(args.quiet, 'Uploading files', nm.put_files)
|
||||
if nm.has(Node.ckey, Node.skey):
|
||||
pretty_run(args.quiet, 'Executing commands and scripts',
|
||||
nm.run_commands, args=(conf['outdir'],
|
||||
args.maxthreads))
|
||||
if nm.has(Node.fkey, Node.flkey):
|
||||
pretty_run(args.quiet, 'Collecting files and filelists',
|
||||
nm.get_files, args=(conf['outdir'], args.maxthreads))
|
||||
if not args.no_archive and nm.has(*Node.conf_archive_general):
|
||||
pretty_run(args.quiet, 'Creating outputs and files archive',
|
||||
nm.create_archive_general, args=(conf['outdir'],
|
||||
main_arc, 60))
|
||||
if args.only_logs or args.getlogs:
|
||||
size = pretty_run('Calculating logs size', nm.calculate_log_size,
|
||||
args=(args.maxthreads,))
|
||||
size = pretty_run(args.quiet, 'Calculating logs size',
|
||||
nm.calculate_log_size, args=(args.maxthreads,))
|
||||
if size == 0:
|
||||
logging.warning('Size zero - no logs to collect.')
|
||||
return
|
||||
enough = pretty_run('Checking free space', nm.is_enough_space,
|
||||
args=(conf['archives'],))
|
||||
enough = pretty_run(args.quiet, 'Checking free space',
|
||||
nm.is_enough_space, args=(conf['archives'],))
|
||||
if enough:
|
||||
pretty_run('Collecting and packing logs', nm.get_logs,
|
||||
pretty_run(args.quiet, 'Collecting and packing logs', nm.get_logs,
|
||||
args=(conf['archives'], conf['compress_timeout']),
|
||||
kwargs={'maxthreads': args.logs_maxthreads,
|
||||
'fake': args.fake_logs})
|
||||
else:
|
||||
logging.warning(('Not enough space for logs in "%s", skipping'
|
||||
'log collection.') %
|
||||
conf['archives'])
|
||||
logging.info("Nodes:\n%s" % nm)
|
||||
print('Run complete. Node information:')
|
||||
print(nm)
|
||||
if not args.quiet:
|
||||
print('Run complete. Node information:')
|
||||
print(nm)
|
||||
if conf['shell_mode']:
|
||||
if args.command:
|
||||
print('Results:')
|
||||
if not args.quiet:
|
||||
print('Results:')
|
||||
for node in nm.nodes.values():
|
||||
for cmd, path in node.mapcmds.items():
|
||||
with open(path, 'r') as f:
|
||||
for line in f.readlines():
|
||||
print('node-%s: %s' % (node.id, line.rstrip('\n')))
|
||||
if args.file:
|
||||
print('Files collected into "%s".' % conf['outdir'])
|
||||
if not args.no_archive:
|
||||
print('Results packed and available in "%s".' % conf['archives'])
|
||||
print('node-%s:\t%s' %
|
||||
(node.id, line.rstrip('\n')))
|
||||
if nm.has(Node.fkey, Node.flkey) and not args.quiet:
|
||||
print('Outputs and files available in "%s".' % conf['outdir'])
|
||||
if all([not args.no_archive, nm.has(*Node.conf_archive_general),
|
||||
not args.quiet]):
|
||||
print('Archives available in "%s".' % conf['archives'])
|
||||
return 0
|
||||
|
||||
if __name__ == '__main__':
|
||||
|
||||
@ -30,6 +30,10 @@ def load_conf(filename):
|
||||
conf['archives'] = os.path.join(gettempdir(), 'timmy', 'archives')
|
||||
conf['cmds_archive'] = ''
|
||||
conf['put'] = []
|
||||
conf['cmds'] = []
|
||||
conf['scripts'] = []
|
||||
conf['files'] = []
|
||||
conf['filelists'] = []
|
||||
conf['logs'] = {'path': '/var/log',
|
||||
'exclude': '[-_]\d{8}$|atop[-_]|\.gz$'}
|
||||
'''Shell mode - only run what was specified via command line.
|
||||
|
||||
@ -39,6 +39,7 @@ class Node(object):
|
||||
pkey = 'put'
|
||||
conf_actionable = [lkey, ckey, skey, fkey, flkey, pkey]
|
||||
conf_appendable = [lkey, ckey, skey, fkey, flkey, pkey]
|
||||
conf_archive_general = [ckey, skey, fkey, flkey]
|
||||
conf_keep_default = [skey, ckey, fkey, flkey]
|
||||
conf_once_prefix = 'once_'
|
||||
conf_match_prefix = 'by_'
|
||||
@ -189,8 +190,8 @@ class Node(object):
|
||||
logging.error("exec_cmd: can't write to file %s" % dfile)
|
||||
return self
|
||||
|
||||
def exec_simple_cmd(self, cmd, infile, outfile, timeout=15,
|
||||
fake=False, ok_codes=None):
|
||||
def exec_simple_cmd(self, cmd, timeout=15, infile=None, outfile=None,
|
||||
fake=False, ok_codes=None, input=None):
|
||||
logging.info('node:%s(%s), exec: %s' % (self.id, self.ip, cmd))
|
||||
if not fake:
|
||||
outs, errs, code = tools.ssh_node(ip=self.ip,
|
||||
@ -199,8 +200,8 @@ class Node(object):
|
||||
env_vars=self.env_vars,
|
||||
timeout=timeout,
|
||||
outputfile=outfile,
|
||||
inputfile=infile,
|
||||
ok_codes=ok_codes)
|
||||
ok_codes=ok_codes,
|
||||
input=input)
|
||||
self.check_code(code, 'exec_simple_cmd', cmd, ok_codes)
|
||||
|
||||
def get_files(self, odir='info', timeout=15):
|
||||
@ -241,7 +242,7 @@ class Node(object):
|
||||
def put_files(self):
|
||||
logging.info('put_files: node: %s, IP: %s' % (self.id, self.ip))
|
||||
for f in self.put:
|
||||
outs, errs, code = tools.get_file_scp(ip=self.ip,
|
||||
outs, errs, code = tools.put_file_scp(ip=self.ip,
|
||||
file=f[0],
|
||||
dest=f[1],
|
||||
recursive=True)
|
||||
@ -453,7 +454,7 @@ class NodeManager(object):
|
||||
self.nodes[node.ip] = node
|
||||
|
||||
def get_version(self):
|
||||
cmd = "awk -F ':' '/release/ {print \$2}' /etc/nailgun/version.yaml"
|
||||
cmd = "awk -F ':' '/release/ {print $2}' /etc/nailgun/version.yaml"
|
||||
fuelnode = self.nodes[self.fuelip]
|
||||
release, err, code = tools.ssh_node(ip=fuelnode.ip,
|
||||
command=cmd,
|
||||
@ -468,7 +469,7 @@ class NodeManager(object):
|
||||
logging.info('release:%s' % (self.version))
|
||||
|
||||
def nodes_get_release(self):
|
||||
cmd = "awk -F ':' '/fuel_version/ {print \$2}' /etc/astute.yaml"
|
||||
cmd = "awk -F ':' '/fuel_version/ {print $2}' /etc/astute.yaml"
|
||||
for node in self.nodes.values():
|
||||
if node.id == 0:
|
||||
# skip master
|
||||
@ -605,7 +606,7 @@ class NodeManager(object):
|
||||
@run_with_lock
|
||||
def get_logs(self, outdir, timeout, fake=False, maxthreads=10, speed=100):
|
||||
if fake:
|
||||
logging.info('archive_logs:skip creating archives(fake:%s)' % fake)
|
||||
logging.info('get_logs: fake = True, skipping' % fake)
|
||||
return
|
||||
txtfl = []
|
||||
speed = self.find_adm_interface_speed(speed)
|
||||
@ -614,31 +615,27 @@ class NodeManager(object):
|
||||
run_items = []
|
||||
for node in [n for n in self.nodes.values() if not n.filtered_out]:
|
||||
if not node.logs_dict():
|
||||
logging.info(("create_archive_logs: node %s - no logs "
|
||||
logging.info(("get_logs: node %s - no logs "
|
||||
"to collect") % node.id)
|
||||
continue
|
||||
node.archivelogsfile = os.path.join(outdir,
|
||||
'logs-node-%s.tar.gz' %
|
||||
str(node.id))
|
||||
tools.mdir(outdir)
|
||||
logslistfile = node.archivelogsfile + '.txt'
|
||||
txtfl.append(logslistfile)
|
||||
try:
|
||||
with open(logslistfile, 'w') as llf:
|
||||
for fn in node.logs_dict():
|
||||
llf.write(fn.lstrip(os.path.abspath(os.sep))+"\0")
|
||||
except:
|
||||
logging.error("create_archive_logs: Can't write to file %s" %
|
||||
logslistfile)
|
||||
continue
|
||||
input = ''
|
||||
for fn in node.logs_dict():
|
||||
input += '%s\0' % fn.lstrip(os.path.abspath(os.sep))
|
||||
with open('test-%s' % node.id, 'w') as fi:
|
||||
fi.write(input)
|
||||
cmd = ("tar --gzip -C %s --create --warning=no-file-changed "
|
||||
" --file - --null --files-from -" % os.path.abspath(os.sep))
|
||||
if not (node.ip == 'localhost' or node.ip.startswith('127.')):
|
||||
cmd = ' '.join([cmd, "| python -c '%s'" % pythonslowpipe])
|
||||
cmd = ' '.join([cmd, "| python -c '%s'; exit ${PIPESTATUS}" %
|
||||
pythonslowpipe])
|
||||
args = {'cmd': cmd,
|
||||
'infile': logslistfile,
|
||||
'outfile': node.archivelogsfile,
|
||||
'timeout': timeout,
|
||||
'outfile': node.archivelogsfile,
|
||||
'input': input,
|
||||
'ok_codes': [0, 1]}
|
||||
run_items.append(tools.RunItem(target=node.exec_simple_cmd,
|
||||
args=args))
|
||||
@ -647,7 +644,7 @@ class NodeManager(object):
|
||||
try:
|
||||
os.remove(tfile)
|
||||
except:
|
||||
logging.error("archive_logs: can't delete file %s" % tfile)
|
||||
logging.error("get_logs: can't delete file %s" % tfile)
|
||||
|
||||
@run_with_lock
|
||||
def get_files(self, odir=Node.fkey, timeout=15):
|
||||
@ -664,6 +661,18 @@ class NodeManager(object):
|
||||
run_items.append(tools.RunItem(target=n.put_files))
|
||||
tools.run_batch(run_items, 10)
|
||||
|
||||
def has(self, *keys):
|
||||
nodes = {}
|
||||
for k in keys:
|
||||
for n in self.nodes.values():
|
||||
if hasattr(n, k):
|
||||
attr = getattr(n, k)
|
||||
if attr:
|
||||
if k not in nodes:
|
||||
nodes[k] = []
|
||||
nodes[k].append(n)
|
||||
return nodes
|
||||
|
||||
|
||||
def main(argv=None):
|
||||
return 0
|
||||
|
||||
@ -28,6 +28,7 @@ import subprocess
|
||||
import yaml
|
||||
from flock import FLock
|
||||
from tempfile import gettempdir
|
||||
from pipes import quote
|
||||
|
||||
|
||||
slowpipe = '''
|
||||
@ -73,19 +74,21 @@ def run_with_lock(f):
|
||||
|
||||
|
||||
class RunItem():
|
||||
def __init__(self, target, args, key=None):
|
||||
def __init__(self, target, args=None, key=None):
|
||||
self.target = target
|
||||
self.args = args
|
||||
self.key = key
|
||||
self.process = None
|
||||
self.queue = None
|
||||
self.key = key
|
||||
|
||||
|
||||
class SemaphoreProcess(Process):
|
||||
def __init__(self, semaphore, target, args, queue=None):
|
||||
def __init__(self, semaphore, target, args=None, queue=None):
|
||||
Process.__init__(self)
|
||||
self.semaphore = semaphore
|
||||
self.target = target
|
||||
if not args:
|
||||
args = {}
|
||||
self.args = args
|
||||
self.queue = queue
|
||||
|
||||
@ -188,12 +191,13 @@ def mdir(directory):
|
||||
|
||||
|
||||
def launch_cmd(cmd, timeout, input=None, ok_codes=None):
|
||||
def _log_msg(cmd, stderr, code, debug=False, stdout=None):
|
||||
message = ('launch_cmd:\n'
|
||||
def _log_msg(cmd, stderr, code, debug=False, stdin=None, stdout=None):
|
||||
message = (u'launch_cmd:\n'
|
||||
'___command: %s\n'
|
||||
'______code: %s\n'
|
||||
'____stderr: %s\n' % (cmd, code, stderr))
|
||||
'____stderr: %s' % (cmd, code, stderr.decode('utf-8')))
|
||||
if debug:
|
||||
message += '\n_____stdin: %s\n' % stdin
|
||||
message += '____stdout: %s' % stdout
|
||||
return message
|
||||
|
||||
@ -228,7 +232,8 @@ def launch_cmd(cmd, timeout, input=None, ok_codes=None):
|
||||
if timeout_killer:
|
||||
timeout_killer.cancel()
|
||||
logging.info(_log_msg(cmd, errs, p.returncode))
|
||||
logging.debug(_log_msg(cmd, errs, p.returncode, debug=True, stdout=outs))
|
||||
logging.debug(_log_msg(cmd, errs, p.returncode, debug=True,
|
||||
stdin=input, stdout=outs))
|
||||
if p.returncode:
|
||||
if not ok_codes or p.returncode not in ok_codes:
|
||||
logging.warning(_log_msg(cmd, errs, p.returncode))
|
||||
@ -237,7 +242,7 @@ def launch_cmd(cmd, timeout, input=None, ok_codes=None):
|
||||
|
||||
def ssh_node(ip, command='', ssh_opts=None, env_vars=None, timeout=15,
|
||||
filename=None, inputfile=None, outputfile=None,
|
||||
ok_codes=None, prefix='nice -n 19 ionice -c 3'):
|
||||
ok_codes=None, input=None, prefix='nice -n 19 ionice -c 3'):
|
||||
if not ssh_opts:
|
||||
ssh_opts = ''
|
||||
if not env_vars:
|
||||
@ -252,21 +257,24 @@ def ssh_node(ip, command='', ssh_opts=None, env_vars=None, timeout=15,
|
||||
env_vars, timeout)
|
||||
else:
|
||||
logging.info("exec ssh")
|
||||
# base cmd str
|
||||
bstr = "timeout '%s' ssh -t -T %s '%s' '%s' " % (
|
||||
timeout, ssh_opts, ip, env_vars)
|
||||
if filename is None:
|
||||
cmd = bstr + '"' + prefix + ' ' + command + '"'
|
||||
cmd = '%s %s' % (bstr, quote(prefix + ' ' + command))
|
||||
if inputfile is not None:
|
||||
'''inputfile and stdin will not work together,
|
||||
give priority to inputfile'''
|
||||
input = None
|
||||
cmd = "%s < '%s'" % (cmd, inputfile)
|
||||
else:
|
||||
cmd = bstr + " '%s bash -s' < '%s'" % (prefix, filename)
|
||||
if inputfile is not None:
|
||||
cmd = bstr + '"' + prefix + " " + command + '" < ' + inputfile
|
||||
cmd = "%s'%s bash -s' < '%s'" % (bstr, prefix, filename)
|
||||
logging.info("ssh_node: inputfile selected, cmd: %s" % cmd)
|
||||
if outputfile is not None:
|
||||
cmd += ' > "' + outputfile + '"'
|
||||
cmd = ("trap 'kill $pid' 15; " +
|
||||
"trap 'kill $pid' 2; " + cmd + '&:; pid=$!; wait $!')
|
||||
return launch_cmd(cmd, timeout, ok_codes=ok_codes)
|
||||
cmd = "%s > '%s'" % (cmd, outputfile)
|
||||
cmd = ("input=\"$(cat | xxd -p)\"; trap 'kill $pid' 15; " +
|
||||
"trap 'kill $pid' 2; echo -n \"$input\" | xxd -r -p | " + cmd +
|
||||
' &:; pid=$!; wait $!')
|
||||
return launch_cmd(cmd, timeout, input=input, ok_codes=ok_codes)
|
||||
|
||||
|
||||
def get_files_rsync(ip, data, ssh_opts, dpath, timeout=15):
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user