x/timmy
Code Issues Proposed changes

node.id unnessesary; improved: output, logging

Browse Source
This commit is contained in:
Aleksandr Dobdin 2016-09-01 13:47:12 +00:00
parent b052cede28
commit 5d362d9452
2 changed files with 59 additions and 48 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
timmy/cli.py
View File

@ -336,9 +336,18 @@ def main(argv=None):
if nm.has(Node.ckey, Node.skey): if nm.has(Node.ckey, Node.skey):
if not args.quiet: if not args.quiet:
print('Results:') print('Results:')
output_dict = {}
maxlength = 0
for node in nm.nodes.values():
name, output = node.get_results(node.mapcmds)
output_dict[node.ip] = {'name': name, 'output': output}
name, output2 = node.get_results(node.mapscr)
output_dict[node.ip]['output'] += output2
maxlength = len(name) if len(name) > maxlength else maxlength
for node in nm.sorted_nodes(): for node in nm.sorted_nodes():
node.print_results(node.mapcmds) for line in output_dict[node.ip]['output']:
node.print_results(node.mapscr) name = output_dict[node.ip]['name'].rjust(maxlength)
print("%s: %s" % (name, line))
if nm.has(Node.ckey, Node.skey, Node.fkey, Node.flkey) and not args.quiet: if nm.has(Node.ckey, Node.skey, Node.fkey, Node.flkey) and not args.quiet:
print('Outputs and/or files available in "%s".' % nm.conf['outdir']) print('Outputs and/or files available in "%s".' % nm.conf['outdir'])
if all([not args.no_archive, nm.has(*Node.conf_archive_general), if all([not args.no_archive, nm.has(*Node.conf_archive_general),

94
timmy/nodes.py
View File

@ -72,7 +72,7 @@ class Node(object):
header = ['node-id', 'env', 'ip', 'mac', 'os', header = ['node-id', 'env', 'ip', 'mac', 'os',
'roles', 'online', 'status', 'name', 'release', 'fqdn'] 'roles', 'online', 'status', 'name', 'release', 'fqdn']
def __init__(self, id, ip, conf, name=None, fqdn=None, mac=None, def __init__(self, ip, conf, id=None, name=None, fqdn=None, mac=None,
cluster=None, roles=None, os_platform=None, cluster=None, roles=None, os_platform=None,
online=True, status="ready", logger=None): online=True, status="ready", logger=None):
self.id = id self.id = id
@ -100,6 +100,10 @@ class Node(object):
self.outputs_timestamp = False self.outputs_timestamp = False
self.outputs_timestamp_dir = None self.outputs_timestamp_dir = None
self.apply_conf(conf) self.apply_conf(conf)
if self.id is not None:
self.repr = "node-%s-%s" % (self.id, self.ip)
else:
self.repr = "node-%s" % self.ip
self.logger = logger or logging.getLogger(__name__) self.logger = logger or logging.getLogger(__name__)
def __str__(self): def __str__(self):
@ -187,28 +191,27 @@ class Node(object):
timeout=self.timeout, timeout=self.timeout,
prefix=self.prefix) prefix=self.prefix)
if code != 0: if code != 0:
self.logger.warning('node: %s: could not determine' self.logger.warning('%s: could not determine'
' MOS release' % self.id) ' MOS release' % self.repr)
release = 'n/a' release = 'n/a'
else: else:
release = release.strip('\n "\'') release = release.strip('\n "\'')
self.logger.info('node: %s, MOS release: %s' % self.logger.info('%s, MOS release: %s' %
(self.id, release)) (self.repr, release))
return release return release
def exec_cmd(self, fake=False, ok_codes=None): def exec_cmd(self, fake=False, ok_codes=None):
sn = 'node-%s' % self.id
cl = 'cluster-%s' % self.cluster cl = 'cluster-%s' % self.cluster
self.logger.debug('%s/%s/%s/%s' % (self.outdir, Node.ckey, cl, sn)) self.logger.debug('%s/%s/%s/%s' %
ddir = os.path.join(self.outdir, Node.ckey, cl, sn) (self.outdir, Node.ckey, cl, self.repr))
if self.cmds: if self.cmds:
ddir = os.path.join(self.outdir, Node.ckey, cl, self.repr)
tools.mdir(ddir) tools.mdir(ddir)
self.cmds = sorted(self.cmds) self.cmds = sorted(self.cmds)
mapcmds = {} mapcmds = {}
for c in self.cmds: for c in self.cmds:
for cmd in c: for cmd in c:
dfile = os.path.join(ddir, 'node-%s-%s-%s' % dfile = os.path.join(ddir, cmd)
(self.id, self.ip, cmd))
if self.outputs_timestamp: if self.outputs_timestamp:
dfile += self.outputs_timestamp_str dfile += self.outputs_timestamp_str
self.logger.info('outfile: %s' % dfile) self.logger.info('outfile: %s' % dfile)
@ -229,8 +232,9 @@ class Node(object):
self.logger.error("can't write to file %s" % self.logger.error("can't write to file %s" %
dfile) dfile)
if self.scripts: if self.scripts:
ddir = os.path.join(self.outdir, Node.skey, cl, self.repr)
tools.mdir(ddir) tools.mdir(ddir)
scripts = sorted(self.scripts) scripts = sorted(self.scripts)
mapscr = {} mapscr = {}
for scr in scripts: for scr in scripts:
if type(scr) is dict: if type(scr) is dict:
@ -242,9 +246,8 @@ class Node(object):
f = scr f = scr
else: else:
f = os.path.join(self.rqdir, Node.skey, scr) f = os.path.join(self.rqdir, Node.skey, scr)
self.logger.debug('node:%s(%s), exec: %s' % (self.id, self.ip, f)) self.logger.debug('%s, exec: %s' % (self.repr, f))
dfile = os.path.join(ddir, 'node-%s-%s-%s' % dfile = os.path.join(ddir, os.path.basename(f))
(self.id, self.ip, os.path.basename(f)))
if self.outputs_timestamp: if self.outputs_timestamp:
dfile += self.outputs_timestamp_str dfile += self.outputs_timestamp_str
self.logger.debug('outfile: %s' % dfile) self.logger.debug('outfile: %s' % dfile)
@ -267,7 +270,7 @@ class Node(object):
def exec_simple_cmd(self, cmd, timeout=15, infile=None, outfile=None, def exec_simple_cmd(self, cmd, timeout=15, infile=None, outfile=None,
fake=False, ok_codes=None, input=None, decode=True): fake=False, ok_codes=None, input=None, decode=True):
self.logger.info('node:%s(%s), exec: %s' % (self.id, self.ip, cmd)) self.logger.info('%s, exec: %s' % (self.repr, cmd))
if not fake: if not fake:
outs, errs, code = tools.ssh_node(ip=self.ip, outs, errs, code = tools.ssh_node(ip=self.ip,
command=cmd, command=cmd,
@ -282,11 +285,10 @@ class Node(object):
self.check_code(code, 'exec_simple_cmd', cmd, errs, ok_codes) self.check_code(code, 'exec_simple_cmd', cmd, errs, ok_codes)
def get_files(self, timeout=15): def get_files(self, timeout=15):
self.logger.info('node: %s, IP: %s' % (self.id, self.ip)) self.logger.info('%s: getting files' % self.repr)
sn = 'node-%s' % self.id
cl = 'cluster-%s' % self.cluster cl = 'cluster-%s' % self.cluster
if self.files or self.filelists: if self.files or self.filelists:
ddir = os.path.join(self.outdir, Node.fkey, cl, sn) ddir = os.path.join(self.outdir, Node.fkey, cl, self.repr)
tools.mdir(ddir) tools.mdir(ddir)
if self.shell_mode: if self.shell_mode:
for f in self.files: for f in self.files:
@ -310,7 +312,7 @@ class Node(object):
except: except:
self.logger.error('could not read file: %s' % fname) self.logger.error('could not read file: %s' % fname)
data += '\n'.join(self.files) data += '\n'.join(self.files)
self.logger.debug('node: %s, data:\n%s' % (self.id, data)) self.logger.debug('%s: data:\n%s' % (self.repr, data))
if data: if data:
o, e, c = tools.get_files_rsync(ip=self.ip, o, e, c = tools.get_files_rsync(ip=self.ip,
data=data, data=data,
@ -320,7 +322,7 @@ class Node(object):
self.check_code(c, 'get_files', 'tools.get_files_rsync', e) self.check_code(c, 'get_files', 'tools.get_files_rsync', e)
def put_files(self): def put_files(self):
self.logger.info('node: %s, IP: %s' % (self.id, self.ip)) self.logger.info('%s: putting files' % self.repr)
for f in self.put: for f in self.put:
outs, errs, code = tools.put_file_scp(ip=self.ip, outs, errs, code = tools.put_file_scp(ip=self.ip,
file=f[0], file=f[0],
@ -380,8 +382,8 @@ class Node(object):
start_param = '' start_param = ''
cmd = ("find '%s' -type f%s -exec du -b {} +" % (item['path'], cmd = ("find '%s' -type f%s -exec du -b {} +" % (item['path'],
start_param)) start_param))
self.logger.info('node: %s, logs du-cmd: %s' % self.logger.info('%s: logs du-cmd: %s' %
(self.id, cmd)) (self.repr, cmd))
outs, errs, code = tools.ssh_node(ip=self.ip, outs, errs, code = tools.ssh_node(ip=self.ip,
command=cmd, command=cmd,
ssh_opts=self.ssh_opts, ssh_opts=self.ssh_opts,
@ -389,9 +391,9 @@ class Node(object):
timeout=timeout, timeout=timeout,
prefix=self.prefix) prefix=self.prefix)
if code == 124: if code == 124:
self.logger.error("node: %s, ip: %s, command: %s, " self.logger.error("%s: command: %s, "
"timeout code: %s, error message: %s" % "timeout code: %s, error message: %s" %
(self.id, self.ip, cmd, code, errs)) (self.repr, cmd, code, errs))
break break
if len(outs): if len(outs):
item['files'] = {} item['files'] = {}
@ -403,8 +405,8 @@ class Node(object):
else: else:
self.logger.debug('log file "%s" excluded' % f) self.logger.debug('log file "%s" excluded' % f)
self.logger.debug('logs: %s' % (item['files'])) self.logger.debug('logs: %s' % (item['files']))
self.logger.info('node: %s, total logs size: %dMB' % self.logger.info('%s: total logs size: %dMB' %
(self.id, (self.repr,
sum(self.logs_dict().values())/1024/1024)) sum(self.logs_dict().values())/1024/1024))
return self.logs return self.logs
@ -422,18 +424,22 @@ class Node(object):
def check_code(self, code, func_name, cmd, err, ok_codes=None): def check_code(self, code, func_name, cmd, err, ok_codes=None):
if code: if code:
if not ok_codes or code not in ok_codes: if not ok_codes or code not in ok_codes:
self.logger.warning("id: %s, fqdn: %s, ip: %s, func: %s, " self.logger.warning("%s: func: %s: "
"cmd: '%s' exited %d, error: %s" % "cmd: '%s' exited %d, error: %s" %
(self.id, self.fqdn, self.ip, (self.repr, func_name, cmd, code, err))
func_name, cmd, code, err))
def print_results(self, result_map): def get_results(self, result_map):
# result_map should be either mapcmds or mapscr # result_map should be either mapcmds or mapscr
if self.id is not None:
short_repr = "node-%s" % self.id
else:
short_repr = self.ip
output = []
for cmd in sorted(result_map): for cmd in sorted(result_map):
with open(result_map[cmd], 'r') as f: with open(result_map[cmd], 'r') as f:
for line in f.readlines(): for line in f.readlines():
print('node-%s:\t%s' % output.append(line.rstrip('\n'))
(self.id, line.rstrip('\n'))) return short_repr, output
class NodeManager(object): class NodeManager(object):
@ -551,7 +557,7 @@ class NodeManager(object):
return '\n'.join(nodestrings) return '\n'.join(nodestrings)
def sorted_nodes(self): def sorted_nodes(self):
s = [n for n in sorted(self.nodes.values(), key=lambda x: x.id)] s = [n for n in sorted(self.nodes.values(), key=lambda x: x.ip)]
return s return s
def import_rq(self): def import_rq(self):
@ -697,8 +703,7 @@ class NodeManager(object):
# set to n/a or may be fuel_version # set to n/a or may be fuel_version
if node.id != 0: if node.id != 0:
node.release = 'n/a' node.release = 'n/a'
self.logger.info('node: %s - release: %s' % (node.id, self.logger.info('%s: release: %s' % (node.repr, node.release))
node.release))
def auth_token(self): def auth_token(self):
'''Get keystone token to access Nailgun API. Requires Fuel 5+''' '''Get keystone token to access Nailgun API. Requires Fuel 5+'''
@ -820,9 +825,8 @@ class NodeManager(object):
roles = str(node_roles).split(', ') roles = str(node_roles).split(', ')
keys = "fqdn name mac os_platform status online ip".split() keys = "fqdn name mac os_platform status online ip".split()
cl = int(node_data['cluster']) if 'cluster' in node_data else None cl = int(node_data['cluster']) if 'cluster' in node_data else None
params = {'id': int(node_data['id']), id = int(node_data['id']) if 'id' in node_data else None
# please do NOT convert cluster id to int type params = {'id': id,
# because None can be valid
'cluster': cl, 'cluster': cl,
'roles': roles, 'roles': roles,
'conf': self.conf} 'conf': self.conf}
@ -848,7 +852,7 @@ class NodeManager(object):
if v == ak: if v == ak:
once_conf = self.conf[k][ak] once_conf = self.conf[k][ak]
node.apply_conf(once_conf, clean=False) node.apply_conf(once_conf, clean=False)
assigned[ak] = node.id assigned[ak] = node.ip
break break
if assigned[ak]: if assigned[ak]:
break break
@ -989,19 +993,17 @@ class NodeManager(object):
run_items = [] run_items = []
for node in [n for n in self.nodes.values() if not n.filtered_out]: for node in [n for n in self.nodes.values() if not n.filtered_out]:
if not node.logs_dict(): if not node.logs_dict():
self.logger.info(("node %s - no logs " self.logger.info(("%s: no logs to collect") % node.repr)
"to collect") % node.id)
continue continue
node.archivelogsfile = os.path.join(self.conf['archive_dir'], node.archivelogsfile = os.path.join(self.conf['archive_dir'],
'logs-node-%s.tar.gz' % 'logs-%s.tar.gz' % node.repr)
str(node.id))
tools.mdir(self.conf['archive_dir']) tools.mdir(self.conf['archive_dir'])
input = '' input = ''
for fn in node.logs_dict(): for fn in node.logs_dict():
input += '%s\0' % fn.lstrip(os.path.abspath(os.sep)) input += '%s\0' % fn.lstrip(os.path.abspath(os.sep))
cmd = ("tar --transform 's,^,node-%d/,' --gzip -C %s --create " cmd = ("tar --transform 's,^,%s/,' --gzip -C %s --create "
"--warning=no-file-changed --file - --null --files-from -" % "--warning=no-file-changed --file - --null --files-from -" %
(node.id, os.path.abspath(os.sep))) (node.repr, os.path.abspath(os.sep)))
if self.conf['logs_speed_limit']: if self.conf['logs_speed_limit']:
if not (node.ip == 'localhost' or node.ip.startswith('127.')): if not (node.ip == 'localhost' or node.ip.startswith('127.')):
cmd = ' '.join([cmd, limitcmd]) cmd = ' '.join([cmd, limitcmd])