From ee44efcd4948590ea99b837d3baaccf9589bd177 Mon Sep 17 00:00:00 2001 From: Aleksandr Dobdin Date: Fri, 23 Dec 2016 13:34:10 +0000 Subject: [PATCH] Added: stderr for mapping scripts Change-Id: If40ea86db849f09ffc2e9936483a2dee3776e65e --- timmy/analyze.py | 34 +++++++++++++++----- timmy/analyze_modules/__example__.py | 9 ++++-- timmy/analyze_modules/df.py | 6 ++-- timmy/analyze_modules/rabbitmq.py | 16 +++++++--- timmy/nodes.py | 46 +++++++++++++++++++++------- 5 files changed, 83 insertions(+), 28 deletions(-) diff --git a/timmy/analyze.py b/timmy/analyze.py index a2d342a..c4564bd 100644 --- a/timmy/analyze.py +++ b/timmy/analyze.py @@ -44,7 +44,7 @@ def analyze(node_manager): module.register(fn_mapping) results = {} - for node in node_manager.nodes.values(): + for node in node_manager.sorted_nodes(): if not node.mapscr: node.generate_mapscr() for script, param in node.mapscr.items(): @@ -53,13 +53,28 @@ def analyze(node_manager): logger.warning('File %s does not exist' % param['output_path']) continue - with open(param['output_path'], 'r') as f: - data = [l.rstrip() for l in f.readlines()] - health, details = fn_mapping[script](data, script, node) + try: + with open(param['output_path'], 'r') as f: + stdout = f.read() + if os.path.exists(param['stderr_path']): + with open(param['stderr_path'], 'r') as f: + stderr = f.read() + exitcode = stderr.splitlines()[0] + exitcode = exitcode.rstrip().split()[1] + else: + stderr = None + exitcode = 0 + except IOError as e: + logger.warning('Could not read from %s' % e.filename) + health, details = fn_mapping[script](stdout, script, node, + stderr=stderr, + exitcode=exitcode) if node.repr not in results: results[node.repr] = [] results[node.repr].append({'script': script, 'output_file': param['output_path'], + 'stderr_file': param['stderr_path'], + 'exitcode': exitcode, 'health': health, 'details': details}) node_manager.analyze_results = results @@ -72,7 +87,7 @@ def analyze_print_results(node_manager): RED: ['RED', '\033[91m']} color_end = '\033[0m' print('Nodes health analysis:') - for node, result in node_manager.analyze_results.items(): + for node, result in sorted(node_manager.analyze_results.items()): node_health = max([x['health'] for x in result]) node_color = code_colors[node_health][1] health_repr = code_colors[node_health][0] @@ -85,8 +100,13 @@ def analyze_print_results(node_manager): color = code_colors[r['health']][1] sys.stdout.write(color) health_repr = code_colors[r['health']][0] - print(' %s: %s' % (r['script'], health_repr)) - print(' %s: %s' % ('output_file', r['output_file'])) + print('%s%s: %s' % (8*' ', r['script'], health_repr)) + print('%s%s: %s' % (12*' ', 'output_file', r['output_file'])) + if ((r['stderr_file'] is not None) and + (os.path.exists(r['stderr_file']))): + print('%s%s: %s' % (12*' ', 'stderr_file', r['stderr_file'])) + if r['exitcode'] != 0: + print('%s%s: %s' % (12*' ', 'exit code', r['exitcode'])) if len(r['details']) > 1: print(' details:') for d in r['details']: diff --git a/timmy/analyze_modules/__example__.py b/timmy/analyze_modules/__example__.py index 1ba8e33..4bce9ed 100644 --- a/timmy/analyze_modules/__example__.py +++ b/timmy/analyze_modules/__example__.py @@ -50,12 +50,14 @@ def register(function_mapping): function_mapping['script-basename'] = parsing_function -def parsing_function(data, script, node): +def parsing_function(stdout, script, node, stderr=None, exitcode=None): ''' each analyzing function should have 3 arguments: - data - list of strings aquired by reading the output file + stdout - string aquired by reading the output file script - path to the script file node - node object + stderr - optional string with error file content + exitcode - optional int with script exit code return should contain 2 values: health - set to one of the imported constants according to the analysis @@ -63,7 +65,8 @@ def parsing_function(data, script, node): lines which were indicative of the issue ''' health = UNKNOWN - line = data[0] # in this example we only look at the first line + # in this example we only look at the first line + line = stdout.splitlines()[0] details = [line] if line.find('error'): health = RED diff --git a/timmy/analyze_modules/df.py b/timmy/analyze_modules/df.py index dc579b6..13ff6c2 100644 --- a/timmy/analyze_modules/df.py +++ b/timmy/analyze_modules/df.py @@ -30,12 +30,13 @@ def register(function_mapping): function_mapping['df-i'] = parse_df_i -def parse_df_m(data, script, node): +def parse_df_m(stdout, script, node, stderr=None, exitcode=None): column_use = "Use%" full = 100 near_full = 80 health = GREEN details = [] + data = [l.rstrip() for l in stdout.splitlines()] if column_use not in data[0]: logger.warning(col_msg % (column_use, script, node.repr)) health = UNKNOWN @@ -59,12 +60,13 @@ def parse_df_m(data, script, node): return health, details -def parse_df_i(data, script, node): +def parse_df_i(stdout, script, node, stderr=None, exitcode=None): column_use = "IUse%" full = 100 near_full = 80 health = GREEN details = [] + data = [l.rstrip() for l in stdout.splitlines()] if column_use not in data[0]: logger.warning(col_msg % (column_use, script, node.repr)) health = UNKNOWN diff --git a/timmy/analyze_modules/rabbitmq.py b/timmy/analyze_modules/rabbitmq.py index 08682b1..034a806 100644 --- a/timmy/analyze_modules/rabbitmq.py +++ b/timmy/analyze_modules/rabbitmq.py @@ -30,11 +30,12 @@ def register(function_mapping): function_mapping['rabbitmqctl-status'] = parse_status -def parse_list_queues(data, script, node): +def parse_list_queues(stdout, script, node, stderr=None, exitcode=None): warning = 100 error = 1000 health = GREEN details = [] + data = [l.rstrip() for l in stdout.splitlines()] for line in data[1:]: elements = line.rstrip().split() if len(elements) < 2: @@ -47,8 +48,8 @@ def parse_list_queues(data, script, node): return health, details -def prepare_status(data): - bad_yaml = ''.join(data[1:]) +def prepare_status(stdout): + bad_yaml = ''.join(stdout.splitlines()[1:]) # quoting string elements bad_yaml = re.sub(r'([,{])([a-z_A-Z]+)([,}])', r'\1"\2"\3', bad_yaml) # changing first element int a key - replacing , with : @@ -116,8 +117,13 @@ def squash_dicts(input_data): return input_data -def parse_status(data, script, node): - status = prepare_status(data) +def parse_status(stdout, script, node, stderr=None, exitcode=None): + status = prepare_status(stdout) + if not status or exitcode: + health = RED + details = ['Failed on getting data from status'] + return health, details + health = GREEN details = [] diff --git a/timmy/nodes.py b/timmy/nodes.py index 72cd0d4..cc83d07 100644 --- a/timmy/nodes.py +++ b/timmy/nodes.py @@ -212,12 +212,14 @@ class Node(object): self.logger.debug('%s, exec: %s' % (self.repr, script_path)) output_path = os.path.join(self.scripts_ddir, os.path.basename(script_path)) + stderr_path = "%s.stderr" % output_path if self.outputs_timestamp: output_path += self.outputs_timestamp_str self.logger.debug('outfile: %s' % output_path) mapscr[scr] = {'env_vars': env_vars, 'script_path': script_path, - 'output_path': output_path} + 'output_path': output_path, + 'stderr_path': stderr_path} self.mapscr = mapscr def exec_cmd(self, fake=False, ok_codes=None): @@ -232,6 +234,8 @@ class Node(object): for c in self.cmds: for cmd in c: dfile = os.path.join(ddir, cmd) + errf = '%s.stderr' % dfile + if self.outputs_timestamp: dfile += self.outputs_timestamp_str self.logger.info('outfile: %s' % dfile) @@ -244,13 +248,22 @@ class Node(object): env_vars=self.env_vars, timeout=self.timeout, prefix=self.prefix) - self.check_code(code, 'exec_cmd', c[cmd], errs, ok_codes) + ec = self.check_code(code, 'exec_cmd', + c[cmd], errs, ok_codes) try: with open(dfile, 'w') as df: df.write(outs.encode('utf-8')) - except: + except IOError: self.logger.error("can't write to file %s" % dfile) + if ec: + try: + with open(errf, 'w') as ef: + ef.write('exitcode: %s\n' % code) + ef.write(errs.encode('utf-8')) + except IOError: + self.logger.error("can't write to file %s" % + errf) if self.scripts: self.generate_mapscr() tools.mdir(self.scripts_ddir) @@ -263,14 +276,23 @@ class Node(object): env_vars=param['env_vars'], timeout=self.timeout, prefix=self.prefix) - self.check_code(code, 'exec_cmd', - 'script %s' % param['script_path'], errs, ok_codes) + ec = self.check_code(code, 'exec_cmd', + ('script %s' % param['script_path']), + errs, ok_codes) try: with open(param['output_path'], 'w') as df: df.write(outs.encode('utf-8')) - except: - self.logger.error("can't write to file %s" - % param['output_path']) + except IOError: + self.logger.error("can't write to file %s" % + param['output_path']) + if not ec: + try: + with open(param['stderr_path'], 'w') as ef: + ef.write('exitcode: %s\n' % code) + ef.write(errs.encode('utf-8')) + except IOError: + self.logger.error("can't write to file %s" % + param['stderr_path']) return mapcmds, self.mapscr def exec_simple_cmd(self, cmd, timeout=15, infile=None, outfile=None, @@ -380,7 +402,7 @@ class Node(object): for line in df: if not line.isspace() and line[0] != '#': data += line - except: + except IOError: self.logger.error('could not read file: %s' % fname) self.logger.debug('%s: data:\n%s' % (self.repr, data)) if data: @@ -496,6 +518,8 @@ class Node(object): self.logger.warning("%s: func: %s: " "cmd: '%s' exited %d, error: %s" % (self.repr, func_name, cmd, code, err)) + return False + return True def get_results(self, result_map): # result_map should be either mapcmds or mapscr @@ -798,7 +822,7 @@ class NodeManager(object): return False try: fs = int(outs.rstrip('\n')) - except: + except ValueError: self.logger.error("can't get free space\nouts: %s" % outs) return False @@ -844,7 +868,7 @@ class NodeManager(object): return self.conf['logs_speed_default'] try: speed = int(out) - except: + except ValueError: speed = self.conf['logs_speed_default'] return speed