x/timmy
Code Issues Proposed changes

fix: simplification log-file argument

Browse Source
This commit is contained in:
adobdin 2016-05-03 06:35:57 +00:00
commit 668d8b8e71
3 changed files with 62 additions and 131 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

17
config.yaml
View File

@ -14,17 +14,14 @@ soft_filter:
status: ['ready'] status: ['ready']
timeout: 15 timeout: 15
compress_timeout: 3600 compress_timeout: 3600
log_files: log_path: '/var/log'
path: '/var/log' log_filter:
filter: include: 'messages'
default: exclude: '[-_]\d{8}$|atop[-_]|\.gz$'
include: '(.)*'
exclude: '[-_]\d{8}$|atop[-_]|\.gz$'
# by_role: # by_role:
# controller: # contrail:
# env_vars: # log_filter:
# OPENRC: '/root/openrc' # include: 'contrail'
# IPTABLES_STR: 'iptables -S'
# by_node_id: # by_node_id:
# 3: # 3:
# env_vars: # env_vars:

6
timmy/conf.py
View File

@ -20,9 +20,9 @@ class Conf(object):
compress_timeout = 3600 compress_timeout = 3600
archives = '/tmp/timmy/archives' archives = '/tmp/timmy/archives'
cmds_archive = '' cmds_archive = ''
log_files = {} log_path = '/var/log'
log_files['filter'] = {'default': {'include': "(.)*", 'exclude': '[-_]\d{8}$|atop[-_]|\.gz$'}} log_filter = {'include': None,
log_files['path'] = '/var/log/' 'exclude': '[-_]\d{8}$|atop[-_]|\.gz$'}
def __init__(self, **entries): def __init__(self, **entries):
self.__dict__.update(entries) self.__dict__.update(entries)

170
timmy/nodes.py
View File

@ -36,7 +36,8 @@ varlogdir = '/var/log'
class Node(object): class Node(object):
conf_fields = ['ssh_opts', 'env_vars', 'log_files'] override_by_id = ['ssh_opts', 'env_vars', 'log_path', 'log_filter']
aggregate_by_role = ['log_path', 'log_filter']
def __init__(self, node_id, mac, cluster, roles, os_platform, def __init__(self, node_id, mac, cluster, roles, os_platform,
online, status, ip, conf): online, status, ip, conf):
@ -53,25 +54,27 @@ class Node(object):
self.logsize = 0 self.logsize = 0
self.flogs = {} self.flogs = {}
self.mapcmds = {} self.mapcmds = {}
self.logs = {}
self.set_conf(conf) self.set_conf(conf)
def override_conf(self, conf): def override_conf(self, conf):
for field in Node.conf_fields: for field in Node.aggregate_by_role:
for role in self.roles: for role in self.roles:
try: try:
setattr(self, field, conf.by_role[role][field]) getattr(self, field).append(conf.by_role[self.role][field])
except: except:
pass pass
for field in Node.override_by_id:
try: try:
setattr(self, field, conf.by_node_id[self.node_id][field]) setattr(self, field, conf.by_node_id[self.node_id][field])
except: except:
pass pass
def set_conf(self, conf): def set_conf(self, conf):
logging.info(conf.ssh_opts)
self.ssh_opts = conf.ssh_opts self.ssh_opts = conf.ssh_opts
self.env_vars = conf.env_vars self.env_vars = conf.env_vars
self.log_files = conf.log_files self.log_path = list([conf.log_path])
self.log_filter = list([conf.log_filter])
self.timeout = conf.timeout self.timeout = conf.timeout
self.override_conf(conf) self.override_conf(conf)
@ -228,107 +231,39 @@ class Node(object):
logging.debug('node: %s, key: %s, data:\n%s' % logging.debug('node: %s, key: %s, data:\n%s' %
(self.node_id, key, self.data[key])) (self.node_id, key, self.data[key]))
def apply_include_filter(self, lfilter): def logs_filter(self):
logging.info('apply_include_filter: node: %s, filter: %s' % (self.node_id, lfilter)) result = {}
flogs = {} for re_pair in self.log_filter:
if 'include' in lfilter and lfilter['include'] is not None: for f, s in self.logs.items():
for f in self.dulogs.splitlines(): if (('include' not in re_pair or re.search(re_pair['include'], f)) and
try: ('exclude' not in re_pair or not re.search(re_pair['exclude'], f))):
if ('include' in lfilter and re.search(lfilter['include'], f)): result[f] = s
flogs[f.split("\t")[1]] = int(f.split("\t")[0]) self.logs = result
else:
logging.debug("filter %s by %s" % (f, lfilter))
except re.error as e:
logging.error('logs_include_filter: filter: %s, str: %s, re.error: %s' %
(lfilter, f, str(e)))
sys.exit(5)
self.flogs.update(flogs) def logs_populate(self, timeout=5):
return True got_logs = False
else: for path in self.log_path:
return False cmd = ("find '%s' -type f -exec du -b {} +" % (path))
logging.info('logs_populate: node: %s, logs du-cmd: %s' %
def apply_exclude_filter(self, lfilter): (self.node_id, cmd))
logging.info('apply_exclude_filter: node: %s, filter: %s' % (self.node_id, lfilter)) outs, errs, code = ssh_node(ip=self.ip,
rflogs = [] command=cmd,
if 'exclude' in lfilter and lfilter['exclude'] is None: ssh_opts=self.ssh_opts,
return True env_vars='',
if 'exclude' in lfilter and lfilter['exclude'] is not None: timeout=timeout)
for f in self.flogs: if code == 124:
try: logging.error("node: %s, ip: %s, command: %s, "
if re.search(lfilter['exclude'], f): "timeout code: %s, error message: %s" %
rflogs.append(f) (self.node_id, self.ip, cmd, code, errs))
logging.info('logs_exclude_filter: %s' % f) break
except re.error as e: if len(outs):
logging.error('logs_include_filter: filter: %s, str: %s, re.error: %s' % got_logs = True
(lfilter, f, str(e))) for line in outs.split('\n'):
sys.exit(5) if '\t' in line:
for f in rflogs: size, filename = line.split('\t')
logging.debug('exclude_filter: node: %s remove: %s' % (self.node_id, f)) self.logs[filename] = int(size)
self.flogs.pop(f, None) logging.debug('logs_populate: logs: %s' % (self.logs))
return True return got_logs
else:
return False
def logs_filter(self, filterconf):
brstr = 'by_role'
logging.info('logs_filter: node: %s, filter: %s' % (self.node_id, filterconf))
bynodeidinc = False
bynodeidexc = False
# need to check the following logic:
if 'by_node_id' in filterconf and self.node_id in filterconf['by_node_id']:
if self.apply_include_filter(filterconf['by_node_id'][self.node_id]):
bynodeidinc = True
if self.apply_exclude_filter(filterconf['by_node_id'][self.node_id]):
bynodeidexc = True
if bynodeidinc:
return
if bynodeidexc:
return
byrole = False
if brstr in filterconf:
for role in self.roles:
if role in filterconf[brstr].keys():
logging.info('logs_filter: apply filter for role %s' % role)
byrole = True
if self.apply_include_filter(filterconf[brstr][role]):
byrole = True
if not byrole:
if 'default' in filterconf:
self.apply_include_filter(filterconf['default'])
else:
# unexpected
logging.warning('default log filter is not defined')
self.flogs = {}
byrole = False
if brstr in filterconf:
for role in self.roles:
if role in filterconf[brstr].keys():
logging.info('logs_filter: apply filter for role %s' % role)
if self.apply_exclude_filter(filterconf[brstr][role]):
byrole = True
if not byrole:
if 'default' in filterconf:
logging.info('logs_filter: apply default exclude filter')
self.apply_exclude_filter(filterconf['default'])
def log_size_from_find(self, path, sshopts, timeout=5):
cmd = ("find '%s' -type f -exec du -b {} +" % (path))
logging.info('log_size_from_find: node: %s, logs du-cmd: %s' % (self.node_id, cmd))
outs, errs, code = ssh_node(ip=self.ip,
command=cmd,
ssh_opts=self.ssh_opts,
env_vars='',
timeout=timeout)
if code == 124:
logging.error("node: %s, ip: %s, command: %s, "
"timeout code: %s, error message: %s" %
(self.node_id, self.ip, cmd, code, errs))
self.dulogs = ""
return False
self.dulogs = outs
logging.info('log_size_from_find: dulogs: %s' % (self.dulogs))
return True
def print_files(self): def print_files(self):
for k in self.files.keys(): for k in self.files.keys():
@ -561,20 +496,19 @@ class Nodes(object):
lock.unlock() lock.unlock()
def calculate_log_size(self, timeout=15): def calculate_log_size(self, timeout=15):
lsize = 0 total_size = 0
for node in [n for n in self.nodes.values() if self.exec_filter(n)]: for node in [n for n in self.nodes.values() if self.exec_filter(n)]:
if not node.log_size_from_find(self.conf.log_files['path'], 5): if not node.logs_populate(5):
logging.warning("can't get log file list from node %s" % node.node_id) logging.warning("can't get log file list from node %s" %
node.node_id)
else: else:
node.logs_filter(self.conf.log_files['filter']) node.logs_filter()
logging.debug('filter logs: node-%s: filtered logs: %s' % logging.debug('filter logs: node-%s: filtered logs: %s' %
(node.node_id, node.flogs)) (node.node_id, node.logs))
for f in node.flogs: total_size += sum(node.logs.values())
lsize += node.flogs[f] logging.info('Full log size on nodes(with fuel): %s bytes' %
for fl in sorted(node.flogs.items(), key=lambda x: x[1]): total_size)
logging.debug(fl) self.alogsize = total_size / 1024
logging.info('Full log size on nodes(with fuel): %s bytes' % lsize)
self.alogsize = lsize / 1024
def is_enough_space(self, directory, coefficient=1.2): def is_enough_space(self, directory, coefficient=1.2):
mdir(directory) mdir(directory)
@ -637,8 +571,8 @@ class Nodes(object):
txtfl.append(logslistfile) txtfl.append(logslistfile)
try: try:
with open(logslistfile, 'w') as llf: with open(logslistfile, 'w') as llf:
for line in node.flogs: for filename in node.logs:
llf.write(line+"\0") llf.write(filename+"\0")
except: except:
logging.error("create_archive_logs: Can't write to file %s" % logging.error("create_archive_logs: Can't write to file %s" %
logslistfile) logslistfile)