x/timmy
Code Issues Proposed changes

added: log filters: new logic

Browse Source
This commit is contained in:
adobdin 2016-04-15 10:22:59 +00:00
parent 2e91f2cd30
commit 6e8b4d14be
1 changed files with 64 additions and 10 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

72
timmy/nodes.py
View File

@ -205,9 +205,9 @@ class Node(object):
logging.debug('node: %s, key: %s, data:\n%s' % logging.debug('node: %s, key: %s, data:\n%s' %
(self.node_id, key, self.data[key])) (self.node_id, key, self.data[key]))
def logs_filter(self, lfilter): def apply_include_filter(self, lfilter):
logging.info('apply_include_filter: node: %s, filter: %s' % (self.node_id, lfilter))
flogs = {} flogs = {}
logging.info('logs_filter: node: %s, filter: %s' % (self.node_id, lfilter))
if 'include' in lfilter and lfilter['include'] is not None: if 'include' in lfilter and lfilter['include'] is not None:
for f in self.dulogs.splitlines(): for f in self.dulogs.splitlines():
try: try:
@ -221,8 +221,15 @@ class Node(object):
sys.exit(5) sys.exit(5)
self.flogs.update(flogs) self.flogs.update(flogs)
return True
else:
return False
def apply_exclude_filter(self, lfilter):
logging.info('apply_exclude_filter: node: %s, filter: %s' % (self.node_id, lfilter))
rflogs = [] rflogs = []
if 'exclude' in lfilter and lfilter['exclude'] is None:
return True
if 'exclude' in lfilter and lfilter['exclude'] is not None: if 'exclude' in lfilter and lfilter['exclude'] is not None:
for f in self.flogs: for f in self.flogs:
try: try:
@ -234,7 +241,54 @@ class Node(object):
(lfilter, f, str(e))) (lfilter, f, str(e)))
sys.exit(5) sys.exit(5)
for f in rflogs: for f in rflogs:
logging.info('apply_exclude_filter: node: %s remove file: %s from log list' % (self.node_id, f ))
self.flogs.pop(f, None) self.flogs.pop(f, None)
return True
else:
return False
def logs_filter(self, filterconf):
brstr = 'by_role'
flogs = {}
logging.info('logs_filter: node: %s, filter: %s' % (self.node_id, filterconf))
bynodeidinc = False
bynodeidexc = False
# need to check the following logic:
#if 'by_node_id' in filterconf and self.node_id in filterconf['by_node_id']:
# if self.apply_include_filter(filterconf['by_node_id'][self.node_id]):
# bynodeidinc = True
# if self.apply_exclude_filter(filterconf['by_node_id'][self.node_id]):
# bynodeidexc = True
#if bynodeidinc:
# return
#if bynodeidexc:
# return
byrole = False
if brstr in filterconf:
for role in self.roles:
if role in filterconf[brstr].keys():
logging.info('logs_filter: apply filter for role %s' % role)
byrole = True
if self.apply_include_filter(filterconf[brstr][role]):
byrole = True
if not byrole:
if 'default' in filterconf:
self.apply_include_filter(filterconf['default'])
else:
# unexpected
logging.warning('default log filter is not defined')
self.flogs = {}
byrole = False
if brstr in filterconf:
for role in self.roles:
if role in filterconf[brstr].keys():
logging.info('logs_filter: apply filter for role %s' % role)
if self.apply_exclude_filter(filterconf[brstr][role]):
byrole = True
if not byrole:
if 'default' in filterconf:
logging.info('logs_filter: apply default exclude filter')
self.apply_exclude_filter(filterconf[brstr])
def log_size_from_find(self, path, sshopts, timeout=5): def log_size_from_find(self, path, sshopts, timeout=5):
cmd = ("find '%s' -type f -exec du -b {} +" % (path)) cmd = ("find '%s' -type f -exec du -b {} +" % (path))
@ -465,13 +519,13 @@ class Nodes(object):
node.cluster != 0): node.cluster != 0):
continue continue
if node.status in self.conf.soft_filter.status and node.online: if node.status in self.conf.soft_filter.status and node.online:
node.logs_filter(self.conf.log_files['filter']['default']) node.logs_filter(self.conf.log_files['filter'])
for role in node.roles: #for role in node.roles:
if ('by_role' in self.conf.log_files['filter'] and # if ('by_role' in self.conf.log_files['filter'] and
role in self.conf.log_files['filter']['by_role'].keys()): # role in self.conf.log_files['filter']['by_role'].keys()):
logging.info('filter_logs: node:%s apply role: %s' % # logging.info('filter_logs: node:%s apply role: %s' %
(node.node_id, role)) # (node.node_id, role))
node.logs_filter(self.conf.log_files['filter']['by_role'][role]) # node.logs_filter(self.conf.log_files['filter']['by_role'][role])
logging.debug('filter logs: node-%s: filtered logs: %s' % logging.debug('filter logs: node-%s: filtered logs: %s' %
(node.node_id, node.flogs)) (node.node_id, node.flogs))