x/slogging
Code Issues Proposed changes

add changes for swift.source

Browse Source
This commit is contained in:
David Goetz 2012-02-29 19:51:23 -08:00
parent 5cced91575
commit 33846a0f38
4 changed files with 26 additions and 16 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
etc/log-processor.conf-sample
View File

@ -31,6 +31,9 @@ class_path = slogging.access_processor.AccessLogProcessor
# server_name = proxy-server
# user = swift
# warn_percent = 0.8
# list of swift.sources (see swift/proxy/server.py posthooklogger)
# that count as service traffic
# service_log_sources =
[log-processor-stats]
# log_dir = /var/log/swift/

26
slogging/access_processor.py
View File

@ -29,12 +29,11 @@ class AccessLogProcessor(object):
def __init__(self, conf):
self.server_name = conf.get('server_name', 'proxy-server')
self.lb_private_ips = [x.strip() for x in \
conf.get('lb_private_ips', '').split(',')\
if x.strip()]
self.service_ips = [x.strip() for x in \
conf.get('service_ips', '').split(',')\
if x.strip()]
for conf_tag in ['lb_private_ips', 'service_ips',
'service_log_sources']:
setattr(self, conf_tag,
[x.strip() for x in conf.get(conf_tag, '').split(',') \
if x.strip()])
self.warn_percent = float(conf.get('warn_percent', '0.8'))
self.logger = get_logger(conf, log_route='access-processor')
@ -42,6 +41,8 @@ class AccessLogProcessor(object):
'''given a raw access log line, return a dict of the good parts'''
d = {}
try:
log_source = None
split_log = raw_log[16:].split(' ')
(unused,
server,
client_ip,
@ -59,8 +60,9 @@ class AccessLogProcessor(object):
etag,
trans_id,
headers,
processing_time) = (unquote(x) for x in
raw_log[16:].split(' ')[:18])
processing_time) = (unquote(x) for x in split_log[:18])
if len(split_log) > 18:
log_source = split_log[18]
except ValueError:
self.logger.debug(_('Bad line data: %s') % repr(raw_log))
return {}
@ -130,6 +132,7 @@ class AccessLogProcessor(object):
d['bytes_out'] = int(d['bytes_out'].replace('-', '0'))
d['bytes_in'] = int(d['bytes_in'].replace('-', '0'))
d['code'] = int(d['code'])
d['log_source'] = log_source
return d
def process(self, obj_stream, data_object_account, data_object_container,
@ -167,14 +170,13 @@ class AccessLogProcessor(object):
aggr_key = (account, year, month, day, hour)
d = hourly_aggr_info.get(aggr_key, {})
if line_data['lb_ip'] in self.lb_private_ips:
if line_data['lb_ip'] in self.lb_private_ips or \
line_data['client_ip'] in self.service_ips or \
line_data['log_source'] in self.service_log_sources:
source = 'service'
else:
source = 'public'
if line_data['client_ip'] in self.service_ips:
source = 'service'
d[(source, 'bytes_out')] = d.setdefault((
source, 'bytes_out'), 0) + bytes_out
d[(source, 'bytes_in')] = d.setdefault((source, 'bytes_in'), 0) + \

10
test_slogging/unit/test_access_processor.py
View File

@ -43,7 +43,9 @@ class TestAccessProcessor(unittest.TestCase):
for param in access_processor.LISTING_PARAMS:
expected[param] = 1
expected['query'] = query
self.assertEquals(res, expected)
self.assertEquals(res['bytes_in'] , expected['bytes_in'])
self.assertEquals(res['bytes_out'] , expected['bytes_out'])
self.assertEquals(res['query'] , expected['query'])
def test_log_line_parser_field_count(self):
p = access_processor.AccessLogProcessor({})
@ -70,7 +72,8 @@ class TestAccessProcessor(unittest.TestCase):
'bytes_out': 13, 'container_name': 'c', 'day': '1',
'minute': '5', 'account': 'a', 'hour': '4',
'referrer': '9', 'request': '/v1/a/c/o',
'user_agent': '10', 'bytes_in': 12, 'lb_ip': '3'}
'user_agent': '10', 'bytes_in': 12, 'lb_ip': '3',
'log_source': None,}
self.assertEquals(res, expected)
# too many fields
log_line = [str(x) for x in range(19)]
@ -86,7 +89,8 @@ class TestAccessProcessor(unittest.TestCase):
'bytes_out': 13, 'container_name': 'c', 'day': '1',
'minute': '5', 'account': 'a', 'hour': '4',
'referrer': '9', 'request': '/v1/a/c/o',
'user_agent': '10', 'bytes_in': 12, 'lb_ip': '3'}
'user_agent': '10', 'bytes_in': 12, 'lb_ip': '3',
'log_source': '18',}
self.assertEquals(res, expected)

3
test_slogging/unit/test_log_processor.py
View File

@ -164,7 +164,8 @@ use = egg:swift#proxy
'request': '/v1/acct/foo/bar',
'user_agent': 'curl',
'bytes_in': 6,
'lb_ip': '4.5.6.7'})
'lb_ip': '4.5.6.7',
'log_source': None})
def test_process_one_access_file(self):
access_proxy_config = self.proxy_config.copy()