From 33846a0f386a4f18be3b412166ef7c83774a3630 Mon Sep 17 00:00:00 2001 From: David Goetz Date: Wed, 29 Feb 2012 19:51:23 -0800 Subject: [PATCH] add changes for swift.source --- etc/log-processor.conf-sample | 3 +++ slogging/access_processor.py | 26 +++++++++++---------- test_slogging/unit/test_access_processor.py | 10 +++++--- test_slogging/unit/test_log_processor.py | 3 ++- 4 files changed, 26 insertions(+), 16 deletions(-) diff --git a/etc/log-processor.conf-sample b/etc/log-processor.conf-sample index 6f8b086..716cadc 100644 --- a/etc/log-processor.conf-sample +++ b/etc/log-processor.conf-sample @@ -31,6 +31,9 @@ class_path = slogging.access_processor.AccessLogProcessor # server_name = proxy-server # user = swift # warn_percent = 0.8 +# list of swift.sources (see swift/proxy/server.py posthooklogger) +# that count as service traffic +# service_log_sources = [log-processor-stats] # log_dir = /var/log/swift/ diff --git a/slogging/access_processor.py b/slogging/access_processor.py index 897c238..13fddfb 100644 --- a/slogging/access_processor.py +++ b/slogging/access_processor.py @@ -29,12 +29,11 @@ class AccessLogProcessor(object): def __init__(self, conf): self.server_name = conf.get('server_name', 'proxy-server') - self.lb_private_ips = [x.strip() for x in \ - conf.get('lb_private_ips', '').split(',')\ - if x.strip()] - self.service_ips = [x.strip() for x in \ - conf.get('service_ips', '').split(',')\ - if x.strip()] + for conf_tag in ['lb_private_ips', 'service_ips', + 'service_log_sources']: + setattr(self, conf_tag, + [x.strip() for x in conf.get(conf_tag, '').split(',') \ + if x.strip()]) self.warn_percent = float(conf.get('warn_percent', '0.8')) self.logger = get_logger(conf, log_route='access-processor') @@ -42,6 +41,8 @@ class AccessLogProcessor(object): '''given a raw access log line, return a dict of the good parts''' d = {} try: + log_source = None + split_log = raw_log[16:].split(' ') (unused, server, client_ip, @@ -59,8 +60,9 @@ class AccessLogProcessor(object): etag, trans_id, headers, - processing_time) = (unquote(x) for x in - raw_log[16:].split(' ')[:18]) + processing_time) = (unquote(x) for x in split_log[:18]) + if len(split_log) > 18: + log_source = split_log[18] except ValueError: self.logger.debug(_('Bad line data: %s') % repr(raw_log)) return {} @@ -130,6 +132,7 @@ class AccessLogProcessor(object): d['bytes_out'] = int(d['bytes_out'].replace('-', '0')) d['bytes_in'] = int(d['bytes_in'].replace('-', '0')) d['code'] = int(d['code']) + d['log_source'] = log_source return d def process(self, obj_stream, data_object_account, data_object_container, @@ -167,14 +170,13 @@ class AccessLogProcessor(object): aggr_key = (account, year, month, day, hour) d = hourly_aggr_info.get(aggr_key, {}) - if line_data['lb_ip'] in self.lb_private_ips: + if line_data['lb_ip'] in self.lb_private_ips or \ + line_data['client_ip'] in self.service_ips or \ + line_data['log_source'] in self.service_log_sources: source = 'service' else: source = 'public' - if line_data['client_ip'] in self.service_ips: - source = 'service' - d[(source, 'bytes_out')] = d.setdefault(( source, 'bytes_out'), 0) + bytes_out d[(source, 'bytes_in')] = d.setdefault((source, 'bytes_in'), 0) + \ diff --git a/test_slogging/unit/test_access_processor.py b/test_slogging/unit/test_access_processor.py index 9b5418d..9302fc6 100644 --- a/test_slogging/unit/test_access_processor.py +++ b/test_slogging/unit/test_access_processor.py @@ -43,7 +43,9 @@ class TestAccessProcessor(unittest.TestCase): for param in access_processor.LISTING_PARAMS: expected[param] = 1 expected['query'] = query - self.assertEquals(res, expected) + self.assertEquals(res['bytes_in'] , expected['bytes_in']) + self.assertEquals(res['bytes_out'] , expected['bytes_out']) + self.assertEquals(res['query'] , expected['query']) def test_log_line_parser_field_count(self): p = access_processor.AccessLogProcessor({}) @@ -70,7 +72,8 @@ class TestAccessProcessor(unittest.TestCase): 'bytes_out': 13, 'container_name': 'c', 'day': '1', 'minute': '5', 'account': 'a', 'hour': '4', 'referrer': '9', 'request': '/v1/a/c/o', - 'user_agent': '10', 'bytes_in': 12, 'lb_ip': '3'} + 'user_agent': '10', 'bytes_in': 12, 'lb_ip': '3', + 'log_source': None,} self.assertEquals(res, expected) # too many fields log_line = [str(x) for x in range(19)] @@ -86,7 +89,8 @@ class TestAccessProcessor(unittest.TestCase): 'bytes_out': 13, 'container_name': 'c', 'day': '1', 'minute': '5', 'account': 'a', 'hour': '4', 'referrer': '9', 'request': '/v1/a/c/o', - 'user_agent': '10', 'bytes_in': 12, 'lb_ip': '3'} + 'user_agent': '10', 'bytes_in': 12, 'lb_ip': '3', + 'log_source': '18',} self.assertEquals(res, expected) diff --git a/test_slogging/unit/test_log_processor.py b/test_slogging/unit/test_log_processor.py index 6d92a48..06c03df 100644 --- a/test_slogging/unit/test_log_processor.py +++ b/test_slogging/unit/test_log_processor.py @@ -164,7 +164,8 @@ use = egg:swift#proxy 'request': '/v1/acct/foo/bar', 'user_agent': 'curl', 'bytes_in': 6, - 'lb_ip': '4.5.6.7'}) + 'lb_ip': '4.5.6.7', + 'log_source': None}) def test_process_one_access_file(self): access_proxy_config = self.proxy_config.copy()