From 6d8ada8447941758c9b05ebbbc2f0777e906ce4b Mon Sep 17 00:00:00 2001 From: Stef T Date: Thu, 4 Oct 2012 19:18:11 +0200 Subject: [PATCH] Changed the dict to a set, thus removing the need for the sentinal value. Also included the speed difference test, but it's in a seperate folder and not part of the normal test-run (for expediancy's sake) --- slogging/access_processor.py | 2 +- test_slogging/speed/test_cidr_speed.py | 61 ++++++++++++++++++++++++++ 2 files changed, 62 insertions(+), 1 deletion(-) create mode 100755 test_slogging/speed/test_cidr_speed.py diff --git a/slogging/access_processor.py b/slogging/access_processor.py index f11336d..193dddd 100644 --- a/slogging/access_processor.py +++ b/slogging/access_processor.py @@ -24,7 +24,7 @@ try: CIDR_support = True def return_ips(conf, conf_tag): - return dict((k, 1) for k in + return set(k for k in IpRangeList(*[x.strip() for x in conf.get(conf_tag, '').split(',') if x.strip()])) def sanitize_ips(line_data): diff --git a/test_slogging/speed/test_cidr_speed.py b/test_slogging/speed/test_cidr_speed.py new file mode 100755 index 0000000..1ebe9b6 --- /dev/null +++ b/test_slogging/speed/test_cidr_speed.py @@ -0,0 +1,61 @@ +#!/usr/bin/env python + +import iptools +import datetime + +import unittest +from slogging import access_processor + + +class TestAccessProcessorSpeed(unittest.TestCase): + def test_CIDR_speed(self): + line = 'Sep 16 20:00:02 srv testsrv 192.%s.119.%s - ' \ + '16/Sep/2012/20/00/02 GET /v1/a/c/o HTTP/1.0 ' \ + '200 - StaticWeb - - 17005 - txn - 0.0095 -' + ips1 = iptools.IpRangeList(*[x.strip() for x in + '127.0.0.1,192.168/16,10/24'.split(',') + if x.strip()]) + ips2 = iptools.IpRangeList(*[x.strip() for x in + '172.168/16,10/30'.split(',') + if x.strip()]) + ips3 = iptools.IpRangeList(*[x.strip() for x in + '127.0.0.1,11/24'.split(',') + if x.strip()]) + + orig_start = datetime.datetime.utcnow() + hit = 0 + for n in range(255): + for a in range(255): + stream = line % (n, a) + data = stream.split(" ") + if data[5] in ips1 or data[5] in ips2 or data[5] in ips3: + hit += 1 + orig_end = datetime.datetime.utcnow() + orig_secs = float("%d.%d" % ((orig_end - orig_start).seconds, + (orig_end - orig_start).microseconds)) + self.assertEqual(hit, 255) + + # now, let's check the speed with pure dicts + dict1 = set(k for k in + iptools.IpRangeList(*[x.strip() for x in + '127.0.0.1,192.168/16,10/24'.split(',') if x.strip()])) + dict2 = set(k for k in + iptools.IpRangeList(*[x.strip() for x in + '172.168/16,10/30'.split(',') if x.strip()])) + dict3 = set(k for k in + iptools.IpRangeList(*[x.strip() for x in + '127.0.0.1,11/24'.split(',') if x.strip()])) + + new_start = datetime.datetime.utcnow() + hit = 0 + for n in range(255): + for a in range(255): + stream = line % (n, a) + data = stream.split(" ") + if data[5] in dict1 or data[5] in dict2 or data[5] in dict3: + hit += 1 + new_end = datetime.datetime.utcnow() + new_secs = float("%d.%d" % ((new_end - new_start).seconds, + (new_end - new_start).microseconds)) + self.assertEqual(hit, 255) + self.assertTrue(new_secs < orig_secs)