x/stackalytics
Code Issues Proposed changes

Refactor read_uri function

Browse Source 
Change-Id: I7a7fbdfaeec7fc8fc0a22021d0640848a5f08e87
This commit is contained in:
Ilya Shakhat 2013-09-15 22:48:33 +04:00
parent 23a26b8be8
commit d14565230b
2 changed files with 8 additions and 15 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

15
stackalytics/processor/mls.py
View File

@ -20,7 +20,6 @@ import StringIO
from email import utils as email_utils from email import utils as email_utils
import re import re
import time import time
import urllib
import urlparse import urlparse
from stackalytics.openstack.common import log as logging from stackalytics.openstack.common import log as logging
@ -55,18 +54,8 @@ TRAILING_RECORD = ('From ishakhat at mirantis.com Tue Sep 17 07:30:43 2013'
'From: ') 'From: ')
def _read_uri(uri):
try:
fd = urllib.urlopen(uri)
raw = fd.read()
fd.close()
return raw
except Exception as e:
LOG.warn('Error while reading uri: %s' % e)
def _get_mail_archive_links(uri): def _get_mail_archive_links(uri):
content = _read_uri(uri) content = utils.read_uri(uri)
links = set(re.findall(r'\shref\s*=\s*[\'"]([^\'"]*\.txt\.gz)', content, links = set(re.findall(r'\shref\s*=\s*[\'"]([^\'"]*\.txt\.gz)', content,
flags=re.IGNORECASE)) flags=re.IGNORECASE))
return [urlparse.urljoin(uri, link) for link in links] return [urlparse.urljoin(uri, link) for link in links]
@ -90,7 +79,7 @@ def _link_content_changed(link, runtime_storage_inst):
def _retrieve_mails(uri): def _retrieve_mails(uri):
LOG.debug('Retrieving mail archive from uri: %s', uri) LOG.debug('Retrieving mail archive from uri: %s', uri)
content = _read_uri(uri) content = utils.read_uri(uri)
gzip_fd = gzip.GzipFile(fileobj=StringIO.StringIO(content)) gzip_fd = gzip.GzipFile(fileobj=StringIO.StringIO(content))
content = gzip_fd.read() content = gzip_fd.read()
LOG.debug('Mail archive is loaded, start processing') LOG.debug('Mail archive is loaded, start processing')

8
stackalytics/processor/utils.py
View File

@ -47,16 +47,20 @@ def check_email_validity(email):
return re.match(r'[\w\d_\.-]+@([\w\d_\.-]+\.)+[\w]+', email) return re.match(r'[\w\d_\.-]+@([\w\d_\.-]+\.)+[\w]+', email)
def read_json_from_uri(uri): def read_uri(uri):
try: try:
fd = urllib.urlopen(uri) fd = urllib.urlopen(uri)
raw = fd.read() raw = fd.read()
fd.close() fd.close()
return json.loads(raw) return raw
except Exception as e: except Exception as e:
LOG.warn('Error while reading uri: %s' % e) LOG.warn('Error while reading uri: %s' % e)
def read_json_from_uri(uri):
return json.loads(read_uri(uri))
def make_range(start, stop, step): def make_range(start, stop, step):
last_full = stop - ((stop - start) % step) last_full = stop - ((stop - start) % step)
for i in xrange(start, last_full, step): for i in xrange(start, last_full, step):