stackforge/satori
Code Issues Proposed changes

Implement New Discovery Schema

Browse Source 
Change-Id: Ibd34c44211920197cf19df535ef038279fff2714
This commit is contained in:
Ziad Sawalha 2014-05-06 09:37:53 -05:00
parent ae4bc4ebb7
commit c382a3e860
12 changed files with 334 additions and 79 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
doc/source/conf.py
View File

@ -28,6 +28,7 @@ sys.path.insert(0, ROOT)
sys.path.insert(0, BASE_DIR)
sys.path = PLUGIN_DIRS + sys.path
#
# Automatically write module docs
#

10
doc/source/index.rst
View File

@ -10,6 +10,7 @@ Satori is a configuration discovery tool for OpenStack and OpenStack tenant host
contributing
releases
terminology
schema
satori
@ -51,8 +52,7 @@ Use Satori
Host:
4.4.4.4 (www.foo.com) is hosted on a Nova Instance
Instance Information:
URI: https://nova.api.somecloud.com/v2/111222/servers/d9119040-f767-414
1-95a4-d4dbf452363a
URI: https://nova.api.somecloud.com/v2/111222/servers/d9119040
Name: sampleserver01.foo.com
ID: d9119040-f767-4141-95a4-d4dbf452363a
ip-addresses:
@ -61,6 +61,12 @@ Use Satori
4.4.4.4
private:
10.1.1.156
Listening Services:
0.0.0.0:6082 varnishd
127.0.0.1:8080 apache2
127.0.0.1:3306 mysqld
Talking to:
10.1.1.71 on 27017
Links

8
doc/source/openstack_resources.rst
View File

@ -31,7 +31,7 @@ Alternatively, the credentials can be passed on the command line:
--os-username yourname \
--os-password yadayadayada \
--os-tenant-name myproject \
--os-auth-url http://...
--os-auth-url http://...
Discovered Host
@ -67,6 +67,12 @@ control plane discovery was possible using the OpenStack credentials.
192.0.2.10
private:
10.1.1.156
Listening Services:
0.0.0.0:6082 varnishd
127.0.0.1:8080 apache2
127.0.0.1:3306 mysqld
Talking to:
10.1.1.71 on 27017
.. _nova: https://github.com/openstack/python-novaclient
.. _OpenStack Nova conventions: https://github.com/openstack/python-novaclient/blob/master/README.rst#id1

33
doc/source/schema.rst Normal file
View File

@ -0,0 +1,33 @@
======
Schema
======
The following list of fields describes the data returned from Satori.
Target
======
Target contains the address suplplied to run the discovery.
Found
=====
All data items discovered are returned under the found key. Keys to resources
discovered are also added under found, but the actual resources are stored
under the resources key.
Resources
=========
All resources (servers, load balancers, DNS domains, etc...) are stored under
the resources key.
Each resource contains the following keys:
* **key**: a globally unique identifier for the resource (could be a URI)
* **id**: the id in the system that hosts the resource
* **type**: the resource type using Heat or Heat-like resource types
* **data**: any additional fields for that resource

11
satori/common/templating.py
View File

@ -76,12 +76,12 @@ def preserve_linefeeds(value):
return value.replace("\n", "\\n").replace("\r", "")
def get_jinja_environment(template, extra_globals=None):
def get_jinja_environment(template, extra_globals=None, **env_vars):
"""Return a sandboxed jinja environment."""
template_map = {'template': template}
env = sandbox.ImmutableSandboxedEnvironment(
loader=jinja2.DictLoader(template_map),
bytecode_cache=CompilerCache())
bytecode_cache=CompilerCache(), **env_vars)
env.filters['prepend'] = do_prepend
env.filters['preserve'] = preserve_linefeeds
env.globals['json'] = json
@ -90,14 +90,17 @@ def get_jinja_environment(template, extra_globals=None):
return env
def parse(template, extra_globals=None, **kwargs):
def parse(template, extra_globals=None, env_vars=None, **kwargs):
"""Parse template.
:param template: the template contents as a string
:param extra_globals: additional globals to include
:param kwargs: extra arguments are passed to the renderer
"""
env = get_jinja_environment(template, extra_globals=extra_globals)
if env_vars is None:
env_vars = {}
env = get_jinja_environment(template, extra_globals=extra_globals,
**env_vars)
minimum_kwargs = {
'data': {},

94
satori/discovery.py
View File

@ -23,7 +23,10 @@ Example usage:
from __future__ import print_function
import ipaddress as ipaddress_module
import sys
import traceback
import ipaddress
from novaclient.v1_1 import client
from pythonwhois import shared
import six
@ -32,48 +35,95 @@ from satori import dns
from satori import utils
def run(address, config=None, interactive=False):
def run(target, config=None, interactive=False):
"""Run discovery and return results."""
if config is None:
config = {}
results = {}
if utils.is_valid_ip_address(address):
ipaddress = address
found = {}
resources = {}
errors = {}
results = {
'target': target,
'created': utils.get_time_string(),
'found': found,
'resources': resources,
}
if utils.is_valid_ip_address(target):
ip_address = target
else:
ipaddress = dns.resolve_hostname(address)
hostname = dns.parse_target_hostname(target)
found['hostname'] = hostname
ip_address = six.text_type(dns.resolve_hostname(hostname))
#TODO(sam): Use ipaddress.ip_address.is_global
# " .is_private
# " .is_unspecified
# " .is_multicast
# To determine address "type"
if not ipaddress_module.ip_address(unicode(ipaddress)).is_loopback:
if not ipaddress.ip_address(ip_address).is_loopback:
try:
results['domain'] = dns.domain_info(address)
domain_info = dns.domain_info(hostname)
resource_type = 'OS::DNS::Domain'
identifier = '%s:%s' % (resource_type, hostname)
resources[identifier] = {
'type': resource_type,
'key': identifier,
}
found['domain-key'] = identifier
resources[identifier]['data'] = domain_info
if 'registered' in domain_info:
found['registered-domain'] = domain_info['registered']
except shared.WhoisException as exc:
results['domain'] = str(exc)
found['ip-address'] = ip_address
results['address'] = ipaddress
host, host_errors = discover_host(ip_address, config,
interactive=interactive)
if host_errors:
errors.update(host_errors)
key = host.get('key') or ip_address
resources[key] = host
found['host-key'] = key
results['updated'] = utils.get_time_string()
return results, errors
results['host'] = host = {'type': 'Undetermined'}
def discover_host(address, config, interactive=False):
"""Discover host by IP address."""
host = {}
errors = {}
if config.get('username'):
server = find_nova_host(ipaddress, config)
server = find_nova_host(address, config)
if server:
host['type'] = 'Nova instance'
host['uri'] = [l['href'] for l in server.links
host['type'] = 'OS::Nova::Instance'
data = {}
host['data'] = data
data['uri'] = [l['href'] for l in server.links
if l['rel'] == 'self'][0]
host['name'] = server.name
host['id'] = server.id
host['addresses'] = server.addresses
data['name'] = server.name
data['id'] = server.id
data['addresses'] = server.addresses
host['key'] = data['uri']
if config.get('system_info'):
module_name = config['system_info'].replace("-", "_")
if '.' not in module_name:
module_name = 'satori.sysinfo.%s' % module_name
system_info_module = utils.import_object(module_name)
result = system_info_module.get_systeminfo(ipaddress, config,
interactive=interactive)
host['system_info'] = result
return results
try:
result = system_info_module.get_systeminfo(
address, config, interactive=interactive)
host.setdefault('data', {})
host['data']['system_info'] = result
except Exception as exc:
exc_traceback = sys.exc_info()[2]
errors['system_info'] = {
'type': "ERROR",
'message': str(exc),
'exception': exc,
'traceback': traceback.format_tb(exc_traceback),
}
return host, errors
def find_nova_host(address, config):
@ -86,6 +136,6 @@ def find_nova_host(address, config):
service_type="compute")
for server in nova.servers.list():
for network_addresses in six.itervalues(server.addresses):
for ipaddress in network_addresses:
if ipaddress['addr'] == address:
for ip_address in network_addresses:
if ip_address['addr'] == address:
return server

27
satori/dns.py
View File

@ -9,7 +9,7 @@
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
#
"""Satori DNS Discovery."""
import datetime
@ -27,21 +27,24 @@ from satori import utils
LOG = logging.getLogger(__name__)
def resolve_hostname(host):
"""Get IP address of hostname or URL."""
def parse_target_hostname(target):
"""Get IP address or FQDN of a target which could be a URL or address."""
if not target:
raise errors.SatoriInvalidNetloc("Target must be supplied.")
try:
if not host:
raise AttributeError("Host must be supplied.")
parsed = urlparse.urlparse(host)
parsed = urlparse.urlparse(target)
except AttributeError as err:
error = "Hostname `%s` is unparseable. Error: %s" % (host, err)
error = "Target `%s` is unparseable. Error: %s" % (target, err)
LOG.exception(error)
raise errors.SatoriInvalidNetloc(error)
# Domain names and IP are in netloc when parsed with a protocol
# they will be in path if parsed without a protocol
hostname = parsed.netloc or parsed.path
return parsed.netloc or parsed.path
def resolve_hostname(hostname):
"""Get IP address of hostname."""
try:
address = socket.gethostbyname(hostname)
except socket.gaierror:
@ -55,13 +58,13 @@ def get_registered_domain(hostname):
return tldextract.extract(hostname).registered_domain
def ip_info(ip):
def ip_info(ip_address):
"""Get as much information as possible for a given ip address."""
if not utils.is_valid_ip_address(ip):
error = "`%s` is an invalid IP address." % ip
if not utils.is_valid_ip_address(ip_address):
error = "`%s` is an invalid IP address." % ip_address
raise errors.SatoriInvalidIP(error)
result = pythonwhois.get_whois(ip)
result = pythonwhois.get_whois(ip_address)
return {
'whois': result['raw']

72
satori/formats/text.jinja
View File

@ -1,20 +1,54 @@
{% if data['address'] != target %}Address:
{{ target }} resolves to IPv4 address {{ data.address }}
{% endif %}{% if data['domain'] %}Domain: {{ data.domain.name }}
Registrar: {{ data.domain.registrar }}{% if data.domain.nameservers %}
Nameservers: {% for nameserver in data.domain.nameservers %}{{nameserver}}{% if not loop.last %}, {% endif %}{% endfor %}{% endif %}{% endif %}{% if data['domain'] and data.domain.days_until_expires %} Expires: {{ data.domain.days_until_expires }} days{% endif %}{% if data['host'] and data.host.type == 'Nova instance' %}Host:
{{ data.address }} ({{ target }}) is hosted on a {{ data.host.type }}
Instance Information:
URI: {{ data.host.uri }}
Name: {{ data.host.name }}
ID: {{ data.host.id }}
ip-addresses:{% for name, addresses in data.host.addresses.items() %}
{{ name }}:{% for address in addresses %}
{{ address.addr }}{% endfor %}{% endfor %}
{% elif data['address'] %}Host:
ip-address: {{ data.address }}
{% else %}Host not found
{% endif %}{% if data['host'] and data.host.system_info %}Packages:
{% for name, package in data.host.system_info.packages.items() %} {{ name }}: {{ package.version }}
{% endfor %}
{% set found = data['found'] | default({}) %}
{% set resources = data['resources'] | default({'n/a': {}}) %}
{% set address = found['ip-address'] %}
{% set hostkey = found['host-key'] | default('n/a') %}
{% set domainkey = found['domain-key'] | default('n/a') %}
{% set server = resources[hostkey] | default(False) %}
{% set domain = resources[domainkey] | default(False) %}
{% if found['ip-address'] != target %}Address:
{{ target }} resolves to IPv4 address {{ found['ip-address'] }}
{%- endif %}
{% if domain %}Domain: {{ domain['data'].name }}
Registrar: {{ domain['data'].registrar }}
{% if domain['data'].nameservers %}
Nameservers: {% for nameserver in domain['data'].nameservers %}{{nameserver}}{% if not loop.last %}, {% endif %}{% endfor %}
{% endif %}
{% if domain['data'].days_until_expires %}
Expires: {{ domain['data'].days_until_expires }} days
{% endif %}
{%- endif %}
{% if server and server.type == 'OS::Nova::Instance' %}
Host:
{{ found['ip-address'] }} ({{ target }}) is hosted on a Nova instance
{% if 'data' in server %} Instance Information:
URI: {{ server['data'].uri | default('n/a') }}
Name: {{ server['data'].name | default('n/a') }}
ID: {{ server['data'].id | default('n/a') }}
{% if 'addresses' in server['data'] %} ip-addresses:
{% for name, addresses in server['data'].addresses.items() %}
{{ name }}:
{% for address in addresses %}
{{ address.addr }}
{% endfor %}
{% endfor %}{% endif %}{% endif %}
{% elif found['ip-address'] %}
Host:
ip-address: {{ found['ip-address'] }}
{% else %}Host not found
{% endif %}
{% if server and 'data' in server and server['data'].system_info %}
{% if 'remote_services' in server['data'].system_info %}
Listening Services:
{% for remote in server['data'].system_info.remote_services | sort %}
{{ remote.ip }}:{{ remote.port }} {{ remote.process }}
{% endfor %}{% endif %}
{% if 'connections' in server['data'].system_info %}
Talking to:
{% for connection in server['data'].system_info.connections | dictsort %}
{{ connection[0] }}{% if connection[1] %} on {% for port in connection[1] %}{{ port }}{% if not loop.last %}, {% endif %}{% endfor %}{% endif %}
{% endfor %}{% endif %}
{% endif %}

20
satori/shell.py
View File

@ -17,7 +17,6 @@
Accept a network location, run through the discovery process and report the
findings back to the user.
"""
from __future__ import print_function
@ -225,9 +224,12 @@ def main(argv=None):
get_template_path(config['format']))
try:
results = discovery.run(config['netloc'], config, interactive=True)
results, errors = discovery.run(config['netloc'], config,
interactive=True)
print(format_output(config['netloc'], results,
template_name=config['format']))
if errors:
sys.stderr.write(format_errors(errors, config))
except Exception as exc: # pylint: disable=W0703
if config['debug']:
LOG.exception(exc)
@ -263,8 +265,20 @@ def format_output(discovered_target, results, template_name="text"):
return(json.dumps(results, indent=2))
else:
template = get_template(template_name)
env_vars = dict(lstrip_blocks=True, trim_blocks=True)
return templating.parse(template, target=discovered_target,
data=results).strip('\n')
data=results, env_vars=env_vars).strip('\n')
def format_errors(errors, config):
"""Format errors for output to console."""
if config['debug']:
return str(errors)
else:
formatted = {}
for key, error in errors.items():
formatted[key] = error['message']
return str(formatted)
if __name__ == "__main__":

2
satori/sysinfo/__init__.py
View File

@ -1 +1 @@
"""."""
"""Modules for Data Plane Discovery."""

4
satori/tests/test_dns.py
View File

@ -136,13 +136,13 @@ class TestDNS(utils.TestCase):
def test_resolve_int_raises_invalid_netloc_error(self):
self.assertRaises(
errors.SatoriInvalidNetloc,
dns.resolve_hostname,
dns.parse_target_hostname,
100)
def test_resolve_none_raises_invalid_netloc_error(self):
self.assertRaises(
errors.SatoriInvalidNetloc,
dns.resolve_hostname,
dns.parse_target_hostname,
None)
def test_registered_domain_subdomain_removed(self):

131
satori/tests/test_formats_text.py
View File

@ -29,20 +29,25 @@ class TestTextTemplate(unittest.TestCase):
def test_no_data(self):
"""Handles response with no host."""
result = templating.parse(self.template, data={})
env_vars = dict(lstrip_blocks=True, trim_blocks=True)
result = templating.parse(self.template, data={}, env_vars=env_vars)
self.assertEqual(result.strip('\n'), 'Host not found')
def test_target_is_ip(self):
"""Handles response when host is just the supplied address."""
env_vars = dict(lstrip_blocks=True, trim_blocks=True)
result = templating.parse(self.template, target='127.0.0.1',
data={'address': '127.0.0.1'})
data={'found': {'ip-address': '127.0.0.1'}},
env_vars=env_vars)
self.assertEqual(result.strip('\n'),
'Host:\n ip-address: 127.0.0.1')
def test_host_not_server(self):
"""Handles response when host is not a nova instance."""
env_vars = dict(lstrip_blocks=True, trim_blocks=True)
result = templating.parse(self.template, target='localhost',
data={'address': '127.0.0.1'})
data={'found': {'ip-address': '127.0.0.1'}},
env_vars=env_vars)
self.assertEqual(result.strip('\n'),
'Address:\n localhost resolves to IPv4 address '
'127.0.0.1\nHost:\n ip-address: 127.0.0.1')
@ -50,20 +55,44 @@ class TestTextTemplate(unittest.TestCase):
def test_host_is_nova_instance(self):
"""Handles response when host is a nova instance."""
data = {
'address': '10.1.1.45',
'host': {
'type': 'Nova instance',
'uri': 'https://servers/path',
'id': '1000B',
'name': 'x',
'addresses': {
'public': [{'type': 'ipv4', 'addr': '10.1.1.45'}]
'found': {
'ip-address': '10.1.1.45',
'hostname': 'x',
'host-key': 'https://servers/path'
},
'target': 'instance.nova.local',
'resources': {
'https://servers/path': {
'type': 'OS::Nova::Instance',
'data': {
'uri': 'https://servers/path',
'id': '1000B',
'name': 'x',
'addresses': {
'public': [{'type': 'ipv4', 'addr': '10.1.1.45'}]
},
'system_info': {
'connections': {
'192.168.2.100': [],
'192.168.2.101': [433],
'192.168.2.102': [8080, 8081]
},
'remote_services': [
{
'ip': '0.0.0.0',
'process': 'nginx',
'port': 80
}
]
}
}
}
}
}
env_vars = dict(lstrip_blocks=True, trim_blocks=True)
result = templating.parse(self.template,
target='instance.nova.local',
data=data)
data=data, env_vars=env_vars)
expected = """\
Address:
instance.nova.local resolves to IPv4 address 10.1.1.45
@ -75,7 +104,83 @@ Host:
ID: 1000B
ip-addresses:
public:
10.1.1.45"""
10.1.1.45
Listening Services:
0.0.0.0:80 nginx
Talking to:
192.168.2.100
192.168.2.101 on 433
192.168.2.102 on 8080, 8081"""
self.assertEqual(result.strip('\n'), expected)
def test_host_has_no_data(self):
"""Handles response when host is a nova instance."""
data = {
'found': {
'ip-address': '10.1.1.45',
'hostname': 'x',
'host-key': 'https://servers/path'
},
'target': 'instance.nova.local',
'resources': {
'https://servers/path': {
'type': 'OS::Nova::Instance'
}
}
}
env_vars = dict(lstrip_blocks=True, trim_blocks=True)
result = templating.parse(self.template,
target='instance.nova.local',
data=data, env_vars=env_vars)
expected = """\
Address:
instance.nova.local resolves to IPv4 address 10.1.1.45
Host:
10.1.1.45 (instance.nova.local) is hosted on a Nova instance"""
self.assertEqual(result.strip('\n'), expected)
def test_host_data_missing_items(self):
"""Handles response when host is a nova instance."""
data = {
'found': {
'ip-address': '10.1.1.45',
'hostname': 'x',
'host-key': 'https://servers/path'
},
'target': 'instance.nova.local',
'resources': {
'https://servers/path': {
'type': 'OS::Nova::Instance',
'data': {
'id': '1000B',
'system_info': {
'remote_services': [
{
'ip': '0.0.0.0',
'process': 'nginx',
'port': 80
}
]
}
}
}
}
}
env_vars = dict(lstrip_blocks=True, trim_blocks=True)
result = templating.parse(self.template,
target='instance.nova.local',
data=data, env_vars=env_vars)
expected = """\
Address:
instance.nova.local resolves to IPv4 address 10.1.1.45
Host:
10.1.1.45 (instance.nova.local) is hosted on a Nova instance
Instance Information:
URI: n/a
Name: n/a
ID: 1000B
Listening Services:
0.0.0.0:80 nginx"""
self.assertEqual(result.strip('\n'), expected)