stackforge/cloud-pydashie
Code Issues Proposed changes

Setting up for generic use with openstack

Browse Source 
Few renames, additional documentation,
setting up default devstack names for conf.

Minor implentation changes and a few reshuffles of classes.

remove now unneeded list from conf.
This commit is contained in:
adriant 2014-12-16 12:34:51 +13:00
parent 8bad81a06a
commit 97fcb2af83
7 changed files with 166 additions and 97 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

64
README.rst
View File

@ -1,10 +1,57 @@
PyDashie
Openstack-PyDashie
########
PyDashy is a port of `Dashing <https://github.com/Shopify/dashing>`_ by `Shopify <http://www.shopify.com/>`_ to Python 2.7
This is simply an implementation of pydashie tailored to showing information about an openstack cluster with nagios/icinga for monitoring. It is primarily for internal use.
It uses the standard python clients for collecting formation from openstack across multiple regions.
The nagios/icinga data is currently collected via ssh but in future might be moved to MKlivestatus as the current method is roundabout.
.. image:: http://evolvedlight.github.com/pydashie/images/mainscreen.png
NOTE: The current layout is hardcoded for 1080p. This might be changed to be configurable by the conf.yaml later. If you need to change the sizing, you can do so by changing the widget dimensions and number of columns within this function:
https://github.com/catalyst/openstack-pydashie/blob/master/pydashie/assets/javascripts/app.js#L336
Setup
############
Configuration is handled via a yaml file as follows:
.. code-block:: yaml
main:
log_file: pydashie.log
openstack:
regions:
- 'region1'
allocation:
region1:
vcpus_allocation_ratio: 2.0
ram_allocation_ratio: 1.0
# remove this amount per node available metric
reserved_ram_per_node: 0
reserved_vcpus_per_node: 0
# total IPs are here as getting this from Neutron is
# far from straightforward
total_floating_ips: 256
auth:
auth_url: 'http://localhost:5000/v2.0'
username: 'admin'
password: 'openstack'
project_name: 'demo'
insecure: False
nagios:
services:
region1:
statfile: './region1-status.dat'
host: 'region1-mon0'
username: 'admin'
Because of differences between allocation per region, and the need for a region list, each region is given it's own allocation data. We use this to know which regions to build clients for and aggregate data over, but in future might try and query a for a full region list and for allocation data from openstack itself.
Installation
############
@ -16,8 +63,17 @@ OR
python setup.py install
And you can run the application by
Running
############
pydashie
You can run the application by:
Goto localhost:5000 to view the sample application in action.
pydashie -c conf.yaml
Goto localhost:5050 to view the application in action.
The port and interface can also be set via the commandline:
pydashie -c conf.yaml -ip 0.0.0.0 -p 5050
Although they default to 0.0.0.0 and 5050 if not manually given.

14
pydashie/conf.yaml
View File

@ -1,25 +1,25 @@
main:
log_file: pydashie.log
openstack:
regions:
- 'region1'
allocation:
region1:
RegionOne:
vcpus_allocation_ratio: 2.0
ram_allocation_ratio: 1.0
# remove this amount per node available metric
reserved_ram_per_node: 0
reserved_vcpus_per_node: 0
# total IPs are here as getting this from Neutron is
# far from straightforward
total_floating_ips: 256
auth:
auth_url: 'http://localhost:5000/v2.0'
username: 'admin'
password: 'openstack'
project_name: 'demo'
insecure: True
insecure: False
nagios:
services:
region1:
statfile: './region1-status.dat'
host: 'region1-mon0'
RegionOne:
statfile: './RegionOne-status.dat'
host: 'RegionOne-mon0'
username: 'admin'

65
pydashie/dashie_sampler.py
View File

@ -1,20 +1,12 @@
import datetime
import json
from math import ceil
from repeated_timer import RepeatedTimer
from novaclient.v1_1 import client as novaclient
from cinderclient.v1 import client as cinderclient
from keystoneclient.v2_0 import client as keystoneclient
from neutronclient.v2_0 import client as neutronclient
class DashieSampler(object):
def __init__(self, app, interval, conf=None, client_cache={}):
def __init__(self, app, interval):
self._app = app
self._os_clients = client_cache
self._conf = conf
self._timer = RepeatedTimer(interval, self._sample)
def stop(self):
@ -45,58 +37,3 @@ class DashieSampler(object):
data = self.sample()
if data:
self._send_event(self.name(), data)
def _convert(self, num):
if num >= 1024 ** 3:
return int(ceil(num / (1024 ** 3))), 'GB'
elif num >= 1024 ** 2:
return int(ceil(num / (1024 ** 2))), 'MB'
elif num >= 1024:
return int(ceil(num / (1024))), 'KB'
else:
return num, 'B'
def _client(self, service, region):
if not self._os_clients.get(region):
self._os_clients[region] = {}
if not self._os_clients[region].get(service):
if service == 'compute':
client = novaclient.Client(
self._conf['auth']['username'],
self._conf['auth']['password'],
self._conf['auth']['project_name'],
self._conf['auth']['auth_url'],
region_name=region,
insecure=self._conf['auth']['insecure'])
self._os_clients[region][service] = client
elif service == 'network':
client = neutronclient.Client(
username=self._conf['auth']['username'],
password=self._conf['auth']['password'],
tenant_name=self._conf['auth']['project_name'],
auth_url=self._conf['auth']['auth_url'],
region_name=region,
insecure=self._conf['auth']['insecure'])
self._os_clients[region][service] = client
elif service == 'storage':
client = cinderclient.Client(
self._conf['auth']['username'],
self._conf['auth']['password'],
self._conf['auth']['project_name'],
self._conf['auth']['auth_url'],
region_name=region,
insecure=self._conf['auth']['insecure'])
self._os_clients[region][service] = client
elif service == 'identity':
client = keystoneclient.Client(
username=self._conf['auth']['username'],
password=self._conf['auth']['password'],
project_name=self._conf['auth']['project_name'],
auth_url=self._conf['auth']['auth_url'],
region_name=region,
insecure=self._conf['auth']['insecure'])
self._os_clients[region][service] = client
return self._os_clients[region][service]

6
pydashie/main.py
View File

@ -183,7 +183,7 @@ def run_sample_app():
a.add_argument("-c", "--config", dest="config", help="Path to config file",
required=True)
a.add_argument("-i", "--interface", dest="ip",
a.add_argument("-ip", "--interface", dest="ip",
help="IP address to serve on.", default="0.0.0.0")
a.add_argument("-p", "--port", help="port to serve on", default="5050")
@ -206,8 +206,8 @@ def run_sample_app():
import SocketServer
SocketServer.BaseServer.handle_error = close_stream
import example_app
example_app.run(args, conf, app, xyzzy)
import openstack_app
openstack_app.run(args, conf, app, xyzzy)
if __name__ == "__main__":

2
pydashie/example_app.py → pydashie/openstack_app.py
View File

@ -1,4 +1,4 @@
from example_samplers import (
from openstack_samplers import (
CPUSampler,
RAMSampler,
IPSampler,

105
pydashie/example_samplers.py → pydashie/openstack_samplers.py
View File

@ -1,11 +1,80 @@
import collections
import random
import nagios
from math import ceil
from dashie_sampler import DashieSampler
from novaclient.v1_1 import client as novaclient
from cinderclient.v1 import client as cinderclient
from keystoneclient.v2_0 import client as keystoneclient
from neutronclient.v2_0 import client as neutronclient
class CPUSampler(DashieSampler):
class BaseOpenstackSampler(DashieSampler):
"""docstring for ClassName"""
def __init__(self, app, interval, conf=None, client_cache={}):
self._os_clients = client_cache
self._conf = conf
super(BaseOpenstackSampler, self).__init__(app, interval)
def _convert(self, num):
if num >= 1024 ** 3:
return int(ceil(num / (1024 ** 3))), 'GB'
elif num >= 1024 ** 2:
return int(ceil(num / (1024 ** 2))), 'MB'
elif num >= 1024:
return int(ceil(num / (1024))), 'KB'
else:
return num, 'B'
def _client(self, service, region):
if not self._os_clients.get(region):
self._os_clients[region] = {}
if not self._os_clients[region].get(service):
if service == 'compute':
client = novaclient.Client(
self._conf['auth']['username'],
self._conf['auth']['password'],
self._conf['auth']['project_name'],
self._conf['auth']['auth_url'],
region_name=region,
insecure=self._conf['auth']['insecure'])
self._os_clients[region][service] = client
elif service == 'network':
client = neutronclient.Client(
username=self._conf['auth']['username'],
password=self._conf['auth']['password'],
tenant_name=self._conf['auth']['project_name'],
auth_url=self._conf['auth']['auth_url'],
region_name=region,
insecure=self._conf['auth']['insecure'])
self._os_clients[region][service] = client
elif service == 'storage':
client = cinderclient.Client(
self._conf['auth']['username'],
self._conf['auth']['password'],
self._conf['auth']['project_name'],
self._conf['auth']['auth_url'],
region_name=region,
insecure=self._conf['auth']['insecure'])
self._os_clients[region][service] = client
elif service == 'identity':
client = keystoneclient.Client(
username=self._conf['auth']['username'],
password=self._conf['auth']['password'],
project_name=self._conf['auth']['project_name'],
auth_url=self._conf['auth']['auth_url'],
region_name=region,
insecure=self._conf['auth']['insecure'])
self._os_clients[region][service] = client
return self._os_clients[region][service]
class CPUSampler(BaseOpenstackSampler):
def __init__(self, *args, **kwargs):
super(CPUSampler, self).__init__(*args, **kwargs)
self._last = 0
@ -17,7 +86,7 @@ class CPUSampler(DashieSampler):
max_cpu = 0
cur_cpu = 0
for region in self._conf['regions']:
for region in self._conf['allocation'].keys():
nova = self._client('compute', region)
stats = nova.hypervisors.statistics()
hypervisors = nova.hypervisors.list()
@ -41,7 +110,7 @@ class CPUSampler(DashieSampler):
return s
class RAMSampler(DashieSampler):
class RAMSampler(BaseOpenstackSampler):
def __init__(self, *args, **kwargs):
super(RAMSampler, self).__init__(*args, **kwargs)
self._last = 0
@ -53,7 +122,7 @@ class RAMSampler(DashieSampler):
max_ram = 0
cur_ram = 0
for region in self._conf['regions']:
for region in self._conf['allocation'].keys():
nova = self._client('compute', region)
stats = nova.hypervisors.statistics()
hypervisors = nova.hypervisors.list()
@ -83,7 +152,7 @@ class RAMSampler(DashieSampler):
return s
class IPSampler(DashieSampler):
class IPSampler(BaseOpenstackSampler):
def __init__(self, *args, **kwargs):
super(IPSampler, self).__init__(*args, **kwargs)
self._last = 0
@ -95,7 +164,7 @@ class IPSampler(DashieSampler):
max_ips = 0
cur_ips = 0
for region in self._conf['regions']:
for region in self._conf['allocation'].keys():
max_ips = (max_ips +
self._conf['allocation'][region]['total_floating_ips'])
@ -121,7 +190,7 @@ class IPSampler(DashieSampler):
return s
class RegionsCPUSampler(DashieSampler):
class RegionsCPUSampler(BaseOpenstackSampler):
def __init__(self, *args, **kwargs):
super(RegionsCPUSampler, self).__init__(*args, **kwargs)
@ -131,7 +200,7 @@ class RegionsCPUSampler(DashieSampler):
def sample(self):
regions = []
for region in self._conf['regions']:
for region in self._conf['allocation'].keys():
nova = self._client('compute', region)
stats = nova.hypervisors.statistics()
hypervisors = nova.hypervisors.list()
@ -151,7 +220,7 @@ class RegionsCPUSampler(DashieSampler):
return {'progress_items': regions}
class RegionsRAMSampler(DashieSampler):
class RegionsRAMSampler(BaseOpenstackSampler):
def __init__(self, *args, **kwargs):
super(RegionsRAMSampler, self).__init__(*args, **kwargs)
@ -161,7 +230,7 @@ class RegionsRAMSampler(DashieSampler):
def sample(self):
regions = []
for region in self._conf['regions']:
for region in self._conf['allocation'].keys():
nova = self._client('compute', region)
stats = nova.hypervisors.statistics()
hypervisors = nova.hypervisors.list()
@ -186,7 +255,7 @@ class RegionsRAMSampler(DashieSampler):
return {'progress_items': regions}
class RegionIPSampler(DashieSampler):
class RegionIPSampler(BaseOpenstackSampler):
def __init__(self, *args, **kwargs):
super(RegionIPSampler, self).__init__(*args, **kwargs)
self._last = 0
@ -197,7 +266,7 @@ class RegionIPSampler(DashieSampler):
def sample(self):
regions = []
for region in self._conf['regions']:
for region in self._conf['allocation'].keys():
neutron = self._client('network', region)
ips = neutron.list_floatingips()
@ -219,7 +288,7 @@ class RegionIPSampler(DashieSampler):
return {'progress_items': regions}
class NagiosSampler(DashieSampler):
class NagiosSampler(BaseOpenstackSampler):
def __init__(self, *args, **kwargs):
super(NagiosSampler, self).__init__(*args, **kwargs)
self._last = 0
@ -252,7 +321,7 @@ class NagiosSampler(DashieSampler):
return s
class NagiosRegionSampler(DashieSampler):
class NagiosRegionSampler(BaseOpenstackSampler):
def name(self):
return 'nagios_regions'
@ -282,7 +351,7 @@ class NagiosRegionSampler(DashieSampler):
return {'criticals': criticals, 'warnings': warnings}
class ResourceSampler(DashieSampler):
class ResourceSampler(BaseOpenstackSampler):
def name(self):
return 'resources'
@ -294,7 +363,7 @@ class ResourceSampler(DashieSampler):
# 'images': 0,
'vpns': 0}
for region in self._conf['regions']:
for region in self._conf['allocation'].keys():
neutron = self._client('network', region)
nova = self._client('compute', region)
# cinder = self._client('storage', region)
@ -325,7 +394,7 @@ class ResourceSampler(DashieSampler):
return {'items': items}
class ConvergenceSampler(DashieSampler):
class ConvergenceSampler(BaseOpenstackSampler):
def name(self):
return 'convergence'
@ -344,7 +413,7 @@ class ConvergenceSampler(DashieSampler):
return {'points': list(self.items)}
class UsageGaugeSampler(DashieSampler):
class UsageGaugeSampler(BaseOpenstackSampler):
def __init__(self, *args, **kwargs):
super(UsageGaugeSampler, self).__init__(*args, **kwargs)

7
requirements.txt
View File

@ -1,3 +1,10 @@
flask>=0.9
CoffeeScript
requests
python-novaclient
python-neutronclient
python-cinderclient
python-keystone
paramiko
pynag