From 722220f77c52fbc27cf69533d331f47ef177e9c8 Mon Sep 17 00:00:00 2001 From: xiaodongwang Date: Sat, 6 Sep 2014 20:02:43 -0700 Subject: [PATCH] initial version to setup compass-monit Change-Id: I04da5b53d1c661ab7b4c1daff4977166daf07cbd --- .gitignore | 42 ++ compass_metrics/__init__.py | 0 compass_metrics/api/__init__.py | 19 + compass_metrics/api/anapi.py | 480 -------------------- compass_metrics/api/{anapi_v2.py => api.py} | 112 +++-- compass_metrics/api/monitor.wsgi | 6 - compass_metrics/utils/__init__.py | 0 compass_metrics/utils/flags.py | 92 ++++ compass_metrics/utils/logsetting.py | 94 ++++ compass_metrics/utils/setting_wrapper.py | 52 +++ conf/compass-monit.conf | 18 + conf/compass_monit.wsgi | 25 + conf/monitor.conf | 18 - conf/setting | 4 + ez_setup.py | 299 ++++++++++++ install/install.sh | 160 +++++++ requirements.txt | 10 + setup.cfg | 4 + setup.py | 103 +++++ test-requirements.txt | 9 + tools/install_venv.py | 73 +++ tools/install_venv_common.py | 172 +++++++ tools/with_venv.sh | 7 + tox.ini | 38 ++ 24 files changed, 1306 insertions(+), 531 deletions(-) create mode 100644 .gitignore create mode 100644 compass_metrics/__init__.py delete mode 100755 compass_metrics/api/anapi.py rename compass_metrics/api/{anapi_v2.py => api.py} (88%) delete mode 100644 compass_metrics/api/monitor.wsgi create mode 100644 compass_metrics/utils/__init__.py create mode 100644 compass_metrics/utils/flags.py create mode 100644 compass_metrics/utils/logsetting.py create mode 100644 compass_metrics/utils/setting_wrapper.py create mode 100644 conf/compass-monit.conf create mode 100644 conf/compass_monit.wsgi delete mode 100644 conf/monitor.conf create mode 100644 conf/setting create mode 100644 ez_setup.py create mode 100755 install/install.sh create mode 100644 requirements.txt create mode 100644 setup.cfg create mode 100644 setup.py create mode 100644 test-requirements.txt create mode 100644 tools/install_venv.py create mode 100644 tools/install_venv_common.py create mode 100755 tools/with_venv.sh create mode 100644 tox.ini diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..5681502 --- /dev/null +++ b/.gitignore @@ -0,0 +1,42 @@ +*.py[cod] +*~ +*.swp + + +# C extensions +*.so + +# log file +*.log + +# Packages +*.egg +*.egg-info +dist +build +eggs +parts +var +sdist +develop-eggs +.installed.cfg +lib +lib64 +__pycache__ + +# Installer logs +pip-log.txt + +# Unit test / coverage reports +.coverage +.tox +nosetests.xml +.testrepository +cover +# Translations +*.mo + +# Mr Developer +.mr.developer.cfg +.project +.pydevproject diff --git a/compass_metrics/__init__.py b/compass_metrics/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/compass_metrics/api/__init__.py b/compass_metrics/api/__init__.py index e69de29..a07f2ab 100644 --- a/compass_metrics/api/__init__.py +++ b/compass_metrics/api/__init__.py @@ -0,0 +1,19 @@ +# Copyright 2014 Huawei Technologies Co. Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from flask import Flask + + +app = Flask(__name__) +app.debug = True diff --git a/compass_metrics/api/anapi.py b/compass_metrics/api/anapi.py deleted file mode 100755 index 1f5b02a..0000000 --- a/compass_metrics/api/anapi.py +++ /dev/null @@ -1,480 +0,0 @@ -#!/usr/bin/env python - -from flask import Flask -from flask.ext import restful -from flask.ext.restful import reqparse -from flask.ext.jsonpify import jsonify -#from flask import request -from flask import redirect, request, current_app -import urllib -import json -import requests -import random -import time - - -app = Flask(__name__) - - -api = restful.Api(app) - -#class MyEncoder1(json.JSONEncoder): -# def default(self, obj): - - -class HelloWorld(restful.Resource): - def get(self): - return {'warning': 'Please use /monit/api'} - -api.add_resource(HelloWorld, '/', '/monit') - -class HelloInfo(restful.Resource): - def get(self): - #return {'info': 'Select host, hostgroup, topology, service from here'} - r = requests.get(ROOT_URL+"/api/v1/tagnames") - return r.json() - -api.add_resource(HelloInfo, '/monit/api') - -class Proxy(restful.Resource): - def get(self, url): - # Get URL Parameters for host and metric info - urlParam = request.url.split('proxy/', 9); - my_url = urlParam[1] - #return my_url - r = requests.get(my_url, stream = True) - #return r.text - return json.loads(r.text) - #return Response(stream_with_context(r.iter_content()), content_type = r.headers['content-type']) - -api.add_resource(Proxy, '/monit/api/proxy/', defaults={'url': '/HelloWorld'}) - -class InvalidUrl(restful.Resource): - def get(self): - return {'warning': 'Please form the URL'} - -api.add_resource(InvalidUrl, '/monit') - -class Hosts(restful.Resource): - def get(self): - return {'hosts': '[\'host1\',\'host2\',\'host3\',\'host4\',\'host5\',\'host6\']'} - r = requests.get(ROOT_URL +"/api/v1/tagvalues") - # weed out just the hosts if possible - return r.json() - -api.add_resource(Hosts, '/monit/api/hosts') - -class Metrics(restful.Resource): - def get(self): - # This is a comprehensive list of metrics not per host due to limitations - r = requests.get(ROOT_URL +"/api/v1/metricnames") - return r.json() - -api.add_resource(Metrics, '/monit/api/metrics') - -class TsGenerateAlarmData: - def __init__(self): - self.params = " " - self.statusStr = "failure" - #self.resultStr = " " - - buildStr = '"alarms":[' - buildStr += '{"id":"critical","name":"critical","data":[' - for j in range(1, 30): - duration = random.randint(0,172800) # number of seconds in 2 days - starttime = long(time.time() - (j * 86400)) # number of seconds in a day - endtime = long(starttime + duration/2) - buildStr += '{"start":' + str(long(starttime*1000)) + ',"end":' + str(long(endtime*1000)) + '},' - - #remove a comma - buildStr = buildStr[:-1] - buildStr += ']},{"id":"minor","name":"minor","data":[' - for j in range(1, 30): - duration = random.randint(0,172800) # number of seconds in 2 days - starttime = long(time.time() - (j * 86400)) # number of seconds in a day - endtime = long(starttime + duration/2) - buildStr += '{"start":' + str(long(starttime*1000)) + ',"end":' + str(long(endtime*1000)) + '},' - - #remove a comma - buildStr = buildStr[:-1] - buildStr += ']},{"id":"positive","name":"positive","data":[' - for j in range(1, 30): - duration = random.randint(0,172800) # number of seconds in 2 days - starttime = long(time.time() - (j * 86400)) # number of seconds in a day - endtime = long(starttime + duration/2) - buildStr += '{"start":' + str(long(starttime*1000)) + ',"end":' + str(long(endtime*1000)) + '},' - - #remove a comma - buildStr = buildStr[:-1] - buildStr += ']},{"id":"info","name":"info","data":[' - for j in range(1, 30): - duration = random.randint(0,172800) # number of seconds in 2 days - starttime = long(time.time() - (j * 86400)) # number of seconds in a day - endtime = long(starttime + duration/2) - buildStr += '{"start":' + str(long(starttime*1000)) + ',"end":' + str(long(endtime*1000)) + '},' - - #remove a comma - buildStr = buildStr[:-1] - - self.resultStr = buildStr + ']}]' - - -class TsQueryBuilder: - def __init__(self, host_s, metric): - self.params = " " - self.statusStr = "failure" - buildStr = '{ "metrics":[' - - repeatStr = '{"tags":{ "host":["HOSTNAME"]},"name":"METRIC","group_by":[{"name":"tag", "tags":["host"]}],"aggregators":[{"name":"sum", "align_sampling": false, "sampling":{ "value": "2", "unit": "minutes"}}]}' - finStr = '],"start_relative": { "value": "17", "unit": "minutes" }}' - repeatStr = repeatStr.replace("METRIC", metric) - - i = len(host_s); - for hostname in host_s: - buildStr += repeatStr.replace("HOSTNAME", host_s[i-1]) - i -= 1 - if i != 0: - buildStr += ',' - self.params = buildStr + finStr - #print "Query: " + self.params - r = requests.post(ROOT_URL +"/api/v1/datapoints/query", data=self.params) - - # check the POST status and return success or fail here - if r.status_code == requests.codes.ok: - self.statusStr = "success" - self.resp_dict = r.json() - -class HostMetric(restful.Resource): - def get(self, hostname, metricname): - # Get URL Parameters for host and metric info - urlParam = request.url.split('/', 9); - MyList = [urlParam[6]] - - # Create and execute the query - qb = TsQueryBuilder(MyList, urlParam[8]) - - # Create JSON Prefix - valStr = '{"result":[{"metrics":[{"id":"' + urlParam[8] + '","data":[' - - # add all time series data - for key, value in qb.resp_dict["queries"][0]["results"][0]["values"]: - # round this value down - digits = str(key) - key = key - (int(digits[8]) * 10000) - key = key - (int(digits[9]) * 1000) - key = key - (int(digits[10]) * 100) - valStr += '{"time":' + str(key) + ',"value":' + str(value) + '},' - - #remove a comma - valStr = valStr[:-1] - - a = TsGenerateAlarmData() - #add final braces - valStr += ']}],' - valStr += a.resultStr - valStr += ',"id":"' + urlParam[6] + '"}],"status":"' + qb.statusStr + '"}' - - #return valStr - return json.loads(valStr) - -api.add_resource( - HostMetric, - '/monit/api/host//metric/', - defaults={'hostname': '', 'metricname': ''} -) - -class HostGroupMetric(restful.Resource): - def get(self, hostgroup, metricname): - # Get URL Parameters for host and metric info - urlParam = request.url.split('/', 9); - hostgroup = urlParam[6] - - # need logic to query hostgroup and convert to list - MyList = ["host1","host2","host3","host4","host5","host6"] - - # Create and execute the query - qb = TsQueryBuilder(MyList, urlParam[8]) - - # Create JSON Prefix - valStr = '{"result":[' - - #return qb.resp_dict - - # add all time series data - i = 0 - for host in MyList: - valStr += '{"metrics":[{"id":"' + urlParam[8] + '","data":[' - for skey, svalue in qb.resp_dict["queries"][i]["results"][0]["values"]: - # round this value down - digits = str(skey) - skey = skey - (int(digits[8]) * 10000) - skey = skey - (int(digits[9]) * 1000) - skey = skey - (int(digits[10]) * 100) - valStr += '{"time":' + str(skey) + ',"value":' + str(svalue) + '},' - #remove a comma - valStr = valStr[:-1] - - #add final recordbraces - #valStr += ']}],"id":"' + MyList[i] + '"},' - - a = TsGenerateAlarmData() - #add final record braces - valStr += ']}],' - valStr += a.resultStr - valStr += ',"id":"' + MyList[i] + '"},' - - i += 1; - - #remove a comma - valStr = valStr[:-1] - - #add final braces - valStr += '],"status":"' + qb.statusStr + '"}' - - #return valStr - return json.loads(valStr) - -api.add_resource( - HostGroupMetric, - '/monit/api/hostgroup//metric/', - defaults={'hostgroup': '', 'metricname': ''} -) - - -class RsHostMetric(restful.Resource): - def get(self, hostname, metricname): - # Get URL Parameters for host and metric info - urlParam = request.url.split('/', 9); - MyList = [urlParam[6]] - - # Create and execute the query - qb = TsQueryBuilder(MyList, urlParam[8]) - - # Create JSON Prefix - valStr = '{"series":[{"metrics":[{"id":"' + urlParam[8] + '","data":[' - - # add all time series data - for key, value in qb.resp_dict["queries"][0]["results"][0]["values"]: - # round this value down - digits = str(key) - key = key - (int(digits[8]) * 10000) - key = key - (int(digits[9]) * 1000) - key = key - (int(digits[10]) * 100) - valStr += '{"time":' + str(key) + ',"value":' + str(value) + '},' - - #remove a comma - valStr = valStr[:-1] - - #add final braces - valStr += ']}],"id":"' + urlParam[6] + '"}],"status":"' + qb.statusStr + '"}' - - return json.loads(valStr) - -api.add_resource( - RsHostMetric, - '/monit/api/rshost//metric/', - defaults={'hostname': '', 'metricname': ''} -) - -class RsHostGroupMetric(restful.Resource): - def get(self, hostgroup, metricname): - # Get URL Parameters for host and metric info - urlParam = request.url.split('/', 9) #; - urlCruft = urlParam[8].split('?', 2) #; - hostgroup = urlParam[6] - themetric = urlCruft[0] - - print themetric - - # need logic to query hostgroup and convert to list - MyList = ["host1","host2","host3","host4","host5","host6"] - - # Create and execute the query - qb = TsQueryBuilder(MyList, themetric) - - # Create JSON Prefix - valStr = '[' - - #return qb.resp_dict - - # add all time series data - i = 0 - for host in MyList: - valStr += '{"name":"' + MyList[i] + ": " + themetric + '","data":[' - for skey, svalue in qb.resp_dict["queries"][i]["results"][0]["values"]: - # round this value down - digits = str(skey) - skey = skey - (int(digits[8]) * 10000) - skey = skey - (int(digits[9]) * 1000) - skey = skey - (int(digits[10]) * 100) - valStr += '{"x":' + str(skey) + ',"y":' + str(svalue) + '},' - - #remove a comma - valStr = valStr[:-1] - - #add final recordbraces - valStr += ']},' - - i += 1; - - #remove a comma - valStr = valStr[:-1] - - #add final braces - valStr += ']' - - callback = request.args.get('callback', False) - if callback: - content = str(callback) + '(' + valStr + ')' - return current_app.response_class(content, mimetype='application/json') - - #return valStr - return json.loads(valStr) - -api.add_resource( - RsHostGroupMetric, - '/monit/api/rshostgroup//metric/', - defaults={'hostgroup': '', 'metricname': ''} -) - -class Alarms(restful.Resource): - def get(self): - alarms = TsGenerateAlarmData() - r = alarms.resultStr - #return r - return json.loads("{"+r+"}") - -api.add_resource(Alarms, '/monit/api/alarms') - -class Services(restful.Resource): - def get(self): - r = requests.get(ROOT_URL +"/api/v1/services") - return r.json() - -api.add_resource(Services, '/monit/api/services') - -class Topology(restful.Resource): - def get(self): - #r = requests.get(ROOT_URL +"/api/v1/topology") - #return r.json() - valAStr = '{"status":"success", "result":[ {"id":"uc-datacenter","name":"uc-datacenter","children":[ {"id":"nova-compute","name":"nova-compute","state":"running","resource":"services","type":"service","children":[ {"id":"compute-server-1.huawei.com","name":"compute-server-1.huawei.com","state":"running","resource":"hosts","type":"server", "children":[ {"id":"nova-compute","name":"nova-compute","state":"running","resource":"services","type":"service","children":[]}, {"id":"nova-consoleauth","name":"nova-consoleauth","state":"running","resource":"services","type":"service","children":[]} ]}, {"id":"compute-server-2.huawei.com","name":"compute-server-2.huawei.com","state":"running","resource":"hosts","type":"server", "children":[ {"id":"nova-novncproxy","name":"nova-novncproxy","state":"running","resource":"services","type":"service","children":[]}, {"id":"ceilometer-agent-compute","name":"ceilometer-agent-compute","state":"running","resource":"services","type":"service","children":[]}, {"id":"neutron-openvswitch-agent","name":"nova-openvswitch-agent","state":"running","resource":"services","type":"service","children":[]} ]} ]} ]} ]}' - - valStr = '{"status":"success", "result":[{"id":"Huawei-Lab-C","name":"Huawei-Lab-C","children":[ {"id":"10.145.81.219","name":"10.145.81.219","state":"warning","resource":"services","type":"service","children":[ {"name":"server-1.huawei.com","id":"host1.huawei.com","state":"running","resource":"hosts","type":"server", "children":[]}, {"name":"server-2.huawei.com","id":"host2.huawei.com","state":"running","resource":"hosts","type":"server", "children":[]}, {"name":"server-3.huawei.com","id":"host3.huawei.com","state":"running","resource":"hosts","type":"server", "children":[]}, {"name":"server-4.huawei.com","id":"host4.huawei.com","state":"running","resource":"hosts","type":"server", "children":[]}, {"name":"server-5.huawei.com","id":"host5.huawei.com","state":"warning","resource":"hosts","type":"server", "children":[]}, {"name":"server-6.huawei.com","id":"host6.huawei.com","state":"running","resource":"hosts","type":"server", "children":[]}, {"name":"server-7.huawei.com","id":"host7.huawei.com","state":"critical","resource":"hosts","type":"server", "children":[]}, {"name":"monit-server-1.huawei.com","id":"10_145_81_205","state":"running","resource":"hosts","type":"server", "children":[]}]}]}]}' - return json.loads(valStr) - - #return valStr - -api.add_resource(Topology, '/monit/api/topologies/1') - -if __name__ == '__main__': - global ROOT_URL - ROOT_URL = "http://10.145.81.205:8080" - app.run(host='0.0.0.0', debug=True, threaded=True) - #app.run(debug=True) - - - -""" -# Topo Structure -{"status":"success", "result":[ - {"id":"uc-datacenter","name":"uc-datacenter","children":[ - {"id":"nova-compute","name":"nova-compute","state":"running","resource":"services","type":"service","children":[ - {"id":"compute-server-1.huawei.com","name":"compute-server-1.huawei.com","state":"running","resource":"hosts","type":"server", "children":[ - {"id":"nova-compute","name":"nova-compute","state":"running","resource":"services","type":"service","children":[]}, - {"id":"nova-consoleauth","name":"nova-consoleauth","state":"running","resource":"services","type":"service","children":[]}, - ]}, - {"id":"compute-server-2.huawei.com","name":"compute-server-2.huawei.com","state":"running","resource":"hosts","type":"server", "children":[ - {"id":"nova-novncproxy","name":"nova-novncproxy","state":"running","resource":"services","type":"service","children":[]}, - {"id":"ceilometer-agent-compute","name":"ceilometer-agent-compute","state":"running","resource":"services","type":"service","children":[]}, - {"id":"neutron-openvswitch-agent","name":"nova-openvswitch-agent","state":"running","resource":"services","type":"service","children":[]} - ]} - ]} - ]} -]} - -#Query structure -{ - "metrics": [ - { - "tags": { - "host": [ "nova-compute_local" ] - }, - "name": "cpu.0.cpu.idle.value", - "aggregators": [ - { - "name": "avg", - "align_sampling": true, - "sampling": { "value": "1", "unit": "minutes" } - } - ] - }, - { - "tags": { - "host": [ "controller_local" ] - }, - "name": "cpu.0.cpu.idle.value", - "aggregators": [ - { - "name": "avg", - "align_sampling": true, - "sampling": { "value": "1", "unit": "minutes" } - } - ] - }, - { - "tags": { - "host": [ "nova-compute_local" ] - }, - "name": "cpu.0.cpu.idle.value", - "aggregators": [ - { - "name": "avg", - "align_sampling": true, - "sampling": { "value": "1", "unit": "minutes" } - } - ] - }, - { - "tags": { - "host": [ "neutron_local" ] - }, - "name": "cpu.0.cpu.idle.value", - "aggregators": [ - { - "name": "avg", - "align_sampling": true, - "sampling": { "value": "1", "unit": "minutes" } - } - ] - } - ], - "cache_time": 0, - "start_relative": { - "value": "1", - "unit": "hours" - } -} - -# Alarms -{?[?{?[?{.. more json here ... - -{"id":"uc-server-1.huawei.com","name":"uc-server-1.huawei.com","resource":"hosts","state":"running","type":"server", "metrics":[], "alarms":[ - {"id":"critical","name":"critical","data":[ - {"start":1406831282409,"end":1406870037149}, - {"start":1406745382748,"end":1406761927670} - ]}, - {"id":"minor","name":"minor","data":[ - {"start":1406873957790,"end":1406886655198}, - {"start":1406774590378,"end":1406850781190} - ]}, - {"id":"positive","name":"positive","data":[ - {"start":1406873957790,"end":1406886655198}, - {"start":1406774590378,"end":1406850781190} - ]}, - {"id":"info","name":"info","data":[ - {"start":1406873957790,"end":1406886655198}, - {"start":1406774590378,"end":1406850781190} - ]} -]} -... more json here...?}?]?}?]?} -""" diff --git a/compass_metrics/api/anapi_v2.py b/compass_metrics/api/api.py similarity index 88% rename from compass_metrics/api/anapi_v2.py rename to compass_metrics/api/api.py index 43c59d2..722dcdd 100755 --- a/compass_metrics/api/anapi_v2.py +++ b/compass_metrics/api/api.py @@ -1,48 +1,65 @@ -#!/usr/bin/env python +# Copyright 2014 Huawei Technologies Co. Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. -from flask import Flask -from flask.ext import restful -from flask.ext.restful import reqparse -from flask.ext.jsonpify import jsonify -#from flask import request -from flask import redirect, request, current_app +"""Define all the RestfulAPI entry points.""" +import logging +import requests +import random +import simplejson as json +import time import urllib -import json -import requests -import random -import time +from flask.ext import restful +from flask import redirect, request, current_app -app = Flask(__name__) +from compass_metrics.api import app + +from compass_metrics.utils import flags +from compass_metrics.utils import logsetting +from compass_metrics.utils import setting_wrapper as setting api = restful.Api(app) -ROOT_URL = "http://metrics:8080" - -#class MyEncoder1(json.JSONEncoder): -# def default(self, obj): +ROOT_URL = setting.ROOT_URL class HelloInfo(restful.Resource): def get(self): - #return {'info': 'Select host, hostgroup, topology, service from here'} - r = requests.get(ROOT_URL+"/api/v1/tagnames") + # return {'info': 'Select host, hostgroup, topology, service from here'} + url = ROOT_URL + '/api/v1/tagnames' + logging.debug('get url %s', url) + r = requests.get(url) + logging.debug('%s response: %s', request.path, r) return r.json() + api.add_resource(HelloInfo, '/') + class Proxy(restful.Resource): def get(self, url): # Get URL Parameters for host and metric info - urlParam = request.url.split('proxy/', 9); + urlParam = request.url.split('proxy/', 9) my_url = urlParam[1] - #return my_url - r = requests.get(my_url, stream = True) - #return r.text + # return my_url + r = requests.get(my_url, stream=True) + # return r.text return json.loads(r.text) - #return Response(stream_with_context(r.iter_content()), content_type = r.headers['content-type']) + # return Response(stream_with_context(r.iter_content()), content_type = r.headers['content-type']) + api.add_resource(Proxy, '/proxy/', defaults={'url': '/HelloWorld'}) @@ -54,16 +71,20 @@ class Hosts(restful.Resource): # weed out just the hosts if possible return r.json() + api.add_resource(Hosts, '/hosts') + class Metrics(restful.Resource): def get(self): # This is a comprehensive list of metrics not per host due to limitations r = requests.get(ROOT_URL +"/api/v1/metricnames") return r.json() + api.add_resource(Metrics, '/metrics') + class TsGenerateAlarmData: def __init__(self): self.params = " " @@ -117,7 +138,28 @@ class TsQueryBuilder: self.statusStr = "failure" buildStr = '{ "metrics":[' - repeatStr = '{"tags":{ "host":["HOSTNAME"]},"name":"METRIC","group_by":[{"name":"tag", "tags":["host"]}],"aggregators":[{"name":"sum", "align_sampling": false, "sampling":{ "value": "2", "unit": "minutes"}}]}' + repeatStr = json.dumps({ + "tags": { + "host": ["HOSTNAME"] + }, + "name": "METRIC", + "group_by": [ + { + "name":"tag", + "tags": ["host"] + } + ], + "aggregators": [ + { + "name": "sum", + "align_sampling": False, + "sampling": { + "value": "2", + "unit": "minutes" + } + } + ] + }) finStr = '],"start_relative": { "value": "17", "unit": "minutes" }}' repeatStr = repeatStr.replace("METRIC", metric) @@ -136,6 +178,7 @@ class TsQueryBuilder: self.statusStr = "success" self.resp_dict = r.json() + class HostMetric(restful.Resource): def get(self, hostname, metricname): # Get URL Parameters for host and metric info @@ -169,12 +212,14 @@ class HostMetric(restful.Resource): #return valStr return json.loads(valStr) + api.add_resource( HostMetric, '/host//metric/', defaults={'hostname': '', 'metricname': ''} ) + class HostGroupMetric(restful.Resource): def get(self, hostgroup, metricname): # Get URL Parameters for host and metric info @@ -226,6 +271,7 @@ class HostGroupMetric(restful.Resource): #return valStr return json.loads(valStr) + api.add_resource( HostGroupMetric, '/hostgroup//metric/', @@ -262,12 +308,14 @@ class RsHostMetric(restful.Resource): return json.loads(valStr) + api.add_resource( RsHostMetric, '/rshost//metric/', defaults={'hostname': '', 'metricname': ''} ) + class RsHostGroupMetric(restful.Resource): def get(self, hostgroup, metricname): # Get URL Parameters for host and metric info @@ -336,15 +384,19 @@ class Alarms(restful.Resource): #return r return json.loads("{"+r+"}") + api.add_resource(Alarms, '/alarms') + class Services(restful.Resource): def get(self): r = requests.get(ROOT_URL +"/api/v1/services") return r.json() + api.add_resource(Services, '/services') + class Topology(restful.Resource): def get(self): #r = requests.get(ROOT_URL +"/api/v1/topology") @@ -358,10 +410,16 @@ class Topology(restful.Resource): api.add_resource(Topology, '/topologies/1') -if __name__ == '__main__': - app.run(host='0.0.0.0', debug=True, threaded=True) - #app.run(debug=True) +def init(): + pass + + +if __name__ == '__main__': + flags.init() + logsetting.init() + init() + app.run(host='0.0.0.0', debug=True, threaded=True) """ diff --git a/compass_metrics/api/monitor.wsgi b/compass_metrics/api/monitor.wsgi deleted file mode 100644 index 704a757..0000000 --- a/compass_metrics/api/monitor.wsgi +++ /dev/null @@ -1,6 +0,0 @@ -from os import sys, path - -sys.path.append(path.dirname(path.abspath(__file__))) - -import anapi_v2 -application = anapi_v2.app diff --git a/compass_metrics/utils/__init__.py b/compass_metrics/utils/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/compass_metrics/utils/flags.py b/compass_metrics/utils/flags.py new file mode 100644 index 0000000..21a74a7 --- /dev/null +++ b/compass_metrics/utils/flags.py @@ -0,0 +1,92 @@ +# Copyright 2014 Huawei Technologies Co. Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Module to load flags. + + .. moduleauthor:: Xiaodong Wang +""" +import sys + +from optparse import OptionParser + + +class Flags(object): + """Class to store flags.""" + + PARSER = OptionParser() + PARSED_OPTIONS = None + + @classmethod + def parse_args(cls): + """parse args.""" + (options, argv) = Flags.PARSER.parse_args() + sys.argv = [sys.argv[0]] + argv + Flags.PARSED_OPTIONS = options + + def __getattr__(self, name): + if Flags.PARSED_OPTIONS and hasattr(Flags.PARSED_OPTIONS, name): + return getattr(Flags.PARSED_OPTIONS, name) + + for option in Flags.PARSER.option_list: + if option.dest == name: + return option.default + + raise AttributeError('Option instance has no attribute %s' % name) + + def __setattr__(self, name, value): + if Flags.PARSED_OPTIONS and hasattr(Flags.PARSED_OPTIONS, name): + setattr(Flags.PARSED_OPTIONS, name, value) + return + + for option in Flags.PARSER.option_list: + if option.dest == name: + option.default = value + return + + object.__setattr__(self, name, value) + + +OPTIONS = Flags() + + +def init(): + """Init flag parsing. + """ + OPTIONS.parse_args() + + +def add(flagname, **kwargs): + """Add a flag name and its setting. + + :param flagname: flag name declared in cmd as --=... + :type flagname: str + """ + Flags.PARSER.add_option('--%s' % flagname, + dest=flagname, **kwargs) + + +def add_bool(flagname, default=True, **kwargs): + """Add a bool flag name and its setting. + + :param flagname: flag name declared in cmd as --[no]. + :type flagname: str + :param default: default value + :type default: bool + """ + Flags.PARSER.add_option('--%s' % flagname, + dest=flagname, default=default, + action="store_true", **kwargs) + Flags.PARSER.add_option('--no%s' % flagname, + dest=flagname, + action="store_false", **kwargs) diff --git a/compass_metrics/utils/logsetting.py b/compass_metrics/utils/logsetting.py new file mode 100644 index 0000000..1d0a53c --- /dev/null +++ b/compass_metrics/utils/logsetting.py @@ -0,0 +1,94 @@ +# Copyright 2014 Huawei Technologies Co. Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Module to setup logging configuration. + + .. moduleauthor:: Xiaodong Wang +""" + +import logging +import logging.handlers +import os +import os.path +import sys + +from compass_metrics.utils import flags +from compass_metrics.utils import setting_wrapper as setting + + +flags.add('loglevel', + help='logging level', default=setting.DEFAULT_LOGLEVEL) +flags.add('logdir', + help='logging directory', default=setting.DEFAULT_LOGDIR) +flags.add('logfile', + help='logging filename', default=None) +flags.add('log_interval', type='int', + help='log interval', default=setting.DEFAULT_LOGINTERVAL) +flags.add('log_interval_unit', + help='log interval unit', default=setting.DEFAULT_LOGINTERVAL_UNIT) +flags.add('log_format', + help='log format', default=setting.DEFAULT_LOGFORMAT) + + +# mapping str setting in flag --loglevel to logging level. +LOGLEVEL_MAPPING = { + 'finest': logging.DEBUG - 2, # more detailed log. + 'fine': logging.DEBUG - 1, # detailed log. + 'debug': logging.DEBUG, + 'info': logging.INFO, + 'warning': logging.WARNING, + 'error': logging.ERROR, + 'critical': logging.CRITICAL, +} + +# disable logging when logsetting.init not called +logging.getLogger().setLevel(logging.CRITICAL) + + +def init(): + """Init loggsetting. It should be called after flags.init.""" + loglevel = flags.OPTIONS.loglevel.lower() + logdir = flags.OPTIONS.logdir + logfile = flags.OPTIONS.logfile + logger = logging.getLogger() + if logger.handlers: + for handler in logger.handlers: + logger.removeHandler(handler) + + if logdir: + if not logfile: + logfile = os.path.basename(sys.argv[0]) + + handler = logging.handlers.TimedRotatingFileHandler( + os.path.join(logdir, logfile), + when=flags.OPTIONS.log_interval_unit, + interval=flags.OPTIONS.log_interval) + else: + if not logfile: + handler = logging.StreamHandler(sys.stderr) + else: + handler = logging.handlers.TimedRotatingFileHandler( + logfile, + when=flags.OPTIONS.log_interval_unit, + interval=flags.OPTIONS.log_interval) + + if loglevel in LOGLEVEL_MAPPING: + logger.setLevel(LOGLEVEL_MAPPING[loglevel]) + handler.setLevel(LOGLEVEL_MAPPING[loglevel]) + + formatter = logging.Formatter( + flags.OPTIONS.log_format) + + handler.setFormatter(formatter) + logger.addHandler(handler) diff --git a/compass_metrics/utils/setting_wrapper.py b/compass_metrics/utils/setting_wrapper.py new file mode 100644 index 0000000..4ed9f55 --- /dev/null +++ b/compass_metrics/utils/setting_wrapper.py @@ -0,0 +1,52 @@ +# Copyright 2014 Huawei Technologies Co. Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""comapss setting wrapper. + + .. moduleauthor:: Xiaodong Wang ,xiaodongwang@huawei.com> +""" +import datetime +import logging +import os +import os.path + + +# default setting +CONFIG_DIR = '/etc/compass_monit' +DEFAULT_LOGLEVEL = 'debug' +DEFAULT_LOGDIR = '/tmp' +DEFAULT_LOGINTERVAL = 1 +DEFAULT_LOGINTERVAL_UNIT = 'h' +DEFAULT_LOGFORMAT = ( + '%(asctime)s - %(filename)s - %(lineno)d - %(levelname)s - %(message)s') +WEB_LOGFILE = 'compass_monit.log' +ROOT_URL = 'http://localhost:8080' + +if ( + 'COMPASS_METRICS_IGNORE_SETTING' in os.environ and + os.environ['COMPASS_METRICS_IGNORE_SETTING'] +): + pass +else: + if 'COMPASS_METRICS_SETTING' in os.environ: + SETTING = os.environ['COMPASS_METRICS_SETTING'] + else: + SETTING = '/etc/compass_monit/setting' + + try: + logging.info('load setting from %s', SETTING) + execfile(SETTING, globals(), locals()) + except Exception as error: + logging.exception(error) + raise error diff --git a/conf/compass-monit.conf b/conf/compass-monit.conf new file mode 100644 index 0000000..c2c20f8 --- /dev/null +++ b/conf/compass-monit.conf @@ -0,0 +1,18 @@ +# Apache config for monitor server +# +# Specify python path if you use virtualenv + +WSGIDaemonProcess compass-monit threads=4 display-name=%{GROUP} +WSGIProcessGroup compass-monit +WSGIScriptAlias /monit/api/v1 /var/www/compass_monit/compass_monit.wsgi +WSGISocketPrefix /var/run/wsgi + + + DocumentRoot /var/www/compass_monit + + + Options Indexes FollowSymLinks + Order allow,deny + Allow from all + + diff --git a/conf/compass_monit.wsgi b/conf/compass_monit.wsgi new file mode 100644 index 0000000..8267902 --- /dev/null +++ b/conf/compass_monit.wsgi @@ -0,0 +1,25 @@ +#!/usr/bin/env python +import site +import sys +import os +import os.path + + +activate_this='$PythonHome/bin/activate_this.py' +execfile(activate_this, dict(__file__=activate_this)) +site.addsitedir('$PythonHome/lib/python2.6/site-packages') +sys.path.append('$PythonHome') +os.environ['PYTHON_EGG_CACHE'] = '/tmp/.egg' + +from compass_metrics.utils import flags +from compass_metrics.utils import logsetting +from compass_metrics.utils import setting_wrapper as setting + +flags.init() +flags.OPTIONS.logfile = setting.WEB_LOGFILE +logsetting.init() + +from compass_metrics.api import api as compass_metrics_api + +compass_metrics_api.init() +application = compass_metrics_api.app diff --git a/conf/monitor.conf b/conf/monitor.conf deleted file mode 100644 index 4c27880..0000000 --- a/conf/monitor.conf +++ /dev/null @@ -1,18 +0,0 @@ -# Apache config for monitor server -# -# Specify python path if you use virtualenv - -WSGIDaemonProcess api threads=4 display-name=%{GROUP} -WSGIProcessGroup api -WSGIScriptAlias /monit/api/v1 /var/www/restApi/monitor.wsgi -WSGISocketPrefix /var/run/wsgi - - - DocumentRoot /var/www/restApi - - - Options Indexes FollowSymLinks - Order allow,deny - Allow from all - - diff --git a/conf/setting b/conf/setting new file mode 100644 index 0000000..32deb14 --- /dev/null +++ b/conf/setting @@ -0,0 +1,4 @@ +DEFAULT_LOGLEVEL = 'debug' +DEFAULT_LOGDIR = '/var/log/compass_monit' +WEB_LOGFILE = 'compass_monit.log' +ROOT_URL = 'http://$ipaddr:8080' diff --git a/ez_setup.py b/ez_setup.py new file mode 100644 index 0000000..41d35d5 --- /dev/null +++ b/ez_setup.py @@ -0,0 +1,299 @@ +#!/usr/bin/python +# +# Copyright 2014 Huawei Technologies Co. Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Bootstrap setuptools installation + +If you want to use setuptools in your package's setup.py, just include this +file in the same directory with it, and add this to the top of your setup.py:: + + from ez_setup import use_setuptools + use_setuptools() + +If you want to require a specific version of setuptools, set a download +mirror, or use an alternate download directory, you can do so by supplying +the appropriate options to ``use_setuptools()``. + +This file can also be run as a script to install or upgrade setuptools. +""" +import optparse +import os +import shutil +import subprocess +import sys +import tarfile +import tempfile + +from distutils import log + + +try: + from site import USER_SITE +except ImportError: + USER_SITE = None + + +DEFAULT_VERSION = "0.9.6" +DEFAULT_URL = "https://pypi.python.org/packages/source/s/setuptools/" + + +def _python_cmd(*args): + """run cmd in python.""" + args = (sys.executable,) + args + return subprocess.call(args) == 0 + + +def _install(tarball, install_args=()): + """install tarball.""" + # extracting the tarball + tmpdir = tempfile.mkdtemp() + log.warn('Extracting in %s', tmpdir) + old_wd = os.getcwd() + try: + os.chdir(tmpdir) + tar = tarfile.open(tarball) + _extractall(tar) + tar.close() + + # going in the directory + subdir = os.path.join(tmpdir, os.listdir(tmpdir)[0]) + os.chdir(subdir) + log.warn('Now working in %s', subdir) + + # installing + log.warn('Installing Setuptools') + if not _python_cmd('setup.py', 'install', *install_args): + log.warn('Something went wrong during the installation.') + log.warn('See the error message above.') + # exitcode will be 2 + return 2 + + finally: + os.chdir(old_wd) + shutil.rmtree(tmpdir) + + +def _build_egg(egg, tarball, to_dir): + """build egg.""" + # extracting the tarball + tmpdir = tempfile.mkdtemp() + log.warn('Extracting in %s', tmpdir) + old_wd = os.getcwd() + try: + os.chdir(tmpdir) + tar = tarfile.open(tarball) + _extractall(tar) + tar.close() + + # going in the directory + subdir = os.path.join(tmpdir, os.listdir(tmpdir)[0]) + os.chdir(subdir) + log.warn('Now working in %s', subdir) + + # building an egg + log.warn('Building a Setuptools egg in %s', to_dir) + _python_cmd('setup.py', '-q', 'bdist_egg', '--dist-dir', to_dir) + + finally: + os.chdir(old_wd) + shutil.rmtree(tmpdir) + + # returning the result + log.warn(egg) + if not os.path.exists(egg): + raise IOError('Could not build the egg.') + + +def _do_download(version, download_base, to_dir, download_delay): + """download package.""" + egg = os.path.join(to_dir, 'setuptools-%s-py%d.%d.egg' + % (version, sys.version_info[0], sys.version_info[1])) + if not os.path.exists(egg): + tarball = download_setuptools(version, download_base, + to_dir, download_delay) + _build_egg(egg, tarball, to_dir) + + sys.path.insert(0, egg) + import setuptools + setuptools.bootstrap_install_from = egg + + +def use_setuptools(version=DEFAULT_VERSION, download_base=DEFAULT_URL, + to_dir=os.curdir, download_delay=15): + """use setuptools to do the setup.""" + # making sure we use the absolute path + to_dir = os.path.abspath(to_dir) + was_imported = 'pkg_resources' in sys.modules or \ + 'setuptools' in sys.modules + try: + import pkg_resources + except ImportError: + return _do_download(version, download_base, to_dir, download_delay) + + try: + pkg_resources.require("setuptools>=" + version) + return + except pkg_resources.DistributionNotFound: + return _do_download(version, download_base, to_dir, + download_delay) + + except pkg_resources.VersionConflict: + error = sys.exc_info()[1] + if was_imported: + sys.stderr.writelines([ + "The required version of setuptools (>=%s) is not available,", + "and can't be installed while this script is running. Please", + "install a more recent version first, using", + "'easy_install -U setuptools'.", + "", + "(Currently using %r)" % (version, error.args[0]), + "", + ]) + sys.exit(2) + else: + del pkg_resources, sys.modules['pkg_resources'] # reload ok + return _do_download(version, download_base, to_dir, + download_delay) + + +def download_setuptools(version=DEFAULT_VERSION, download_base=DEFAULT_URL, + to_dir=os.curdir, delay=15): + """Download setuptools from a specified location and return its filename + + `version` should be a valid setuptools version number that is available + as an egg for download under the `download_base` URL (which should end + with a '/'). `to_dir` is the directory where the egg will be downloaded. + `delay` is the number of seconds to pause before an actual download + attempt. + """ + # making sure we use the absolute path + to_dir = os.path.abspath(to_dir) + try: + from urllib.request import urlopen + except ImportError: + from urllib2 import urlopen + + tgz_name = "setuptools-%s.tar.gz" % version + url = download_base + tgz_name + saveto = os.path.join(to_dir, tgz_name) + src = dst = None + if not os.path.exists(saveto): # Avoid repeated downloads + try: + log.warn("Downloading %s", url) + src = urlopen(url) + # Read/write all in one block, so we don't create a corrupt file + # if the download is interrupted. + data = src.read() + dst = open(saveto, "wb") + dst.write(data) + finally: + if src: + src.close() + if dst: + dst.close() + + return os.path.realpath(saveto) + + +def _extractall(self, path=".", members=None): + """Extract all members from the archive to the current working + directory and set owner, modification time and permissions on + directories afterwards. `path' specifies a different directory + to extract to. `members' is optional and must be a subset of the + list returned by getmembers(). + """ + import copy + import operator + from tarfile import ExtractError + directories = [] + + if members is None: + members = self + + for tarinfo in members: + if tarinfo.isdir(): + # Extract directories with a safe mode. + directories.append(tarinfo) + tarinfo = copy.copy(tarinfo) + tarinfo.mode = 448 # decimal for oct 0700 + + self.extract(tarinfo, path) + + # Reverse sort directories. + if sys.version_info < (2, 4): + def sorter(dir1, dir2): + """sort dir""" + return cmp(dir1.name, dir2.name) + + directories.sort(sorter) + directories.reverse() + else: + directories.sort(key=operator.attrgetter('name'), reverse=True) + + # Set correct owner, mtime and filemode on directories. + for tarinfo in directories: + dirpath = os.path.join(path, tarinfo.name) + try: + self.chown(tarinfo, dirpath) + self.utime(tarinfo, dirpath) + self.chmod(tarinfo, dirpath) + except ExtractError: + error = sys.exc_info()[1] + if self.errorlevel > 1: + raise + else: + self._dbg(1, "tarfile: %s" % error) + + +def _build_install_args(options): + """Build install args + + Build the arguments to 'python setup.py install' on the setuptools package + """ + install_args = [] + if options.user_install: + if sys.version_info < (2, 6): + log.warn("--user requires Python 2.6 or later") + raise SystemExit(1) + + install_args.append('--user') + + return install_args + + +def _parse_args(): + """Parse the command line for options.""" + parser = optparse.OptionParser() + parser.add_option( + '--user', dest='user_install', action='store_true', default=False, + help='install in user site package (requires Python 2.6 or later)') + parser.add_option( + '--download-base', dest='download_base', metavar="URL", + default=DEFAULT_URL, + help='alternative URL from where to download the setuptools package') + options, _ = parser.parse_args() + # positional arguments are ignored + return options + + +def main(version=DEFAULT_VERSION): + """Install or upgrade setuptools and EasyInstall.""" + options = _parse_args() + tarball = download_setuptools(download_base=options.download_base) + return _install(tarball, _build_install_args(options)) + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/install/install.sh b/install/install.sh new file mode 100755 index 0000000..6069b3d --- /dev/null +++ b/install/install.sh @@ -0,0 +1,160 @@ +#!/bin/bash +# + +### Log the script all outputs locally +exec > >(sudo tee install.log) +exec 2>&1 + +### Creat a lock to avoid running multiple instances of script. +LOCKFILE="/tmp/`basename $0`" +LOCKFD=99 + +# PRIVATE +_lock() { flock -$1 $LOCKFD; } +_no_more_locking() { _lock u; _lock xn && rm -f $LOCKFILE; } +_prepare_locking() { eval "exec $LOCKFD>\"$LOCKFILE\""; trap _no_more_locking EXIT; } + +# ON START +_prepare_locking + +# PUBLIC +exlock_now() { _lock xn; } # obtain an exclusive lock immediately or fail + +exlock_now || exit 1 + +### BEGIN OF SCRIPT ### +DIR=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd ) + +### Trap any error code with related filename and line. +errtrap() +{ + FILE=${BASH_SOURCE[1]:-$BASH_SOURCE[0]} + echo "[FILE: "$(basename $FILE)", LINE: $1] Error: Command or function exited with status $2" +} + +if [[ "$-" == *x* ]]; then +trap 'errtrap $LINENO $?' ERR +fi + +# Install figlet +sudo yum -y install figlet >& /dev/null +if [[ "$?" != "0" ]]; then + echo "failed to install figlet" + exit 1 +else + echo "figlet is installed" +fi +figlet -ctf slant Compass Metrics Installer + +while [ $1 ]; do + flags=$1 + param=${flags/'--'/''} + var=$(echo $param | cut -d"=" -f1) + val=$(echo $param | cut -d"=" -f2) + export $var=$val + shift +done + +# Load variables +loadvars() +{ + varname=${1,,} + eval var=\$$(echo $1) + + if [[ -z $var ]]; then + echo -e "\x1b[32mPlease enter the $varname (Example: $2):\x1b[37m" + while read input + do + if [ "$input" == "" ]; then + echo "Default $varname '$2' chosen" + export $(echo $1)="$2" + break + else + echo "You have entered $input" + export $(echo $1)="$input" + break + fi + done + fi +} + +loadvars NIC "eth0" +sudo ifconfig $NIC +if [ $? -ne 0 ]; then + echo "There is no nic '$NIC' yet" + exit 1 +fi +sudo ifconfig $NIC | grep 'inet addr:' >& /dev/null +if [ $? -ne 0 ]; then + echo "There is not any IP address assigned to the NIC '$NIC' yet, please assign an IP address first." + exit 1 +fi + +export ipaddr=$(ifconfig $NIC | grep 'inet addr:' | cut -d: -f2 | awk '{ print $1}') + +export SCRIPT_DIR=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd ) +export COMPASS_METTRICS_DIR=${SCRIPT_DIR}/.. + +echo 'Installing Required packages for Compass monit...' +sudo yum clean all +sudo yum update -y --skip-broken + +sudo yum install -y python python-devel git wget syslinux mod_wsgi httpd yum-utils python-virtualenv +if [[ "$?" != "0" ]]; then + echo "failed to install yum dependency" + exit 1 +fi + +sudo easy_install --upgrade pip +if [[ "$?" != "0" ]]; then + echo "failed to install easy install" + exit 1 +fi + +sudo pip install virtualenvwrapper + +sudo chkconfig httpd on + +sudo mkdir -p /var/www/compass_monit +sudo mkdir -p /var/log/compass_monit +sudo mkdir -p /etc/compass_monit +sudo +sudo chmod -R 777 /var/log/compass_monit + + +cp -rf ${COMPASS_METTRICS_DIR}/conf/compass-monit.conf /etc/httpd/conf.d/compass-monit.conf +sudo cp -rf ${COMPASS_METTRICS_DIR}/conf/compass_monit.wsgi /var/www/compass_monit/compass_monit.wsgi +sudo cp -rf ${COMPASS_METTRICS_DIR}/conf/setting /etc/compass_monit/setting + +cd ${COMPASS_METTRICS_DIR} +source `which virtualenvwrapper.sh` +if ! lsvirtualenv |grep compass-monit>/dev/null; then + mkvirtualenv compass-monit +fi +workon compass-monit +python setup.py install +if [[ "$?" != "0" ]]; then + echo "failed to install compass-monit package" + deactivate + exit 1 +else + echo "compass-monit package is installed in virtualenv under current dir" +fi + +sudo sed -e 's|$PythonHome|'$VIRTUAL_ENV'|' -i /var/www/compass_monit/compass_monit.wsgi +sudo sed -i "s/\$ipaddr/$ipaddr/g" /etc/compass_monit/setting + +deactivate + +sudo service httpd restart +sleep 10 +sudo service httpd status +if [[ "$?" != "0" ]]; then + echo "httpd is not started" + exit 1 +else + echo "httpd has already started" +fi + +figlet -ctf slant Installation Complete! +echo -e "It takes\x1b[32m $SECONDS \x1b[0mseconds during the installation." diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..cb179eb --- /dev/null +++ b/requirements.txt @@ -0,0 +1,10 @@ +argparse +flask +flask-script +flask-restful +flask-sqlalchemy +flask-login +simplejson +requests +flask-wtf +itsdangerous diff --git a/setup.cfg b/setup.cfg new file mode 100644 index 0000000..d32affe --- /dev/null +++ b/setup.cfg @@ -0,0 +1,4 @@ +[egg_info] +tag_build= +tag_date=0 + diff --git a/setup.py b/setup.py new file mode 100644 index 0000000..7910394 --- /dev/null +++ b/setup.py @@ -0,0 +1,103 @@ +#!/usr/bin/python +# +# Copyright 2014 Huawei Technologies Co. Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Bootstrap setuptools installation + +If you want to use setuptools in your package's setup.py, just include this +file in the same directory with it, and add this to the top of your setup.py:: + + from ez_setup import use_setuptools + use_setuptools() + +If you want to require a specific version of setuptools, set a download +mirror, or use an alternate download directory, you can do so by supplying +the appropriate options to ``use_setuptools()``. + +This file can also be run as a script to install or upgrade setuptools. +""" + +"""setup script.""" +try: + from setuptools import find_packages +except ImportError: + from ez_setup import use_setuptools + use_setuptools() + + +from setuptools.command.test import test as TestCommand +from setuptools import setup + + +import os +import sys + + +class Tox(TestCommand): + """Tox to do the setup.""" + + def finalize_options(self): + TestCommand.finalize_options(self) + self.test_args = [] + self.test_suite = True + + def run_tests(self): + import tox + errno = tox.cmdline(self.test_args) + sys.exit(errno) + + +INSTALL_REQUIRES_DIR = os.path.join( + os.path.dirname(__file__), 'requirements.txt') + + +with open(INSTALL_REQUIRES_DIR, 'r') as requires_file: + REQUIREMENTS = [line.strip() for line in requires_file if line != '\n'] + + +setup( + name='compass_metrics', + version='0.1.0', + + # general info + description='Monitoring for Open Deployment System', + long_description='Monitoring for Open Deployment System', + author='Compass Dev Group, Huawei Cloud', + author_email='shuo.yang@huawei.com', + url='https://github.com/stackforge/compass-monit', + download_url='', + + # dependency + install_requires=REQUIREMENTS, + packages=find_packages(exclude=['compass_metrics.tests']), + include_package_data=True, + #TODO login UI will be replaced by compass_metrics's own templates later + package_data={'compass_metrics': ['templates/*.jinja', 'static/js/*.js', + 'static/css/*.css', 'static/img/*.png']}, + classifiers=[ + 'Development Status :: 5 - Production/Stable', + 'Environment :: Console', + 'Intended Audience :: Developers', + 'Intended Audience :: Information Technology', + 'Intended Audience :: System Administrators', + 'License :: OSI Approved :: Apache Software License', + 'Operating System :: POSIX :: Linux', + 'Programming Language :: Python :: 2.6', + 'Programming Language :: Python :: 2.7', + ], + # test, + tests_require=['tox'], + cmdclass={'test': Tox}, +) diff --git a/test-requirements.txt b/test-requirements.txt new file mode 100644 index 0000000..82584bd --- /dev/null +++ b/test-requirements.txt @@ -0,0 +1,9 @@ +discover +mock +unittest2 +testtools>=0.9.32 +testrepository>=0.0.17 +mimeparse +coverage>=3.6 +hacking>=0.8.0,<0.9 +virtualenv diff --git a/tools/install_venv.py b/tools/install_venv.py new file mode 100644 index 0000000..1385a3e --- /dev/null +++ b/tools/install_venv.py @@ -0,0 +1,73 @@ +# Copyright 2010 United States Government as represented by the +# Administrator of the National Aeronautics and Space Administration. +# All Rights Reserved. +# +# Copyright 2010 OpenStack Foundation +# Copyright 2013 IBM Corp. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +# Shamelessly copied from horizon project and renamed to Compass + +import os +import sys + +import install_venv_common as install_venv # noqa + + +def print_help(venv, root): + help = """ + Compass development environment setup is complete. + + Compass development uses virtualenv to track and manage Python + dependencies while in development and testing. + + To activate the Compass virtualenv for the extent of your current shell + session you can run: + + $ source %s/bin/activate + + Or, if you prefer, you can run commands in the virtualenv on a case by case + basis by running: + + $ %s/tools/with_venv.sh + + Also, make test will automatically use the virtualenv. + """ + print(help % (venv, root)) + + +def main(argv): + root = os.path.dirname(os.path.dirname(os.path.realpath(__file__))) + + if os.environ.get('tools_path'): + root = os.environ['tools_path'] + venv = os.path.join(root, '.venv') + if os.environ.get('venv'): + venv = os.environ['venv'] + + pip_requires = os.path.join(root, 'requirements.txt') + test_requires = os.path.join(root, 'test-requirements.txt') + py_version = "python%s.%s" % (sys.version_info[0], sys.version_info[1]) + project = 'Compass' + install = install_venv.InstallVenv(root, venv, pip_requires, test_requires, + py_version, project) + options = install.parse_args(argv) + install.check_python_version() + install.check_dependencies() + install.create_virtualenv(no_site_packages=options.no_site_packages) + install.install_dependencies() + print_help(venv, root) + +if __name__ == '__main__': + main(sys.argv) diff --git a/tools/install_venv_common.py b/tools/install_venv_common.py new file mode 100644 index 0000000..46822e3 --- /dev/null +++ b/tools/install_venv_common.py @@ -0,0 +1,172 @@ +# Copyright 2013 OpenStack Foundation +# Copyright 2013 IBM Corp. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +"""Provides methods needed by installation script for OpenStack development +virtual environments. + +Since this script is used to bootstrap a virtualenv from the system's Python +environment, it should be kept strictly compatible with Python 2.6. + +Synced in from openstack-common +""" + +from __future__ import print_function + +import optparse +import os +import subprocess +import sys + + +class InstallVenv(object): + + def __init__(self, root, venv, requirements, + test_requirements, py_version, + project): + self.root = root + self.venv = venv + self.requirements = requirements + self.test_requirements = test_requirements + self.py_version = py_version + self.project = project + + def die(self, message, *args): + print(message % args, file=sys.stderr) + sys.exit(1) + + def check_python_version(self): + if sys.version_info < (2, 6): + self.die("Need Python Version >= 2.6") + + def run_command_with_code(self, cmd, redirect_output=True, + check_exit_code=True): + """Runs a command in an out-of-process shell. + + Returns the output of that command. Working directory is self.root. + """ + if redirect_output: + stdout = subprocess.PIPE + else: + stdout = None + + proc = subprocess.Popen(cmd, cwd=self.root, stdout=stdout) + output = proc.communicate()[0] + if check_exit_code and proc.returncode != 0: + self.die('Command "%s" failed.\n%s', ' '.join(cmd), output) + return (output, proc.returncode) + + def run_command(self, cmd, redirect_output=True, check_exit_code=True): + return self.run_command_with_code(cmd, redirect_output, + check_exit_code)[0] + + def get_distro(self): + if (os.path.exists('/etc/fedora-release') or + os.path.exists('/etc/redhat-release')): + return Fedora( + self.root, self.venv, self.requirements, + self.test_requirements, self.py_version, self.project) + else: + return Distro( + self.root, self.venv, self.requirements, + self.test_requirements, self.py_version, self.project) + + def check_dependencies(self): + self.get_distro().install_virtualenv() + + def create_virtualenv(self, no_site_packages=True): + """Creates the virtual environment and installs PIP. + + Creates the virtual environment and installs PIP only into the + virtual environment. + """ + if not os.path.isdir(self.venv): + print('Creating venv...', end=' ') + if no_site_packages: + self.run_command(['virtualenv', '-q', '--no-site-packages', + self.venv]) + else: + self.run_command(['virtualenv', '-q', self.venv]) + print('done.') + else: + print("venv already exists...") + pass + + def pip_install(self, *args): + self.run_command(['tools/with_venv.sh', + 'pip', 'install', '--upgrade'] + list(args), + redirect_output=False) + + def install_dependencies(self): + print('Installing dependencies with pip (this can take a while)...') + + # First things first, make sure our venv has the latest pip and + # setuptools and pbr + self.pip_install('pip>=1.4') + self.pip_install('setuptools') + self.pip_install('pbr') + + self.pip_install('-r', self.requirements, '-r', self.test_requirements) + + def parse_args(self, argv): + """Parses command-line arguments.""" + parser = optparse.OptionParser() + parser.add_option('-n', '--no-site-packages', + action='store_true', + help="Do not inherit packages from global Python " + "install") + return parser.parse_args(argv[1:])[0] + + +class Distro(InstallVenv): + + def check_cmd(self, cmd): + return bool(self.run_command(['which', cmd], + check_exit_code=False).strip()) + + def install_virtualenv(self): + if self.check_cmd('virtualenv'): + return + + if self.check_cmd('easy_install'): + print('Installing virtualenv via easy_install...', end=' ') + if self.run_command(['easy_install', 'virtualenv']): + print('Succeeded') + return + else: + print('Failed') + + self.die('ERROR: virtualenv not found.\n\n%s development' + ' requires virtualenv, please install it using your' + ' favorite package management tool' % self.project) + + +class Fedora(Distro): + """This covers all Fedora-based distributions. + + Includes: Fedora, RHEL, CentOS, Scientific Linux + """ + + def check_pkg(self, pkg): + return self.run_command_with_code(['rpm', '-q', pkg], + check_exit_code=False)[1] == 0 + + def install_virtualenv(self): + if self.check_cmd('virtualenv'): + return + + if not self.check_pkg('python-virtualenv'): + self.die("Please install 'python-virtualenv'.") + + super(Fedora, self).install_virtualenv() diff --git a/tools/with_venv.sh b/tools/with_venv.sh new file mode 100755 index 0000000..7303990 --- /dev/null +++ b/tools/with_venv.sh @@ -0,0 +1,7 @@ +#!/bin/bash +TOOLS_PATH=${TOOLS_PATH:-$(dirname $0)} +VENV_PATH=${VENV_PATH:-${TOOLS_PATH}} +VENV_DIR=${VENV_NAME:-/../.venv} +TOOLS=${TOOLS_PATH} +VENV=${VENV:-${VENV_PATH}/${VENV_DIR}} +source ${VENV}/bin/activate && "$@" diff --git a/tox.ini b/tox.ini new file mode 100644 index 0000000..d05e7b0 --- /dev/null +++ b/tox.ini @@ -0,0 +1,38 @@ +[tox] +minversion = 1.6 +skipsdist = True +envlist = py26,py27,pep8 + +[testenv] +setenv = VIRTUAL_ENV={envdir} + LANG=en_US.UTF-8 + LANGUAGE=en_US:en + LC_ALL=C + COMPASS_IGNORE_SETTING=1 + +deps = -r{toxinidir}/requirements.txt + -r{toxinidir}/test-requirements.txt +install_command = pip install -U {opts} {packages} +usedevelop = True +commands = python setup.py testr --slowest --testr-args='{posargs}' +distribute = false + +[testenv:pep8] +commands = flake8 +distribute = false + +[testenv:venv] +commands = {posargs} + +[testenv:cover] +commands = python setup.py testr --coverage --testr-args='{posargs}' + +[tox:jenkins] +downloadcache = ~/cache/pip + +[flake8] +ignore = H302,H233,H803,F401 +show-source = true +builtins = _ +exclude=.venv,.git,.tox,dist,doc,*lib/python*,*egg,tools,build +