From 8163483591e60ffe4f93a269789963422e505363 Mon Sep 17 00:00:00 2001 From: Timur Nurlygayanov Date: Mon, 18 Mar 2013 19:38:37 +0400 Subject: [PATCH 01/37] Sync --- dashboard/api/windc.py | 6 ++-- dashboard/windc/tables.py | 35 +++++++++---------- .../windc/templates/windc/_services_tabs.html | 1 + dashboard/windc/views.py | 13 +++---- 4 files changed, 27 insertions(+), 28 deletions(-) diff --git a/dashboard/api/windc.py b/dashboard/api/windc.py index 5fcac63..0739722 100644 --- a/dashboard/api/windc.py +++ b/dashboard/api/windc.py @@ -78,8 +78,10 @@ def services_create(request, datacenter, parameters): def services_list(request, datacenter_id): session_id = None sessions = windcclient(request).sessions.list(datacenter_id) + LOG.critical('DC ID: ' + str(datacenter_id)) + for s in sessions: - if s.state in ['open', 'deployed', 'deploying']: + if s.state in ['open', 'deploying']: session_id = s.id if session_id is None: @@ -94,7 +96,7 @@ def services_list(request, datacenter_id): def services_get(request, datacenter_id, service_id): services = services_list(request, datacenter_id) - + for service in services: if service.id is service_id: return service diff --git a/dashboard/windc/tables.py b/dashboard/windc/tables.py index db29e6a..791fb11 100644 --- a/dashboard/windc/tables.py +++ b/dashboard/windc/tables.py @@ -134,15 +134,19 @@ class UpdateDCRow(tables.Row): def get_data(self, request, datacenter_id): return api.windc.datacenters_get(request, datacenter_id) - - + + class UpdateServiceRow(tables.Row): ajax = True def get_data(self, request, service_id): link = request.__dict__['META']['HTTP_REFERER'] datacenter_id = re.search('windc/(\S+)', link).group(0)[6:-1] - + + LOG.critical('////////////////') + LOG.critical(datacenter_id) + LOG.critical('////////////////') + return api.windc.services_get(request, datacenter_id, service_id) @@ -153,6 +157,13 @@ STATUS_DISPLAY_CHOICES = ( ('finished', 'Active') ) +STATUS_DISPLAY_CHOICES = ( + ('draft', 'Ready to deploy'), + ('pending', 'Wait for configuration'), + ('inprogress', 'Deploy in progress'), + ('finished', 'Active') +) + class WinDCTable(tables.DataTable): @@ -162,6 +173,7 @@ class WinDCTable(tables.DataTable): ('Active', True) ) + name = tables.Column('name', link=('horizon:project:windc:services'), verbose_name=_('Name')) @@ -182,7 +194,7 @@ class WinDCTable(tables.DataTable): class WinServicesTable(tables.DataTable): - + STATUS_CHOICES = ( (None, True), ('Ready to deploy', True), @@ -193,7 +205,7 @@ class WinServicesTable(tables.DataTable): link=('horizon:project:windc:service_details'),) _type = tables.Column('service_type', verbose_name=_('Type')) - + status = tables.Column('status', verbose_name=_('Status'), status=True, status_choices=STATUS_CHOICES, @@ -204,18 +216,5 @@ class WinServicesTable(tables.DataTable): verbose_name = _('Services') row_class = UpdateServiceRow status_columns = ['status'] - row_actions = (ShowDataCenterServices, DeleteDataCenter, - DeployDataCenter) - - -class WinServicesTable(tables.DataTable): - - name = tables.Column('name', verbose_name=_('Name'), - link=('horizon:project:windc:service_details'),) - _type = tables.Column('service_type', verbose_name=_('Type')) - - class Meta: - name = 'services' - verbose_name = _('Services') table_actions = (CreateService,) row_actions = (DeleteService,) diff --git a/dashboard/windc/templates/windc/_services_tabs.html b/dashboard/windc/templates/windc/_services_tabs.html index 49b3d24..a58eac8 100644 --- a/dashboard/windc/templates/windc/_services_tabs.html +++ b/dashboard/windc/templates/windc/_services_tabs.html @@ -51,6 +51,7 @@ {% block modal-footer %} {% if wizard.steps.prev %} + {% else %} {% endif %} diff --git a/dashboard/windc/views.py b/dashboard/windc/views.py index d6d9e2d..48bfb0d 100644 --- a/dashboard/windc/views.py +++ b/dashboard/windc/views.py @@ -66,10 +66,7 @@ class Wizard(ModalFormMixin, SessionWizardView, generic.FormView): parameters['configuration'] = 'standalone' parameters['name'] = str(form_list[1].data.get('1-dc_name', 'noname')) - - # Fix Me in orchestrator - parameters['domain'] = parameters['name'] - + parameters['domain'] = parameters['name'] # Fix Me in orchestrator parameters['adminPassword'] = \ str(form_list[1].data.get('1-adm_password', '')) dc_count = int(form_list[1].data.get('1-dc_count', 1)) @@ -91,10 +88,12 @@ class Wizard(ModalFormMixin, SessionWizardView, generic.FormView): dc_pass = form_list[1].data.get('1-domain_user_password', '') parameters['name'] = str(form_list[1].data.get('1-iis_name', 'noname')) - + parameters['domain'] = parameters['name'] parameters['credentials'] = {'username': 'Administrator', 'password': password} parameters['domain'] = str(domain) + # 'username': str(dc_user), + # 'password': str(dc_pass)} parameters['location'] = 'west-dc' parameters['units'] = [] @@ -140,9 +139,6 @@ class IndexView(tables.DataTableView): def get_data(self): try: data_centers = api.windc.datacenters_list(self.request) - for dc in data_centers: - dc.status = api.windc.datacenters_get_status(self.request, - dc.id) except: data_centers = [] exceptions.handle(self.request, @@ -162,6 +158,7 @@ class WinServices(tables.DataTableView): def get_data(self): try: dc_id = self.kwargs['data_center_id'] + self.datacenter_id = dc_id datacenter = api.windc.datacenters_get(self.request, dc_id) self.dc_name = datacenter.name services = api.windc.services_list(self.request, dc_id) From 91c9cc43ec72e5d151713faef15252fe7b91c76b Mon Sep 17 00:00:00 2001 From: Timur Nurlygayanov Date: Mon, 18 Mar 2013 19:42:38 +0400 Subject: [PATCH 02/37] Sync --- dashboard/windc/tables.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/dashboard/windc/tables.py b/dashboard/windc/tables.py index 791fb11..10b518a 100644 --- a/dashboard/windc/tables.py +++ b/dashboard/windc/tables.py @@ -142,10 +142,6 @@ class UpdateServiceRow(tables.Row): def get_data(self, request, service_id): link = request.__dict__['META']['HTTP_REFERER'] datacenter_id = re.search('windc/(\S+)', link).group(0)[6:-1] - - LOG.critical('////////////////') - LOG.critical(datacenter_id) - LOG.critical('////////////////') return api.windc.services_get(request, datacenter_id, service_id) From 4ad5e3b8853c583da8053ba7dbb7bcac5d4199c4 Mon Sep 17 00:00:00 2001 From: Timur Nurlygayanov Date: Mon, 18 Mar 2013 19:43:15 +0400 Subject: [PATCH 03/37] Sync --- dashboard/windc/tables.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dashboard/windc/tables.py b/dashboard/windc/tables.py index 10b518a..c8c6646 100644 --- a/dashboard/windc/tables.py +++ b/dashboard/windc/tables.py @@ -190,7 +190,7 @@ class WinDCTable(tables.DataTable): class WinServicesTable(tables.DataTable): - + STATUS_CHOICES = ( (None, True), ('Ready to deploy', True), @@ -201,7 +201,7 @@ class WinServicesTable(tables.DataTable): link=('horizon:project:windc:service_details'),) _type = tables.Column('service_type', verbose_name=_('Type')) - + status = tables.Column('status', verbose_name=_('Status'), status=True, status_choices=STATUS_CHOICES, From 09df742d00bb298d7839b53fe7244d6e29beeacb Mon Sep 17 00:00:00 2001 From: Timur Nurlygayanov Date: Mon, 18 Mar 2013 19:50:45 +0400 Subject: [PATCH 04/37] Fixed typo. --- dashboard/windc/templates/windc/_services_tabs.html | 1 - 1 file changed, 1 deletion(-) diff --git a/dashboard/windc/templates/windc/_services_tabs.html b/dashboard/windc/templates/windc/_services_tabs.html index a58eac8..49b3d24 100644 --- a/dashboard/windc/templates/windc/_services_tabs.html +++ b/dashboard/windc/templates/windc/_services_tabs.html @@ -51,7 +51,6 @@ {% block modal-footer %} {% if wizard.steps.prev %} - {% else %} {% endif %} From 67dca5c46ed9e1cc42a3c153afecf36260d81b3b Mon Sep 17 00:00:00 2001 From: Dmitry Teselkin Date: Tue, 19 Mar 2013 17:35:10 +0400 Subject: [PATCH 05/37] Write-Host replaced by Write-Log --- conductor/data/templates/agent/CreatePrimaryDC.template | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conductor/data/templates/agent/CreatePrimaryDC.template b/conductor/data/templates/agent/CreatePrimaryDC.template index b181dba..f3b6867 100644 --- a/conductor/data/templates/agent/CreatePrimaryDC.template +++ b/conductor/data/templates/agent/CreatePrimaryDC.template @@ -1,6 +1,6 @@ { "Scripts": [ - "RnVuY3Rpb24gU2V0LUxvY2FsVXNlclBhc3N3b3JkIHsNCiAgICBwYXJhbSAoDQogICAgICAgIFtTdHJpbmddICRVc2VyTmFtZSwNCiAgICAgICAgW1N0cmluZ10gJFBhc3N3b3JkLA0KICAgICAgICBbU3dpdGNoXSAkRm9yY2UNCiAgICApDQogICAgDQogICAgdHJhcCB7IFN0b3AtRXhlY3V0aW9uICRfIH0NCiAgICANCiAgICBpZiAoKEdldC1XbWlPYmplY3QgV2luMzJfVXNlckFjY291bnQgLUZpbHRlciAiTG9jYWxBY2NvdW50ID0gJ1RydWUnIEFORCBOYW1lPSckVXNlck5hbWUnIikgLWVxICRudWxsKSB7DQogICAgICAgIHRocm93ICJVbmFibGUgdG8gZmluZCBsb2NhbCB1c2VyIGFjY291bnQgJyRVc2VyTmFtZSciDQogICAgfQ0KICAgIA0KICAgIGlmICgkRm9yY2UpIHsNCiAgICAgICAgV3JpdGUtTG9nICJDaGFuZ2luZyBwYXNzd29yZCBmb3IgdXNlciAnJFVzZXJOYW1lJyB0byAnKioqKionIiAjIDopDQogICAgICAgIChbQURTSV0gIldpbk5UOi8vLi8kVXNlck5hbWUiKS5TZXRQYXNzd29yZCgkUGFzc3dvcmQpDQogICAgfQ0KICAgIGVsc2Ugew0KICAgICAgICBXcml0ZS1Mb2dXYXJuaW5nICJZb3UgYXJlIHRyeWluZyB0byBjaGFuZ2UgcGFzc3dvcmQgZm9yIHVzZXIgJyRVc2VyTmFtZScuIFRvIGRvIHRoaXMgcGxlYXNlIHJ1biB0aGUgY29tbWFuZCBhZ2FpbiB3aXRoIC1Gb3JjZSBwYXJhbWV0ZXIuIg0KICAgICAgICAkVXNlckFjY291bnQNCiAgICB9DQp9DQoNCg0KDQpGdW5jdGlvbiBJbnN0YWxsLVJvbGVQcmltYXJ5RG9tYWluQ29udHJvbGxlcg0Kew0KPCMNCi5TWU5PUFNJUw0KQ29uZmlndXJlIG5vZGUncyBuZXR3b3JrIGFkYXB0ZXJzLg0KQ3JlYXRlIGZpcnN0IGRvbWFpbiBjb250cm9sbGVyIGluIHRoZSBmb3Jlc3QuDQoNCi5FWEFNUExFDQpQUz4gSW5zdGFsbC1Sb2xlUHJpbWFyeURvbWFpbkNvbnRyb2xsZXIgLURvbWFpbk5hbWUgYWNtZS5sb2NhbCAtU2FmZU1vZGVQYXNzd29yZCAiUEBzc3cwcmQiDQoNCkluc3RhbGwgRE5TIGFuZCBBRERTLCBjcmVhdGUgZm9yZXN0IGFuZCBkb21haW4gJ2FjbWUubG9jYWwnLg0KU2V0IERDIHJlY292ZXJ5IG1vZGUgcGFzc3dvcmQgdG8gJ1BAc3N3MHJkJy4NCiM+DQoJDQoJcGFyYW0NCgkoDQoJCVtTdHJpbmddDQoJCSMgTmV3IGRvbWFpbiBuYW1lLg0KCQkkRG9tYWluTmFtZSwNCgkJDQoJCVtTdHJpbmddDQoJCSMgRG9tYWluIGNvbnRyb2xsZXIgcmVjb3ZlcnkgbW9kZSBwYXNzd29yZC4NCgkJJFNhZmVNb2RlUGFzc3dvcmQNCgkpDQoNCgl0cmFwIHsgU3RvcC1FeGVjdXRpb24gJF8gfQ0KDQogICAgICAgICMgQWRkIHJlcXVpcmVkIHdpbmRvd3MgZmVhdHVyZXMNCglBZGQtV2luZG93c0ZlYXR1cmVXcmFwcGVyIGANCgkJLU5hbWUgIkROUyIsIkFELURvbWFpbi1TZXJ2aWNlcyIsIlJTQVQtREZTLU1nbXQtQ29uIiBgDQoJCS1JbmNsdWRlTWFuYWdlbWVudFRvb2xzIGANCiAgICAgICAgLU5vdGlmeVJlc3RhcnQNCg0KDQoJV3JpdGUtTG9nICJDcmVhdGluZyBmaXJzdCBkb21haW4gY29udHJvbGxlciAuLi4iDQoJCQ0KCSRTTUFQID0gQ29udmVydFRvLVNlY3VyZVN0cmluZyAtU3RyaW5nICRTYWZlTW9kZVBhc3N3b3JkIC1Bc1BsYWluVGV4dCAtRm9yY2UNCgkJDQoJSW5zdGFsbC1BRERTRm9yZXN0IGANCgkJLURvbWFpbk5hbWUgJERvbWFpbk5hbWUgYA0KCQktU2FmZU1vZGVBZG1pbmlzdHJhdG9yUGFzc3dvcmQgJFNNQVAgYA0KCQktRG9tYWluTW9kZSBEZWZhdWx0IGANCgkJLUZvcmVzdE1vZGUgRGVmYXVsdCBgDQoJCS1Ob1JlYm9vdE9uQ29tcGxldGlvbiBgDQoJCS1Gb3JjZSBgDQoJCS1FcnJvckFjdGlvbiBTdG9wIHwgT3V0LU51bGwNCg0KCVdyaXRlLUhvc3QgIldhaXRpbmcgZm9yIHJlYm9vdCAuLi4iCQkNCiMJU3RvcC1FeGVjdXRpb24gLUV4aXRDb2RlIDMwMTAgLUV4aXRTdHJpbmcgIkNvbXB1dGVyIG11c3QgYmUgcmVzdGFydGVkIHRvIGZpbmlzaCBkb21haW4gY29udHJvbGxlciBwcm9tb3Rpb24uIg0KIwlXcml0ZS1Mb2cgIlJlc3RhcmluZyBjb21wdXRlciAuLi4iDQojCVJlc3RhcnQtQ29tcHV0ZXIgLUZvcmNlDQp9DQo=" + "RnVuY3Rpb24gU2V0LUxvY2FsVXNlclBhc3N3b3JkIHsKICAgIHBhcmFtICgKICAgICAgICBbU3RyaW5nXSAkVXNlck5hbWUsCiAgICAgICAgW1N0cmluZ10gJFBhc3N3b3JkLAogICAgICAgIFtTd2l0Y2hdICRGb3JjZQogICAgKQogICAgCiAgICB0cmFwIHsgU3RvcC1FeGVjdXRpb24gJF8gfQogICAgCiAgICBpZiAoKEdldC1XbWlPYmplY3QgV2luMzJfVXNlckFjY291bnQgLUZpbHRlciAiTG9jYWxBY2NvdW50ID0gJ1RydWUnIEFORCBOYW1lPSckVXNlck5hbWUnIikgLWVxICRudWxsKSB7CiAgICAgICAgdGhyb3cgIlVuYWJsZSB0byBmaW5kIGxvY2FsIHVzZXIgYWNjb3VudCAnJFVzZXJOYW1lJyIKICAgIH0KICAgIAogICAgaWYgKCRGb3JjZSkgewogICAgICAgIFdyaXRlLUxvZyAiQ2hhbmdpbmcgcGFzc3dvcmQgZm9yIHVzZXIgJyRVc2VyTmFtZScgdG8gJyoqKioqJyIgIyA6KQogICAgICAgIChbQURTSV0gIldpbk5UOi8vLi8kVXNlck5hbWUiKS5TZXRQYXNzd29yZCgkUGFzc3dvcmQpCiAgICB9CiAgICBlbHNlIHsKICAgICAgICBXcml0ZS1Mb2dXYXJuaW5nICJZb3UgYXJlIHRyeWluZyB0byBjaGFuZ2UgcGFzc3dvcmQgZm9yIHVzZXIgJyRVc2VyTmFtZScuIFRvIGRvIHRoaXMgcGxlYXNlIHJ1biB0aGUgY29tbWFuZCBhZ2FpbiB3aXRoIC1Gb3JjZSBwYXJhbWV0ZXIuIgogICAgICAgICRVc2VyQWNjb3VudAogICAgfQp9CgoKCkZ1bmN0aW9uIEluc3RhbGwtUm9sZVByaW1hcnlEb21haW5Db250cm9sbGVyCnsKPCMKLlNZTk9QU0lTCkNvbmZpZ3VyZSBub2RlJ3MgbmV0d29yayBhZGFwdGVycy4KQ3JlYXRlIGZpcnN0IGRvbWFpbiBjb250cm9sbGVyIGluIHRoZSBmb3Jlc3QuCgouRVhBTVBMRQpQUz4gSW5zdGFsbC1Sb2xlUHJpbWFyeURvbWFpbkNvbnRyb2xsZXIgLURvbWFpbk5hbWUgYWNtZS5sb2NhbCAtU2FmZU1vZGVQYXNzd29yZCAiUEBzc3cwcmQiCgpJbnN0YWxsIEROUyBhbmQgQUREUywgY3JlYXRlIGZvcmVzdCBhbmQgZG9tYWluICdhY21lLmxvY2FsJy4KU2V0IERDIHJlY292ZXJ5IG1vZGUgcGFzc3dvcmQgdG8gJ1BAc3N3MHJkJy4KIz4KCQoJcGFyYW0KCSgKCQlbU3RyaW5nXQoJCSMgTmV3IGRvbWFpbiBuYW1lLgoJCSREb21haW5OYW1lLAoJCQoJCVtTdHJpbmddCgkJIyBEb21haW4gY29udHJvbGxlciByZWNvdmVyeSBtb2RlIHBhc3N3b3JkLgoJCSRTYWZlTW9kZVBhc3N3b3JkCgkpCgoJdHJhcCB7IFN0b3AtRXhlY3V0aW9uICRfIH0KCiAgICAgICAgIyBBZGQgcmVxdWlyZWQgd2luZG93cyBmZWF0dXJlcwoJQWRkLVdpbmRvd3NGZWF0dXJlV3JhcHBlciBgCgkJLU5hbWUgIkROUyIsIkFELURvbWFpbi1TZXJ2aWNlcyIsIlJTQVQtREZTLU1nbXQtQ29uIiBgCgkJLUluY2x1ZGVNYW5hZ2VtZW50VG9vbHMgYAogICAgICAgIC1Ob3RpZnlSZXN0YXJ0CgoKCVdyaXRlLUxvZyAiQ3JlYXRpbmcgZmlyc3QgZG9tYWluIGNvbnRyb2xsZXIgLi4uIgoJCQoJJFNNQVAgPSBDb252ZXJ0VG8tU2VjdXJlU3RyaW5nIC1TdHJpbmcgJFNhZmVNb2RlUGFzc3dvcmQgLUFzUGxhaW5UZXh0IC1Gb3JjZQoJCQoJSW5zdGFsbC1BRERTRm9yZXN0IGAKCQktRG9tYWluTmFtZSAkRG9tYWluTmFtZSBgCgkJLVNhZmVNb2RlQWRtaW5pc3RyYXRvclBhc3N3b3JkICRTTUFQIGAKCQktRG9tYWluTW9kZSBEZWZhdWx0IGAKCQktRm9yZXN0TW9kZSBEZWZhdWx0IGAKCQktTm9SZWJvb3RPbkNvbXBsZXRpb24gYAoJCS1Gb3JjZSBgCgkJLUVycm9yQWN0aW9uIFN0b3AgfCBPdXQtTnVsbAoKCVdyaXRlLUxvZyAiV2FpdGluZyBmb3IgcmVib290IC4uLiIJCQojCVN0b3AtRXhlY3V0aW9uIC1FeGl0Q29kZSAzMDEwIC1FeGl0U3RyaW5nICJDb21wdXRlciBtdXN0IGJlIHJlc3RhcnRlZCB0byBmaW5pc2ggZG9tYWluIGNvbnRyb2xsZXIgcHJvbW90aW9uLiIKIwlXcml0ZS1Mb2cgIlJlc3RhcmluZyBjb21wdXRlciAuLi4iCiMJUmVzdGFydC1Db21wdXRlciAtRm9yY2UKfQo=" ], "Commands": [ { From 7f5f8bcc9f742c1249d19d3142580f01c269c36a Mon Sep 17 00:00:00 2001 From: Dmitry Teselkin Date: Tue, 19 Mar 2013 17:37:56 +0400 Subject: [PATCH 06/37] Old devstack scripts removed --- Deployment/devstack-scripts/localrc | 36 ------------- Deployment/devstack-scripts/post-stack.sh | 57 --------------------- Deployment/devstack-scripts/post-unstack.sh | 28 ---------- 3 files changed, 121 deletions(-) delete mode 100644 Deployment/devstack-scripts/localrc delete mode 100644 Deployment/devstack-scripts/post-stack.sh delete mode 100644 Deployment/devstack-scripts/post-unstack.sh diff --git a/Deployment/devstack-scripts/localrc b/Deployment/devstack-scripts/localrc deleted file mode 100644 index ef8a92e..0000000 --- a/Deployment/devstack-scripts/localrc +++ /dev/null @@ -1,36 +0,0 @@ -# lab_id = ( 100 | 101 | 102 ) -lab_id= -lab_password=K#er0P@ssw0rd - -if [ -z "$lab_id" ] ; then - echo "Please specify 'lab_id' parameter in 'localrc' file." - exit -fi - -#-------------------------------------- - -HOST_IP=172.18.124.${lab_id} - -FIXED_RANGE=10.0.${lab_id}.0/24 -NETWORK_GATEWAY=10.0.${lab_id}.1 - -FLAT_INTERFACE=eth1 -PUBLIC_INTERFACE=eth0 - - -ADMIN_PASSWORD=$lab_password -MYSQL_PASSWORD=$lab_password -RABBIT_PASSWORD=$lab_password -SERVICE_PASSWORD=$lab_password -SERVICE_TOKEN=tokentoken - - -ENABLED_SERVICES+=,heat,h-api,h-api-cfn,h-api-cw,h-eng - - -LOGFILE=/opt/stack/devstack/stack.sh.log -SCREEN_LOGDIR=/var/log/devstack -#SCREEN_LOGDIR=/dev/null - - -EXTRA_OPTS=(force_config_drive=true) diff --git a/Deployment/devstack-scripts/post-stack.sh b/Deployment/devstack-scripts/post-stack.sh deleted file mode 100644 index 2ca9945..0000000 --- a/Deployment/devstack-scripts/post-stack.sh +++ /dev/null @@ -1,57 +0,0 @@ -#!/bin/bash - -source openrc admin admin - -if [ -z "$(sudo rabbitmqctl list_users | grep keero)" ] ; then - echo "Adding RabbitMQ 'keero' user" - sudo rabbitmqctl add_user keero keero -else - echo "User 'Keero' already exists." -fi - - -if [ -z "$(sudo rabbitmq-plugins list -e | grep rabbitmq_management)" ] ; then - echo "Enabling RabbitMQ management plugin" - sudo rabbitmq-plugins enable rabbitmq_management -else - echo "RabbitMQ management plugin already enabled." -fi - - -echo "Restarting RabbitMQ ..." -sudo service rabbitmq-server restart - - -echo "* Removing nova flavors ..." -for id in $(nova flavor-list | awk '$2 ~ /[[:digit:]]/ {print $2}') ; do - echo "** Removing flavor '$id'" - nova flavor-delete $id -done - - -echo "* Creating new flavors ..." -nova flavor-create m1.small auto 2048 40 1 -nova flavor-create m1.medium auto 4096 60 2 -nova flavor-create m1.large auto 8192 80 4 - - -if [ -z "$(nova keypair-list | grep keero_key)" ] ; then - echo "Creating keypair 'keero_key' ..." - nova keypair-add keero_key -else - echo "Keypair 'keero_key' already exists" -fi - - -echo "Removing existing image" -glance image-delete ws-2012-full-agent - - -echo "* Importing image into glance ..." -glance image-create \ - --name ws-2012-full-agent \ - --disk-format qcow2 \ - --container-format ovf \ - --is-public true \ - --location http://172.18.124.100:8888/ws-2012-full-agent.qcow2 -# --file /opt/keero/iso/ws-2012-full-agent.qcow2 diff --git a/Deployment/devstack-scripts/post-unstack.sh b/Deployment/devstack-scripts/post-unstack.sh deleted file mode 100644 index 8e53265..0000000 --- a/Deployment/devstack-scripts/post-unstack.sh +++ /dev/null @@ -1,28 +0,0 @@ -#!/bin/bash - -source openrc admin admin - -if [ -z "$TOP_DIR" ] ; then - echo "Environment variable TOP_DIR is not set." - exit -fi - -echo "Devstack installed in '$TOP_DIR'" - -#Remove certificates -echo "* Removing certificate files ..." -for file in $(sudo find $TOP_DIR/accrc/ -type f -regex ".+.pem.*") ; do - echo "Removing file '$file'" - sudo rm -f "$file" -done - -# Remove logs -echo "* Removing 'devstack' logs ..." -sudo rm -f /var/log/devstack/* - - -echo "* Removing 'apache2' logs ..." -for file in $(sudo find /var/log/apache2 -type f) ; do - echo "Removing file '$file'" - sudo rm -f "$file" -done From 998b210043d958daf7262c5239b88febf376c973 Mon Sep 17 00:00:00 2001 From: Dmitry Teselkin Date: Tue, 19 Mar 2013 17:45:41 +0400 Subject: [PATCH 07/37] New devstack scripts added --- Deployment/devstack-scripts/devstack.localrc | 27 +++++ Deployment/devstack-scripts/functions.sh | 75 ++++++++++++++ Deployment/devstack-scripts/localrc | 15 +++ Deployment/devstack-scripts/post-stack.sh | 72 ++++++++++++++ Deployment/devstack-scripts/post-unstack.sh | 25 +++++ Deployment/devstack-scripts/pre-stack.sh | 99 +++++++++++++++++++ Deployment/devstack-scripts/pre-unstack.sh | 6 ++ Deployment/devstack-scripts/start-devstack.sh | 25 +++++ Deployment/devstack-scripts/start-vm.sh | 22 +++++ Deployment/devstack-scripts/stop-devstack.sh | 22 +++++ 10 files changed, 388 insertions(+) create mode 100644 Deployment/devstack-scripts/devstack.localrc create mode 100644 Deployment/devstack-scripts/functions.sh create mode 100644 Deployment/devstack-scripts/localrc create mode 100644 Deployment/devstack-scripts/post-stack.sh create mode 100644 Deployment/devstack-scripts/post-unstack.sh create mode 100644 Deployment/devstack-scripts/pre-stack.sh create mode 100644 Deployment/devstack-scripts/pre-unstack.sh create mode 100644 Deployment/devstack-scripts/start-devstack.sh create mode 100644 Deployment/devstack-scripts/start-vm.sh create mode 100644 Deployment/devstack-scripts/stop-devstack.sh diff --git a/Deployment/devstack-scripts/devstack.localrc b/Deployment/devstack-scripts/devstack.localrc new file mode 100644 index 0000000..758db5f --- /dev/null +++ b/Deployment/devstack-scripts/devstack.localrc @@ -0,0 +1,27 @@ +lab_id=101 +lab_password=swordfish + +HOST_IP=172.18.124.${lab_id} +#PUBLIC_INTERFACE=eth1 + +FIXED_RANGE=10.0.${lab_id}.0/24 +NETWORK_GATEWAY=10.0.${lab_id}.1 + +#PUBLIC_INTERFACE=eth0 +FLAT_INTERFACE=eth1 + +ADMIN_PASSWORD=$lab_password +MYSQL_PASSWORD=$lab_password +RABBIT_PASSWORD=$lab_password +SERVICE_PASSWORD=$lab_password +SERVICE_TOKEN=tokentoken +ENABLED_SERVICES+=,heat,h-api,h-api-cfn,h-api-cw,h-eng + +LOGFILE=/opt/stack/devstack/stack.sh.log +SCREEN_LOGDIR=/opt/stack/log/ +#SCREEN_LOGDIR=/dev/null + +API_RATE_LIMIT=False + +EXTRA_OPTS=(force_config_drive=true libvirt_images_type=qcow2 force_raw_images=false) + diff --git a/Deployment/devstack-scripts/functions.sh b/Deployment/devstack-scripts/functions.sh new file mode 100644 index 0000000..1e4f545 --- /dev/null +++ b/Deployment/devstack-scripts/functions.sh @@ -0,0 +1,75 @@ +#!/bin/bash + + + +# Test if the named environment variable is set and not zero length +# is_set env-var +#function is_set() { +# local var=\$"$1" +# eval "[ -n \"$var\" ]" # For ex.: sh -c "[ -n \"$var\" ]" would be better, but several exercises depends on this +#} + + + +# Prints "message" and exits +# die "message" +#function die() { +# local exitcode=$? +# if [ $exitcode == 0 ]; then +# exitcode=1 +# fi +# set +o xtrace +# local msg="[ERROR] $0:$1 $2" +# echo $msg 1>&2; +# if [[ -n ${SCREEN_LOGDIR} ]]; then +# echo $msg >> "${SCREEN_LOGDIR}/error.log" +# fi +# exit $exitcode +#} + + + +# Checks an environment variable is not set or has length 0 OR if the +# exit code is non-zero and prints "message" and exits +# NOTE: env-var is the variable name without a '$' +# die_if_not_set env-var "message" +function die_if_not_set() { + local exitcode=$? + set +o xtrace + local evar=$1; shift + if ! is_set $evar || [ $exitcode != 0 ]; then + if [[ -z "$1" ]] ; then + die "Env var '$evar' is not set!" + else + die $@ + fi + fi +} + + + +function restart_service { + while [[ -n "$1" ]] ; do + echo "Restarting service '$1' ..." + sudo service $1 restart + shift 1 + done +} + + + +# Normalize config values to True or False +# Accepts as False: 0 no false False FALSE +# Accepts as True: 1 yes true True TRUE +# VAR=$(trueorfalse default-value test-value) +#function trueorfalse() { +# local default=$1 +# local testval=$2 +# +# [[ -z "$testval" ]] && { echo "$default"; return; } +# [[ "0 no false False FALSE" =~ "$testval" ]] && { echo "False"; return; } +# [[ "1 yes true True TRUE" =~ "$testval" ]] && { echo "True"; return; } +# echo "$default" +#} + + diff --git a/Deployment/devstack-scripts/localrc b/Deployment/devstack-scripts/localrc new file mode 100644 index 0000000..2658117 --- /dev/null +++ b/Deployment/devstack-scripts/localrc @@ -0,0 +1,15 @@ +#!/bin/bash + +DEVSTACK_DIR=/home/stack/devstack + +MYSQL_DB_TMPFS=true +MYSQL_DB_TMPFS_SIZE=128M + +NOVA_CACHE_TMPFS=true +NOVA_CACHE_TMPFS_SIZE=24G + + +#====================================== +source $DEVSTACK_DIR/openrc admin admin +source ./functions.sh + diff --git a/Deployment/devstack-scripts/post-stack.sh b/Deployment/devstack-scripts/post-stack.sh new file mode 100644 index 0000000..840568f --- /dev/null +++ b/Deployment/devstack-scripts/post-stack.sh @@ -0,0 +1,72 @@ +#!/bin/bash + +if [ -z "$1" ] ; then + source ./localrc +fi + + +function glance_image_create { + local __image_name=$1 + + if [[ -z "$__image_name" ]] ; then + echo "No image name provided!" + return + fi + + echo "Importing image '$__image_name' into Glance..." + glance image-delete "$__image_name" + glance image-create \ + --name "$__image_name" \ + --disk-format qcow2 \ + --container-format bare \ + --is-public true \ + --copy-from "http://172.18.124.100:8888/$__image_name.qcow2" +} + + +# Executing post-stack actions +#=============================================================================== + +if [ -z "$(sudo rabbitmqctl list_users | grep keero)" ] ; then + echo "Adding RabbitMQ 'keero' user" + sudo rabbitmqctl add_user keero keero +else + echo "User 'Keero' already exists." +fi + + +if [ -z "$(sudo rabbitmq-plugins list -e | grep rabbitmq_management)" ] ; then + echo "Enabling RabbitMQ management plugin" + sudo rabbitmq-plugins enable rabbitmq_management +else + echo "RabbitMQ management plugin already enabled." +fi + + +echo "Restarting RabbitMQ ..." +restart_service rabbitmq-server + + +echo "* Removing nova flavors ..." +for id in $(nova flavor-list | awk '$2 ~ /[[:digit:]]/ {print $2}') ; do + echo "** Removing flavor '$id'" + nova flavor-delete $id +done + + +echo "* Creating new flavors ..." +nova flavor-create m1.small auto 1024 40 1 +nova flavor-create m1.medium auto 2048 40 2 +nova flavor-create m1.large auto 4096 40 4 + + +if [ -z "$(nova keypair-list | grep keero_key)" ] ; then + echo "Creating keypair 'keero_key' ..." + nova keypair-add keero_key +else + echo "Keypair 'keero_key' already exists" +fi + +#=============================================================================== + +glance_image_create "ws-2012-full" diff --git a/Deployment/devstack-scripts/post-unstack.sh b/Deployment/devstack-scripts/post-unstack.sh new file mode 100644 index 0000000..962c731 --- /dev/null +++ b/Deployment/devstack-scripts/post-unstack.sh @@ -0,0 +1,25 @@ +#!/bin/bash + +if [[ -z "$1" ]] ; then + source ./localrc +fi + + +#Remove certificates +echo "* Removing old certificate files" +for file in $(sudo find $DEVSTACK_DIR/accrc/ -type f -regex ".+.pem.*") ; do + echo "Removing file '$file'" + sudo rm -f "$file" +done + +# Remove logs +echo Removing 'devstack' logs +sudo rm -f /var/log/devstack/* +#sudo rm -f /opt/stack/devstack/stack.sh.log + +echo "* Removing 'apache2' logs" +for file in $(sudo find /var/log/apache2 -type f) ; do + echo "Removing file '$file'" + sudo rm -f "$file" +done + diff --git a/Deployment/devstack-scripts/pre-stack.sh b/Deployment/devstack-scripts/pre-stack.sh new file mode 100644 index 0000000..c90f8f3 --- /dev/null +++ b/Deployment/devstack-scripts/pre-stack.sh @@ -0,0 +1,99 @@ +#!/bin/bash + +if [ -z "$1" ] ; then + source ./localrc +fi + + +# Executing pre-stack actions +#=============================================================================== + +# Executing checks +#----------------- +die_if_not_set DEVSTACK_DIR +die_if_not_set MYSQL_DB_TMPFS_SIZE +die_if_not_set NOVA_CACHE_TMPFS_SIZE +#----------------- + + +restart_service dbus rabbitmq-server + + +# Moving MySQL database to tmpfs +#------------------------------- +if [[ $(trueorfalse True $MYSQL_DB_TMPFS) = "True" ]] ; then + die_if_not_set MYSQL_DB_TMPFS_SIZE + mount_dir=/var/lib/mysql + sudo -s << EOF + echo "Stopping MySQL Server" + service mysql stop + + umount $mount_dir + mount -t tmpfs -o size=$MYSQL_DB_TMPFS_SIZE tmpfs $mount_dir + chmod 700 $mount_dir + chown mysql:mysql $mount_dir + + mysql_install_db + + /usr/bin/mysqld_safe --skip-grant-tables & + sleep 5 +EOF + + sudo mysql << EOF + FLUSH PRIVILEGES; + SET PASSWORD FOR 'root'@'localhost' = PASSWORD('swordfish'); + SET PASSWORD FOR 'root'@'127.0.0.1' = PASSWORD('swordfish'); +EOF + + sudo -s << EOF + killall mysqld + sleep 5 + + echo "Starting MySQL Server" + service mysql start +EOF +else + echo "MYSQL_DB_TMPFS = '$MYSQL_DB_TMPFS'" +fi +#------------------------------- + + +# Devstack log folder +#-------------------- +sudo -s << EOF + mkdir -p $SCREEN_LOGDIR + chown stack:stack $SCREEN_LOGDIR +EOF +#-------------------- + + +# Moving nova images cache to tmpfs +#---------------------------------- +if [[ $(trueorfalse True $NOVA_CACHE_TMPFS) = "True" ]] ; then + die_if_not_set NOVA_CACHE_TMPFS_SIZE + mount_dir=/opt/stack/data/nova/instances + sudo -s << EOF + umount $mount_dir + mount -t tmpfs -o size=$NOVA_CACHE_TMPFS_SIZE tmpfs $mount_dir + chmod 775 $mount_dir + chown stack:stack $mount_dir +EOF +else + echo "NOVA_CACHE_TMPFS = '$NOVA_CACHE_TMPFS'" +fi + +#---------------------------------- + + +# Replacing devstack's localrc config +#------------------------------------ +if [[ -f "devstack.localrc" ]] ; then + rm -f "$DEVSTACK_DIR/localrc" + cp devstack.localrc "$DEVSTACK_DIR/localrc" +else + echo "File 'devstack.localrc' not found!" +fi +#------------------------------------ + +#=============================================================================== + diff --git a/Deployment/devstack-scripts/pre-unstack.sh b/Deployment/devstack-scripts/pre-unstack.sh new file mode 100644 index 0000000..e311369 --- /dev/null +++ b/Deployment/devstack-scripts/pre-unstack.sh @@ -0,0 +1,6 @@ +#!/bin/bash + +if [[ -z "$1" ]] ; then + source ./localrc +fi + diff --git a/Deployment/devstack-scripts/start-devstack.sh b/Deployment/devstack-scripts/start-devstack.sh new file mode 100644 index 0000000..305200d --- /dev/null +++ b/Deployment/devstack-scripts/start-devstack.sh @@ -0,0 +1,25 @@ +#!/bin/bash + +source ./localrc + + +# Executing pre-stack actions +#=============================================================================== +source ./pre-stack.sh no-localrc +#=============================================================================== + + + +# Creating stack +#=============================================================================== +$DEVSTACK_DIR/stack.sh +#=============================================================================== + + + +# Executing post-stack actions +#=============================================================================== +source ./post-stack.sh no-localrc +#=============================================================================== + + diff --git a/Deployment/devstack-scripts/start-vm.sh b/Deployment/devstack-scripts/start-vm.sh new file mode 100644 index 0000000..a71b63e --- /dev/null +++ b/Deployment/devstack-scripts/start-vm.sh @@ -0,0 +1,22 @@ +#!/bin/bash + +image_file=$1 + +function die { + echo "$@" + exit 1 +} + +[ -z "$image_file" ] && die "VM name MUST be provided!" +[ -f "$image_file" ] || die "File '$image_file' not found." + +echo "Starting VM '$image_file' ..." + +kvm \ + -m 2048 \ + -drive file="$image_file",if=virtio \ + -redir tcp:3389::3389 -redir tcp:3390::3390 \ + -nographic \ + -usbdevice tablet \ + -vnc :20 + diff --git a/Deployment/devstack-scripts/stop-devstack.sh b/Deployment/devstack-scripts/stop-devstack.sh new file mode 100644 index 0000000..af35304 --- /dev/null +++ b/Deployment/devstack-scripts/stop-devstack.sh @@ -0,0 +1,22 @@ +#!/bin/bash + +source ./localrc + + +# Executing pre-unstack actions +#=============================================================================== +source ./pre-unstack.sh no-localrc +#=============================================================================== + + +# Executing unstack.sh +#=============================================================================== +$DEVSTACK_DIR/unstack.sh +#=============================================================================== + + +# Executing post-unstack actions +#=============================================================================== +source ./post-unstack.sh no-localrc +#=============================================================================== + From 3c641a3828ae1dd4b9e390f6b6cbf8a02d512d3e Mon Sep 17 00:00:00 2001 From: Timur Nurlygayanov Date: Wed, 20 Mar 2013 16:35:47 +0400 Subject: [PATCH 08/37] Added unit tests for REST API client. Fixed pep8. --- .../portasclient/common/base.py | 13 +- .../portasclient/common/utils.py | 1 - python-portasclient/portasclient/shell.py | 56 +++--- .../portasclient/v1/services.py | 4 +- python-portasclient/tests/test_sanity.py | 169 ++++++++++++++++++ python-portasclient/tools/test-requires | 2 + 6 files changed, 210 insertions(+), 35 deletions(-) create mode 100644 python-portasclient/tests/test_sanity.py diff --git a/python-portasclient/portasclient/common/base.py b/python-portasclient/portasclient/common/base.py index 4115a84..e92def9 100644 --- a/python-portasclient/portasclient/common/base.py +++ b/python-portasclient/portasclient/common/base.py @@ -49,14 +49,16 @@ class Manager(object): def __init__(self, api): self.api = api - def _list(self, url, response_key=None, obj_class=None, body=None, headers={}): + def _list(self, url, response_key=None, obj_class=None, + body=None, headers={}): + resp, body = self.api.json_request('GET', url, headers=headers) if obj_class is None: obj_class = self.resource_class if response_key: - if not body.has_key(response_key): + if not response_key in body: body[response_key] = [] data = body[response_key] else: @@ -74,9 +76,12 @@ class Manager(object): return self.resource_class(self, body[response_key]) return self.resource_class(self, body) - def _create(self, url, body=None, response_key=None, return_raw=False, headers={}): + def _create(self, url, body=None, response_key=None, + return_raw=False, headers={}): + if body: - resp, body = self.api.json_request('POST', url, body=body, headers=headers) + resp, body = self.api.json_request('POST', url, + body=body, headers=headers) else: resp, body = self.api.json_request('POST', url, headers=headers) if return_raw: diff --git a/python-portasclient/portasclient/common/utils.py b/python-portasclient/portasclient/common/utils.py index 549a3b4..3b4d49d 100644 --- a/python-portasclient/portasclient/common/utils.py +++ b/python-portasclient/portasclient/common/utils.py @@ -22,7 +22,6 @@ import prettytable from portasclient.openstack.common import importutils - # Decorator for cli-args def arg(*args, **kwargs): def _decorator(func): diff --git a/python-portasclient/portasclient/shell.py b/python-portasclient/portasclient/shell.py index d57c0f4..124f38b 100644 --- a/python-portasclient/portasclient/shell.py +++ b/python-portasclient/portasclient/shell.py @@ -41,8 +41,7 @@ class PortasShell(object): # Global arguments parser.add_argument('-h', '--help', action='store_true', - help=argparse.SUPPRESS, - ) + help=argparse.SUPPRESS,) parser.add_argument('-d', '--debug', default=bool(utils.env('PORTASCLIENT_DEBUG')), @@ -60,24 +59,24 @@ class PortasShell(object): "\"insecure\" SSL (https) requests. " "The server's certificate will " "not be verified against any certificate " - "authorities. This option should be used with " - "caution.") + "authorities. This option should be used " + "with caution.") parser.add_argument('--cert-file', help='Path of certificate file to use in SSL ' - 'connection. This file can optionally be prepended' - ' with the private key.') + 'connection. This file can optionally be ' + 'prepended with the private key.') parser.add_argument('--key-file', help='Path of client key to use in SSL connection.' - ' This option is not necessary if your key is ' - 'prepended to your cert file.') + ' This option is not necessary if your ' + 'key is prepended to your cert file.') parser.add_argument('--ca-file', help='Path of CA SSL certificate(s) used to verify' - ' the remote server certificate. Without this ' - 'option glance looks for the default system ' - 'CA certificates.') + ' the remote server certificate. Without ' + 'this option glance looks for the default ' + 'system CA certificates.') parser.add_argument('--timeout', default=600, @@ -226,24 +225,24 @@ class PortasShell(object): endpoint = args.portas_url else: if not args.os_username: - raise exceptions.CommandError("You must provide a username via " - "either --os-username or via " - "env[OS_USERNAME]") + raise exceptions.CommandError("You must provide a username " + "via either --os-username " + "or via env[OS_USERNAME]") if not args.os_password: - raise exceptions.CommandError("You must provide a password via " - "either --os-password or via " - "env[OS_PASSWORD]") + raise exceptions.CommandError("You must provide a password " + "via either --os-password " + "or via env[OS_PASSWORD]") if not (args.os_tenant_id or args.os_tenant_name): - raise exceptions.CommandError("You must provide a tenant_id via " - "either --os-tenant-id or via " - "env[OS_TENANT_ID]") + raise exceptions.CommandError("You must provide a tenant_id " + "via either --os-tenant-id " + "or via env[OS_TENANT_ID]") if not args.os_auth_url: - raise exceptions.CommandError("You must provide an auth url via " - "either --os-auth-url or via " - "env[OS_AUTH_URL]") + raise exceptions.CommandError("You must provide an auth url " + "via either --os-auth-url or " + "via env[OS_AUTH_URL]") kwargs = { 'username': args.os_username, 'password': args.os_password, @@ -257,8 +256,8 @@ class PortasShell(object): _ksclient = self._get_ksclient(**kwargs) token = args.os_auth_token or _ksclient.auth_token - endpoint = args.portas_url or \ - self._get_endpoint(_ksclient, **kwargs) + url = args.portas_url + endpoint = url or self._get_endpoint(_ksclient, **kwargs) kwargs = { 'token': token, @@ -274,7 +273,8 @@ class PortasShell(object): try: args.func(client, args) except exceptions.Unauthorized: - raise exceptions.CommandError("Invalid OpenStack Identity credentials.") + msg = "Invalid OpenStack Identity credentials." + raise exceptions.CommandError(msg) @utils.arg('command', metavar='', nargs='?', help='Display help for ') @@ -286,8 +286,8 @@ class PortasShell(object): if args.command in self.subcommands: self.subcommands[args.command].print_help() else: - raise exceptions.CommandError("'%s' is not a valid subcommand" % - args.command) + msg = "'%s' is not a valid subcommand" + raise exceptions.CommandError(msg % args.command) else: self.parser.print_help() diff --git a/python-portasclient/portasclient/v1/services.py b/python-portasclient/portasclient/v1/services.py index dd65448..d195c6c 100644 --- a/python-portasclient/portasclient/v1/services.py +++ b/python-portasclient/portasclient/v1/services.py @@ -50,8 +50,8 @@ class ActiveDirectoryManager(base.Manager): headers = {'X-Configuration-Session': session_id} path = 'environments/{id}/activeDirectories/{active_directory_id}' - return self._delete(patch.format(id=environment_id, - active_directory_id=service_id), + return self._delete(path.format(id=environment_id, + active_directory_id=service_id), headers=headers) diff --git a/python-portasclient/tests/test_sanity.py b/python-portasclient/tests/test_sanity.py new file mode 100644 index 0000000..63d9e37 --- /dev/null +++ b/python-portasclient/tests/test_sanity.py @@ -0,0 +1,169 @@ +import unittest +import logging +from mock import MagicMock + +from portasclient.v1 import Client +import portasclient.v1.environments as environments +import portasclient.v1.services as services +import portasclient.v1.sessions as sessions + +LOG = logging.getLogger('Unit tests') + +def my_mock(*a, **b): + return [a, b] + +api = MagicMock(json_request=my_mock) + +class SanityUnitTests(unittest.TestCase): + + def test_create_client_instance(self): + + endpoint = 'http://localhost:8001' + test_client = Client(endpoint=endpoint, token='1', timeout=10) + + assert test_client.environments is not None + assert test_client.sessions is not None + assert test_client.activeDirectories is not None + assert test_client.webServers is not None + + def test_env_manager_list(self): + + manager = environments.EnvironmentManager(api) + + result = manager.list() + + assert result == [] + + def test_env_manager_create(self): + + manager = environments.EnvironmentManager(api) + + result = manager.create('test') + + assert result.headers == {} + assert result.body == {'name': 'test'} + + def test_env_manager_delete(self): + + manager = environments.EnvironmentManager(api) + + result = manager.delete('test') + + assert result is None + + def test_env_manager_update(self): + + manager = environments.EnvironmentManager(api) + + result = manager.update('1', 'test') + + assert result.body == {'name': 'test'} + + def test_env_manager_get(self): + + manager = environments.EnvironmentManager(api) + + result = manager.get('test') + ## WTF? + assert result.manager is not None + + def test_ad_manager_list(self): + + manager = services.ActiveDirectoryManager(api) + + result = manager.list('datacenter1') + + assert result == [] + + def test_ad_manager_create(self): + + manager = services.ActiveDirectoryManager(api) + + result = manager.create('datacenter1', 'session1', 'test') + + assert result.headers == {'X-Configuration-Session': 'session1'} + assert result.body == 'test' + + #@unittest.skip("https://mirantis.jira.com/browse/KEERO-218") + def test_ad_manager_delete(self): + + manager = services.ActiveDirectoryManager(api) + + result = manager.delete('datacenter1', 'session1', 'test') + + assert result is None + + def test_iis_manager_list(self): + + manager = services.WebServerManager(api) + + result = manager.list('datacenter1') + + assert result == [] + + def test_iis_manager_create(self): + + manager = services.WebServerManager(api) + + result = manager.create('datacenter1', 'session1', 'test') + + assert result.headers == {'X-Configuration-Session': 'session1'} + assert result.body == 'test' + + #@unittest.skip("https://mirantis.jira.com/browse/KEERO-218") + def test_iis_manager_delete(self): + + manager = services.WebServerManager(api) + + result = manager.delete('datacenter1', 'session1', 'test') + + assert result is None + + def test_session_manager_list(self): + + manager = sessions.SessionManager(api) + + result = manager.list('datacenter1') + + assert result == [] + + def test_session_manager_delete(self): + + manager = sessions.SessionManager(api) + + result = manager.delete('datacenter1', 'session1') + + assert result is None + + def test_session_manager_get(self): + + manager = sessions.SessionManager(api) + + result = manager.get('datacenter1', 'session1') + # WTF? + assert result.manager is not None + + def test_session_manager_configure(self): + + manager = sessions.SessionManager(api) + + result = manager.configure('datacenter1') + + assert result.headers == {} + + def test_session_manager_deploy(self): + + manager = sessions.SessionManager(api) + + result = manager.deploy('datacenter1', '1') + + assert result is None + + @unittest.skip("https://mirantis.jira.com/browse/KEERO-219") + def test_session_manager_reports(self): + + manager = sessions.SessionManager(api) + + result = manager.reports('datacenter1', '1') + + assert result == [] \ No newline at end of file diff --git a/python-portasclient/tools/test-requires b/python-portasclient/tools/test-requires index c39eaf1..6114c87 100644 --- a/python-portasclient/tools/test-requires +++ b/python-portasclient/tools/test-requires @@ -1,5 +1,7 @@ distribute>=0.6.24 +mock +anyjson mox nose nose-exclude From 3e1ade0c721235a9c943350924437bc9a78ab65f Mon Sep 17 00:00:00 2001 From: Timur Nurlygayanov Date: Wed, 20 Mar 2013 16:50:46 +0400 Subject: [PATCH 09/37] Fix and unit test for issue: https://mirantis.jira.com/browse/KEERO-219 --- python-portasclient/portasclient/v1/sessions.py | 2 +- python-portasclient/tests/test_sanity.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/python-portasclient/portasclient/v1/sessions.py b/python-portasclient/portasclient/v1/sessions.py index 200078b..f5b3247 100644 --- a/python-portasclient/portasclient/v1/sessions.py +++ b/python-portasclient/portasclient/v1/sessions.py @@ -60,7 +60,7 @@ class SessionManager(base.Manager): path.format(id=environment_id, session_id=session_id)) - data = body['reports'] + data = body.get('reports', []) return [Status(self, res, loaded=True) for res in data if res] def delete(self, environment_id, session_id): diff --git a/python-portasclient/tests/test_sanity.py b/python-portasclient/tests/test_sanity.py index 63d9e37..89d3628 100644 --- a/python-portasclient/tests/test_sanity.py +++ b/python-portasclient/tests/test_sanity.py @@ -159,7 +159,7 @@ class SanityUnitTests(unittest.TestCase): assert result is None - @unittest.skip("https://mirantis.jira.com/browse/KEERO-219") + #@unittest.skip("https://mirantis.jira.com/browse/KEERO-219") def test_session_manager_reports(self): manager = sessions.SessionManager(api) From 2d27f4f5054f34982ed67da2bf4b35c8ac1558d3 Mon Sep 17 00:00:00 2001 From: Stan Lagun Date: Thu, 21 Mar 2013 13:23:18 +0400 Subject: [PATCH 10/37] Issues #195 and #199 --- conductor/bin/app.py | 36 +- conductor/conductor/app.py | 79 +- conductor/conductor/cloud_formation.py | 6 +- .../conductor/commands/cloud_formation.py | 35 +- conductor/conductor/commands/command.py | 2 +- conductor/conductor/commands/dispatcher.py | 26 +- conductor/conductor/commands/windows_agent.py | 58 +- conductor/conductor/config.py | 188 +++++ conductor/conductor/openstack/__init__.py | 0 .../conductor/openstack/common/__init__.py | 0 .../openstack/common/eventlet_backdoor.py | 87 ++ .../conductor/openstack/common/exception.py | 142 ++++ .../openstack/common/gettextutils.py | 33 + .../conductor/openstack/common/importutils.py | 67 ++ .../conductor/openstack/common/jsonutils.py | 141 ++++ conductor/conductor/openstack/common/local.py | 48 ++ conductor/conductor/openstack/common/log.py | 522 ++++++++++++ .../conductor/openstack/common/loopingcall.py | 95 +++ .../conductor/openstack/common/service.py | 332 ++++++++ conductor/conductor/openstack/common/setup.py | 359 ++++++++ .../conductor/openstack/common/sslutils.py | 80 ++ .../conductor/openstack/common/threadgroup.py | 114 +++ .../conductor/openstack/common/timeutils.py | 186 ++++ .../conductor/openstack/common/uuidutils.py | 39 + .../conductor/openstack/common/version.py | 94 +++ conductor/conductor/openstack/common/wsgi.py | 797 ++++++++++++++++++ .../conductor/openstack/common/xmlutils.py | 74 ++ conductor/conductor/rabbitmq.py | 197 +++-- conductor/conductor/reporting.py | 15 +- conductor/conductor/version.py | 20 + conductor/conductor/windows_agent.py | 9 +- .../templates/agent-config/Default.template | 2 +- conductor/etc/app.config | 5 - conductor/etc/conductor-paste.ini | 0 conductor/etc/conductor.conf | 10 + conductor/logs/.gitignore | 4 + conductor/openstack-common.conf | 7 + conductor/tools/pip-requires | 12 +- 38 files changed, 3721 insertions(+), 200 deletions(-) create mode 100644 conductor/conductor/openstack/__init__.py create mode 100644 conductor/conductor/openstack/common/__init__.py create mode 100644 conductor/conductor/openstack/common/eventlet_backdoor.py create mode 100644 conductor/conductor/openstack/common/exception.py create mode 100644 conductor/conductor/openstack/common/gettextutils.py create mode 100644 conductor/conductor/openstack/common/importutils.py create mode 100644 conductor/conductor/openstack/common/jsonutils.py create mode 100644 conductor/conductor/openstack/common/local.py create mode 100644 conductor/conductor/openstack/common/log.py create mode 100644 conductor/conductor/openstack/common/loopingcall.py create mode 100644 conductor/conductor/openstack/common/service.py create mode 100644 conductor/conductor/openstack/common/setup.py create mode 100644 conductor/conductor/openstack/common/sslutils.py create mode 100644 conductor/conductor/openstack/common/threadgroup.py create mode 100644 conductor/conductor/openstack/common/timeutils.py create mode 100644 conductor/conductor/openstack/common/uuidutils.py create mode 100644 conductor/conductor/openstack/common/version.py create mode 100644 conductor/conductor/openstack/common/wsgi.py create mode 100644 conductor/conductor/openstack/common/xmlutils.py create mode 100644 conductor/conductor/version.py delete mode 100644 conductor/etc/app.config create mode 100644 conductor/etc/conductor-paste.ini create mode 100644 conductor/etc/conductor.conf create mode 100644 conductor/logs/.gitignore create mode 100644 conductor/openstack-common.conf diff --git a/conductor/bin/app.py b/conductor/bin/app.py index 3c2619b..8c74b5c 100644 --- a/conductor/bin/app.py +++ b/conductor/bin/app.py @@ -1,3 +1,37 @@ #!/usr/bin/env python +# vim: tabstop=4 shiftwidth=4 softtabstop=4 -from conductor import app \ No newline at end of file +# Copyright 2011 OpenStack LLC. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import os +import sys + + +from conductor import config +from conductor.openstack.common import log +from conductor.openstack.common import service +from conductor.app import ConductorWorkflowService + +if __name__ == '__main__': + try: + config.parse_args() + log.setup('conductor') + launcher = service.ServiceLauncher() + launcher.launch_service(ConductorWorkflowService()) + launcher.wait() + except RuntimeError, e: + sys.stderr.write("ERROR: %s\n" % e) + sys.exit(1) diff --git a/conductor/conductor/app.py b/conductor/conductor/app.py index f4cf9a7..52d8b2a 100644 --- a/conductor/conductor/app.py +++ b/conductor/conductor/app.py @@ -1,64 +1,69 @@ import datetime import glob import json -import time import sys -import tornado.ioloop -import rabbitmq +from conductor.openstack.common import service from workflow import Workflow import cloud_formation import windows_agent from commands.dispatcher import CommandDispatcher from config import Config import reporting +import rabbitmq config = Config(sys.argv[1] if len(sys.argv) > 1 else None) -rmqclient = rabbitmq.RabbitMqClient( - virtual_host=config.get_setting('rabbitmq', 'vhost', '/'), - login=config.get_setting('rabbitmq', 'login', 'guest'), - password=config.get_setting('rabbitmq', 'password', 'guest'), - host=config.get_setting('rabbitmq', 'host', 'localhost')) - - -def schedule(callback, *args, **kwargs): - tornado.ioloop.IOLoop.instance().add_timeout(time.time() + 0.1, - lambda args=args, kwargs=kwargs: callback(*args, **kwargs)) - - def task_received(task, message_id): - print 'Starting at', datetime.datetime.now() - reporter = reporting.Reporter(rmqclient, message_id, task['id']) + with rabbitmq.RmqClient() as rmqclient: + print 'Starting at', datetime.datetime.now() + reporter = reporting.Reporter(rmqclient, message_id, task['id']) - command_dispatcher = CommandDispatcher(task['name'], rmqclient) - workflows = [] - for path in glob.glob("data/workflows/*.xml"): - print "loading", path - workflow = Workflow(path, task, command_dispatcher, config, reporter) - workflows.append(workflow) + command_dispatcher = CommandDispatcher(task['name'], rmqclient) + workflows = [] + for path in glob.glob("data/workflows/*.xml"): + print "loading", path + workflow = Workflow(path, task, command_dispatcher, config, reporter) + workflows.append(workflow) - def loop(callback): - for workflow in workflows: - workflow.execute() - if not command_dispatcher.execute_pending(lambda: schedule(loop, callback)): - callback() + while True: + for workflow in workflows: + workflow.execute() + if not command_dispatcher.execute_pending(): + break - def shutdown(): command_dispatcher.close() - rmqclient.send('task-results', json.dumps(task), message_id=message_id) + result_msg = rabbitmq.Message() + result_msg.body = task + result_msg.id = message_id + + rmqclient.send(message=result_msg, key='task-results') print 'Finished at', datetime.datetime.now() - loop(shutdown) -def message_received(body, message_id, **kwargs): - task_received(json.loads(body), message_id) +class ConductorWorkflowService(service.Service): + def __init__(self): + super(ConductorWorkflowService, self).__init__() + def start(self): + super(ConductorWorkflowService, self).start() + self.tg.add_thread(self._start_rabbitmq) -def start(): - rmqclient.subscribe("tasks", message_received) + def stop(self): + super(ConductorWorkflowService, self).stop() -rmqclient.start(start) -tornado.ioloop.IOLoop.instance().start() + def _start_rabbitmq(self): + while True: + try: + with rabbitmq.RmqClient() as rmq: + rmq.declare('tasks', 'tasks') + rmq.declare('task-results', 'tasks') + with rmq.open('tasks') as subscription: + while True: + msg = subscription.get_message() + self.tg.add_thread( + task_received, msg.body, msg.id) + except Exception as ex: + print ex diff --git a/conductor/conductor/cloud_formation.py b/conductor/conductor/cloud_formation.py index 3ef3b41..1642714 100644 --- a/conductor/conductor/cloud_formation.py +++ b/conductor/conductor/cloud_formation.py @@ -22,9 +22,13 @@ def prepare_user_data(context, template='Default', **kwargs): with open('data/templates/agent-config/%s.template' % template) as template_file: init_script = init_script_file.read() - template_data = template_file.read().replace( + template_data = template_file.read() + template_data = template_data.replace( '%RABBITMQ_HOST%', config.get_setting('rabbitmq', 'host') or 'localhost') + template_data = template_data.replace( + '%RESULT_QUEUE%', + '-execution-results-%s' % str(context['/dataSource']['name'])) return init_script.replace( '%WINDOWS_AGENT_CONFIG_BASE64%', diff --git a/conductor/conductor/commands/cloud_formation.py b/conductor/conductor/commands/cloud_formation.py index 0d12083..d0225ed 100644 --- a/conductor/conductor/commands/cloud_formation.py +++ b/conductor/conductor/commands/cloud_formation.py @@ -1,4 +1,4 @@ -import json +import anyjson import os import uuid @@ -17,7 +17,7 @@ class HeatExecutor(CommandBase): template_data = template_file.read() template_data = conductor.helpers.transform_json( - json.loads(template_data), mappings) + anyjson.loads(template_data), mappings) self._pending_list.append({ 'template': template_data, @@ -28,7 +28,7 @@ class HeatExecutor(CommandBase): def has_pending_commands(self): return len(self._pending_list) > 0 - def execute_pending(self, callback): + def execute_pending(self): if not self.has_pending_commands(): return False @@ -40,7 +40,7 @@ class HeatExecutor(CommandBase): arguments = conductor.helpers.merge_dicts( arguments, t['arguments'], max_levels=1) - print 'Executing heat template', json.dumps(template), \ + print 'Executing heat template', anyjson.dumps(template), \ 'with arguments', arguments, 'on stack', self._stack if not os.path.exists("tmp"): @@ -48,28 +48,23 @@ class HeatExecutor(CommandBase): file_name = "tmp/" + str(uuid.uuid4()) print "Saving template to", file_name with open(file_name, "w") as f: - f.write(json.dumps(template)) + f.write(anyjson.dumps(template)) arguments_str = ';'.join(['%s=%s' % (key, value) for (key, value) in arguments.items()]) - call([ - "./heat_run", "stack-create", - "-f" + file_name, - "-P" + arguments_str, - self._stack - ]) - - - callbacks = [] - for t in self._pending_list: - if t['callback']: - callbacks.append(t['callback']) + # call([ + # "./heat_run", "stack-create", + # "-f" + file_name, + # "-P" + arguments_str, + # self._stack + # ]) + pending_list = self._pending_list self._pending_list = [] - for cb in callbacks: - cb(True) + for item in pending_list: + item['callback'](True) + - callback() return True diff --git a/conductor/conductor/commands/command.py b/conductor/conductor/commands/command.py index ca9d144..ad2d469 100644 --- a/conductor/conductor/commands/command.py +++ b/conductor/conductor/commands/command.py @@ -2,7 +2,7 @@ class CommandBase(object): def execute(self, **kwargs): pass - def execute_pending(self, callback): + def execute_pending(self): return False def has_pending_commands(self): diff --git a/conductor/conductor/commands/dispatcher.py b/conductor/conductor/commands/dispatcher.py index b815ddb..606266e 100644 --- a/conductor/conductor/commands/dispatcher.py +++ b/conductor/conductor/commands/dispatcher.py @@ -4,33 +4,22 @@ import windows_agent class CommandDispatcher(command.CommandBase): - def __init__(self, environment_name, rmqclient): + def __init__(self, environment_id, rmqclient): self._command_map = { - 'cf': cloud_formation.HeatExecutor(environment_name), + 'cf': cloud_formation.HeatExecutor(environment_id), 'agent': windows_agent.WindowsAgentExecutor( - environment_name, rmqclient) + environment_id, rmqclient, environment_id) } def execute(self, name, **kwargs): self._command_map[name].execute(**kwargs) - def execute_pending(self, callback): - result = 0 - count = [0] - - def on_result(): - count[0] -= 1 - if not count[0]: - callback() - + def execute_pending(self): + result = False for command in self._command_map.values(): - count[0] += 1 - result += 1 - if not command.execute_pending(on_result): - count[0] -= 1 - result -= 1 + result |= command.execute_pending() - return result > 0 + return result def has_pending_commands(self): @@ -40,6 +29,7 @@ class CommandDispatcher(command.CommandBase): return result + def close(self): for t in self._command_map.values(): t.close() diff --git a/conductor/conductor/commands/windows_agent.py b/conductor/conductor/commands/windows_agent.py index c4747b6..978ddd2 100644 --- a/conductor/conductor/commands/windows_agent.py +++ b/conductor/conductor/commands/windows_agent.py @@ -1,66 +1,60 @@ import json import uuid +from conductor.rabbitmq import Message import conductor.helpers from command import CommandBase class WindowsAgentExecutor(CommandBase): - def __init__(self, stack, rmqclient): + def __init__(self, stack, rmqclient, environment): self._stack = stack self._rmqclient = rmqclient - self._callback = None self._pending_list = [] - self._current_pending_list = [] - rmqclient.subscribe('-execution-results', self._on_message) + self._results_queue = '-execution-results-%s' % str(environment) + rmqclient.declare(self._results_queue) def execute(self, template, mappings, host, callback): with open('data/templates/agent/%s.template' % template) as template_file: template_data = template_file.read() - template_data = json.dumps(conductor.helpers.transform_json( - json.loads(template_data), mappings)) + template_data = conductor.helpers.transform_json( + json.loads(template_data), mappings) + id = str(uuid.uuid4()).lower() + host = ('%s-%s' % (self._stack, host)).lower().replace(' ', '-') self._pending_list.append({ - 'id': str(uuid.uuid4()).lower(), - 'template': template_data, - 'host': ('%s-%s' % (self._stack, host)).lower().replace(' ', '-'), + 'id': id, 'callback': callback }) - def _on_message(self, body, message_id, **kwargs): - msg_id = message_id.lower() - item, index = conductor.helpers.find(lambda t: t['id'] == msg_id, - self._current_pending_list) - if item: - self._current_pending_list.pop(index) - item['callback'](json.loads(body)) - if self._callback and not self._current_pending_list: - cb = self._callback - self._callback = None - cb() + msg = Message() + msg.body = template_data + msg.id = id + self._rmqclient.declare(host) + self._rmqclient.send(message=msg, key=host) + print 'Sending RMQ message %s to %s' % ( + template_data, host) def has_pending_commands(self): return len(self._pending_list) > 0 - def execute_pending(self, callback): + def execute_pending(self): if not self.has_pending_commands(): return False - self._current_pending_list = self._pending_list - self._pending_list = [] + with self._rmqclient.open(self._results_queue) as subscription: + while self.has_pending_commands(): + msg = subscription.get_message() + msg_id = msg.id.lower() + item, index = conductor.helpers.find( + lambda t: t['id'] == msg_id, self._pending_list) + if item: + self._pending_list.pop(index) + item['callback'](msg.body) - self._callback = callback - - for rec in self._current_pending_list: - self._rmqclient.send( - queue=rec['host'], data=rec['template'], message_id=rec['id']) - print 'Sending RMQ message %s to %s' % ( - rec['template'], rec['host']) return True - def close(self): - self._rmqclient.unsubscribe('-execution-results') diff --git a/conductor/conductor/config.py b/conductor/conductor/config.py index 881d4ad..1e42cad 100644 --- a/conductor/conductor/config.py +++ b/conductor/conductor/config.py @@ -1,5 +1,193 @@ +#!/usr/bin/env python +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright 2011 OpenStack LLC. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +""" +Routines for configuring Glance +""" + +import logging +import logging.config +import logging.handlers +import os +import sys + +from oslo.config import cfg +from paste import deploy + +from conductor.version import version_info as version from ConfigParser import SafeConfigParser +paste_deploy_opts = [ + cfg.StrOpt('flavor'), + cfg.StrOpt('config_file'), +] + +rabbit_opts = [ + cfg.StrOpt('host', default='localhost'), + cfg.IntOpt('port', default=5672), + cfg.StrOpt('login', default='guest'), + cfg.StrOpt('password', default='guest'), + cfg.StrOpt('virtual_host', default='/'), +] + +CONF = cfg.CONF +CONF.register_opts(paste_deploy_opts, group='paste_deploy') +CONF.register_opts(rabbit_opts, group='rabbitmq') + + +CONF.import_opt('verbose', 'conductor.openstack.common.log') +CONF.import_opt('debug', 'conductor.openstack.common.log') +CONF.import_opt('log_dir', 'conductor.openstack.common.log') +CONF.import_opt('log_file', 'conductor.openstack.common.log') +CONF.import_opt('log_config', 'conductor.openstack.common.log') +CONF.import_opt('log_format', 'conductor.openstack.common.log') +CONF.import_opt('log_date_format', 'conductor.openstack.common.log') +CONF.import_opt('use_syslog', 'conductor.openstack.common.log') +CONF.import_opt('syslog_log_facility', 'conductor.openstack.common.log') + + +def parse_args(args=None, usage=None, default_config_files=None): + CONF(args=args, + project='conductor', + version=version.cached_version_string(), + usage=usage, + default_config_files=default_config_files) + +def setup_logging(): + """ + Sets up the logging options for a log with supplied name + """ + + if CONF.log_config: + # Use a logging configuration file for all settings... + if os.path.exists(CONF.log_config): + logging.config.fileConfig(CONF.log_config) + return + else: + raise RuntimeError("Unable to locate specified logging " + "config file: %s" % CONF.log_config) + + root_logger = logging.root + if CONF.debug: + root_logger.setLevel(logging.DEBUG) + elif CONF.verbose: + root_logger.setLevel(logging.INFO) + else: + root_logger.setLevel(logging.WARNING) + + formatter = logging.Formatter(CONF.log_format, CONF.log_date_format) + + if CONF.use_syslog: + try: + facility = getattr(logging.handlers.SysLogHandler, + CONF.syslog_log_facility) + except AttributeError: + raise ValueError(_("Invalid syslog facility")) + + handler = logging.handlers.SysLogHandler(address='/dev/log', + facility=facility) + elif CONF.log_file: + logfile = CONF.log_file + if CONF.log_dir: + logfile = os.path.join(CONF.log_dir, logfile) + handler = logging.handlers.WatchedFileHandler(logfile) + else: + handler = logging.StreamHandler(sys.stdout) + + handler.setFormatter(formatter) + root_logger.addHandler(handler) + + +def _get_deployment_flavor(): + """ + Retrieve the paste_deploy.flavor config item, formatted appropriately + for appending to the application name. + """ + flavor = CONF.paste_deploy.flavor + return '' if not flavor else ('-' + flavor) + + +def _get_paste_config_path(): + paste_suffix = '-paste.ini' + conf_suffix = '.conf' + if CONF.config_file: + # Assume paste config is in a paste.ini file corresponding + # to the last config file + path = CONF.config_file[-1].replace(conf_suffix, paste_suffix) + else: + path = CONF.prog + '-paste.ini' + return CONF.find_file(os.path.basename(path)) + + +def _get_deployment_config_file(): + """ + Retrieve the deployment_config_file config item, formatted as an + absolute pathname. + """ + path = CONF.paste_deploy.config_file + if not path: + path = _get_paste_config_path() + if not path: + msg = "Unable to locate paste config file for %s." % CONF.prog + raise RuntimeError(msg) + return os.path.abspath(path) + + +def load_paste_app(app_name=None): + """ + Builds and returns a WSGI app from a paste config file. + + We assume the last config file specified in the supplied ConfigOpts + object is the paste config file. + + :param app_name: name of the application to load + + :raises RuntimeError when config file cannot be located or application + cannot be loaded from config file + """ + if app_name is None: + app_name = CONF.prog + + # append the deployment flavor to the application name, + # in order to identify the appropriate paste pipeline + app_name += _get_deployment_flavor() + + conf_file = _get_deployment_config_file() + + try: + logger = logging.getLogger(__name__) + logger.debug(_("Loading %(app_name)s from %(conf_file)s"), + {'conf_file': conf_file, 'app_name': app_name}) + + app = deploy.loadapp("config:%s" % conf_file, name=app_name) + + # Log the options used when starting if we're in debug mode... + if CONF.debug: + CONF.log_opt_values(logger, logging.DEBUG) + + return app + except (LookupError, ImportError), e: + msg = _("Unable to load %(app_name)s from " + "configuration file %(conf_file)s." + "\nGot: %(e)r") % locals() + logger.error(msg) + raise RuntimeError(msg) + class Config(object): CONFIG_PATH = './etc/app.config' diff --git a/conductor/conductor/openstack/__init__.py b/conductor/conductor/openstack/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/conductor/conductor/openstack/common/__init__.py b/conductor/conductor/openstack/common/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/conductor/conductor/openstack/common/eventlet_backdoor.py b/conductor/conductor/openstack/common/eventlet_backdoor.py new file mode 100644 index 0000000..7605e26 --- /dev/null +++ b/conductor/conductor/openstack/common/eventlet_backdoor.py @@ -0,0 +1,87 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright (c) 2012 OpenStack Foundation. +# Administrator of the National Aeronautics and Space Administration. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import gc +import pprint +import sys +import traceback + +import eventlet +import eventlet.backdoor +import greenlet +from oslo.config import cfg + +eventlet_backdoor_opts = [ + cfg.IntOpt('backdoor_port', + default=None, + help='port for eventlet backdoor to listen') +] + +CONF = cfg.CONF +CONF.register_opts(eventlet_backdoor_opts) + + +def _dont_use_this(): + print "Don't use this, just disconnect instead" + + +def _find_objects(t): + return filter(lambda o: isinstance(o, t), gc.get_objects()) + + +def _print_greenthreads(): + for i, gt in enumerate(_find_objects(greenlet.greenlet)): + print i, gt + traceback.print_stack(gt.gr_frame) + print + + +def _print_nativethreads(): + for threadId, stack in sys._current_frames().items(): + print threadId + traceback.print_stack(stack) + print + + +def initialize_if_enabled(): + backdoor_locals = { + 'exit': _dont_use_this, # So we don't exit the entire process + 'quit': _dont_use_this, # So we don't exit the entire process + 'fo': _find_objects, + 'pgt': _print_greenthreads, + 'pnt': _print_nativethreads, + } + + if CONF.backdoor_port is None: + return None + + # NOTE(johannes): The standard sys.displayhook will print the value of + # the last expression and set it to __builtin__._, which overwrites + # the __builtin__._ that gettext sets. Let's switch to using pprint + # since it won't interact poorly with gettext, and it's easier to + # read the output too. + def displayhook(val): + if val is not None: + pprint.pprint(val) + sys.displayhook = displayhook + + sock = eventlet.listen(('localhost', CONF.backdoor_port)) + port = sock.getsockname()[1] + eventlet.spawn_n(eventlet.backdoor.backdoor_server, sock, + locals=backdoor_locals) + return port diff --git a/conductor/conductor/openstack/common/exception.py b/conductor/conductor/openstack/common/exception.py new file mode 100644 index 0000000..beabfcd --- /dev/null +++ b/conductor/conductor/openstack/common/exception.py @@ -0,0 +1,142 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright 2011 OpenStack Foundation. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +""" +Exceptions common to OpenStack projects +""" + +import logging + +from conductor.openstack.common.gettextutils import _ + +_FATAL_EXCEPTION_FORMAT_ERRORS = False + + +class Error(Exception): + def __init__(self, message=None): + super(Error, self).__init__(message) + + +class ApiError(Error): + def __init__(self, message='Unknown', code='Unknown'): + self.message = message + self.code = code + super(ApiError, self).__init__('%s: %s' % (code, message)) + + +class NotFound(Error): + pass + + +class UnknownScheme(Error): + + msg = "Unknown scheme '%s' found in URI" + + def __init__(self, scheme): + msg = self.__class__.msg % scheme + super(UnknownScheme, self).__init__(msg) + + +class BadStoreUri(Error): + + msg = "The Store URI %s was malformed. Reason: %s" + + def __init__(self, uri, reason): + msg = self.__class__.msg % (uri, reason) + super(BadStoreUri, self).__init__(msg) + + +class Duplicate(Error): + pass + + +class NotAuthorized(Error): + pass + + +class NotEmpty(Error): + pass + + +class Invalid(Error): + pass + + +class BadInputError(Exception): + """Error resulting from a client sending bad input to a server""" + pass + + +class MissingArgumentError(Error): + pass + + +class DatabaseMigrationError(Error): + pass + + +class ClientConnectionError(Exception): + """Error resulting from a client connecting to a server""" + pass + + +def wrap_exception(f): + def _wrap(*args, **kw): + try: + return f(*args, **kw) + except Exception, e: + if not isinstance(e, Error): + #exc_type, exc_value, exc_traceback = sys.exc_info() + logging.exception(_('Uncaught exception')) + #logging.error(traceback.extract_stack(exc_traceback)) + raise Error(str(e)) + raise + _wrap.func_name = f.func_name + return _wrap + + +class OpenstackException(Exception): + """ + Base Exception + + To correctly use this class, inherit from it and define + a 'message' property. That message will get printf'd + with the keyword arguments provided to the constructor. + """ + message = "An unknown exception occurred" + + def __init__(self, **kwargs): + try: + self._error_string = self.message % kwargs + + except Exception as e: + if _FATAL_EXCEPTION_FORMAT_ERRORS: + raise e + else: + # at least get the core message out if something happened + self._error_string = self.message + + def __str__(self): + return self._error_string + + +class MalformedRequestBody(OpenstackException): + message = "Malformed message body: %(reason)s" + + +class InvalidContentType(OpenstackException): + message = "Invalid content type %(content_type)s" diff --git a/conductor/conductor/openstack/common/gettextutils.py b/conductor/conductor/openstack/common/gettextutils.py new file mode 100644 index 0000000..000d23b --- /dev/null +++ b/conductor/conductor/openstack/common/gettextutils.py @@ -0,0 +1,33 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright 2012 Red Hat, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +""" +gettext for openstack-common modules. + +Usual usage in an openstack.common module: + + from conductor.openstack.common.gettextutils import _ +""" + +import gettext + + +t = gettext.translation('openstack-common', 'locale', fallback=True) + + +def _(msg): + return t.ugettext(msg) diff --git a/conductor/conductor/openstack/common/importutils.py b/conductor/conductor/openstack/common/importutils.py new file mode 100644 index 0000000..6862ca8 --- /dev/null +++ b/conductor/conductor/openstack/common/importutils.py @@ -0,0 +1,67 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright 2011 OpenStack Foundation. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +""" +Import related utilities and helper functions. +""" + +import sys +import traceback + + +def import_class(import_str): + """Returns a class from a string including module and class""" + mod_str, _sep, class_str = import_str.rpartition('.') + try: + __import__(mod_str) + return getattr(sys.modules[mod_str], class_str) + except (ValueError, AttributeError): + raise ImportError('Class %s cannot be found (%s)' % + (class_str, + traceback.format_exception(*sys.exc_info()))) + + +def import_object(import_str, *args, **kwargs): + """Import a class and return an instance of it.""" + return import_class(import_str)(*args, **kwargs) + + +def import_object_ns(name_space, import_str, *args, **kwargs): + """ + Import a class and return an instance of it, first by trying + to find the class in a default namespace, then failing back to + a full path if not found in the default namespace. + """ + import_value = "%s.%s" % (name_space, import_str) + try: + return import_class(import_value)(*args, **kwargs) + except ImportError: + return import_class(import_str)(*args, **kwargs) + + +def import_module(import_str): + """Import a module.""" + __import__(import_str) + return sys.modules[import_str] + + +def try_import(import_str, default=None): + """Try to import a module and if it fails return default.""" + try: + return import_module(import_str) + except ImportError: + return default diff --git a/conductor/conductor/openstack/common/jsonutils.py b/conductor/conductor/openstack/common/jsonutils.py new file mode 100644 index 0000000..c281b6a --- /dev/null +++ b/conductor/conductor/openstack/common/jsonutils.py @@ -0,0 +1,141 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright 2010 United States Government as represented by the +# Administrator of the National Aeronautics and Space Administration. +# Copyright 2011 Justin Santa Barbara +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +''' +JSON related utilities. + +This module provides a few things: + + 1) A handy function for getting an object down to something that can be + JSON serialized. See to_primitive(). + + 2) Wrappers around loads() and dumps(). The dumps() wrapper will + automatically use to_primitive() for you if needed. + + 3) This sets up anyjson to use the loads() and dumps() wrappers if anyjson + is available. +''' + + +import datetime +import functools +import inspect +import itertools +import json +import xmlrpclib + +from conductor.openstack.common import timeutils + + +def to_primitive(value, convert_instances=False, convert_datetime=True, + level=0, max_depth=3): + """Convert a complex object into primitives. + + Handy for JSON serialization. We can optionally handle instances, + but since this is a recursive function, we could have cyclical + data structures. + + To handle cyclical data structures we could track the actual objects + visited in a set, but not all objects are hashable. Instead we just + track the depth of the object inspections and don't go too deep. + + Therefore, convert_instances=True is lossy ... be aware. + + """ + nasty = [inspect.ismodule, inspect.isclass, inspect.ismethod, + inspect.isfunction, inspect.isgeneratorfunction, + inspect.isgenerator, inspect.istraceback, inspect.isframe, + inspect.iscode, inspect.isbuiltin, inspect.isroutine, + inspect.isabstract] + for test in nasty: + if test(value): + return unicode(value) + + # value of itertools.count doesn't get caught by inspects + # above and results in infinite loop when list(value) is called. + if type(value) == itertools.count: + return unicode(value) + + # FIXME(vish): Workaround for LP bug 852095. Without this workaround, + # tests that raise an exception in a mocked method that + # has a @wrap_exception with a notifier will fail. If + # we up the dependency to 0.5.4 (when it is released) we + # can remove this workaround. + if getattr(value, '__module__', None) == 'mox': + return 'mock' + + if level > max_depth: + return '?' + + # The try block may not be necessary after the class check above, + # but just in case ... + try: + recursive = functools.partial(to_primitive, + convert_instances=convert_instances, + convert_datetime=convert_datetime, + level=level, + max_depth=max_depth) + # It's not clear why xmlrpclib created their own DateTime type, but + # for our purposes, make it a datetime type which is explicitly + # handled + if isinstance(value, xmlrpclib.DateTime): + value = datetime.datetime(*tuple(value.timetuple())[:6]) + + if isinstance(value, (list, tuple)): + return [recursive(v) for v in value] + elif isinstance(value, dict): + return dict((k, recursive(v)) for k, v in value.iteritems()) + elif convert_datetime and isinstance(value, datetime.datetime): + return timeutils.strtime(value) + elif hasattr(value, 'iteritems'): + return recursive(dict(value.iteritems()), level=level + 1) + elif hasattr(value, '__iter__'): + return recursive(list(value)) + elif convert_instances and hasattr(value, '__dict__'): + # Likely an instance of something. Watch for cycles. + # Ignore class member vars. + return recursive(value.__dict__, level=level + 1) + else: + return value + except TypeError: + # Class objects are tricky since they may define something like + # __iter__ defined but it isn't callable as list(). + return unicode(value) + + +def dumps(value, default=to_primitive, **kwargs): + return json.dumps(value, default=default, **kwargs) + + +def loads(s): + return json.loads(s) + + +def load(s): + return json.load(s) + + +try: + import anyjson +except ImportError: + pass +else: + anyjson._modules.append((__name__, 'dumps', TypeError, + 'loads', ValueError, 'load')) + anyjson.force_implementation(__name__) diff --git a/conductor/conductor/openstack/common/local.py b/conductor/conductor/openstack/common/local.py new file mode 100644 index 0000000..9b1d22a --- /dev/null +++ b/conductor/conductor/openstack/common/local.py @@ -0,0 +1,48 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright 2011 OpenStack Foundation. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +"""Greenthread local storage of variables using weak references""" + +import weakref + +from eventlet import corolocal + + +class WeakLocal(corolocal.local): + def __getattribute__(self, attr): + rval = corolocal.local.__getattribute__(self, attr) + if rval: + # NOTE(mikal): this bit is confusing. What is stored is a weak + # reference, not the value itself. We therefore need to lookup + # the weak reference and return the inner value here. + rval = rval() + return rval + + def __setattr__(self, attr, value): + value = weakref.ref(value) + return corolocal.local.__setattr__(self, attr, value) + + +# NOTE(mikal): the name "store" should be deprecated in the future +store = WeakLocal() + +# A "weak" store uses weak references and allows an object to fall out of scope +# when it falls out of scope in the code that uses the thread local storage. A +# "strong" store will hold a reference to the object so that it never falls out +# of scope. +weak_store = WeakLocal() +strong_store = corolocal.local diff --git a/conductor/conductor/openstack/common/log.py b/conductor/conductor/openstack/common/log.py new file mode 100644 index 0000000..31ed37f --- /dev/null +++ b/conductor/conductor/openstack/common/log.py @@ -0,0 +1,522 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright 2011 OpenStack Foundation. +# Copyright 2010 United States Government as represented by the +# Administrator of the National Aeronautics and Space Administration. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +"""Openstack logging handler. + +This module adds to logging functionality by adding the option to specify +a context object when calling the various log methods. If the context object +is not specified, default formatting is used. Additionally, an instance uuid +may be passed as part of the log message, which is intended to make it easier +for admins to find messages related to a specific instance. + +It also allows setting of formatting information through conf. + +""" + +import cStringIO +import inspect +import itertools +import logging +import logging.config +import logging.handlers +import os +import stat +import sys +import traceback + +from oslo.config import cfg + +from conductor.openstack.common.gettextutils import _ +from conductor.openstack.common import jsonutils +from conductor.openstack.common import local +from conductor.openstack.common import notifier + + +_DEFAULT_LOG_FORMAT = "%(asctime)s %(levelname)8s [%(name)s] %(message)s" +_DEFAULT_LOG_DATE_FORMAT = "%Y-%m-%d %H:%M:%S" + +common_cli_opts = [ + cfg.BoolOpt('debug', + short='d', + default=False, + help='Print debugging output (set logging level to ' + 'DEBUG instead of default WARNING level).'), + cfg.BoolOpt('verbose', + short='v', + default=False, + help='Print more verbose output (set logging level to ' + 'INFO instead of default WARNING level).'), +] + +logging_cli_opts = [ + cfg.StrOpt('log-config', + metavar='PATH', + help='If this option is specified, the logging configuration ' + 'file specified is used and overrides any other logging ' + 'options specified. Please see the Python logging module ' + 'documentation for details on logging configuration ' + 'files.'), + cfg.StrOpt('log-format', + default=_DEFAULT_LOG_FORMAT, + metavar='FORMAT', + help='A logging.Formatter log message format string which may ' + 'use any of the available logging.LogRecord attributes. ' + 'Default: %(default)s'), + cfg.StrOpt('log-date-format', + default=_DEFAULT_LOG_DATE_FORMAT, + metavar='DATE_FORMAT', + help='Format string for %%(asctime)s in log records. ' + 'Default: %(default)s'), + cfg.StrOpt('log-file', + metavar='PATH', + deprecated_name='logfile', + help='(Optional) Name of log file to output to. ' + 'If not set, logging will go to stdout.'), + cfg.StrOpt('log-dir', + deprecated_name='logdir', + help='(Optional) The directory to keep log files in ' + '(will be prepended to --log-file)'), + cfg.BoolOpt('use-syslog', + default=False, + help='Use syslog for logging.'), + cfg.StrOpt('syslog-log-facility', + default='LOG_USER', + help='syslog facility to receive log lines') +] + +generic_log_opts = [ + cfg.BoolOpt('use_stderr', + default=True, + help='Log output to standard error'), + cfg.StrOpt('logfile_mode', + default='0644', + help='Default file mode used when creating log files'), +] + +log_opts = [ + cfg.StrOpt('logging_context_format_string', + default='%(asctime)s.%(msecs)03d %(levelname)s %(name)s ' + '[%(request_id)s %(user)s %(tenant)s] %(instance)s' + '%(message)s', + help='format string to use for log messages with context'), + cfg.StrOpt('logging_default_format_string', + default='%(asctime)s.%(msecs)03d %(process)d %(levelname)s ' + '%(name)s [-] %(instance)s%(message)s', + help='format string to use for log messages without context'), + cfg.StrOpt('logging_debug_format_suffix', + default='%(funcName)s %(pathname)s:%(lineno)d', + help='data to append to log format when level is DEBUG'), + cfg.StrOpt('logging_exception_prefix', + default='%(asctime)s.%(msecs)03d %(process)d TRACE %(name)s ' + '%(instance)s', + help='prefix each line of exception output with this format'), + cfg.ListOpt('default_log_levels', + default=[ + 'amqplib=WARN', + 'sqlalchemy=WARN', + 'boto=WARN', + 'suds=INFO', + 'keystone=INFO', + 'eventlet.wsgi.server=WARN' + ], + help='list of logger=LEVEL pairs'), + cfg.BoolOpt('publish_errors', + default=False, + help='publish error events'), + cfg.BoolOpt('fatal_deprecations', + default=False, + help='make deprecations fatal'), + + # NOTE(mikal): there are two options here because sometimes we are handed + # a full instance (and could include more information), and other times we + # are just handed a UUID for the instance. + cfg.StrOpt('instance_format', + default='[instance: %(uuid)s] ', + help='If an instance is passed with the log message, format ' + 'it like this'), + cfg.StrOpt('instance_uuid_format', + default='[instance: %(uuid)s] ', + help='If an instance UUID is passed with the log message, ' + 'format it like this'), +] + +CONF = cfg.CONF +CONF.register_cli_opts(common_cli_opts) +CONF.register_cli_opts(logging_cli_opts) +CONF.register_opts(generic_log_opts) +CONF.register_opts(log_opts) + +# our new audit level +# NOTE(jkoelker) Since we synthesized an audit level, make the logging +# module aware of it so it acts like other levels. +logging.AUDIT = logging.INFO + 1 +logging.addLevelName(logging.AUDIT, 'AUDIT') + + +try: + NullHandler = logging.NullHandler +except AttributeError: # NOTE(jkoelker) NullHandler added in Python 2.7 + class NullHandler(logging.Handler): + def handle(self, record): + pass + + def emit(self, record): + pass + + def createLock(self): + self.lock = None + + +def _dictify_context(context): + if context is None: + return None + if not isinstance(context, dict) and getattr(context, 'to_dict', None): + context = context.to_dict() + return context + + +def _get_binary_name(): + return os.path.basename(inspect.stack()[-1][1]) + + +def _get_log_file_path(binary=None): + logfile = CONF.log_file + logdir = CONF.log_dir + + if logfile and not logdir: + return logfile + + if logfile and logdir: + return os.path.join(logdir, logfile) + + if logdir: + binary = binary or _get_binary_name() + return '%s.log' % (os.path.join(logdir, binary),) + + +class ContextAdapter(logging.LoggerAdapter): + warn = logging.LoggerAdapter.warning + + def __init__(self, logger, project_name, version_string): + self.logger = logger + self.project = project_name + self.version = version_string + + def audit(self, msg, *args, **kwargs): + self.log(logging.AUDIT, msg, *args, **kwargs) + + def deprecated(self, msg, *args, **kwargs): + stdmsg = _("Deprecated: %s") % msg + if CONF.fatal_deprecations: + self.critical(stdmsg, *args, **kwargs) + raise DeprecatedConfig(msg=stdmsg) + else: + self.warn(stdmsg, *args, **kwargs) + + def process(self, msg, kwargs): + if 'extra' not in kwargs: + kwargs['extra'] = {} + extra = kwargs['extra'] + + context = kwargs.pop('context', None) + if not context: + context = getattr(local.store, 'context', None) + if context: + extra.update(_dictify_context(context)) + + instance = kwargs.pop('instance', None) + instance_extra = '' + if instance: + instance_extra = CONF.instance_format % instance + else: + instance_uuid = kwargs.pop('instance_uuid', None) + if instance_uuid: + instance_extra = (CONF.instance_uuid_format + % {'uuid': instance_uuid}) + extra.update({'instance': instance_extra}) + + extra.update({"project": self.project}) + extra.update({"version": self.version}) + extra['extra'] = extra.copy() + return msg, kwargs + + +class JSONFormatter(logging.Formatter): + def __init__(self, fmt=None, datefmt=None): + # NOTE(jkoelker) we ignore the fmt argument, but its still there + # since logging.config.fileConfig passes it. + self.datefmt = datefmt + + def formatException(self, ei, strip_newlines=True): + lines = traceback.format_exception(*ei) + if strip_newlines: + lines = [itertools.ifilter( + lambda x: x, + line.rstrip().splitlines()) for line in lines] + lines = list(itertools.chain(*lines)) + return lines + + def format(self, record): + message = {'message': record.getMessage(), + 'asctime': self.formatTime(record, self.datefmt), + 'name': record.name, + 'msg': record.msg, + 'args': record.args, + 'levelname': record.levelname, + 'levelno': record.levelno, + 'pathname': record.pathname, + 'filename': record.filename, + 'module': record.module, + 'lineno': record.lineno, + 'funcname': record.funcName, + 'created': record.created, + 'msecs': record.msecs, + 'relative_created': record.relativeCreated, + 'thread': record.thread, + 'thread_name': record.threadName, + 'process_name': record.processName, + 'process': record.process, + 'traceback': None} + + if hasattr(record, 'extra'): + message['extra'] = record.extra + + if record.exc_info: + message['traceback'] = self.formatException(record.exc_info) + + return jsonutils.dumps(message) + + +class PublishErrorsHandler(logging.Handler): + def emit(self, record): + if ('conductor.openstack.common.notifier.log_notifier' in + CONF.notification_driver): + return + notifier.api.notify(None, 'error.publisher', + 'error_notification', + notifier.api.ERROR, + dict(error=record.msg)) + + +def _create_logging_excepthook(product_name): + def logging_excepthook(type, value, tb): + extra = {} + if CONF.verbose: + extra['exc_info'] = (type, value, tb) + getLogger(product_name).critical(str(value), **extra) + return logging_excepthook + + +def setup(product_name): + """Setup logging.""" + if CONF.log_config: + logging.config.fileConfig(CONF.log_config) + else: + _setup_logging_from_conf() + sys.excepthook = _create_logging_excepthook(product_name) + + +def set_defaults(logging_context_format_string): + cfg.set_defaults(log_opts, + logging_context_format_string= + logging_context_format_string) + + +def _find_facility_from_conf(): + facility_names = logging.handlers.SysLogHandler.facility_names + facility = getattr(logging.handlers.SysLogHandler, + CONF.syslog_log_facility, + None) + + if facility is None and CONF.syslog_log_facility in facility_names: + facility = facility_names.get(CONF.syslog_log_facility) + + if facility is None: + valid_facilities = facility_names.keys() + consts = ['LOG_AUTH', 'LOG_AUTHPRIV', 'LOG_CRON', 'LOG_DAEMON', + 'LOG_FTP', 'LOG_KERN', 'LOG_LPR', 'LOG_MAIL', 'LOG_NEWS', + 'LOG_AUTH', 'LOG_SYSLOG', 'LOG_USER', 'LOG_UUCP', + 'LOG_LOCAL0', 'LOG_LOCAL1', 'LOG_LOCAL2', 'LOG_LOCAL3', + 'LOG_LOCAL4', 'LOG_LOCAL5', 'LOG_LOCAL6', 'LOG_LOCAL7'] + valid_facilities.extend(consts) + raise TypeError(_('syslog facility must be one of: %s') % + ', '.join("'%s'" % fac + for fac in valid_facilities)) + + return facility + + +def _setup_logging_from_conf(): + log_root = getLogger(None).logger + for handler in log_root.handlers: + log_root.removeHandler(handler) + + if CONF.use_syslog: + facility = _find_facility_from_conf() + syslog = logging.handlers.SysLogHandler(address='/dev/log', + facility=facility) + log_root.addHandler(syslog) + + logpath = _get_log_file_path() + if logpath: + filelog = logging.handlers.WatchedFileHandler(logpath) + log_root.addHandler(filelog) + + mode = int(CONF.logfile_mode, 8) + st = os.stat(logpath) + if st.st_mode != (stat.S_IFREG | mode): + os.chmod(logpath, mode) + + if CONF.use_stderr: + streamlog = ColorHandler() + log_root.addHandler(streamlog) + + elif not CONF.log_file: + # pass sys.stdout as a positional argument + # python2.6 calls the argument strm, in 2.7 it's stream + streamlog = logging.StreamHandler(sys.stdout) + log_root.addHandler(streamlog) + + if CONF.publish_errors: + log_root.addHandler(PublishErrorsHandler(logging.ERROR)) + + for handler in log_root.handlers: + datefmt = CONF.log_date_format + if CONF.log_format: + handler.setFormatter(logging.Formatter(fmt=CONF.log_format, + datefmt=datefmt)) + else: + handler.setFormatter(LegacyFormatter(datefmt=datefmt)) + + if CONF.debug: + log_root.setLevel(logging.DEBUG) + elif CONF.verbose: + log_root.setLevel(logging.INFO) + else: + log_root.setLevel(logging.WARNING) + + level = logging.NOTSET + for pair in CONF.default_log_levels: + mod, _sep, level_name = pair.partition('=') + level = logging.getLevelName(level_name) + logger = logging.getLogger(mod) + logger.setLevel(level) + for handler in log_root.handlers: + logger.addHandler(handler) + +_loggers = {} + + +def getLogger(name='unknown', version='unknown'): + if name not in _loggers: + _loggers[name] = ContextAdapter(logging.getLogger(name), + name, + version) + return _loggers[name] + + +class WritableLogger(object): + """A thin wrapper that responds to `write` and logs.""" + + def __init__(self, logger, level=logging.INFO): + self.logger = logger + self.level = level + + def write(self, msg): + self.logger.log(self.level, msg) + + +class LegacyFormatter(logging.Formatter): + """A context.RequestContext aware formatter configured through flags. + + The flags used to set format strings are: logging_context_format_string + and logging_default_format_string. You can also specify + logging_debug_format_suffix to append extra formatting if the log level is + debug. + + For information about what variables are available for the formatter see: + http://docs.python.org/library/logging.html#formatter + + """ + + def format(self, record): + """Uses contextstring if request_id is set, otherwise default.""" + # NOTE(sdague): default the fancier formating params + # to an empty string so we don't throw an exception if + # they get used + for key in ('instance', 'color'): + if key not in record.__dict__: + record.__dict__[key] = '' + + if record.__dict__.get('request_id', None): + self._fmt = CONF.logging_context_format_string + else: + self._fmt = CONF.logging_default_format_string + + if (record.levelno == logging.DEBUG and + CONF.logging_debug_format_suffix): + self._fmt += " " + CONF.logging_debug_format_suffix + + # Cache this on the record, Logger will respect our formated copy + if record.exc_info: + record.exc_text = self.formatException(record.exc_info, record) + return logging.Formatter.format(self, record) + + def formatException(self, exc_info, record=None): + """Format exception output with CONF.logging_exception_prefix.""" + if not record: + return logging.Formatter.formatException(self, exc_info) + + stringbuffer = cStringIO.StringIO() + traceback.print_exception(exc_info[0], exc_info[1], exc_info[2], + None, stringbuffer) + lines = stringbuffer.getvalue().split('\n') + stringbuffer.close() + + if CONF.logging_exception_prefix.find('%(asctime)') != -1: + record.asctime = self.formatTime(record, self.datefmt) + + formatted_lines = [] + for line in lines: + pl = CONF.logging_exception_prefix % record.__dict__ + fl = '%s%s' % (pl, line) + formatted_lines.append(fl) + return '\n'.join(formatted_lines) + + +class ColorHandler(logging.StreamHandler): + LEVEL_COLORS = { + logging.DEBUG: '\033[00;32m', # GREEN + logging.INFO: '\033[00;36m', # CYAN + logging.AUDIT: '\033[01;36m', # BOLD CYAN + logging.WARN: '\033[01;33m', # BOLD YELLOW + logging.ERROR: '\033[01;31m', # BOLD RED + logging.CRITICAL: '\033[01;31m', # BOLD RED + } + + def format(self, record): + record.color = self.LEVEL_COLORS[record.levelno] + return logging.StreamHandler.format(self, record) + + +class DeprecatedConfig(Exception): + message = _("Fatal call to deprecated config: %(msg)s") + + def __init__(self, msg): + super(Exception, self).__init__(self.message % dict(msg=msg)) diff --git a/conductor/conductor/openstack/common/loopingcall.py b/conductor/conductor/openstack/common/loopingcall.py new file mode 100644 index 0000000..238fab9 --- /dev/null +++ b/conductor/conductor/openstack/common/loopingcall.py @@ -0,0 +1,95 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright 2010 United States Government as represented by the +# Administrator of the National Aeronautics and Space Administration. +# Copyright 2011 Justin Santa Barbara +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import sys + +from eventlet import event +from eventlet import greenthread + +from conductor.openstack.common.gettextutils import _ +from conductor.openstack.common import log as logging +from conductor.openstack.common import timeutils + +LOG = logging.getLogger(__name__) + + +class LoopingCallDone(Exception): + """Exception to break out and stop a LoopingCall. + + The poll-function passed to LoopingCall can raise this exception to + break out of the loop normally. This is somewhat analogous to + StopIteration. + + An optional return-value can be included as the argument to the exception; + this return-value will be returned by LoopingCall.wait() + + """ + + def __init__(self, retvalue=True): + """:param retvalue: Value that LoopingCall.wait() should return.""" + self.retvalue = retvalue + + +class LoopingCall(object): + def __init__(self, f=None, *args, **kw): + self.args = args + self.kw = kw + self.f = f + self._running = False + + def start(self, interval, initial_delay=None): + self._running = True + done = event.Event() + + def _inner(): + if initial_delay: + greenthread.sleep(initial_delay) + + try: + while self._running: + start = timeutils.utcnow() + self.f(*self.args, **self.kw) + end = timeutils.utcnow() + if not self._running: + break + delay = interval - timeutils.delta_seconds(start, end) + if delay <= 0: + LOG.warn(_('task run outlasted interval by %s sec') % + -delay) + greenthread.sleep(delay if delay > 0 else 0) + except LoopingCallDone, e: + self.stop() + done.send(e.retvalue) + except Exception: + LOG.exception(_('in looping call')) + done.send_exception(*sys.exc_info()) + return + else: + done.send(True) + + self.done = done + + greenthread.spawn_n(_inner) + return self.done + + def stop(self): + self._running = False + + def wait(self): + return self.done.wait() diff --git a/conductor/conductor/openstack/common/service.py b/conductor/conductor/openstack/common/service.py new file mode 100644 index 0000000..df48769 --- /dev/null +++ b/conductor/conductor/openstack/common/service.py @@ -0,0 +1,332 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright 2010 United States Government as represented by the +# Administrator of the National Aeronautics and Space Administration. +# Copyright 2011 Justin Santa Barbara +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +"""Generic Node base class for all workers that run on hosts.""" + +import errno +import os +import random +import signal +import sys +import time + +import eventlet +import logging as std_logging +from oslo.config import cfg + +from conductor.openstack.common import eventlet_backdoor +from conductor.openstack.common.gettextutils import _ +from conductor.openstack.common import importutils +from conductor.openstack.common import log as logging +from conductor.openstack.common import threadgroup + + +rpc = importutils.try_import('conductor.openstack.common.rpc') +CONF = cfg.CONF +LOG = logging.getLogger(__name__) + + +class Launcher(object): + """Launch one or more services and wait for them to complete.""" + + def __init__(self): + """Initialize the service launcher. + + :returns: None + + """ + self._services = threadgroup.ThreadGroup() + eventlet_backdoor.initialize_if_enabled() + + @staticmethod + def run_service(service): + """Start and wait for a service to finish. + + :param service: service to run and wait for. + :returns: None + + """ + service.start() + service.wait() + + def launch_service(self, service): + """Load and start the given service. + + :param service: The service you would like to start. + :returns: None + + """ + self._services.add_thread(self.run_service, service) + + def stop(self): + """Stop all services which are currently running. + + :returns: None + + """ + self._services.stop() + + def wait(self): + """Waits until all services have been stopped, and then returns. + + :returns: None + + """ + self._services.wait() + + +class SignalExit(SystemExit): + def __init__(self, signo, exccode=1): + super(SignalExit, self).__init__(exccode) + self.signo = signo + + +class ServiceLauncher(Launcher): + def _handle_signal(self, signo, frame): + # Allow the process to be killed again and die from natural causes + signal.signal(signal.SIGTERM, signal.SIG_DFL) + signal.signal(signal.SIGINT, signal.SIG_DFL) + + raise SignalExit(signo) + + def wait(self): + signal.signal(signal.SIGTERM, self._handle_signal) + signal.signal(signal.SIGINT, self._handle_signal) + + LOG.debug(_('Full set of CONF:')) + CONF.log_opt_values(LOG, std_logging.DEBUG) + + status = None + try: + super(ServiceLauncher, self).wait() + except SignalExit as exc: + signame = {signal.SIGTERM: 'SIGTERM', + signal.SIGINT: 'SIGINT'}[exc.signo] + LOG.info(_('Caught %s, exiting'), signame) + status = exc.code + except SystemExit as exc: + status = exc.code + finally: + if rpc: + rpc.cleanup() + self.stop() + return status + + +class ServiceWrapper(object): + def __init__(self, service, workers): + self.service = service + self.workers = workers + self.children = set() + self.forktimes = [] + + +class ProcessLauncher(object): + def __init__(self): + self.children = {} + self.sigcaught = None + self.running = True + rfd, self.writepipe = os.pipe() + self.readpipe = eventlet.greenio.GreenPipe(rfd, 'r') + + signal.signal(signal.SIGTERM, self._handle_signal) + signal.signal(signal.SIGINT, self._handle_signal) + + def _handle_signal(self, signo, frame): + self.sigcaught = signo + self.running = False + + # Allow the process to be killed again and die from natural causes + signal.signal(signal.SIGTERM, signal.SIG_DFL) + signal.signal(signal.SIGINT, signal.SIG_DFL) + + def _pipe_watcher(self): + # This will block until the write end is closed when the parent + # dies unexpectedly + self.readpipe.read() + + LOG.info(_('Parent process has died unexpectedly, exiting')) + + sys.exit(1) + + def _child_process(self, service): + # Setup child signal handlers differently + def _sigterm(*args): + signal.signal(signal.SIGTERM, signal.SIG_DFL) + raise SignalExit(signal.SIGTERM) + + signal.signal(signal.SIGTERM, _sigterm) + # Block SIGINT and let the parent send us a SIGTERM + signal.signal(signal.SIGINT, signal.SIG_IGN) + + # Reopen the eventlet hub to make sure we don't share an epoll + # fd with parent and/or siblings, which would be bad + eventlet.hubs.use_hub() + + # Close write to ensure only parent has it open + os.close(self.writepipe) + # Create greenthread to watch for parent to close pipe + eventlet.spawn_n(self._pipe_watcher) + + # Reseed random number generator + random.seed() + + launcher = Launcher() + launcher.run_service(service) + + def _start_child(self, wrap): + if len(wrap.forktimes) > wrap.workers: + # Limit ourselves to one process a second (over the period of + # number of workers * 1 second). This will allow workers to + # start up quickly but ensure we don't fork off children that + # die instantly too quickly. + if time.time() - wrap.forktimes[0] < wrap.workers: + LOG.info(_('Forking too fast, sleeping')) + time.sleep(1) + + wrap.forktimes.pop(0) + + wrap.forktimes.append(time.time()) + + pid = os.fork() + if pid == 0: + # NOTE(johannes): All exceptions are caught to ensure this + # doesn't fallback into the loop spawning children. It would + # be bad for a child to spawn more children. + status = 0 + try: + self._child_process(wrap.service) + except SignalExit as exc: + signame = {signal.SIGTERM: 'SIGTERM', + signal.SIGINT: 'SIGINT'}[exc.signo] + LOG.info(_('Caught %s, exiting'), signame) + status = exc.code + except SystemExit as exc: + status = exc.code + except BaseException: + LOG.exception(_('Unhandled exception')) + status = 2 + finally: + wrap.service.stop() + + os._exit(status) + + LOG.info(_('Started child %d'), pid) + + wrap.children.add(pid) + self.children[pid] = wrap + + return pid + + def launch_service(self, service, workers=1): + wrap = ServiceWrapper(service, workers) + + LOG.info(_('Starting %d workers'), wrap.workers) + while self.running and len(wrap.children) < wrap.workers: + self._start_child(wrap) + + def _wait_child(self): + try: + # Don't block if no child processes have exited + pid, status = os.waitpid(0, os.WNOHANG) + if not pid: + return None + except OSError as exc: + if exc.errno not in (errno.EINTR, errno.ECHILD): + raise + return None + + if os.WIFSIGNALED(status): + sig = os.WTERMSIG(status) + LOG.info(_('Child %(pid)d killed by signal %(sig)d'), + dict(pid=pid, sig=sig)) + else: + code = os.WEXITSTATUS(status) + LOG.info(_('Child %(pid)s exited with status %(code)d'), + dict(pid=pid, code=code)) + + if pid not in self.children: + LOG.warning(_('pid %d not in child list'), pid) + return None + + wrap = self.children.pop(pid) + wrap.children.remove(pid) + return wrap + + def wait(self): + """Loop waiting on children to die and respawning as necessary""" + + LOG.debug(_('Full set of CONF:')) + CONF.log_opt_values(LOG, std_logging.DEBUG) + + while self.running: + wrap = self._wait_child() + if not wrap: + # Yield to other threads if no children have exited + # Sleep for a short time to avoid excessive CPU usage + # (see bug #1095346) + eventlet.greenthread.sleep(.01) + continue + + while self.running and len(wrap.children) < wrap.workers: + self._start_child(wrap) + + if self.sigcaught: + signame = {signal.SIGTERM: 'SIGTERM', + signal.SIGINT: 'SIGINT'}[self.sigcaught] + LOG.info(_('Caught %s, stopping children'), signame) + + for pid in self.children: + try: + os.kill(pid, signal.SIGTERM) + except OSError as exc: + if exc.errno != errno.ESRCH: + raise + + # Wait for children to die + if self.children: + LOG.info(_('Waiting on %d children to exit'), len(self.children)) + while self.children: + self._wait_child() + + +class Service(object): + """Service object for binaries running on hosts.""" + + def __init__(self, threads=1000): + self.tg = threadgroup.ThreadGroup(threads) + + def start(self): + pass + + def stop(self): + self.tg.stop() + + def wait(self): + self.tg.wait() + + +def launch(service, workers=None): + if workers: + launcher = ProcessLauncher() + launcher.launch_service(service, workers=workers) + else: + launcher = ServiceLauncher() + launcher.launch_service(service) + return launcher diff --git a/conductor/conductor/openstack/common/setup.py b/conductor/conductor/openstack/common/setup.py new file mode 100644 index 0000000..0e9ff88 --- /dev/null +++ b/conductor/conductor/openstack/common/setup.py @@ -0,0 +1,359 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright 2011 OpenStack Foundation. +# Copyright 2012-2013 Hewlett-Packard Development Company, L.P. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +""" +Utilities with minimum-depends for use in setup.py +""" + +import email +import os +import re +import subprocess +import sys + +from setuptools.command import sdist + + +def parse_mailmap(mailmap='.mailmap'): + mapping = {} + if os.path.exists(mailmap): + with open(mailmap, 'r') as fp: + for l in fp: + try: + canonical_email, alias = re.match( + r'[^#]*?(<.+>).*(<.+>).*', l).groups() + except AttributeError: + continue + mapping[alias] = canonical_email + return mapping + + +def _parse_git_mailmap(git_dir, mailmap='.mailmap'): + mailmap = os.path.join(os.path.dirname(git_dir), mailmap) + return parse_mailmap(mailmap) + + +def canonicalize_emails(changelog, mapping): + """Takes in a string and an email alias mapping and replaces all + instances of the aliases in the string with their real email. + """ + for alias, email_address in mapping.iteritems(): + changelog = changelog.replace(alias, email_address) + return changelog + + +# Get requirements from the first file that exists +def get_reqs_from_files(requirements_files): + for requirements_file in requirements_files: + if os.path.exists(requirements_file): + with open(requirements_file, 'r') as fil: + return fil.read().split('\n') + return [] + + +def parse_requirements(requirements_files=['requirements.txt', + 'tools/pip-requires']): + requirements = [] + for line in get_reqs_from_files(requirements_files): + # For the requirements list, we need to inject only the portion + # after egg= so that distutils knows the package it's looking for + # such as: + # -e git://github.com/openstack/nova/master#egg=nova + if re.match(r'\s*-e\s+', line): + requirements.append(re.sub(r'\s*-e\s+.*#egg=(.*)$', r'\1', + line)) + # such as: + # http://github.com/openstack/nova/zipball/master#egg=nova + elif re.match(r'\s*https?:', line): + requirements.append(re.sub(r'\s*https?:.*#egg=(.*)$', r'\1', + line)) + # -f lines are for index locations, and don't get used here + elif re.match(r'\s*-f\s+', line): + pass + # argparse is part of the standard library starting with 2.7 + # adding it to the requirements list screws distro installs + elif line == 'argparse' and sys.version_info >= (2, 7): + pass + else: + requirements.append(line) + + return requirements + + +def parse_dependency_links(requirements_files=['requirements.txt', + 'tools/pip-requires']): + dependency_links = [] + # dependency_links inject alternate locations to find packages listed + # in requirements + for line in get_reqs_from_files(requirements_files): + # skip comments and blank lines + if re.match(r'(\s*#)|(\s*$)', line): + continue + # lines with -e or -f need the whole line, minus the flag + if re.match(r'\s*-[ef]\s+', line): + dependency_links.append(re.sub(r'\s*-[ef]\s+', '', line)) + # lines that are only urls can go in unmolested + elif re.match(r'\s*https?:', line): + dependency_links.append(line) + return dependency_links + + +def _run_shell_command(cmd, throw_on_error=False): + if os.name == 'nt': + output = subprocess.Popen(["cmd.exe", "/C", cmd], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + else: + output = subprocess.Popen(["/bin/sh", "-c", cmd], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + out = output.communicate() + if output.returncode and throw_on_error: + raise Exception("%s returned %d" % cmd, output.returncode) + if len(out) == 0: + return None + if len(out[0].strip()) == 0: + return None + return out[0].strip() + + +def _get_git_directory(): + parent_dir = os.path.dirname(__file__) + while True: + git_dir = os.path.join(parent_dir, '.git') + if os.path.exists(git_dir): + return git_dir + parent_dir, child = os.path.split(parent_dir) + if not child: # reached to root dir + return None + + +def write_git_changelog(): + """Write a changelog based on the git changelog.""" + new_changelog = 'ChangeLog' + git_dir = _get_git_directory() + if not os.getenv('SKIP_WRITE_GIT_CHANGELOG'): + if git_dir: + git_log_cmd = 'git --git-dir=%s log' % git_dir + changelog = _run_shell_command(git_log_cmd) + mailmap = _parse_git_mailmap(git_dir) + with open(new_changelog, "w") as changelog_file: + changelog_file.write(canonicalize_emails(changelog, mailmap)) + else: + open(new_changelog, 'w').close() + + +def generate_authors(): + """Create AUTHORS file using git commits.""" + jenkins_email = 'jenkins@review.(openstack|stackforge).org' + old_authors = 'AUTHORS.in' + new_authors = 'AUTHORS' + git_dir = _get_git_directory() + if not os.getenv('SKIP_GENERATE_AUTHORS'): + if git_dir: + # don't include jenkins email address in AUTHORS file + git_log_cmd = ("git --git-dir=" + git_dir + + " log --format='%aN <%aE>' | sort -u | " + "egrep -v '" + jenkins_email + "'") + changelog = _run_shell_command(git_log_cmd) + mailmap = _parse_git_mailmap(git_dir) + with open(new_authors, 'w') as new_authors_fh: + new_authors_fh.write(canonicalize_emails(changelog, mailmap)) + if os.path.exists(old_authors): + with open(old_authors, "r") as old_authors_fh: + new_authors_fh.write('\n' + old_authors_fh.read()) + else: + open(new_authors, 'w').close() + + +_rst_template = """%(heading)s +%(underline)s + +.. automodule:: %(module)s + :members: + :undoc-members: + :show-inheritance: +""" + + +def get_cmdclass(): + """Return dict of commands to run from setup.py.""" + + cmdclass = dict() + + def _find_modules(arg, dirname, files): + for filename in files: + if filename.endswith('.py') and filename != '__init__.py': + arg["%s.%s" % (dirname.replace('/', '.'), + filename[:-3])] = True + + class LocalSDist(sdist.sdist): + """Builds the ChangeLog and Authors files from VC first.""" + + def run(self): + write_git_changelog() + generate_authors() + # sdist.sdist is an old style class, can't use super() + sdist.sdist.run(self) + + cmdclass['sdist'] = LocalSDist + + # If Sphinx is installed on the box running setup.py, + # enable setup.py to build the documentation, otherwise, + # just ignore it + try: + from sphinx.setup_command import BuildDoc + + class LocalBuildDoc(BuildDoc): + + builders = ['html', 'man'] + + def generate_autoindex(self): + print "**Autodocumenting from %s" % os.path.abspath(os.curdir) + modules = {} + option_dict = self.distribution.get_option_dict('build_sphinx') + source_dir = os.path.join(option_dict['source_dir'][1], 'api') + if not os.path.exists(source_dir): + os.makedirs(source_dir) + for pkg in self.distribution.packages: + if '.' not in pkg: + os.path.walk(pkg, _find_modules, modules) + module_list = modules.keys() + module_list.sort() + autoindex_filename = os.path.join(source_dir, 'autoindex.rst') + with open(autoindex_filename, 'w') as autoindex: + autoindex.write(""".. toctree:: + :maxdepth: 1 + +""") + for module in module_list: + output_filename = os.path.join(source_dir, + "%s.rst" % module) + heading = "The :mod:`%s` Module" % module + underline = "=" * len(heading) + values = dict(module=module, heading=heading, + underline=underline) + + print "Generating %s" % output_filename + with open(output_filename, 'w') as output_file: + output_file.write(_rst_template % values) + autoindex.write(" %s.rst\n" % module) + + def run(self): + if not os.getenv('SPHINX_DEBUG'): + self.generate_autoindex() + + for builder in self.builders: + self.builder = builder + self.finalize_options() + self.project = self.distribution.get_name() + self.version = self.distribution.get_version() + self.release = self.distribution.get_version() + BuildDoc.run(self) + + class LocalBuildLatex(LocalBuildDoc): + builders = ['latex'] + + cmdclass['build_sphinx'] = LocalBuildDoc + cmdclass['build_sphinx_latex'] = LocalBuildLatex + except ImportError: + pass + + return cmdclass + + +def _get_revno(git_dir): + """Return the number of commits since the most recent tag. + + We use git-describe to find this out, but if there are no + tags then we fall back to counting commits since the beginning + of time. + """ + describe = _run_shell_command( + "git --git-dir=%s describe --always" % git_dir) + if "-" in describe: + return describe.rsplit("-", 2)[-2] + + # no tags found + revlist = _run_shell_command( + "git --git-dir=%s rev-list --abbrev-commit HEAD" % git_dir) + return len(revlist.splitlines()) + + +def _get_version_from_git(pre_version): + """Return a version which is equal to the tag that's on the current + revision if there is one, or tag plus number of additional revisions + if the current revision has no tag.""" + + git_dir = _get_git_directory() + if git_dir: + if pre_version: + try: + return _run_shell_command( + "git --git-dir=" + git_dir + " describe --exact-match", + throw_on_error=True).replace('-', '.') + except Exception: + sha = _run_shell_command( + "git --git-dir=" + git_dir + " log -n1 --pretty=format:%h") + return "%s.a%s.g%s" % (pre_version, _get_revno(git_dir), sha) + else: + return _run_shell_command( + "git --git-dir=" + git_dir + " describe --always").replace( + '-', '.') + return None + + +def _get_version_from_pkg_info(package_name): + """Get the version from PKG-INFO file if we can.""" + try: + pkg_info_file = open('PKG-INFO', 'r') + except (IOError, OSError): + return None + try: + pkg_info = email.message_from_file(pkg_info_file) + except email.MessageError: + return None + # Check to make sure we're in our own dir + if pkg_info.get('Name', None) != package_name: + return None + return pkg_info.get('Version', None) + + +def get_version(package_name, pre_version=None): + """Get the version of the project. First, try getting it from PKG-INFO, if + it exists. If it does, that means we're in a distribution tarball or that + install has happened. Otherwise, if there is no PKG-INFO file, pull the + version from git. + + We do not support setup.py version sanity in git archive tarballs, nor do + we support packagers directly sucking our git repo into theirs. We expect + that a source tarball be made from our git repo - or that if someone wants + to make a source tarball from a fork of our repo with additional tags in it + that they understand and desire the results of doing that. + """ + version = os.environ.get("OSLO_PACKAGE_VERSION", None) + if version: + return version + version = _get_version_from_pkg_info(package_name) + if version: + return version + version = _get_version_from_git(pre_version) + if version: + return version + raise Exception("Versioning for this project requires either an sdist" + " tarball, or access to an upstream git repository.") diff --git a/conductor/conductor/openstack/common/sslutils.py b/conductor/conductor/openstack/common/sslutils.py new file mode 100644 index 0000000..4168e87 --- /dev/null +++ b/conductor/conductor/openstack/common/sslutils.py @@ -0,0 +1,80 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright 2013 IBM +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import os +import ssl + +from oslo.config import cfg + +from conductor.openstack.common.gettextutils import _ + + +ssl_opts = [ + cfg.StrOpt('ca_file', + default=None, + help="CA certificate file to use to verify " + "connecting clients"), + cfg.StrOpt('cert_file', + default=None, + help="Certificate file to use when starting " + "the server securely"), + cfg.StrOpt('key_file', + default=None, + help="Private key file to use when starting " + "the server securely"), +] + + +CONF = cfg.CONF +CONF.register_opts(ssl_opts, "ssl") + + +def is_enabled(): + cert_file = CONF.ssl.cert_file + key_file = CONF.ssl.key_file + ca_file = CONF.ssl.ca_file + use_ssl = cert_file or key_file + + if cert_file and not os.path.exists(cert_file): + raise RuntimeError(_("Unable to find cert_file : %s") % cert_file) + + if ca_file and not os.path.exists(ca_file): + raise RuntimeError(_("Unable to find ca_file : %s") % ca_file) + + if key_file and not os.path.exists(key_file): + raise RuntimeError(_("Unable to find key_file : %s") % key_file) + + if use_ssl and (not cert_file or not key_file): + raise RuntimeError(_("When running server in SSL mode, you must " + "specify both a cert_file and key_file " + "option value in your configuration file")) + + return use_ssl + + +def wrap(sock): + ssl_kwargs = { + 'server_side': True, + 'certfile': CONF.ssl.cert_file, + 'keyfile': CONF.ssl.key_file, + 'cert_reqs': ssl.CERT_NONE, + } + + if CONF.ssl.ca_file: + ssl_kwargs['ca_certs'] = CONF.ssl.ca_file + ssl_kwargs['cert_reqs'] = ssl.CERT_REQUIRED + + return ssl.wrap_socket(sock, **ssl_kwargs) diff --git a/conductor/conductor/openstack/common/threadgroup.py b/conductor/conductor/openstack/common/threadgroup.py new file mode 100644 index 0000000..c8d0d9e --- /dev/null +++ b/conductor/conductor/openstack/common/threadgroup.py @@ -0,0 +1,114 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright 2012 Red Hat, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from eventlet import greenlet +from eventlet import greenpool +from eventlet import greenthread + +from conductor.openstack.common import log as logging +from conductor.openstack.common import loopingcall + + +LOG = logging.getLogger(__name__) + + +def _thread_done(gt, *args, **kwargs): + """ Callback function to be passed to GreenThread.link() when we spawn() + Calls the :class:`ThreadGroup` to notify if. + + """ + kwargs['group'].thread_done(kwargs['thread']) + + +class Thread(object): + """ Wrapper around a greenthread, that holds a reference to the + :class:`ThreadGroup`. The Thread will notify the :class:`ThreadGroup` when + it has done so it can be removed from the threads list. + """ + def __init__(self, thread, group): + self.thread = thread + self.thread.link(_thread_done, group=group, thread=self) + + def stop(self): + self.thread.kill() + + def wait(self): + return self.thread.wait() + + +class ThreadGroup(object): + """ The point of the ThreadGroup classis to: + + * keep track of timers and greenthreads (making it easier to stop them + when need be). + * provide an easy API to add timers. + """ + def __init__(self, thread_pool_size=10): + self.pool = greenpool.GreenPool(thread_pool_size) + self.threads = [] + self.timers = [] + + def add_timer(self, interval, callback, initial_delay=None, + *args, **kwargs): + pulse = loopingcall.LoopingCall(callback, *args, **kwargs) + pulse.start(interval=interval, + initial_delay=initial_delay) + self.timers.append(pulse) + + def add_thread(self, callback, *args, **kwargs): + gt = self.pool.spawn(callback, *args, **kwargs) + th = Thread(gt, self) + self.threads.append(th) + + def thread_done(self, thread): + self.threads.remove(thread) + + def stop(self): + current = greenthread.getcurrent() + for x in self.threads: + if x is current: + # don't kill the current thread. + continue + try: + x.stop() + except Exception as ex: + LOG.exception(ex) + + for x in self.timers: + try: + x.stop() + except Exception as ex: + LOG.exception(ex) + self.timers = [] + + def wait(self): + for x in self.timers: + try: + x.wait() + except greenlet.GreenletExit: + pass + except Exception as ex: + LOG.exception(ex) + current = greenthread.getcurrent() + for x in self.threads: + if x is current: + continue + try: + x.wait() + except greenlet.GreenletExit: + pass + except Exception as ex: + LOG.exception(ex) diff --git a/conductor/conductor/openstack/common/timeutils.py b/conductor/conductor/openstack/common/timeutils.py new file mode 100644 index 0000000..35cdf8c --- /dev/null +++ b/conductor/conductor/openstack/common/timeutils.py @@ -0,0 +1,186 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright 2011 OpenStack Foundation. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +""" +Time related utilities and helper functions. +""" + +import calendar +import datetime + +import iso8601 + + +# ISO 8601 extended time format with microseconds +_ISO8601_TIME_FORMAT_SUBSECOND = '%Y-%m-%dT%H:%M:%S.%f' +_ISO8601_TIME_FORMAT = '%Y-%m-%dT%H:%M:%S' +PERFECT_TIME_FORMAT = _ISO8601_TIME_FORMAT_SUBSECOND + + +def isotime(at=None, subsecond=False): + """Stringify time in ISO 8601 format""" + if not at: + at = utcnow() + st = at.strftime(_ISO8601_TIME_FORMAT + if not subsecond + else _ISO8601_TIME_FORMAT_SUBSECOND) + tz = at.tzinfo.tzname(None) if at.tzinfo else 'UTC' + st += ('Z' if tz == 'UTC' else tz) + return st + + +def parse_isotime(timestr): + """Parse time from ISO 8601 format""" + try: + return iso8601.parse_date(timestr) + except iso8601.ParseError as e: + raise ValueError(e.message) + except TypeError as e: + raise ValueError(e.message) + + +def strtime(at=None, fmt=PERFECT_TIME_FORMAT): + """Returns formatted utcnow.""" + if not at: + at = utcnow() + return at.strftime(fmt) + + +def parse_strtime(timestr, fmt=PERFECT_TIME_FORMAT): + """Turn a formatted time back into a datetime.""" + return datetime.datetime.strptime(timestr, fmt) + + +def normalize_time(timestamp): + """Normalize time in arbitrary timezone to UTC naive object""" + offset = timestamp.utcoffset() + if offset is None: + return timestamp + return timestamp.replace(tzinfo=None) - offset + + +def is_older_than(before, seconds): + """Return True if before is older than seconds.""" + if isinstance(before, basestring): + before = parse_strtime(before).replace(tzinfo=None) + return utcnow() - before > datetime.timedelta(seconds=seconds) + + +def is_newer_than(after, seconds): + """Return True if after is newer than seconds.""" + if isinstance(after, basestring): + after = parse_strtime(after).replace(tzinfo=None) + return after - utcnow() > datetime.timedelta(seconds=seconds) + + +def utcnow_ts(): + """Timestamp version of our utcnow function.""" + return calendar.timegm(utcnow().timetuple()) + + +def utcnow(): + """Overridable version of utils.utcnow.""" + if utcnow.override_time: + try: + return utcnow.override_time.pop(0) + except AttributeError: + return utcnow.override_time + return datetime.datetime.utcnow() + + +def iso8601_from_timestamp(timestamp): + """Returns a iso8601 formated date from timestamp""" + return isotime(datetime.datetime.utcfromtimestamp(timestamp)) + + +utcnow.override_time = None + + +def set_time_override(override_time=datetime.datetime.utcnow()): + """ + Override utils.utcnow to return a constant time or a list thereof, + one at a time. + """ + utcnow.override_time = override_time + + +def advance_time_delta(timedelta): + """Advance overridden time using a datetime.timedelta.""" + assert(not utcnow.override_time is None) + try: + for dt in utcnow.override_time: + dt += timedelta + except TypeError: + utcnow.override_time += timedelta + + +def advance_time_seconds(seconds): + """Advance overridden time by seconds.""" + advance_time_delta(datetime.timedelta(0, seconds)) + + +def clear_time_override(): + """Remove the overridden time.""" + utcnow.override_time = None + + +def marshall_now(now=None): + """Make an rpc-safe datetime with microseconds. + + Note: tzinfo is stripped, but not required for relative times.""" + if not now: + now = utcnow() + return dict(day=now.day, month=now.month, year=now.year, hour=now.hour, + minute=now.minute, second=now.second, + microsecond=now.microsecond) + + +def unmarshall_time(tyme): + """Unmarshall a datetime dict.""" + return datetime.datetime(day=tyme['day'], + month=tyme['month'], + year=tyme['year'], + hour=tyme['hour'], + minute=tyme['minute'], + second=tyme['second'], + microsecond=tyme['microsecond']) + + +def delta_seconds(before, after): + """ + Compute the difference in seconds between two date, time, or + datetime objects (as a float, to microsecond resolution). + """ + delta = after - before + try: + return delta.total_seconds() + except AttributeError: + return ((delta.days * 24 * 3600) + delta.seconds + + float(delta.microseconds) / (10 ** 6)) + + +def is_soon(dt, window): + """ + Determines if time is going to happen in the next window seconds. + + :params dt: the time + :params window: minimum seconds to remain to consider the time not soon + + :return: True if expiration is within the given duration + """ + soon = (utcnow() + datetime.timedelta(seconds=window)) + return normalize_time(dt) <= soon diff --git a/conductor/conductor/openstack/common/uuidutils.py b/conductor/conductor/openstack/common/uuidutils.py new file mode 100644 index 0000000..fff6309 --- /dev/null +++ b/conductor/conductor/openstack/common/uuidutils.py @@ -0,0 +1,39 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright (c) 2012 Intel Corporation. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +""" +UUID related utilities and helper functions. +""" + +import uuid + + +def generate_uuid(): + return str(uuid.uuid4()) + + +def is_uuid_like(val): + """Returns validation of a value as a UUID. + + For our purposes, a UUID is a canonical form string: + aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa + + """ + try: + return str(uuid.UUID(val)) == val + except (TypeError, ValueError, AttributeError): + return False diff --git a/conductor/conductor/openstack/common/version.py b/conductor/conductor/openstack/common/version.py new file mode 100644 index 0000000..0b80a60 --- /dev/null +++ b/conductor/conductor/openstack/common/version.py @@ -0,0 +1,94 @@ + +# Copyright 2012 OpenStack Foundation +# Copyright 2012-2013 Hewlett-Packard Development Company, L.P. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +""" +Utilities for consuming the version from pkg_resources. +""" + +import pkg_resources + + +class VersionInfo(object): + + def __init__(self, package): + """Object that understands versioning for a package + :param package: name of the python package, such as glance, or + python-glanceclient + """ + self.package = package + self.release = None + self.version = None + self._cached_version = None + + def __str__(self): + """Make the VersionInfo object behave like a string.""" + return self.version_string() + + def __repr__(self): + """Include the name.""" + return "VersionInfo(%s:%s)" % (self.package, self.version_string()) + + def _get_version_from_pkg_resources(self): + """Get the version of the package from the pkg_resources record + associated with the package.""" + try: + requirement = pkg_resources.Requirement.parse(self.package) + provider = pkg_resources.get_provider(requirement) + return provider.version + except pkg_resources.DistributionNotFound: + # The most likely cause for this is running tests in a tree + # produced from a tarball where the package itself has not been + # installed into anything. Revert to setup-time logic. + from conductor.openstack.common import setup + return setup.get_version(self.package) + + def release_string(self): + """Return the full version of the package including suffixes indicating + VCS status. + """ + if self.release is None: + self.release = self._get_version_from_pkg_resources() + + return self.release + + def version_string(self): + """Return the short version minus any alpha/beta tags.""" + if self.version is None: + parts = [] + for part in self.release_string().split('.'): + if part[0].isdigit(): + parts.append(part) + else: + break + self.version = ".".join(parts) + + return self.version + + # Compatibility functions + canonical_version_string = version_string + version_string_with_vcs = release_string + + def cached_version_string(self, prefix=""): + """Generate an object which will expand in a string context to + the results of version_string(). We do this so that don't + call into pkg_resources every time we start up a program when + passing version information into the CONF constructor, but + rather only do the calculation when and if a version is requested + """ + if not self._cached_version: + self._cached_version = "%s%s" % (prefix, + self.version_string()) + return self._cached_version diff --git a/conductor/conductor/openstack/common/wsgi.py b/conductor/conductor/openstack/common/wsgi.py new file mode 100644 index 0000000..c367224 --- /dev/null +++ b/conductor/conductor/openstack/common/wsgi.py @@ -0,0 +1,797 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright 2011 OpenStack Foundation. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +"""Utility methods for working with WSGI servers.""" + +import eventlet +eventlet.patcher.monkey_patch(all=False, socket=True) + +import datetime +import errno +import socket +import sys +import time + +import eventlet.wsgi +from oslo.config import cfg +import routes +import routes.middleware +import webob.dec +import webob.exc +from xml.dom import minidom +from xml.parsers import expat + +from conductor.openstack.common import exception +from conductor.openstack.common.gettextutils import _ +from conductor.openstack.common import jsonutils +from conductor.openstack.common import log as logging +from conductor.openstack.common import service +from conductor.openstack.common import sslutils +from conductor.openstack.common import xmlutils + +socket_opts = [ + cfg.IntOpt('backlog', + default=4096, + help="Number of backlog requests to configure the socket with"), + cfg.IntOpt('tcp_keepidle', + default=600, + help="Sets the value of TCP_KEEPIDLE in seconds for each " + "server socket. Not supported on OS X."), +] + +CONF = cfg.CONF +CONF.register_opts(socket_opts) + +LOG = logging.getLogger(__name__) + + +def run_server(application, port): + """Run a WSGI server with the given application.""" + sock = eventlet.listen(('0.0.0.0', port)) + eventlet.wsgi.server(sock, application) + + +class Service(service.Service): + """ + Provides a Service API for wsgi servers. + + This gives us the ability to launch wsgi servers with the + Launcher classes in service.py. + """ + + def __init__(self, application, port, + host='0.0.0.0', backlog=4096, threads=1000): + self.application = application + self._port = port + self._host = host + self._backlog = backlog if backlog else CONF.backlog + super(Service, self).__init__(threads) + + def _get_socket(self, host, port, backlog): + # TODO(dims): eventlet's green dns/socket module does not actually + # support IPv6 in getaddrinfo(). We need to get around this in the + # future or monitor upstream for a fix + info = socket.getaddrinfo(host, + port, + socket.AF_UNSPEC, + socket.SOCK_STREAM)[0] + family = info[0] + bind_addr = info[-1] + + sock = None + retry_until = time.time() + 30 + while not sock and time.time() < retry_until: + try: + sock = eventlet.listen(bind_addr, + backlog=backlog, + family=family) + if sslutils.is_enabled(): + sock = sslutils.wrap(sock) + + except socket.error, err: + if err.args[0] != errno.EADDRINUSE: + raise + eventlet.sleep(0.1) + if not sock: + raise RuntimeError(_("Could not bind to %(host)s:%(port)s " + "after trying for 30 seconds") % + {'host': host, 'port': port}) + sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + # sockets can hang around forever without keepalive + sock.setsockopt(socket.SOL_SOCKET, socket.SO_KEEPALIVE, 1) + + # This option isn't available in the OS X version of eventlet + if hasattr(socket, 'TCP_KEEPIDLE'): + sock.setsockopt(socket.IPPROTO_TCP, + socket.TCP_KEEPIDLE, + CONF.tcp_keepidle) + + return sock + + def start(self): + """Start serving this service using the provided server instance. + + :returns: None + + """ + super(Service, self).start() + self._socket = self._get_socket(self._host, self._port, self._backlog) + self.tg.add_thread(self._run, self.application, self._socket) + + @property + def backlog(self): + return self._backlog + + @property + def host(self): + return self._socket.getsockname()[0] if self._socket else self._host + + @property + def port(self): + return self._socket.getsockname()[1] if self._socket else self._port + + def stop(self): + """Stop serving this API. + + :returns: None + + """ + super(Service, self).stop() + + def _run(self, application, socket): + """Start a WSGI server in a new green thread.""" + logger = logging.getLogger('eventlet.wsgi') + eventlet.wsgi.server(socket, + application, + custom_pool=self.tg.pool, + log=logging.WritableLogger(logger)) + + +class Middleware(object): + """ + Base WSGI middleware wrapper. These classes require an application to be + initialized that will be called next. By default the middleware will + simply call its wrapped app, or you can override __call__ to customize its + behavior. + """ + + def __init__(self, application): + self.application = application + + def process_request(self, req): + """ + Called on each request. + + If this returns None, the next application down the stack will be + executed. If it returns a response then that response will be returned + and execution will stop here. + """ + return None + + def process_response(self, response): + """Do whatever you'd like to the response.""" + return response + + @webob.dec.wsgify + def __call__(self, req): + response = self.process_request(req) + if response: + return response + response = req.get_response(self.application) + return self.process_response(response) + + +class Debug(Middleware): + """ + Helper class that can be inserted into any WSGI application chain + to get information about the request and response. + """ + + @webob.dec.wsgify + def __call__(self, req): + print ("*" * 40) + " REQUEST ENVIRON" + for key, value in req.environ.items(): + print key, "=", value + print + resp = req.get_response(self.application) + + print ("*" * 40) + " RESPONSE HEADERS" + for (key, value) in resp.headers.iteritems(): + print key, "=", value + print + + resp.app_iter = self.print_generator(resp.app_iter) + + return resp + + @staticmethod + def print_generator(app_iter): + """ + Iterator that prints the contents of a wrapper string iterator + when iterated. + """ + print ("*" * 40) + " BODY" + for part in app_iter: + sys.stdout.write(part) + sys.stdout.flush() + yield part + print + + +class Router(object): + + """ + WSGI middleware that maps incoming requests to WSGI apps. + """ + + def __init__(self, mapper): + """ + Create a router for the given routes.Mapper. + + Each route in `mapper` must specify a 'controller', which is a + WSGI app to call. You'll probably want to specify an 'action' as + well and have your controller be a wsgi.Controller, who will route + the request to the action method. + + Examples: + mapper = routes.Mapper() + sc = ServerController() + + # Explicit mapping of one route to a controller+action + mapper.connect(None, "/svrlist", controller=sc, action="list") + + # Actions are all implicitly defined + mapper.resource("server", "servers", controller=sc) + + # Pointing to an arbitrary WSGI app. You can specify the + # {path_info:.*} parameter so the target app can be handed just that + # section of the URL. + mapper.connect(None, "/v1.0/{path_info:.*}", controller=BlogApp()) + """ + self.map = mapper + self._router = routes.middleware.RoutesMiddleware(self._dispatch, + self.map) + + @webob.dec.wsgify + def __call__(self, req): + """ + Route the incoming request to a controller based on self.map. + If no match, return a 404. + """ + return self._router + + @staticmethod + @webob.dec.wsgify + def _dispatch(req): + """ + Called by self._router after matching the incoming request to a route + and putting the information into req.environ. Either returns 404 + or the routed WSGI app's response. + """ + match = req.environ['wsgiorg.routing_args'][1] + if not match: + return webob.exc.HTTPNotFound() + app = match['controller'] + return app + + +class Request(webob.Request): + """Add some Openstack API-specific logic to the base webob.Request.""" + + default_request_content_types = ('application/json', 'application/xml') + default_accept_types = ('application/json', 'application/xml') + default_accept_type = 'application/json' + + def best_match_content_type(self, supported_content_types=None): + """Determine the requested response content-type. + + Based on the query extension then the Accept header. + Defaults to default_accept_type if we don't find a preference + + """ + supported_content_types = (supported_content_types or + self.default_accept_types) + + parts = self.path.rsplit('.', 1) + if len(parts) > 1: + ctype = 'application/{0}'.format(parts[1]) + if ctype in supported_content_types: + return ctype + + bm = self.accept.best_match(supported_content_types) + return bm or self.default_accept_type + + def get_content_type(self, allowed_content_types=None): + """Determine content type of the request body. + + Does not do any body introspection, only checks header + + """ + if "Content-Type" not in self.headers: + return None + + content_type = self.content_type + allowed_content_types = (allowed_content_types or + self.default_request_content_types) + + if content_type not in allowed_content_types: + raise exception.InvalidContentType(content_type=content_type) + return content_type + + +class Resource(object): + """ + WSGI app that handles (de)serialization and controller dispatch. + + Reads routing information supplied by RoutesMiddleware and calls + the requested action method upon its deserializer, controller, + and serializer. Those three objects may implement any of the basic + controller action methods (create, update, show, index, delete) + along with any that may be specified in the api router. A 'default' + method may also be implemented to be used in place of any + non-implemented actions. Deserializer methods must accept a request + argument and return a dictionary. Controller methods must accept a + request argument. Additionally, they must also accept keyword + arguments that represent the keys returned by the Deserializer. They + may raise a webob.exc exception or return a dict, which will be + serialized by requested content type. + """ + def __init__(self, controller, deserializer=None, serializer=None): + """ + :param controller: object that implement methods created by routes lib + :param deserializer: object that supports webob request deserialization + through controller-like actions + :param serializer: object that supports webob response serialization + through controller-like actions + """ + self.controller = controller + self.serializer = serializer or ResponseSerializer() + self.deserializer = deserializer or RequestDeserializer() + + @webob.dec.wsgify(RequestClass=Request) + def __call__(self, request): + """WSGI method that controls (de)serialization and method dispatch.""" + + try: + action, action_args, accept = self.deserialize_request(request) + except exception.InvalidContentType: + msg = _("Unsupported Content-Type") + return webob.exc.HTTPUnsupportedMediaType(explanation=msg) + except exception.MalformedRequestBody: + msg = _("Malformed request body") + return webob.exc.HTTPBadRequest(explanation=msg) + + action_result = self.execute_action(action, request, **action_args) + try: + return self.serialize_response(action, action_result, accept) + # return unserializable result (typically a webob exc) + except Exception: + return action_result + + def deserialize_request(self, request): + return self.deserializer.deserialize(request) + + def serialize_response(self, action, action_result, accept): + return self.serializer.serialize(action_result, accept, action) + + def execute_action(self, action, request, **action_args): + return self.dispatch(self.controller, action, request, **action_args) + + def dispatch(self, obj, action, *args, **kwargs): + """Find action-specific method on self and call it.""" + try: + method = getattr(obj, action) + except AttributeError: + method = getattr(obj, 'default') + + return method(*args, **kwargs) + + def get_action_args(self, request_environment): + """Parse dictionary created by routes library.""" + try: + args = request_environment['wsgiorg.routing_args'][1].copy() + except Exception: + return {} + + try: + del args['controller'] + except KeyError: + pass + + try: + del args['format'] + except KeyError: + pass + + return args + + +class ActionDispatcher(object): + """Maps method name to local methods through action name.""" + + def dispatch(self, *args, **kwargs): + """Find and call local method.""" + action = kwargs.pop('action', 'default') + action_method = getattr(self, str(action), self.default) + return action_method(*args, **kwargs) + + def default(self, data): + raise NotImplementedError() + + +class DictSerializer(ActionDispatcher): + """Default request body serialization""" + + def serialize(self, data, action='default'): + return self.dispatch(data, action=action) + + def default(self, data): + return "" + + +class JSONDictSerializer(DictSerializer): + """Default JSON request body serialization""" + + def default(self, data): + def sanitizer(obj): + if isinstance(obj, datetime.datetime): + _dtime = obj - datetime.timedelta(microseconds=obj.microsecond) + return _dtime.isoformat() + return unicode(obj) + return jsonutils.dumps(data, default=sanitizer) + + +class XMLDictSerializer(DictSerializer): + + def __init__(self, metadata=None, xmlns=None): + """ + :param metadata: information needed to deserialize xml into + a dictionary. + :param xmlns: XML namespace to include with serialized xml + """ + super(XMLDictSerializer, self).__init__() + self.metadata = metadata or {} + self.xmlns = xmlns + + def default(self, data): + # We expect data to contain a single key which is the XML root. + root_key = data.keys()[0] + doc = minidom.Document() + node = self._to_xml_node(doc, self.metadata, root_key, data[root_key]) + + return self.to_xml_string(node) + + def to_xml_string(self, node, has_atom=False): + self._add_xmlns(node, has_atom) + return node.toprettyxml(indent=' ', encoding='UTF-8') + + #NOTE (ameade): the has_atom should be removed after all of the + # xml serializers and view builders have been updated to the current + # spec that required all responses include the xmlns:atom, the has_atom + # flag is to prevent current tests from breaking + def _add_xmlns(self, node, has_atom=False): + if self.xmlns is not None: + node.setAttribute('xmlns', self.xmlns) + if has_atom: + node.setAttribute('xmlns:atom', "http://www.w3.org/2005/Atom") + + def _to_xml_node(self, doc, metadata, nodename, data): + """Recursive method to convert data members to XML nodes.""" + result = doc.createElement(nodename) + + # Set the xml namespace if one is specified + # TODO(justinsb): We could also use prefixes on the keys + xmlns = metadata.get('xmlns', None) + if xmlns: + result.setAttribute('xmlns', xmlns) + + #TODO(bcwaldon): accomplish this without a type-check + if type(data) is list: + collections = metadata.get('list_collections', {}) + if nodename in collections: + metadata = collections[nodename] + for item in data: + node = doc.createElement(metadata['item_name']) + node.setAttribute(metadata['item_key'], str(item)) + result.appendChild(node) + return result + singular = metadata.get('plurals', {}).get(nodename, None) + if singular is None: + if nodename.endswith('s'): + singular = nodename[:-1] + else: + singular = 'item' + for item in data: + node = self._to_xml_node(doc, metadata, singular, item) + result.appendChild(node) + #TODO(bcwaldon): accomplish this without a type-check + elif type(data) is dict: + collections = metadata.get('dict_collections', {}) + if nodename in collections: + metadata = collections[nodename] + for k, v in data.items(): + node = doc.createElement(metadata['item_name']) + node.setAttribute(metadata['item_key'], str(k)) + text = doc.createTextNode(str(v)) + node.appendChild(text) + result.appendChild(node) + return result + attrs = metadata.get('attributes', {}).get(nodename, {}) + for k, v in data.items(): + if k in attrs: + result.setAttribute(k, str(v)) + else: + node = self._to_xml_node(doc, metadata, k, v) + result.appendChild(node) + else: + # Type is atom + node = doc.createTextNode(str(data)) + result.appendChild(node) + return result + + def _create_link_nodes(self, xml_doc, links): + link_nodes = [] + for link in links: + link_node = xml_doc.createElement('atom:link') + link_node.setAttribute('rel', link['rel']) + link_node.setAttribute('href', link['href']) + if 'type' in link: + link_node.setAttribute('type', link['type']) + link_nodes.append(link_node) + return link_nodes + + +class ResponseHeadersSerializer(ActionDispatcher): + """Default response headers serialization""" + + def serialize(self, response, data, action): + self.dispatch(response, data, action=action) + + def default(self, response, data): + response.status_int = 200 + + +class ResponseSerializer(object): + """Encode the necessary pieces into a response object""" + + def __init__(self, body_serializers=None, headers_serializer=None): + self.body_serializers = { + 'application/xml': XMLDictSerializer(), + 'application/json': JSONDictSerializer(), + } + self.body_serializers.update(body_serializers or {}) + + self.headers_serializer = (headers_serializer or + ResponseHeadersSerializer()) + + def serialize(self, response_data, content_type, action='default'): + """Serialize a dict into a string and wrap in a wsgi.Request object. + + :param response_data: dict produced by the Controller + :param content_type: expected mimetype of serialized response body + + """ + response = webob.Response() + self.serialize_headers(response, response_data, action) + self.serialize_body(response, response_data, content_type, action) + return response + + def serialize_headers(self, response, data, action): + self.headers_serializer.serialize(response, data, action) + + def serialize_body(self, response, data, content_type, action): + response.headers['Content-Type'] = content_type + if data is not None: + serializer = self.get_body_serializer(content_type) + response.body = serializer.serialize(data, action) + + def get_body_serializer(self, content_type): + try: + return self.body_serializers[content_type] + except (KeyError, TypeError): + raise exception.InvalidContentType(content_type=content_type) + + +class RequestHeadersDeserializer(ActionDispatcher): + """Default request headers deserializer""" + + def deserialize(self, request, action): + return self.dispatch(request, action=action) + + def default(self, request): + return {} + + +class RequestDeserializer(object): + """Break up a Request object into more useful pieces.""" + + def __init__(self, body_deserializers=None, headers_deserializer=None, + supported_content_types=None): + + self.supported_content_types = supported_content_types + + self.body_deserializers = { + 'application/xml': XMLDeserializer(), + 'application/json': JSONDeserializer(), + } + self.body_deserializers.update(body_deserializers or {}) + + self.headers_deserializer = (headers_deserializer or + RequestHeadersDeserializer()) + + def deserialize(self, request): + """Extract necessary pieces of the request. + + :param request: Request object + :returns: tuple of (expected controller action name, dictionary of + keyword arguments to pass to the controller, the expected + content type of the response) + + """ + action_args = self.get_action_args(request.environ) + action = action_args.pop('action', None) + + action_args.update(self.deserialize_headers(request, action)) + action_args.update(self.deserialize_body(request, action)) + + accept = self.get_expected_content_type(request) + + return (action, action_args, accept) + + def deserialize_headers(self, request, action): + return self.headers_deserializer.deserialize(request, action) + + def deserialize_body(self, request, action): + if not len(request.body) > 0: + LOG.debug(_("Empty body provided in request")) + return {} + + try: + content_type = request.get_content_type() + except exception.InvalidContentType: + LOG.debug(_("Unrecognized Content-Type provided in request")) + raise + + if content_type is None: + LOG.debug(_("No Content-Type provided in request")) + return {} + + try: + deserializer = self.get_body_deserializer(content_type) + except exception.InvalidContentType: + LOG.debug(_("Unable to deserialize body as provided Content-Type")) + raise + + return deserializer.deserialize(request.body, action) + + def get_body_deserializer(self, content_type): + try: + return self.body_deserializers[content_type] + except (KeyError, TypeError): + raise exception.InvalidContentType(content_type=content_type) + + def get_expected_content_type(self, request): + return request.best_match_content_type(self.supported_content_types) + + def get_action_args(self, request_environment): + """Parse dictionary created by routes library.""" + try: + args = request_environment['wsgiorg.routing_args'][1].copy() + except Exception: + return {} + + try: + del args['controller'] + except KeyError: + pass + + try: + del args['format'] + except KeyError: + pass + + return args + + +class TextDeserializer(ActionDispatcher): + """Default request body deserialization""" + + def deserialize(self, datastring, action='default'): + return self.dispatch(datastring, action=action) + + def default(self, datastring): + return {} + + +class JSONDeserializer(TextDeserializer): + + def _from_json(self, datastring): + try: + return jsonutils.loads(datastring) + except ValueError: + msg = _("cannot understand JSON") + raise exception.MalformedRequestBody(reason=msg) + + def default(self, datastring): + return {'body': self._from_json(datastring)} + + +class XMLDeserializer(TextDeserializer): + + def __init__(self, metadata=None): + """ + :param metadata: information needed to deserialize xml into + a dictionary. + """ + super(XMLDeserializer, self).__init__() + self.metadata = metadata or {} + + def _from_xml(self, datastring): + plurals = set(self.metadata.get('plurals', {})) + + try: + node = xmlutils.safe_minidom_parse_string(datastring).childNodes[0] + return {node.nodeName: self._from_xml_node(node, plurals)} + except expat.ExpatError: + msg = _("cannot understand XML") + raise exception.MalformedRequestBody(reason=msg) + + def _from_xml_node(self, node, listnames): + """Convert a minidom node to a simple Python type. + + :param listnames: list of XML node names whose subnodes should + be considered list items. + + """ + + if len(node.childNodes) == 1 and node.childNodes[0].nodeType == 3: + return node.childNodes[0].nodeValue + elif node.nodeName in listnames: + return [self._from_xml_node(n, listnames) for n in node.childNodes] + else: + result = dict() + for attr in node.attributes.keys(): + result[attr] = node.attributes[attr].nodeValue + for child in node.childNodes: + if child.nodeType != node.TEXT_NODE: + result[child.nodeName] = self._from_xml_node(child, + listnames) + return result + + def find_first_child_named(self, parent, name): + """Search a nodes children for the first child with a given name""" + for node in parent.childNodes: + if node.nodeName == name: + return node + return None + + def find_children_named(self, parent, name): + """Return all of a nodes children who have the given name""" + for node in parent.childNodes: + if node.nodeName == name: + yield node + + def extract_text(self, node): + """Get the text field contained by the given node""" + if len(node.childNodes) == 1: + child = node.childNodes[0] + if child.nodeType == child.TEXT_NODE: + return child.nodeValue + return "" + + def default(self, datastring): + return {'body': self._from_xml(datastring)} diff --git a/conductor/conductor/openstack/common/xmlutils.py b/conductor/conductor/openstack/common/xmlutils.py new file mode 100644 index 0000000..ae7c077 --- /dev/null +++ b/conductor/conductor/openstack/common/xmlutils.py @@ -0,0 +1,74 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright 2013 IBM +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from xml.dom import minidom +from xml.parsers import expat +from xml import sax +from xml.sax import expatreader + + +class ProtectedExpatParser(expatreader.ExpatParser): + """An expat parser which disables DTD's and entities by default.""" + + def __init__(self, forbid_dtd=True, forbid_entities=True, + *args, **kwargs): + # Python 2.x old style class + expatreader.ExpatParser.__init__(self, *args, **kwargs) + self.forbid_dtd = forbid_dtd + self.forbid_entities = forbid_entities + + def start_doctype_decl(self, name, sysid, pubid, has_internal_subset): + raise ValueError("Inline DTD forbidden") + + def entity_decl(self, entityName, is_parameter_entity, value, base, + systemId, publicId, notationName): + raise ValueError(" entity declaration forbidden") + + def unparsed_entity_decl(self, name, base, sysid, pubid, notation_name): + # expat 1.2 + raise ValueError(" unparsed entity forbidden") + + def external_entity_ref(self, context, base, systemId, publicId): + raise ValueError(" external entity forbidden") + + def notation_decl(self, name, base, sysid, pubid): + raise ValueError(" notation forbidden") + + def reset(self): + expatreader.ExpatParser.reset(self) + if self.forbid_dtd: + self._parser.StartDoctypeDeclHandler = self.start_doctype_decl + self._parser.EndDoctypeDeclHandler = None + if self.forbid_entities: + self._parser.EntityDeclHandler = self.entity_decl + self._parser.UnparsedEntityDeclHandler = self.unparsed_entity_decl + self._parser.ExternalEntityRefHandler = self.external_entity_ref + self._parser.NotationDeclHandler = self.notation_decl + try: + self._parser.SkippedEntityHandler = None + except AttributeError: + # some pyexpat versions do not support SkippedEntity + pass + + +def safe_minidom_parse_string(xml_string): + """Parse an XML string using minidom safely. + + """ + try: + return minidom.parseString(xml_string, parser=ProtectedExpatParser()) + except sax.SAXParseException: + raise expat.ExpatError() diff --git a/conductor/conductor/rabbitmq.py b/conductor/conductor/rabbitmq.py index d7c3351..6a6de5e 100644 --- a/conductor/conductor/rabbitmq.py +++ b/conductor/conductor/rabbitmq.py @@ -1,72 +1,125 @@ -import uuid -import pika -from pika.adapters import TornadoConnection -import time - -try: - import tornado.ioloop - - IOLoop = tornado.ioloop.IOLoop -except ImportError: - IOLoop = None - - -class RabbitMqClient(object): - def __init__(self, host='localhost', login='guest', - password='guest', virtual_host='/'): - credentials = pika.PlainCredentials(login, password) - self._connection_parameters = pika.ConnectionParameters( - credentials=credentials, host=host, virtual_host=virtual_host) - self._subscriptions = {} - - def _create_connection(self): - self.connection = TornadoConnection( - parameters=self._connection_parameters, - on_open_callback=self._on_connected) - - def _on_connected(self, connection): - self._channel = connection.channel(self._on_channel_open) - - def _on_channel_open(self, channel): - self._channel = channel - if self._started_callback: - self._started_callback() - - def _on_queue_declared(self, frame, queue, callback, ctag): - def invoke_callback(ch, method_frame, header_frame, body): - callback(body=body, - message_id=header_frame.message_id or "") - - self._channel.basic_consume(invoke_callback, queue=queue, - no_ack=True, consumer_tag=ctag) - - def subscribe(self, queue, callback): - ctag = str(uuid.uuid4()) - self._subscriptions[queue] = ctag - - self._channel.queue_declare( - queue=queue, durable=True, - callback=lambda frame, ctag=ctag: self._on_queue_declared( - frame, queue, callback, ctag)) - - def unsubscribe(self, queue): - self._channel.basic_cancel(consumer_tag=self._subscriptions[queue]) - del self._subscriptions[queue] - - def start(self, callback=None): - if IOLoop is None: raise ImportError("Tornado not installed") - self._started_callback = callback - ioloop = IOLoop.instance() - self.timeout_id = ioloop.add_timeout(time.time() + 0.1, - self._create_connection) - - def send(self, queue, data, exchange="", message_id=""): - properties = pika.BasicProperties(message_id=message_id) - self._channel.queue_declare( - queue=queue, durable=True, - callback=lambda frame: self._channel.basic_publish( - exchange=exchange, routing_key=queue, - body=data, properties=properties)) - - - +from eventlet import patcher +puka = patcher.import_patched('puka') +#import puka +import anyjson +import config + +class RmqClient(object): + def __init__(self): + settings = config.CONF.rabbitmq + self._client = puka.Client('amqp://{0}:{1}@{2}:{3}/{4}'.format( + settings.login, + settings.password, + settings.host, + settings.port, + settings.virtual_host + )) + self._connected = False + + def __enter__(self): + self.connect() + return self + + def __exit__(self, exc_type, exc_val, exc_tb): + self.close() + return False + + def connect(self): + if not self._connected: + promise = self._client.connect() + self._client.wait(promise, timeout=10000) + self._connected = True + + def close(self): + if self._connected: + self._client.close() + self._connected = False + + def declare(self, queue, exchange=None): + promise = self._client.queue_declare(str(queue), durable=True) + self._client.wait(promise) + + if exchange: + promise = self._client.exchange_declare(str(exchange), durable=True) + self._client.wait(promise) + promise = self._client.queue_bind( + str(queue), str(exchange), routing_key=str(queue)) + self._client.wait(promise) + + def send(self, message, key, exchange='', timeout=None): + if not self._connected: + raise RuntimeError('Not connected to RabbitMQ') + + headers = { 'message_id': message.id } + + promise = self._client.basic_publish( + exchange=str(exchange), + routing_key=str(key), + body=anyjson.dumps(message.body), + headers=headers) + self._client.wait(promise, timeout=timeout) + + def open(self, queue): + if not self._connected: + raise RuntimeError('Not connected to RabbitMQ') + + return Subscription(self._client, queue) + +class Subscription(object): + def __init__(self, client, queue): + self._client = client + self._queue = queue + self._promise = None + self._lastMessage = None + + def __enter__(self): + self._promise = self._client.basic_consume( + queue=self._queue, + prefetch_count=1) + return self + + def __exit__(self, exc_type, exc_val, exc_tb): + self._ack_last() + promise = self._client.basic_cancel(self._promise) + self._client.wait(promise) + return False + + def _ack_last(self): + if self._lastMessage: + self._client.basic_ack(self._lastMessage) + self._lastMessage = None + + def get_message(self, timeout=None): + if not self._promise: + raise RuntimeError( + "Subscription object must be used within 'with' block") + self._ack_last() + self._lastMessage = self._client.wait(self._promise, timeout=timeout) + #print self._lastMessage + msg = Message() + msg.body = anyjson.loads(self._lastMessage['body']) + msg.id = self._lastMessage['headers'].get('message_id') + return msg + + +class Message(object): + def __init__(self): + self._body = {} + self._id = '' + + @property + def body(self): + return self._body + + @body.setter + def body(self, value): + self._body = value + + @property + def id(self): + return self._id + + @id.setter + def id(self, value): + self._id = value or '' + diff --git a/conductor/conductor/reporting.py b/conductor/conductor/reporting.py index 4dbef12..1f37b8c 100644 --- a/conductor/conductor/reporting.py +++ b/conductor/conductor/reporting.py @@ -1,22 +1,29 @@ import xml_code_engine import json - +import rabbitmq class Reporter(object): def __init__(self, rmqclient, task_id, environment_id): self._rmqclient = rmqclient self._task_id = task_id self._environment_id = environment_id + rmqclient.declare('task-reports') def _report_func(self, id, entity, text, **kwargs): - msg = json.dumps({ + body = { 'id': id, 'entity': entity, 'text': text, 'environment_id': self._environment_id - }) + } + + msg = rabbitmq.Message() + msg.body = body + msg.id = self._task_id + self._rmqclient.send( - queue='task-reports', data=msg, message_id=self._task_id) + message=msg, + key='task-reports') def _report_func(context, id, entity, text, **kwargs): reporter = context['/reporter'] diff --git a/conductor/conductor/version.py b/conductor/conductor/version.py new file mode 100644 index 0000000..736f240 --- /dev/null +++ b/conductor/conductor/version.py @@ -0,0 +1,20 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright 2012 OpenStack Foundation +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + + +from conductor.openstack.common import version as common_version + +version_info = common_version.VersionInfo('conductor') diff --git a/conductor/conductor/windows_agent.py b/conductor/conductor/windows_agent.py index 287abb0..2e7761c 100644 --- a/conductor/conductor/windows_agent.py +++ b/conductor/conductor/windows_agent.py @@ -7,7 +7,8 @@ def send_command(engine, context, body, template, host, mappings=None, command_dispatcher = context['/commandDispatcher'] def callback(result_value): - print "Received result for %s: %s. Body is %s" % (template, result_value, body) + print "Received result for %s: %s. Body is %s" % \ + (template, result_value, body) if result is not None: context[result] = result_value['Result'] @@ -16,10 +17,8 @@ def send_command(engine, context, body, template, host, mappings=None, engine.evaluate_content(success_handler, context) command_dispatcher.execute(name='agent', - template=template, - mappings=mappings, - host=host, - callback=callback) + template=template, mappings=mappings, + host=host, callback=callback) xml_code_engine.XmlCodeEngine.register_function(send_command, "send-command") \ No newline at end of file diff --git a/conductor/data/templates/agent-config/Default.template b/conductor/data/templates/agent-config/Default.template index 54d9cb9..179bfb8 100644 --- a/conductor/data/templates/agent-config/Default.template +++ b/conductor/data/templates/agent-config/Default.template @@ -23,7 +23,7 @@ - + diff --git a/conductor/etc/app.config b/conductor/etc/app.config deleted file mode 100644 index f69fe45..0000000 --- a/conductor/etc/app.config +++ /dev/null @@ -1,5 +0,0 @@ -[rabbitmq] -host = localhost -vhost = keero -login = keero -password = keero \ No newline at end of file diff --git a/conductor/etc/conductor-paste.ini b/conductor/etc/conductor-paste.ini new file mode 100644 index 0000000..e69de29 diff --git a/conductor/etc/conductor.conf b/conductor/etc/conductor.conf new file mode 100644 index 0000000..189eeed --- /dev/null +++ b/conductor/etc/conductor.conf @@ -0,0 +1,10 @@ +[DEFAULT] +log_file = logs/conductor.log + + +[rabbitmq] +host = localhost +port = 5672 +virtual_host = keero +login = keero +password = keero \ No newline at end of file diff --git a/conductor/logs/.gitignore b/conductor/logs/.gitignore new file mode 100644 index 0000000..44c5ea8 --- /dev/null +++ b/conductor/logs/.gitignore @@ -0,0 +1,4 @@ +# Ignore everything in this directory +* +# Except this file +!.gitignore \ No newline at end of file diff --git a/conductor/openstack-common.conf b/conductor/openstack-common.conf new file mode 100644 index 0000000..0437737 --- /dev/null +++ b/conductor/openstack-common.conf @@ -0,0 +1,7 @@ +[DEFAULT] + +# The list of modules to copy from openstack-common +modules=setup,wsgi,config,exception,gettextutils,importutils,jsonutils,log,xmlutils,sslutils,service,notifier,local,install_venv_common,version,timeutils,eventlet_backdoor,threadgroup,loopingcall,uuidutils + +# The base module to hold the copy of openstack.common +base=conductor \ No newline at end of file diff --git a/conductor/tools/pip-requires b/conductor/tools/pip-requires index a7bcbfe..816ab98 100644 --- a/conductor/tools/pip-requires +++ b/conductor/tools/pip-requires @@ -1,3 +1,9 @@ -pika -tornado -jsonpath \ No newline at end of file +anyjson +eventlet>=0.9.12 +jsonpath +puka +Paste +PasteDeploy +iso8601>=0.1.4 + +http://tarballs.openstack.org/oslo-config/oslo-config-2013.1b4.tar.gz#egg=oslo-config From c61d333c30c0966f3cb012ebc356f31c64e87057 Mon Sep 17 00:00:00 2001 From: Stan Lagun Date: Thu, 21 Mar 2013 14:05:45 +0400 Subject: [PATCH 11/37] Entry point script renamed 2d27f4f5054f34982ed67da2bf4b35c8ac1558d3 --- conductor/bin/{app.py => conductor} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename conductor/bin/{app.py => conductor} (100%) diff --git a/conductor/bin/app.py b/conductor/bin/conductor similarity index 100% rename from conductor/bin/app.py rename to conductor/bin/conductor From 2c208062452b2e289a0b15630348538e1754dcca Mon Sep 17 00:00:00 2001 From: Dmitry Teselkin Date: Thu, 21 Mar 2013 17:26:49 +0400 Subject: [PATCH 12/37] Userdata script updated to support computer renaming functionality --- conductor/data/init.ps1 | 36 +++++++++++++++++++++++++++++------- 1 file changed, 29 insertions(+), 7 deletions(-) diff --git a/conductor/data/init.ps1 b/conductor/data/init.ps1 index 620792c..12c2453 100644 --- a/conductor/data/init.ps1 +++ b/conductor/data/init.ps1 @@ -3,12 +3,34 @@ $WindowsAgentConfigBase64 = '%WINDOWS_AGENT_CONFIG_BASE64%' $WindowsAgentConfigFile = "C:\Keero\Agent\WindowsAgent.exe.config" +$NewComputerName = '%INTERNAL_HOSTNAME%' + +$RestartRequired = $false + Import-Module CoreFunctions -Stop-Service "Keero Agent" -Backup-File $WindowsAgentConfigFile -Remove-Item $WindowsAgentConfigFile -Force -ConvertFrom-Base64String -Base64String $WindowsAgentConfigBase64 -Path $WindowsAgentConfigFile -Exec sc.exe 'config','"Keero Agent"','start=','delayed-auto' -Start-Service 'Keero Agent' -Write-Log 'All done!' \ No newline at end of file +if ( $WindowsAgentConfigBase64 -ne '%WINDOWS_AGENT_CONFIG_BASE64%' ) { + Write-Log "Updating Keero Windows Agent." + Stop-Service "Keero Agent" + Backup-File $WindowsAgentConfigFile + Remove-Item $WindowsAgentConfigFile -Force + ConvertFrom-Base64String -Base64String $WindowsAgentConfigBase64 -Path $WindowsAgentConfigFile + Exec sc.exe 'config','"Keero Agent"','start=','delayed-auto' + Write-Log "Service has been updated." +} + +if ( $NewComputerName -ne '%INTERNAL_HOSTNAME%' ) { + Write-Log "Renaming computer ..." + Rename-Computer -NewName $NewComputerName | Out-Null + Write-Log "New name assigned, restart required." + $RestartRequired = $true +} + +Write-Log 'All done!' +if ( $RestartRequired ) { + Write-Log "Restarting computer ..." + Restart-Computer -Force +} +else { + Start-Service 'Keero Agent' +} From f202ac273b45b2aac06da3d9195a27c4ca11da44 Mon Sep 17 00:00:00 2001 From: Dmitry Teselkin Date: Thu, 21 Mar 2013 17:28:12 +0400 Subject: [PATCH 13/37] All Cloudbase-Init plugins disabled except UserDataPlugin --- Deployment/cloudbase-init/config/cloudbase-init.conf | 1 + 1 file changed, 1 insertion(+) diff --git a/Deployment/cloudbase-init/config/cloudbase-init.conf b/Deployment/cloudbase-init/config/cloudbase-init.conf index 57d1faa..0b167a0 100644 --- a/Deployment/cloudbase-init/config/cloudbase-init.conf +++ b/Deployment/cloudbase-init/config/cloudbase-init.conf @@ -8,3 +8,4 @@ config_drive_cdrom=false verbose=true logdir=C:\Program Files (x86)\Cloudbase Solutions\Cloudbase-Init\log\ logfile=cloudbase-init.log +plugins=cloudbaseinit.plugins.windows.userdata.UserDataPlugin \ No newline at end of file From 9964cd217592f280d02753698d4856e2a51edff8 Mon Sep 17 00:00:00 2001 From: Timur Nurlygayanov Date: Thu, 21 Mar 2013 18:05:46 +0400 Subject: [PATCH 14/37] Added deployment script for automated tests. --- tests/deploy.sh | 49 ++++++++++++++++++++++++------------------------- 1 file changed, 24 insertions(+), 25 deletions(-) mode change 100644 => 100755 tests/deploy.sh diff --git a/tests/deploy.sh b/tests/deploy.sh old mode 100644 new mode 100755 index e57207d..cad10a2 --- a/tests/deploy.sh +++ b/tests/deploy.sh @@ -24,10 +24,20 @@ expect "*$*" send -- "./unstack.sh\n" expect "*$*" send -- "./stack.sh\n" -expect "*Would you like to start it now?*" +expect "*/usr/bin/service: 123: exec: status: not found*" send -- "y\n" expect "*stack.sh completed*" +send -- "sudo rabbitmq-plugins enable rabbitmq_management\n" +expect "*$*" +send -- "sudo service rabbitmq-server restart\n" +expect "*$*" +send -- "sudo rabbitmqctl add_user keero keero\n" +expect "*$*" +send -- "sudo rabbitmqctl set_user_tags keero administrator\n" +expect "*$*" + + send -- "source openrc admin admin\n" expect "*$*" @@ -37,7 +47,7 @@ expect "*$*" send -- "nova keypair-add keero-linux-keys > heat_key.priv\n" expect "*$*" -send -- "glance image-create --name 'ws-2012-full-agent' --is-public true --container-format ovf --disk-format qcow2 < ws-2012-full-agent.qcow2\n" +send -- "glance image-create --name 'ws-2012-full' --is-public true --container-format ovf --disk-format qcow2 < ws-2012-full.qcow2\n" expect "*$*" send -- "cd ~/keero\n" @@ -50,33 +60,22 @@ send -- "cp -Rf ~/keero/dashboard/windc /opt/stack/horizon/openstack_dashboard/d expect "*$*" send -- "cp -f ~/keero/dashboard/api/windc.py /opt/stack/horizon/openstack_dashboard/api/\n" expect "*$*" -send -- "cp -Rf ~/keero/dashboard/windcclient /opt/stack/horizon/\n" +send -- "cd ~/keero/python-portasclient\n" expect "*$*" -send -- "cd ~/keero/windc\n" +send -- "sudo python setup.py install\n" expect "*$*" -send -- "rm -rf windc.sqlite\n" +send -- "cd ~/keero/portas\n" expect "*$*" -send -- "./tools/with_venv.sh ./bin/windc-api --config-file=./etc/windc-api-paste.ini --dbsync\n" +send -- "./tools/with_venv.sh ./bin/portas-api --config-file=./etc/portas-api.conf & > ~/APIservice.log\n" +sleep 10 +send -- "\n" +expect "*$*" +send -- "cd ~/keero/conductor\n" +expect "*$*" +send -- "./tools/with_venv.sh ./bin/app.py & > ~/conductor.log\n" +sleep 10 +send -- "\n" expect "*$*" send -- "logout\n" expect "*#*" -send -- "rabbitmq-plugins enable rabbitmq_management\n" -expect "*#*" -send -- "service rabbitmq-server restart\n" -expect "*#*" -send -- "rabbitmqctl add_user keero keero\n" -expect "*#*" -send -- "rabbitmqctl set_user_tags keero administrator\n" -expect "*#*" - -send -- "su - stack\n" -expect "*$*" -send -- "cd /opt/stack/devstack\n" -expect "*$*" -send -- "source openrc admin admin\n" -expect "*$*" -send -- "cd /opt/stack/keero/windc\n" -expect "*$*" -send -- "sudo ./tools/with_venv.sh ./bin/windc-api --config-file=./etc/windc-api-paste.ini > /opt/stack/tests_windc_daemon.log &\n" -expect "*$*" From e6e9dda6778b9377f922c78e6a71b9f6e15704b9 Mon Sep 17 00:00:00 2001 From: Timur Nurlygayanov Date: Thu, 21 Mar 2013 18:09:13 +0400 Subject: [PATCH 15/37] Fixed: changed the run mode for install venv script --- conductor/tools/with_venv.sh | 0 1 file changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 conductor/tools/with_venv.sh diff --git a/conductor/tools/with_venv.sh b/conductor/tools/with_venv.sh old mode 100644 new mode 100755 From c8a9858ca21d1c9dd8435bc08ae49822b11618f0 Mon Sep 17 00:00:00 2001 From: Dmitry Teselkin Date: Fri, 22 Mar 2013 13:06:59 +0400 Subject: [PATCH 16/37] Code to start\stop keero components using devstack functions --- Deployment/devstack-scripts/devstack.localrc | 1 + Deployment/devstack-scripts/localrc | 2 +- Deployment/devstack-scripts/start-devstack.sh | 1 + Deployment/devstack-scripts/start-keero.sh | 25 +++++++++++++++++++ Deployment/devstack-scripts/stop-devstack.sh | 1 + Deployment/devstack-scripts/stop-keero.sh | 12 +++++++++ 6 files changed, 41 insertions(+), 1 deletion(-) create mode 100644 Deployment/devstack-scripts/start-keero.sh create mode 100644 Deployment/devstack-scripts/stop-keero.sh diff --git a/Deployment/devstack-scripts/devstack.localrc b/Deployment/devstack-scripts/devstack.localrc index 758db5f..3295cdf 100644 --- a/Deployment/devstack-scripts/devstack.localrc +++ b/Deployment/devstack-scripts/devstack.localrc @@ -16,6 +16,7 @@ RABBIT_PASSWORD=$lab_password SERVICE_PASSWORD=$lab_password SERVICE_TOKEN=tokentoken ENABLED_SERVICES+=,heat,h-api,h-api-cfn,h-api-cw,h-eng +ENABLED_SERVICES+=,conductor,portas LOGFILE=/opt/stack/devstack/stack.sh.log SCREEN_LOGDIR=/opt/stack/log/ diff --git a/Deployment/devstack-scripts/localrc b/Deployment/devstack-scripts/localrc index 2658117..13ce3cd 100644 --- a/Deployment/devstack-scripts/localrc +++ b/Deployment/devstack-scripts/localrc @@ -1,6 +1,7 @@ #!/bin/bash DEVSTACK_DIR=/home/stack/devstack +INSTALL_DIR=/opt/stack MYSQL_DB_TMPFS=true MYSQL_DB_TMPFS_SIZE=128M @@ -8,7 +9,6 @@ MYSQL_DB_TMPFS_SIZE=128M NOVA_CACHE_TMPFS=true NOVA_CACHE_TMPFS_SIZE=24G - #====================================== source $DEVSTACK_DIR/openrc admin admin source ./functions.sh diff --git a/Deployment/devstack-scripts/start-devstack.sh b/Deployment/devstack-scripts/start-devstack.sh index 305200d..e206d87 100644 --- a/Deployment/devstack-scripts/start-devstack.sh +++ b/Deployment/devstack-scripts/start-devstack.sh @@ -20,6 +20,7 @@ $DEVSTACK_DIR/stack.sh # Executing post-stack actions #=============================================================================== source ./post-stack.sh no-localrc +source ./start-keero.sh no-localrc #=============================================================================== diff --git a/Deployment/devstack-scripts/start-keero.sh b/Deployment/devstack-scripts/start-keero.sh new file mode 100644 index 0000000..0c4aa5e --- /dev/null +++ b/Deployment/devstack-scripts/start-keero.sh @@ -0,0 +1,25 @@ +#!/bin/bash + +if [[ -z "$1" ]] ; then + source ./localrc +fi + +die_if_not_set INSTALL_DIR + +# Starting Portas +#================ +if [[ ! -d "$INSTALL_DIR/portas" ]] ; then + mkdir -p "$INSTALL_DIR/portas" +fi + +cp "$INSTALL_DIR/keero/portas/etc" "$INSTALL_DIR/portas/etc" + +screen_it portas "cd $INSTALL_DIR/portas && portas-api --config-file=$INSTALL_DIR/portas/etc/portas-api.conf" +#================ + + + +# Starting Conductor +#=================== +screen_it conductor "cd $INSTALL_DIR/keero/conductor && bash ./tools/with_venv.sh ./bin/app.py" +#=================== diff --git a/Deployment/devstack-scripts/stop-devstack.sh b/Deployment/devstack-scripts/stop-devstack.sh index af35304..32b3d36 100644 --- a/Deployment/devstack-scripts/stop-devstack.sh +++ b/Deployment/devstack-scripts/stop-devstack.sh @@ -18,5 +18,6 @@ $DEVSTACK_DIR/unstack.sh # Executing post-unstack actions #=============================================================================== source ./post-unstack.sh no-localrc +source ./stop-keero.sh no-localrc #=============================================================================== diff --git a/Deployment/devstack-scripts/stop-keero.sh b/Deployment/devstack-scripts/stop-keero.sh new file mode 100644 index 0000000..bf8af5f --- /dev/null +++ b/Deployment/devstack-scripts/stop-keero.sh @@ -0,0 +1,12 @@ +#!/bin/bash + +if [[ -z "$1" ]] ; then + source ./localrc +fi + +# Stopping Keero components +#========================== +for serv in conductor portas ; do + screen -S $SCREEN_NAME -p $serv -X kill +done +#========================== From eaddba5640012947e9062bef35f14303e21e0ad3 Mon Sep 17 00:00:00 2001 From: Timur Nurlygayanov Date: Mon, 25 Mar 2013 11:52:03 +0400 Subject: [PATCH 17/37] Fixed automated tests for WebUI. Added class Page with simple objects. --- tests/selenium/datacenters_page.py | 52 ++++-------- tests/selenium/login_page.py | 25 ++---- tests/selenium/page.py | 130 +++++++++++++++++++++++++++++ tests/selenium/services_page.py | 54 ++++-------- 4 files changed, 169 insertions(+), 92 deletions(-) create mode 100644 tests/selenium/page.py diff --git a/tests/selenium/datacenters_page.py b/tests/selenium/datacenters_page.py index e48a9c1..7c32cce 100644 --- a/tests/selenium/datacenters_page.py +++ b/tests/selenium/datacenters_page.py @@ -1,56 +1,36 @@ import re -from login_page import LoginPage from services_page import ServicesPage +import page -class DataCentersPage(): +class DataCentersPage(page.Page): page = None - def __init__(self): - start_page = LoginPage() - self.page = start_page.login() - self.page.find_element_by_link_text('Project').click() - self.page.find_element_by_link_text('Windows Data Centers').click() - def create_data_center(self, name): - button_text = 'Create Windows Data Center' - self.page.find_element_by_link_text(button_text).click() - name_field = self.page.find_element_by_id('id_name') + self.Button('Create Windows Data Center').Click() + + self.EditBox('id_name').Set(name) xpath = "//input[@value='Create']" - button = self.page.find_element_by_xpath(xpath) + self.Button(xpath).Click() - name_field.clear() - name_field.send_keys(name) - - button.click() - - return self.page - - def find_data_center(self, name): - return self.page.find_element_by_link_text(name) + return self def delete_data_center(self, name): - datacenter = self.find_data_center(name) - link = datacenter.get_attribute('href') + link = self.Link(name).Address() datacenter_id = re.search('windc/(\S+)', link).group(0)[6:-1] xpath = ".//*[@id='windc__row__%s']/td[3]/div/a[2]" % datacenter_id - more_button = self.page.find_element_by_xpath(xpath) + self.Button(xpath).Click() - more_button.click() + button_id = "windc__row_%s__action_delete" % datacenter_id + self.Button(button_id).Click() - delete_button_id = "windc__row_%s__action_delete" % datacenter_id - delete_button = self.page.find_element_by_id(delete_button_id) + self.Button("Delete Data Center").Click() - delete_button.click() - - self.page.find_element_by_link_text("Delete Data Center").click() - - return self.page + return self def select_data_center(self, name): - datacenter = self.page.find_data_center(name) - datacenter.click() - self.page = ServicesPage(self.page) - return self.page + self.Link(name).Click() + page = ServicesPage(self.driver) + return page diff --git a/tests/selenium/login_page.py b/tests/selenium/login_page.py index f0f357c..b22c9b1 100644 --- a/tests/selenium/login_page.py +++ b/tests/selenium/login_page.py @@ -1,8 +1,8 @@ import ConfigParser -from selenium import webdriver +import page -class LoginPage(): +class LoginPage(page.Page): def login(self): config = ConfigParser.RawConfigParser() @@ -11,20 +11,11 @@ class LoginPage(): user = config.get('server', 'user') password = config.get('server', 'password') - page = webdriver.Firefox() - page.set_page_load_timeout(30) - page.implicitly_wait(30) - page.get(url) - name = page.find_element_by_name('username') - pwd = page.find_element_by_name('password') + self.Open(url) + + self.EditBox('username').Set(user) + self.EditBox('password').Set(password) xpath = "//button[@type='submit']" - button = page.find_element_by_xpath(xpath) + self.Button(xpath).Click() - name.clear() - name.send_keys(user) - pwd.clear() - pwd.send_keys(password) - - button.click() - - return page + return self diff --git a/tests/selenium/page.py b/tests/selenium/page.py new file mode 100644 index 0000000..e2dcb56 --- /dev/null +++ b/tests/selenium/page.py @@ -0,0 +1,130 @@ +from selenium import webdriver + + +class ButtonClass: + button = None + + def __init__(self, object): + self.button = object + + def Click(self): + self.button.click() + + def isPresented(self): + if self.button != None: + return True + return False + + +class LinkClass: + link = None + + def __init__(self, object): + self.link = object + + def Click(self): + self.link.click() + + def isPresented(self): + if self.link != None: + return True + return False + + def Address(self): + return self.link.get_attribute('href') + + +class EditBoxClass: + edit = None + + def __init__(self, object): + self.edit = object + + def isPresented(self): + if self.edit != None: + return True + return False + + def Set(self, value): + self.edit.clear() + self.edit.send_keys(value) + + def Text(self): + return self.edit.get_text() + + +class DropDownListClass: + select = None + + def __init__(self, object): + self.select = object + + def isPresented(self): + if self.select != None: + return True + return False + + def Set(self, value): + self.select.select_by_visible_text(value) + + def Text(self): + return self.select.get_text() + + +class Page: + + driver = None + timeout = 30 + + def __init__(self, driver): + driver.set_page_load_timeout(30) + driver.implicitly_wait(30) + self.driver = driver + + def _find_element(self, parameter): + object = None + try: + object = self.driver.find_element_by_name(parameter) + return object + except: + pass + try: + object = self.driver.find_element_by_id(parameter) + return object + except: + pass + try: + object = self.driver.find_element_by_xpath(parameter) + return object + except: + pass + try: + object = self.driver.find_element_by_link_text(parameter) + return object + except: + pass + + return object + + def Open(self, url): + self.driver.get(url) + + def Button(self, name): + object = self._find_element(name) + button = ButtonClass(object) + return button + + def Link(self, name): + object = self._find_element(name) + link = LinkClass(object) + return link + + def EditBox(self, name): + object = self._find_element(name) + edit = EditBoxClass(object) + return edit + + def DropDownList(self, name): + object = self._find_element(name) + select = DropDownListClass(object) + return select \ No newline at end of file diff --git a/tests/selenium/services_page.py b/tests/selenium/services_page.py index fb43043..e8ea9f8 100644 --- a/tests/selenium/services_page.py +++ b/tests/selenium/services_page.py @@ -1,57 +1,33 @@ -import ConfigParser -from selenium import webdriver +import page +import re -class ServicesPage(): - page = None - - def __init__(self, page): - self.page = page +class ServicesPage(page.Page): def create_service(self, service_type, parameters): - button_id = 'services__action_CreateService' - button = self.page.find_element_by_id(button_id) - button.click() - - self.select_type_of_service(service_type) + self.Button('services__action_CreateService').Click() + self.DropDownList('0-service').Set(service_type) + self.Button('wizard_goto_step').Click() for parameter in parameters: - field = self.page.find_element_by_name(parameter.key) - field.clear() - field.send_keys(parameter.value) + self.EditBox(parameter.key).Set(parameter.value) - xpath = "//input[@value='Deploy']" - deploy_button = self.page.find_element_by_xpath(xpath) - deploy_button.click() + self.Button("//input[@value='Create']").Click() - return page - - def select_type_of_service(self, service_type): - type_field = self.page.find_element_by_name('0-service') - type_field.select_by_visible_text(service_type) - next_button = self.page.find_element_by_name('wizard_goto_step') - next_button.click() - return self.page - - def find_service(self, name): - return self.page.find_element_by_link_text(name) + return self def delete_service(self, name): - service = self.find_data_center(name) - link = service.get_attribute('href') + link = self.Link(name).Address() service_id = re.search('windc/(\S+)', link).group(0)[6:-1] xpath = ".//*[@id='services__row__%s']/td[5]/div/a[2]" % service_id - more_button = self.page.find_element_by_xpath(xpath) - more_button.click() + self.Button(xpath).Click() - delete_button_id = "services__row_%s__action_delete" % datacenter_id - delete_button = self.page.find_element_by_id(delete_button_id) + button_id = "services__row_%s__action_delete" % service_id + self.Button(button_id).Click() - delete_button.click() + self.Button("Delete Service").Click() - self.page.find_element_by_link_text("Delete Service").click() - - return self.page + return self From c13879946562f8fb6a2b66251871e5a57cb365d5 Mon Sep 17 00:00:00 2001 From: Timur Nurlygayanov Date: Mon, 25 Mar 2013 15:04:10 +0400 Subject: [PATCH 18/37] Fixed automated tests for web UI. --- tests/selenium/conf.ini | 2 +- tests/selenium/datacenters_page.py | 4 +- tests/selenium/page.py | 111 +++++++++++++++-------------- tests/selenium/test.py | 36 ++++++---- 4 files changed, 84 insertions(+), 69 deletions(-) diff --git a/tests/selenium/conf.ini b/tests/selenium/conf.ini index 8b43288..437aeba 100644 --- a/tests/selenium/conf.ini +++ b/tests/selenium/conf.ini @@ -1,4 +1,4 @@ [server] address=http://172.18.124.101 user=admin -password=AkvareL707 \ No newline at end of file +password=swordfish \ No newline at end of file diff --git a/tests/selenium/datacenters_page.py b/tests/selenium/datacenters_page.py index 7c32cce..f0e74a8 100644 --- a/tests/selenium/datacenters_page.py +++ b/tests/selenium/datacenters_page.py @@ -20,7 +20,9 @@ class DataCentersPage(page.Page): link = self.Link(name).Address() datacenter_id = re.search('windc/(\S+)', link).group(0)[6:-1] - xpath = ".//*[@id='windc__row__%s']/td[3]/div/a[2]" % datacenter_id + print 'Data Center ID: ' + datacenter_id + + xpath = ".//*[@id='windc__row__%s']/td[4]/div/a[2]" % datacenter_id self.Button(xpath).Click() button_id = "windc__row_%s__action_delete" % datacenter_id diff --git a/tests/selenium/page.py b/tests/selenium/page.py index e2dcb56..80085ab 100644 --- a/tests/selenium/page.py +++ b/tests/selenium/page.py @@ -1,11 +1,10 @@ -from selenium import webdriver class ButtonClass: button = None - def __init__(self, object): - self.button = object + def __init__(self, obj): + self.button = obj def Click(self): self.button.click() @@ -19,8 +18,8 @@ class ButtonClass: class LinkClass: link = None - def __init__(self, object): - self.link = object + def __init__(self, obj): + self.link = obj def Click(self): self.link.click() @@ -35,10 +34,9 @@ class LinkClass: class EditBoxClass: - edit = None - def __init__(self, object): - self.edit = object + def __init__(self, obj): + self.edit = obj def isPresented(self): if self.edit != None: @@ -56,8 +54,8 @@ class EditBoxClass: class DropDownListClass: select = None - def __init__(self, object): - self.select = object + def __init__(self, obj): + self.select = obj def isPresented(self): if self.select != None: @@ -72,59 +70,68 @@ class DropDownListClass: class Page: - + driver = None - timeout = 30 - + timeout = 300 + def __init__(self, driver): driver.set_page_load_timeout(30) - driver.implicitly_wait(30) + driver.implicitly_wait(0.01) self.driver = driver - + def _find_element(self, parameter): - object = None - try: - object = self.driver.find_element_by_name(parameter) - return object - except: - pass - try: - object = self.driver.find_element_by_id(parameter) - return object - except: - pass - try: - object = self.driver.find_element_by_xpath(parameter) - return object - except: - pass - try: - object = self.driver.find_element_by_link_text(parameter) - return object - except: - pass - - return object - + obj = None + k = 0 + while (obj == None and k < self.timeout): + k += 1 + try: + obj = self.driver.find_element_by_name(parameter) + return obj + except: + pass + try: + obj = self.driver.find_element_by_id(parameter) + return obj + except: + pass + try: + obj = self.driver.find_element_by_xpath(parameter) + return obj + except: + pass + try: + obj = self.driver.find_element_by_partial_link_text(parameter) + return obj + except: + pass + + return obj + def Open(self, url): self.driver.get(url) - + def Button(self, name): - object = self._find_element(name) - button = ButtonClass(object) + obj = self._find_element(name) + button = ButtonClass(obj) return button - + def Link(self, name): - object = self._find_element(name) - link = LinkClass(object) + obj = self._find_element(name) + link = LinkClass(obj) return link - + def EditBox(self, name): - object = self._find_element(name) - edit = EditBoxClass(object) + obj = self._find_element(name) + edit = EditBoxClass(obj) return edit - + def DropDownList(self, name): - object = self._find_element(name) - select = DropDownListClass(object) - return select \ No newline at end of file + obj = self._find_element(name) + select = DropDownListClass(obj) + return select + + def Navigate(self, path): + steps = path.split(':') + + for step in steps: + self.Button(step).Click() diff --git a/tests/selenium/test.py b/tests/selenium/test.py index 6c8389c..4391a0b 100644 --- a/tests/selenium/test.py +++ b/tests/selenium/test.py @@ -1,54 +1,60 @@ # -*- coding: utf-8 -*- -import untitest2 +import unittest2 +from login_page import LoginPage from datacenters_page import DataCentersPage +from selenium import webdriver class SanityTests(unittest2.TestCase): def setUp(self): - self.page = DataCentersPage() + driver = webdriver.Firefox() + self.page = LoginPage(driver) + self.page.login() + self.page.Navigate('Project:Windows Data Centers') + self.page = DataCentersPage(driver) def tearDown(self): - self.page.close() + self.page.driver.close() def test_01_create_data_center(self): self.page.create_data_center('dc1') - assert self.page.find_data_center('dc1') is not None + assert self.page.Link('dc1').isPresented() def test_02_delete_data_center(self): self.page.delete_data_center('dc1') - assert self.page.find_data_center('dc1') is None + assert not self.page.Link('dc1').isPresented() def test_03_create_data_centers(self): for i in range(1, 20): name = 'datacenter' + str(i) self.page.create_data_center(name) - assert self.page.find_data_center(name) is not None + assert self.page.Link(name).isPresented() def test_04_delete_data_centers(self): self.page.delete_data_center('datacenter1') self.page.delete_data_center('datacenter20') - assert self.page.find_data_center('datacenter1') is None - assert self.page.find_data_center('datacenter20') is None + assert not self.page.Link('datacenter1').isPresented() + assert not self.page.Link('datacenter20').isPresented() for i in range(2, 19): name = 'datacenter' + str(i) - assert self.page.find_data_center(name) is not None + assert self.page.Link(name).isPresented() def test_05_create_service_ad(self): name = 'dc001.local' - self.page.create_data_center(name) - self.select_data_center(name) - + self.page.create_data_center('test') + self.page.select_data_center('test') + ad_parameters = {'1-dc_name': name, '1-dc_count': 1, '1-adm_password': 'AkvareL707!', '1-recovery_password': 'AkvareL707!'} self.page.create_service('Active Directory', ad_parameters) - - assert self.page.find_service(name) is not None + + assert self.page.Link(name).isPresented() if __name__ == '__main__': - unittest2.main() \ No newline at end of file + unittest2.main() From bb3e62bb4d5060c344e171f304194ce8bd8c2fc9 Mon Sep 17 00:00:00 2001 From: Timur Nurlygayanov Date: Tue, 26 Mar 2013 06:35:35 +0700 Subject: [PATCH 19/37] Fixed all pep8. --- conductor/bin/app.py | 2 +- conductor/conductor/app.py | 7 +- conductor/conductor/cloud_formation.py | 1 - .../conductor/commands/cloud_formation.py | 1 - conductor/conductor/commands/dispatcher.py | 1 - conductor/conductor/commands/windows_agent.py | 1 - conductor/conductor/function_context.py | 2 +- conductor/conductor/helpers.py | 1 + conductor/conductor/rabbitmq.py | 6 +- conductor/conductor/reporting.py | 5 +- conductor/conductor/windows_agent.py | 8 +- conductor/conductor/workflow.py | 3 +- conductor/conductor/xml_code_engine.py | 3 +- portas/portas/api/middleware/context.py | 4 +- portas/portas/api/v1/__init__.py | 28 ++++--- portas/portas/api/v1/active_directories.py | 30 ++++--- portas/portas/api/v1/environments.py | 5 +- portas/portas/api/v1/router.py | 17 ++-- portas/portas/api/v1/sessions.py | 55 ++++++++---- portas/portas/api/v1/webservers.py | 24 ++++-- portas/portas/common/config.py | 1 + portas/portas/common/exception.py | 3 +- portas/portas/common/service.py | 22 +++-- portas/portas/common/uuidutils.py | 2 +- portas/portas/context.py | 6 +- portas/portas/db/__init__.py | 2 +- .../versions/001_add_initial_tables.py | 4 +- .../versions/002_add_session_table.py | 5 +- .../versions/003_add_status_table.py | 5 +- .../005_remove_obsolete_service_table.py | 17 ++-- portas/portas/db/models.py | 3 +- portas/portas/utils.py | 6 +- python-portasclient/portasclient/version.py | 2 +- python-portasclient/tests/test_sanity.py | 84 ++++++++----------- 34 files changed, 203 insertions(+), 163 deletions(-) diff --git a/conductor/bin/app.py b/conductor/bin/app.py index 3c2619b..bb7624b 100644 --- a/conductor/bin/app.py +++ b/conductor/bin/app.py @@ -1,3 +1,3 @@ #!/usr/bin/env python -from conductor import app \ No newline at end of file +from conductor import app diff --git a/conductor/conductor/app.py b/conductor/conductor/app.py index f4cf9a7..5e241c7 100644 --- a/conductor/conductor/app.py +++ b/conductor/conductor/app.py @@ -41,12 +41,14 @@ def task_received(task, message_id): def loop(callback): for workflow in workflows: workflow.execute() - if not command_dispatcher.execute_pending(lambda: schedule(loop, callback)): + func = lambda: schedule(loop, callback) + if not command_dispatcher.execute_pending(func): callback() def shutdown(): command_dispatcher.close() - rmqclient.send('task-results', json.dumps(task), message_id=message_id) + rmqclient.send('task-results', json.dumps(task), + message_id=message_id) print 'Finished at', datetime.datetime.now() loop(shutdown) @@ -61,4 +63,3 @@ def start(): rmqclient.start(start) tornado.ioloop.IOLoop.instance().start() - diff --git a/conductor/conductor/cloud_formation.py b/conductor/conductor/cloud_formation.py index 3ef3b41..95fb9e4 100644 --- a/conductor/conductor/cloud_formation.py +++ b/conductor/conductor/cloud_formation.py @@ -36,4 +36,3 @@ xml_code_engine.XmlCodeEngine.register_function( xml_code_engine.XmlCodeEngine.register_function( prepare_user_data, "prepare_user_data") - diff --git a/conductor/conductor/commands/cloud_formation.py b/conductor/conductor/commands/cloud_formation.py index 0d12083..262a1f2 100644 --- a/conductor/conductor/commands/cloud_formation.py +++ b/conductor/conductor/commands/cloud_formation.py @@ -59,7 +59,6 @@ class HeatExecutor(CommandBase): self._stack ]) - callbacks = [] for t in self._pending_list: if t['callback']: diff --git a/conductor/conductor/commands/dispatcher.py b/conductor/conductor/commands/dispatcher.py index b815ddb..97db0ef 100644 --- a/conductor/conductor/commands/dispatcher.py +++ b/conductor/conductor/commands/dispatcher.py @@ -32,7 +32,6 @@ class CommandDispatcher(command.CommandBase): return result > 0 - def has_pending_commands(self): result = False for command in self._command_map.values(): diff --git a/conductor/conductor/commands/windows_agent.py b/conductor/conductor/commands/windows_agent.py index c4747b6..b0abbcd 100644 --- a/conductor/conductor/commands/windows_agent.py +++ b/conductor/conductor/commands/windows_agent.py @@ -63,4 +63,3 @@ class WindowsAgentExecutor(CommandBase): def close(self): self._rmqclient.unsubscribe('-execution-results') - diff --git a/conductor/conductor/function_context.py b/conductor/conductor/function_context.py index 237f23e..e27b6db 100644 --- a/conductor/conductor/function_context.py +++ b/conductor/conductor/function_context.py @@ -48,4 +48,4 @@ class Context(object): return str(self._data) if self._parent: return str(self._parent) - return str({}) \ No newline at end of file + return str({}) diff --git a/conductor/conductor/helpers.py b/conductor/conductor/helpers.py index 4128e16..435a35b 100644 --- a/conductor/conductor/helpers.py +++ b/conductor/conductor/helpers.py @@ -38,6 +38,7 @@ def merge_dicts(dict1, dict2, max_levels=0): result[key] = value return result + def find(f, seq): """Return first item in sequence where f(item) == True.""" index = 0 diff --git a/conductor/conductor/rabbitmq.py b/conductor/conductor/rabbitmq.py index d7c3351..8a9cebe 100644 --- a/conductor/conductor/rabbitmq.py +++ b/conductor/conductor/rabbitmq.py @@ -54,7 +54,8 @@ class RabbitMqClient(object): del self._subscriptions[queue] def start(self, callback=None): - if IOLoop is None: raise ImportError("Tornado not installed") + if IOLoop is None: + raise ImportError("Tornado not installed") self._started_callback = callback ioloop = IOLoop.instance() self.timeout_id = ioloop.add_timeout(time.time() + 0.1, @@ -67,6 +68,3 @@ class RabbitMqClient(object): callback=lambda frame: self._channel.basic_publish( exchange=exchange, routing_key=queue, body=data, properties=properties)) - - - diff --git a/conductor/conductor/reporting.py b/conductor/conductor/reporting.py index 4dbef12..7d1a3d4 100644 --- a/conductor/conductor/reporting.py +++ b/conductor/conductor/reporting.py @@ -18,12 +18,9 @@ class Reporter(object): self._rmqclient.send( queue='task-reports', data=msg, message_id=self._task_id) + def _report_func(context, id, entity, text, **kwargs): reporter = context['/reporter'] return reporter._report_func(id, entity, text, **kwargs) xml_code_engine.XmlCodeEngine.register_function(_report_func, "report") - - - - diff --git a/conductor/conductor/windows_agent.py b/conductor/conductor/windows_agent.py index 287abb0..ba916ba 100644 --- a/conductor/conductor/windows_agent.py +++ b/conductor/conductor/windows_agent.py @@ -3,11 +3,13 @@ import xml_code_engine def send_command(engine, context, body, template, host, mappings=None, result=None, **kwargs): - if not mappings: mappings = {} + if not mappings: + mappings = {} command_dispatcher = context['/commandDispatcher'] def callback(result_value): - print "Received result for %s: %s. Body is %s" % (template, result_value, body) + msg = "Received result for %s: %s. Body is %s" + print msg % (template, result_value, body) if result is not None: context[result] = result_value['Result'] @@ -22,4 +24,4 @@ def send_command(engine, context, body, template, host, mappings=None, callback=callback) -xml_code_engine.XmlCodeEngine.register_function(send_command, "send-command") \ No newline at end of file +xml_code_engine.XmlCodeEngine.register_function(send_command, "send-command") diff --git a/conductor/conductor/workflow.py b/conductor/conductor/workflow.py index a39a7da..46d1d3d 100644 --- a/conductor/conductor/workflow.py +++ b/conductor/conductor/workflow.py @@ -5,6 +5,7 @@ import re import xml_code_engine import function_context + class Workflow(object): def __init__(self, filename, data, command_dispatcher, config, reporter): self._data = data @@ -84,7 +85,6 @@ class Workflow(object): else: return position + suffix.split('.') - @staticmethod def _select_func(context, path='', source=None, **kwargs): @@ -102,7 +102,6 @@ class Workflow(object): context['/dataSource'], Workflow._correct_position(path, context)) - @staticmethod def _set_func(path, context, body, engine, target=None, **kwargs): body_data = engine.evaluate_content(body, context) diff --git a/conductor/conductor/xml_code_engine.py b/conductor/conductor/xml_code_engine.py index fe676b0..42c18a1 100644 --- a/conductor/conductor/xml_code_engine.py +++ b/conductor/conductor/xml_code_engine.py @@ -61,7 +61,8 @@ class XmlCodeEngine(object): return_value = result if len(result) == 0: return_value = ''.join(parts) - if do_strip: return_value = return_value.strip() + if do_strip: + return_value = return_value.strip() elif len(result) == 1: return_value = result[0] diff --git a/portas/portas/api/middleware/context.py b/portas/portas/api/middleware/context.py index cd0d6a7..fc81c5c 100644 --- a/portas/portas/api/middleware/context.py +++ b/portas/portas/api/middleware/context.py @@ -75,7 +75,7 @@ class ContextMiddleware(BaseContextMiddleware): 'auth_tok': req.headers.get('X-Auth-Token', deprecated_token), 'service_catalog': service_catalog, 'session': req.headers.get('X-Configuration-Session') - } + } req.context = portas.context.RequestContext(**kwargs) else: raise webob.exc.HTTPUnauthorized() @@ -84,4 +84,4 @@ class ContextMiddleware(BaseContextMiddleware): def factory(cls, global_conf, **local_conf): def filter(app): return cls(app) - return filter \ No newline at end of file + return filter diff --git a/portas/portas/api/v1/__init__.py b/portas/portas/api/v1/__init__.py index 6fa090e..2c08d56 100644 --- a/portas/portas/api/v1/__init__.py +++ b/portas/portas/api/v1/__init__.py @@ -27,15 +27,19 @@ def get_env_status(environment_id, session_id): unit = get_session() if not session_id: - session = unit.query(Session).filter( - Session.environment_id == environment_id and Session.state.in_(['open', 'deploying'])).first() + variants = ['open', 'deploying'] + session = unit.query(Session).filter(Session.environment_id == + environment_id and + Session.state.in_(variants) + ).first() if session: session_id = session.id else: return status session_state = unit.query(Session).get(session_id).state - reports_count = unit.query(Status).filter_by(environment_id=environment_id, session_id=session_id).count() + reports_count = unit.query(Status).filter_by(environment_id=environment_id, + session_id=session_id).count() if session_state == 'deployed': status = 'finished' @@ -50,13 +54,16 @@ def get_env_status(environment_id, session_id): def get_statuses(type): if type in draft['services']: - return [get_service_status(environment_id, session_id, service) for service in - draft['services'][type]] + services = draft['services'][type] + return [get_service_status(environment_id, + session_id, + service) for service in services] else: return [] is_inprogress = filter(lambda item: item == 'inprogress', - get_statuses('activeDirectories') + get_statuses('webServers')) + get_statuses('activeDirectories') + + get_statuses('webServers')) if session_state == 'deploying' and is_inprogress > 1: status = 'inprogress' @@ -71,10 +78,11 @@ def get_service_status(environment_id, session_id, service): session_state = unit.query(Session).get(session_id).state entities = [u['id'] for u in service['units']] - reports_count = unit.query(Status).filter(Status.environment_id == environment_id - and Status.session_id == session_id - and Status.entity_id.in_(entities)) \ - .count() + reports_count = unit.query(Status).filter( + Status.environment_id == environment_id + and Status.session_id == session_id + and Status.entity_id.in_(entities) + ).count() if session_state == 'deployed': status = 'finished' diff --git a/portas/portas/api/v1/active_directories.py b/portas/portas/api/v1/active_directories.py index 2b57a67..bacdb2a 100644 --- a/portas/portas/api/v1/active_directories.py +++ b/portas/portas/api/v1/active_directories.py @@ -9,18 +9,23 @@ log = logging.getLogger(__name__) class Controller(object): def index(self, request, environment_id): - log.debug(_('ActiveDirectory:Index '.format(environment_id))) + log.debug(_('ActiveDirectory:Index '. + format(environment_id))) - draft = prepare_draft(get_draft(environment_id, request.context.session)) + draft = prepare_draft(get_draft(environment_id, + request.context.session)) for dc in draft['services']['activeDirectories']: - dc['status'] = get_service_status(environment_id, request.context.session, dc) + dc['status'] = get_service_status(environment_id, + request.context.session, + dc) return {'activeDirectories': draft['services']['activeDirectories']} @utils.verify_session def create(self, request, environment_id, body): - log.debug(_('ActiveDirectory:Create '.format(environment_id, body))) + log.debug(_('ActiveDirectory:Create '. + format(environment_id, body))) draft = get_draft(session_id=request.context.session) @@ -33,7 +38,8 @@ class Controller(object): for unit in active_directory['units']: unit_count += 1 unit['id'] = uuidutils.generate_uuid() - unit['name'] = 'dc{0}{1}'.format(unit_count, active_directory['id'][:4]) + unit['name'] = 'dc{0}{1}'.format(unit_count, + active_directory['id'][:4]) draft = prepare_draft(draft) draft['services']['activeDirectories'].append(active_directory) @@ -42,23 +48,25 @@ class Controller(object): return active_directory def delete(self, request, environment_id, active_directory_id): - log.debug(_('ActiveDirectory:Delete '.format(environment_id, active_directory_id))) + log.debug(_('ActiveDirectory:Delete '. + format(environment_id, active_directory_id))) draft = get_draft(request.context.session) - draft['services']['activeDirectories'] = [service for service in draft['services']['activeDirectories'] if - service['id'] != active_directory_id] + items = [service for service in draft['services']['activeDirectories'] + if service['id'] != active_directory_id] + draft['services']['activeDirectories'] = items save_draft(request.context.session, draft) def prepare_draft(draft): - if not draft.has_key('services'): + if not 'services' in draft: draft['services'] = {} - if not draft['services'].has_key('activeDirectories'): + if not 'activeDirectories' in draft['services']: draft['services']['activeDirectories'] = [] return draft def create_resource(): - return wsgi.Resource(Controller()) \ No newline at end of file + return wsgi.Resource(Controller()) diff --git a/portas/portas/api/v1/environments.py b/portas/portas/api/v1/environments.py index e8c82ee..4129bbf 100644 --- a/portas/portas/api/v1/environments.py +++ b/portas/portas/api/v1/environments.py @@ -61,7 +61,8 @@ class Controller(object): return env def update(self, request, environment_id, body): - log.debug(_('Environments:Update '.format(environment_id, body))) + log.debug(_('Environments:Update '. + format(environment_id, body))) session = get_session() environment = session.query(Environment).get(environment_id) @@ -88,4 +89,4 @@ class Controller(object): def create_resource(): - return wsgi.Resource(Controller()) \ No newline at end of file + return wsgi.Resource(Controller()) diff --git a/portas/portas/api/v1/router.py b/portas/portas/api/v1/router.py index 5a18601..ed0da25 100644 --- a/portas/portas/api/v1/router.py +++ b/portas/portas/api/v1/router.py @@ -16,7 +16,8 @@ # under the License. import routes from portas.openstack.common import wsgi -from portas.api.v1 import environments, sessions, active_directories, webservers +from portas.api.v1 import (environments, sessions, + active_directories, webservers) class API(wsgi.Router): @@ -64,11 +65,13 @@ class API(wsgi.Router): controller=sessions_resource, action='delete', conditions={'method': ['DELETE']}) - mapper.connect('/environments/{environment_id}/sessions/{session_id}/reports', + mapper.connect('/environments/{environment_id}/sessions/' + '{session_id}/reports', controller=sessions_resource, action='reports', conditions={'method': ['GET']}) - mapper.connect('/environments/{environment_id}/sessions/{session_id}/deploy', + mapper.connect('/environments/{environment_id}/sessions/' + '{session_id}/deploy', controller=sessions_resource, action='deploy', conditions={'method': ['POST']}) @@ -82,7 +85,8 @@ class API(wsgi.Router): controller=activeDirectories_resource, action='create', conditions={'method': ['POST']}) - mapper.connect('/environments/{environment_id}/activeDirectories/{active_directory_id}', + mapper.connect('/environments/{environment_id}/activeDirectories/' + '{active_directory_id}', controller=activeDirectories_resource, action='delete', conditions={'method': ['DELETE']}) @@ -96,8 +100,9 @@ class API(wsgi.Router): controller=webServers_resource, action='create', conditions={'method': ['POST']}) - mapper.connect('/environments/{environment_id}/webServers/{web_server_id}', + mapper.connect('/environments/{environment_id}/webServers/' + '{web_server_id}', controller=webServers_resource, action='delete', conditions={'method': ['DELETE']}) - super(API, self).__init__(mapper) \ No newline at end of file + super(API, self).__init__(mapper) diff --git a/portas/portas/api/v1/sessions.py b/portas/portas/api/v1/sessions.py index dd12cea..5bf52b0 100644 --- a/portas/portas/api/v1/sessions.py +++ b/portas/portas/api/v1/sessions.py @@ -17,34 +17,44 @@ log = logging.getLogger(__name__) class Controller(object): def __init__(self): self.write_lock = Semaphore(1) - connection = amqp.Connection('{0}:{1}'.format(rabbitmq.host, rabbitmq.port), virtual_host=rabbitmq.virtual_host, - userid=rabbitmq.userid, password=rabbitmq.password, + connection = amqp.Connection('{0}:{1}'. + format(rabbitmq.host, rabbitmq.port), + virtual_host=rabbitmq.virtual_host, + userid=rabbitmq.userid, + password=rabbitmq.password, ssl=rabbitmq.use_ssl, insist=True) self.ch = connection.channel() - self.ch.exchange_declare('tasks', 'direct', durable=True, auto_delete=False) + self.ch.exchange_declare('tasks', 'direct', durable=True, + auto_delete=False) def index(self, request, environment_id): log.debug(_('Session:List '.format(environment_id))) - filters = {'environment_id': environment_id, 'user_id': request.context.user} + filters = {'environment_id': environment_id, + 'user_id': request.context.user} unit = get_session() configuration_sessions = unit.query(Session).filter_by(**filters) - return {"sessions": [session.to_dict() for session in configuration_sessions if - session.environment.tenant_id == request.context.tenant]} + sessions = [session.to_dict() for session in configuration_sessions if + session.environment.tenant_id == request.context.tenant] + return {"sessions": sessions} def configure(self, request, environment_id): log.debug(_('Session:Configure '.format(environment_id))) - params = {'environment_id': environment_id, 'user_id': request.context.user, 'state': 'open'} + params = {'environment_id': environment_id, + 'user_id': request.context.user, + 'state': 'open'} session = Session() session.update(params) unit = get_session() - if unit.query(Session).filter(Session.environment_id == environment_id and Session.state.in_( - ['open', 'deploing'])).first(): + if unit.query(Session).filter(Session.environment_id == environment_id + and + Session.state.in_(['open', 'deploing']) + ).first(): log.info('There is already open session for this environment') raise exc.HTTPConflict @@ -58,7 +68,8 @@ class Controller(object): return session.to_dict() def show(self, request, environment_id, session_id): - log.debug(_('Session:Show '.format(environment_id, session_id))) + log.debug(_('Session:Show '. + format(environment_id, session_id))) unit = get_session() session = unit.query(Session).get(session_id) @@ -70,14 +81,16 @@ class Controller(object): return session.to_dict() def delete(self, request, environment_id, session_id): - log.debug(_('Session:Delete '.format(environment_id, session_id))) + log.debug(_('Session:Delete '. + format(environment_id, session_id))) unit = get_session() session = unit.query(Session).get(session_id) + comment = 'Session object in \'deploying\' state could not be deleted' if session.state == 'deploying': - log.info('Session is in \'deploying\' state. Could not be deleted.') - raise exc.HTTPForbidden(comment='Session object in \'deploying\' state could not be deleted') + log.info(comment) + raise exc.HTTPForbidden(comment=comment) with unit.begin(): unit.delete(session) @@ -85,7 +98,8 @@ class Controller(object): return None def reports(self, request, environment_id, session_id): - log.debug(_('Session:Reports '.format(environment_id, session_id))) + log.debug(_('Session:Reports '. + format(environment_id, session_id))) unit = get_session() statuses = unit.query(Status).filter_by(session_id=session_id) @@ -93,20 +107,25 @@ class Controller(object): return {'reports': [status.to_dict() for status in statuses]} def deploy(self, request, environment_id, session_id): - log.debug(_('Session:Deploy '.format(environment_id, session_id))) + log.debug(_('Session:Deploy '. + format(environment_id, session_id))) unit = get_session() session = unit.query(Session).get(session_id) + msg = _('Could not deploy session. Session is already ' + 'deployed or in deployment state') if session.state != 'open': - log.warn(_('Could not deploy session. Session is already deployed or in deployment state')) + log.warn(msg) session.state = 'deploying' session.save(unit) with self.write_lock: - self.ch.basic_publish(Message(body=anyjson.serialize(session.description)), 'tasks', 'tasks') + self.ch.basic_publish(Message(body=anyjson. + serialize(session.description)), + 'tasks', 'tasks') def create_resource(): - return wsgi.Resource(Controller()) \ No newline at end of file + return wsgi.Resource(Controller()) diff --git a/portas/portas/api/v1/webservers.py b/portas/portas/api/v1/webservers.py index d1af6bc..ef1344f 100644 --- a/portas/portas/api/v1/webservers.py +++ b/portas/portas/api/v1/webservers.py @@ -11,16 +11,19 @@ class Controller(object): def index(self, request, environment_id): log.debug(_('WebServer:List '.format(environment_id))) - draft = prepare_draft(get_draft(environment_id, request.context.session)) + draft = prepare_draft(get_draft(environment_id, + request.context.session)) for dc in draft['services']['webServers']: - dc['status'] = get_service_status(environment_id, request.context.session, dc) + dc['status'] = get_service_status(environment_id, + request.context.session, dc) return {'webServers': draft['services']['webServers']} @utils.verify_session def create(self, request, environment_id, body): - log.debug(_('WebServer:Create '.format(environment_id, body))) + log.debug(_('WebServer:Create '. + format(environment_id, body))) draft = get_draft(session_id=request.context.session) @@ -43,23 +46,26 @@ class Controller(object): @utils.verify_session def delete(self, request, environment_id, web_server_id): - log.debug(_('WebServer:Delete '.format(environment_id, web_server_id))) + log.debug(_('WebServer:Delete '. + format(environment_id, web_server_id))) draft = get_draft(session_id=request.context.session) - draft['services']['webServers'] = [service for service in draft['services']['webServers'] if - service['id'] != web_server_id] + + elements = [service for service in draft['services']['webServers'] if + service['id'] != web_server_id] + draft['services']['webServers'] = elements save_draft(request.context.session, draft) def prepare_draft(draft): - if not draft.has_key('services'): + if not 'services' in draft: draft['services'] = {} - if not draft['services'].has_key('webServers'): + if not 'webServers' in draft['services']: draft['services']['webServers'] = [] return draft def create_resource(): - return wsgi.Resource(Controller()) \ No newline at end of file + return wsgi.Resource(Controller()) diff --git a/portas/portas/common/config.py b/portas/portas/common/config.py index c0ea7d0..58a6378 100644 --- a/portas/portas/common/config.py +++ b/portas/portas/common/config.py @@ -82,6 +82,7 @@ def parse_args(args=None, usage=None, default_config_files=None): usage=usage, default_config_files=default_config_files) + def setup_logging(): """ Sets up the logging options for a log with supplied name diff --git a/portas/portas/common/exception.py b/portas/portas/common/exception.py index 59bb0ee..7f338e8 100644 --- a/portas/portas/common/exception.py +++ b/portas/portas/common/exception.py @@ -45,6 +45,7 @@ class PortasException(Exception): super(PortasException, self).__init__(message) + class SchemaLoadError(PortasException): message = _("Unable to load schema: %(reason)s") @@ -52,5 +53,3 @@ class SchemaLoadError(PortasException): class InvalidObject(PortasException): message = _("Provided object does not match schema " "'%(schema)s': %(reason)s") - - diff --git a/portas/portas/common/service.py b/portas/portas/common/service.py index 143a358..2d8f38e 100644 --- a/portas/portas/common/service.py +++ b/portas/portas/common/service.py @@ -27,14 +27,18 @@ class TaskResultHandlerService(service.Service): super(TaskResultHandlerService, self).stop() def _handle_results(self): - connection = amqp.Connection('{0}:{1}'.format(rabbitmq.host, rabbitmq.port), virtual_host=rabbitmq.virtual_host, - userid=rabbitmq.userid, password=rabbitmq.password, + connection = amqp.Connection('{0}:{1}'. + format(rabbitmq.host, rabbitmq.port), + virtual_host=rabbitmq.virtual_host, + userid=rabbitmq.userid, + password=rabbitmq.password, ssl=rabbitmq.use_ssl, insist=True) ch = connection.channel() def bind(exchange, queue): if not exchange: - ch.exchange_declare(exchange, 'direct', durable=True, auto_delete=False) + ch.exchange_declare(exchange, 'direct', durable=True, + auto_delete=False) ch.queue_declare(queue, durable=True, auto_delete=False) if not exchange: ch.queue_bind(queue, exchange, queue) @@ -43,13 +47,15 @@ class TaskResultHandlerService(service.Service): bind(conf.reports_exchange, conf.reports_queue) ch.basic_consume(conf.results_exchange, callback=handle_result) - ch.basic_consume(conf.reports_exchange, callback=handle_report, no_ack=True) + ch.basic_consume(conf.reports_exchange, callback=handle_report, + no_ack=True) while ch.callbacks: ch.wait() def handle_report(msg): - log.debug(_('Got report message from orchestration engine:\n{0}'.format(msg.body))) + log.debug(_('Got report message from orchestration engine:\n{0}'. + format(msg.body))) params = anyjson.deserialize(msg.body) params['entity_id'] = params['id'] @@ -61,7 +67,8 @@ def handle_report(msg): session = get_session() #connect with session conf_session = session.query(Session).filter_by( - **{'environment_id': status.environment_id, 'state': 'deploying'}).first() + **{'environment_id': status.environment_id, + 'state': 'deploying'}).first() status.session_id = conf_session.id with session.begin(): @@ -69,7 +76,8 @@ def handle_report(msg): def handle_result(msg): - log.debug(_('Got result message from orchestration engine:\n{0}'.format(msg.body))) + log.debug(_('Got result message from orchestration engine:\n{0}'. + format(msg.body))) environment_result = anyjson.deserialize(msg.body) diff --git a/portas/portas/common/uuidutils.py b/portas/portas/common/uuidutils.py index f3513ac..cf00651 100644 --- a/portas/portas/common/uuidutils.py +++ b/portas/portas/common/uuidutils.py @@ -2,4 +2,4 @@ import uuid def generate_uuid(): - return str(uuid.uuid4()).replace('-', '') \ No newline at end of file + return str(uuid.uuid4()).replace('-', '') diff --git a/portas/portas/context.py b/portas/portas/context.py index 9fb9c56..c541829 100644 --- a/portas/portas/context.py +++ b/portas/portas/context.py @@ -24,7 +24,9 @@ class RequestContext(object): accesses the system, as well as additional request information. """ - def __init__(self, auth_tok=None, user=None, tenant=None, roles=None, service_catalog=None, session=None): + def __init__(self, auth_tok=None, user=None, tenant=None, + roles=None, service_catalog=None, session=None): + self.auth_tok = auth_tok self.user = user self.tenant = tenant @@ -55,4 +57,4 @@ class RequestContext(object): @classmethod def from_dict(cls, values): - return cls(**values) \ No newline at end of file + return cls(**values) diff --git a/portas/portas/db/__init__.py b/portas/portas/db/__init__.py index 75fd934..2070644 100644 --- a/portas/portas/db/__init__.py +++ b/portas/portas/db/__init__.py @@ -9,4 +9,4 @@ sql_connection_opt = cfg.StrOpt('sql_connection', 'Default: %(default)s') CONF = cfg.CONF -CONF.register_opt(sql_connection_opt) \ No newline at end of file +CONF.register_opt(sql_connection_opt) diff --git a/portas/portas/db/migrate_repo/versions/001_add_initial_tables.py b/portas/portas/db/migrate_repo/versions/001_add_initial_tables.py index f2262f2..52a6d16 100644 --- a/portas/portas/db/migrate_repo/versions/001_add_initial_tables.py +++ b/portas/portas/db/migrate_repo/versions/001_add_initial_tables.py @@ -11,7 +11,7 @@ Table('environment', meta, Column('updated', DateTime(), nullable=False), Column('tenant_id', String(32), nullable=False), Column('description', Text(), nullable=False), -) + ) Table('service', meta, Column('id', String(32), primary_key=True), @@ -21,7 +21,7 @@ Table('service', meta, Column('created', DateTime, nullable=False), Column('updated', DateTime, nullable=False), Column('description', Text(), nullable=False), -) + ) def upgrade(migrate_engine): diff --git a/portas/portas/db/migrate_repo/versions/002_add_session_table.py b/portas/portas/db/migrate_repo/versions/002_add_session_table.py index d9ca59c..30f89c8 100644 --- a/portas/portas/db/migrate_repo/versions/002_add_session_table.py +++ b/portas/portas/db/migrate_repo/versions/002_add_session_table.py @@ -5,12 +5,13 @@ meta = MetaData() session = Table('session', meta, Column('id', String(32), primary_key=True), - Column('environment_id', String(32), ForeignKey('environment.id')), + Column('environment_id', String(32), + ForeignKey('environment.id')), Column('created', DateTime, nullable=False), Column('updated', DateTime, nullable=False), Column('user_id', String(32), nullable=False), Column('state', Text(), nullable=False), -) + ) def upgrade(migrate_engine): diff --git a/portas/portas/db/migrate_repo/versions/003_add_status_table.py b/portas/portas/db/migrate_repo/versions/003_add_status_table.py index 6bacb9c..1ee86b2 100644 --- a/portas/portas/db/migrate_repo/versions/003_add_status_table.py +++ b/portas/portas/db/migrate_repo/versions/003_add_status_table.py @@ -8,10 +8,11 @@ status = Table('status', meta, Column('created', DateTime, nullable=False), Column('updated', DateTime, nullable=False), Column('entity', String(10), nullable=False), - Column('environment_id', String(32), ForeignKey('environment.id')), + Column('environment_id', String(32), + ForeignKey('environment.id')), Column('session_id', String(32), ForeignKey('session.id')), Column('text', Text(), nullable=False), -) + ) def upgrade(migrate_engine): diff --git a/portas/portas/db/migrate_repo/versions/005_remove_obsolete_service_table.py b/portas/portas/db/migrate_repo/versions/005_remove_obsolete_service_table.py index 2692829..8c3bfc1 100644 --- a/portas/portas/db/migrate_repo/versions/005_remove_obsolete_service_table.py +++ b/portas/portas/db/migrate_repo/versions/005_remove_obsolete_service_table.py @@ -5,14 +5,15 @@ from sqlalchemy.types import String, Text, DateTime meta = MetaData() service = Table('service', meta, - Column('id', String(32), primary_key=True), - Column('name', String(255), nullable=False), - Column('type', String(40), nullable=False), - Column('environment_id', String(32), ForeignKey('environment.id')), - Column('created', DateTime, nullable=False), - Column('updated', DateTime, nullable=False), - Column('description', Text(), nullable=False), -) + Column('id', String(32), primary_key=True), + Column('name', String(255), nullable=False), + Column('type', String(40), nullable=False), + Column('environment_id', String(32), + ForeignKey('environment.id')), + Column('created', DateTime, nullable=False), + Column('updated', DateTime, nullable=False), + Column('description', Text(), nullable=False), + ) def upgrade(migrate_engine): diff --git a/portas/portas/db/models.py b/portas/portas/db/models.py index 412d1fb..9491bab 100644 --- a/portas/portas/db/models.py +++ b/portas/portas/db/models.py @@ -83,7 +83,8 @@ class ModelBase(object): def to_dict(self): dictionary = self.__dict__.copy() - return {k: v for k, v in dictionary.iteritems() if k != '_sa_instance_state'} + return {k: v for k, v in dictionary.iteritems() + if k != '_sa_instance_state'} class JsonBlob(TypeDecorator): diff --git a/portas/portas/utils.py b/portas/portas/utils.py index 393c6fb..30ae919 100644 --- a/portas/portas/utils.py +++ b/portas/portas/utils.py @@ -11,8 +11,8 @@ def verify_session(func): @functools.wraps(func) def __inner(self, request, *args, **kwargs): if hasattr(request, 'context') and request.context.session: - uw = get_session() - configuration_session = uw.query(Session).get(request.context.session) + uw = get_session().query(Session) + configuration_session = uw.get(request.context.session) if configuration_session.state != 'open': log.info('Session is already deployed') @@ -22,5 +22,3 @@ def verify_session(func): raise exc.HTTPUnauthorized return func(self, request, *args, **kwargs) return __inner - - diff --git a/python-portasclient/portasclient/version.py b/python-portasclient/portasclient/version.py index 53823b3..c140d8d 100644 --- a/python-portasclient/portasclient/version.py +++ b/python-portasclient/portasclient/version.py @@ -15,6 +15,6 @@ # under the License. -from portas.openstack.common import version as common_version +from portasclient.openstack.common import version as common_version version_info = common_version.VersionInfo('python-portasclient') diff --git a/python-portasclient/tests/test_sanity.py b/python-portasclient/tests/test_sanity.py index 89d3628..87da75d 100644 --- a/python-portasclient/tests/test_sanity.py +++ b/python-portasclient/tests/test_sanity.py @@ -1,19 +1,29 @@ +import os import unittest import logging from mock import MagicMock +from mock import patch +from portasclient.client import Client as CommonClient from portasclient.v1 import Client import portasclient.v1.environments as environments import portasclient.v1.services as services import portasclient.v1.sessions as sessions +import portasclient.shell as shell +import portasclient.common.http as http + + LOG = logging.getLogger('Unit tests') + def my_mock(*a, **b): return [a, b] + api = MagicMock(json_request=my_mock) + class SanityUnitTests(unittest.TestCase): def test_create_client_instance(self): @@ -25,145 +35,121 @@ class SanityUnitTests(unittest.TestCase): assert test_client.sessions is not None assert test_client.activeDirectories is not None assert test_client.webServers is not None + + def test_common_client(self): + endpoint = 'http://localhost:8001' + test_client = CommonClient('1', endpoint=endpoint, token='1', timeout=10) + + assert test_client.environments is not None + assert test_client.sessions is not None + assert test_client.activeDirectories is not None + assert test_client.webServers is not None def test_env_manager_list(self): - manager = environments.EnvironmentManager(api) - result = manager.list() - assert result == [] def test_env_manager_create(self): - manager = environments.EnvironmentManager(api) - result = manager.create('test') - assert result.headers == {} assert result.body == {'name': 'test'} def test_env_manager_delete(self): - manager = environments.EnvironmentManager(api) - result = manager.delete('test') - assert result is None def test_env_manager_update(self): - manager = environments.EnvironmentManager(api) - result = manager.update('1', 'test') - assert result.body == {'name': 'test'} def test_env_manager_get(self): - manager = environments.EnvironmentManager(api) - result = manager.get('test') ## WTF? assert result.manager is not None + + def test_env(self): + environment = environments.Environment(api, api) + assert environment.data() is not None def test_ad_manager_list(self): - manager = services.ActiveDirectoryManager(api) - result = manager.list('datacenter1') - + assert result == [] + result = manager.list('datacenter1', '1') assert result == [] def test_ad_manager_create(self): - manager = services.ActiveDirectoryManager(api) - result = manager.create('datacenter1', 'session1', 'test') - assert result.headers == {'X-Configuration-Session': 'session1'} assert result.body == 'test' #@unittest.skip("https://mirantis.jira.com/browse/KEERO-218") def test_ad_manager_delete(self): - manager = services.ActiveDirectoryManager(api) - result = manager.delete('datacenter1', 'session1', 'test') - assert result is None def test_iis_manager_list(self): - manager = services.WebServerManager(api) - result = manager.list('datacenter1') - + assert result == [] + result = manager.list('datacenter1', '1') assert result == [] def test_iis_manager_create(self): - manager = services.WebServerManager(api) - result = manager.create('datacenter1', 'session1', 'test') - assert result.headers == {'X-Configuration-Session': 'session1'} assert result.body == 'test' #@unittest.skip("https://mirantis.jira.com/browse/KEERO-218") def test_iis_manager_delete(self): - manager = services.WebServerManager(api) - result = manager.delete('datacenter1', 'session1', 'test') - assert result is None + def test_service_ad(self): + service_ad = services.ActiveDirectory(api, api) + assert service_ad.data() is not None + + def test_service_iis(self): + service_iis = services.ActiveDirectory(api, api) + assert service_iis.data() is not None + def test_session_manager_list(self): - manager = sessions.SessionManager(api) - result = manager.list('datacenter1') - assert result == [] def test_session_manager_delete(self): - manager = sessions.SessionManager(api) - result = manager.delete('datacenter1', 'session1') - assert result is None def test_session_manager_get(self): - manager = sessions.SessionManager(api) - result = manager.get('datacenter1', 'session1') # WTF? assert result.manager is not None def test_session_manager_configure(self): - manager = sessions.SessionManager(api) - result = manager.configure('datacenter1') - assert result.headers == {} def test_session_manager_deploy(self): - manager = sessions.SessionManager(api) - result = manager.deploy('datacenter1', '1') - assert result is None #@unittest.skip("https://mirantis.jira.com/browse/KEERO-219") def test_session_manager_reports(self): - manager = sessions.SessionManager(api) - result = manager.reports('datacenter1', '1') - assert result == [] \ No newline at end of file From 29da07cda8e456a7ffa678c1b28ae3f1a800a116 Mon Sep 17 00:00:00 2001 From: Timur Nurlygayanov Date: Tue, 26 Mar 2013 06:36:57 +0700 Subject: [PATCH 20/37] Added initial unit tests for RestAPI service. --- portas/portas/tests/sanity_tests.py | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) create mode 100644 portas/portas/tests/sanity_tests.py diff --git a/portas/portas/tests/sanity_tests.py b/portas/portas/tests/sanity_tests.py new file mode 100644 index 0000000..d6258f2 --- /dev/null +++ b/portas/portas/tests/sanity_tests.py @@ -0,0 +1,24 @@ +import unittest2 +from mock import MagicMock + +import portas.api.v1.router as router + +def my_mock(link, controller, action, conditions): + return [link, controller, action, conditions] + +def func_mock(): + return True + +class SanityUnitTests(unittest2.TestCase): + + def test_api(self): + router.webservers = MagicMock(create_resource=func_mock) + router.sessions = MagicMock(create_resource=func_mock) + router.active_directories = MagicMock(create_resource=func_mock) + router.environments = MagicMock(create_resource=func_mock) + mapper = MagicMock(connect=my_mock) + + object = router.API(mapper) + + assert object._router is not None + \ No newline at end of file From b59edd7ad2d242bb1c011e833e4e872ee5e8d1dc Mon Sep 17 00:00:00 2001 From: Serg Melikyan Date: Tue, 26 Mar 2013 12:34:22 +0400 Subject: [PATCH 21/37] #KEERO-220 Send X-Auth-Token to Conductor --- portas/portas/api/v1/sessions.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/portas/portas/api/v1/sessions.py b/portas/portas/api/v1/sessions.py index 5bf52b0..b1584d0 100644 --- a/portas/portas/api/v1/sessions.py +++ b/portas/portas/api/v1/sessions.py @@ -121,9 +121,13 @@ class Controller(object): session.state = 'deploying' session.save(unit) + #Set X-Auth-Tokenconductor for conductor + env = session.description + env['token'] = request.context.auth_token + with self.write_lock: self.ch.basic_publish(Message(body=anyjson. - serialize(session.description)), + serialize(env)), 'tasks', 'tasks') From e4abc9b157bce2741024c0225778ca2575de50c9 Mon Sep 17 00:00:00 2001 From: Timur Nurlygayanov Date: Tue, 26 Mar 2013 13:24:37 +0400 Subject: [PATCH 22/37] Fixed WebUI tests. Added new tests. --- tests/selenium/datacenters_page.py | 30 +++++-- tests/selenium/page.py | 85 ++++++++++++++---- tests/selenium/services_page.py | 11 ++- tests/selenium/test.py | 135 +++++++++++++++++++++++++---- 4 files changed, 214 insertions(+), 47 deletions(-) diff --git a/tests/selenium/datacenters_page.py b/tests/selenium/datacenters_page.py index f0e74a8..03e4e26 100644 --- a/tests/selenium/datacenters_page.py +++ b/tests/selenium/datacenters_page.py @@ -4,9 +4,9 @@ import page class DataCentersPage(page.Page): - page = None def create_data_center(self, name): + self.Refresh() self.Button('Create Windows Data Center').Click() @@ -14,14 +14,12 @@ class DataCentersPage(page.Page): xpath = "//input[@value='Create']" self.Button(xpath).Click() - return self - def delete_data_center(self, name): + self.Refresh() + link = self.Link(name).Address() datacenter_id = re.search('windc/(\S+)', link).group(0)[6:-1] - print 'Data Center ID: ' + datacenter_id - xpath = ".//*[@id='windc__row__%s']/td[4]/div/a[2]" % datacenter_id self.Button(xpath).Click() @@ -30,9 +28,27 @@ class DataCentersPage(page.Page): self.Button("Delete Data Center").Click() - return self - def select_data_center(self, name): self.Link(name).Click() page = ServicesPage(self.driver) return page + + def deploy_data_center(self, name): + self.Refresh() + + link = self.Link(name).Address() + datacenter_id = re.search('windc/(\S+)', link).group(0)[6:-1] + + xpath = ".//*[@id='windc__row__%s']/td[4]/div/a[2]" % datacenter_id + self.Button(xpath).Click() + + button_id = "windc__row_%s__action_deploy" % datacenter_id + self.Button(button_id).Click() + + def get_datacenter_status(self, name): + self.Navigate('Windows Data Centers') + link = self.Link(name).Address() + datacenter_id = re.search('windc/(\S+)', link).group(0)[6:-1] + + xpath = ".//*[@id='windc__row__%s']/td[3]" % datacenter_id + return self.TableCell(xpath).Text() diff --git a/tests/selenium/page.py b/tests/selenium/page.py index 80085ab..43ed7cc 100644 --- a/tests/selenium/page.py +++ b/tests/selenium/page.py @@ -1,16 +1,38 @@ +import logging +from selenium.webdriver.support.ui import Select +logging.basicConfig() +LOG = logging.getLogger(' Page object: ') + + +class TableCellClass: + table = None + + def __init__(self, obj): + if not obj: + LOG.error('TableCell does not found') + self.table = obj + + def Text(self): + if self.table: + return self.table.text() + else: + return '' class ButtonClass: button = None def __init__(self, obj): + if not obj: + LOG.error('Button does not found') self.button = obj def Click(self): - self.button.click() + if self.button: + self.button.click() def isPresented(self): - if self.button != None: + if self.button: return True return False @@ -19,60 +41,84 @@ class LinkClass: link = None def __init__(self, obj): + if not obj: + LOG.error('Link does not found') self.link = obj def Click(self): - self.link.click() + if self.link: + self.link.click() def isPresented(self): - if self.link != None: + if self.link: return True return False def Address(self): - return self.link.get_attribute('href') + if self.link: + return self.link.get_attribute('href') + else: + return '' class EditBoxClass: def __init__(self, obj): + if not obj: + LOG.error('EditBox does not found') self.edit = obj def isPresented(self): - if self.edit != None: + if self.edit: return True return False def Set(self, value): - self.edit.clear() - self.edit.send_keys(value) + if self.edit: + self.edit.clear() + self.edit.send_keys(value) def Text(self): - return self.edit.get_text() + if self.edit: + return self.edit.get_text() + else: + return '' class DropDownListClass: select = None def __init__(self, obj): + if not obj: + LOG.error('DropDownList does not found') self.select = obj def isPresented(self): - if self.select != None: + if self.select is not None: return True return False def Set(self, value): - self.select.select_by_visible_text(value) + if self.select: + Select(self.select).select_by_visible_text(value) def Text(self): - return self.select.get_text() + if self.select: + return self.select.get_text() + else: + return '' + + +error_msg = """ + Object with parameter: %s + does not found on page. + """ class Page: driver = None - timeout = 300 + timeout = 30 def __init__(self, driver): driver.set_page_load_timeout(30) @@ -82,7 +128,7 @@ class Page: def _find_element(self, parameter): obj = None k = 0 - while (obj == None and k < self.timeout): + while (obj is None and k < self.timeout): k += 1 try: obj = self.driver.find_element_by_name(parameter) @@ -105,11 +151,20 @@ class Page: except: pass - return obj + LOG.error(error_msg % parameter) + return None def Open(self, url): self.driver.get(url) + def Refresh(self): + self.driver.refresh() + + def TableCell(self, name): + obj = self._find_element(name) + table = TableCellClass(obj) + return table + def Button(self, name): obj = self._find_element(name) button = ButtonClass(obj) diff --git a/tests/selenium/services_page.py b/tests/selenium/services_page.py index e8ea9f8..772b9df 100644 --- a/tests/selenium/services_page.py +++ b/tests/selenium/services_page.py @@ -5,19 +5,20 @@ import re class ServicesPage(page.Page): def create_service(self, service_type, parameters): + self.Refresh() self.Button('services__action_CreateService').Click() self.DropDownList('0-service').Set(service_type) self.Button('wizard_goto_step').Click() - for parameter in parameters: - self.EditBox(parameter.key).Set(parameter.value) + for key in parameters: + self.EditBox(key).Set(parameters[key]) self.Button("//input[@value='Create']").Click() - return self - def delete_service(self, name): + self.Refresh() + link = self.Link(name).Address() service_id = re.search('windc/(\S+)', link).group(0)[6:-1] @@ -29,5 +30,3 @@ class ServicesPage(page.Page): self.Button(button_id).Click() self.Button("Delete Service").Click() - - return self diff --git a/tests/selenium/test.py b/tests/selenium/test.py index 4391a0b..1af0699 100644 --- a/tests/selenium/test.py +++ b/tests/selenium/test.py @@ -5,19 +5,7 @@ from login_page import LoginPage from datacenters_page import DataCentersPage from selenium import webdriver - -class SanityTests(unittest2.TestCase): - - def setUp(self): - driver = webdriver.Firefox() - self.page = LoginPage(driver) - self.page.login() - self.page.Navigate('Project:Windows Data Centers') - self.page = DataCentersPage(driver) - - def tearDown(self): - self.page.driver.close() - +""" def test_01_create_data_center(self): self.page.create_data_center('dc1') assert self.page.Link('dc1').isPresented() @@ -27,25 +15,42 @@ class SanityTests(unittest2.TestCase): assert not self.page.Link('dc1').isPresented() def test_03_create_data_centers(self): - for i in range(1, 20): + for i in range(1, 10): name = 'datacenter' + str(i) self.page.create_data_center(name) assert self.page.Link(name).isPresented() def test_04_delete_data_centers(self): self.page.delete_data_center('datacenter1') - self.page.delete_data_center('datacenter20') + self.page.delete_data_center('datacenter9') assert not self.page.Link('datacenter1').isPresented() - assert not self.page.Link('datacenter20').isPresented() + assert not self.page.Link('datacenter9').isPresented() - for i in range(2, 19): + for i in range(2, 9): name = 'datacenter' + str(i) assert self.page.Link(name).isPresented() +""" + + +class SanityTests(unittest2.TestCase): + + @classmethod + def setUpClass(self): + driver = webdriver.Firefox() + self.page = LoginPage(driver) + self.page.login() + self.page.Navigate('Project:Windows Data Centers') + self.page = DataCentersPage(driver) + + @classmethod + def tearDownClass(self): + self.page.driver.close() def test_05_create_service_ad(self): name = 'dc001.local' - self.page.create_data_center('test') - self.page.select_data_center('test') + self.page.Navigate('Windows Data Centers') + self.page.create_data_center('test05') + self.page = self.page.select_data_center('test05') ad_parameters = {'1-dc_name': name, '1-dc_count': 1, @@ -54,7 +59,99 @@ class SanityTests(unittest2.TestCase): self.page.create_service('Active Directory', ad_parameters) assert self.page.Link(name).isPresented() +""" + def test_06_create_service_ad_two_instances(self): + name = 'dc002.local' + self.page.Navigate('Windows Data Centers') + self.page.create_data_center('test06') + self.page = self.page.select_data_center('test06') + ad_parameters = {'1-dc_name': name, + '1-dc_count': 2, + '1-adm_password': 'P@ssw0rd2', + '1-recovery_password': 'P@ssw0rd'} + self.page.create_service('Active Directory', ad_parameters) + + assert self.page.Link(name).isPresented() + + def test_07_create_service_ad_with_iis(self): + ad_name = 'dc003.local' + self.page.Navigate('Windows Data Centers') + self.page.create_data_center('test07') + self.page = self.page.select_data_center('test07') + + ad_parameters = {'1-dc_name': ad_name, + '1-dc_count': 3, + '1-adm_password': 'P@ssw0rd', + '1-recovery_password': 'P@ssw0rd2'} + self.page.create_service('Active Directory', ad_parameters) + + assert self.page.Link(ad_name).isPresented() + + iis_name = 'iis_server1' + iis_parameters = {'1-iis_name': iis_name, + '1-adm_password': 'P@ssw0rd', + '1-iis_domain': 'dc003.local', + '1-domain_user_name': 'Administrator', + '1-domain_user_password': 'P@ssw0rd'} + self.page.create_service('Internet Information Services', + iis_parameters) + + assert self.page.Link(iis_name).isPresented() + + iis_name = 'iis_server2' + iis_parameters = {'1-iis_name': iis_name, + '1-adm_password': 'P@ssw0rd', + '1-iis_domain': 'dc003.local', + '1-domain_user_name': 'Administrator', + '1-domain_user_password': 'P@ssw0rd'} + self.page.create_service('Internet Information Services', + iis_parameters) + + assert self.page.Link(iis_name).isPresented() + + iis_name = 'iis_server3' + iis_parameters = {'1-iis_name': iis_name, + '1-adm_password': 'P@ssw0rd', + '1-iis_domain': 'dc003.local', + '1-domain_user_name': 'Administrator', + '1-domain_user_password': 'P@ssw0rd'} + self.page.create_service('Internet Information Services', + iis_parameters) + + assert self.page.Link(iis_name).isPresented() + + def test_08_deploy_data_center(self): + ad_name = 'AD.net' + self.page.Navigate('Windows Data Centers') + self.page.create_data_center('test08') + self.page = self.page.select_data_center('test08') + + ad_parameters = {'1-dc_name': ad_name, + '1-dc_count': 2, + '1-adm_password': 'P@ssw0rd', + '1-recovery_password': 'P@ssw0rd2'} + self.page.create_service('Active Directory', ad_parameters) + + assert self.page.Link(ad_name).isPresented() + + iis_parameters = {'1-iis_name': 'iis_server', + '1-adm_password': 'P@ssw0rd', + '1-iis_domain': 'AD.net', + '1-domain_user_name': 'Administrator', + '1-domain_user_password': 'P@ssw0rd'} + self.page.create_service('Internet Information Services', + iis_parameters) + + assert self.page.Link('iis_server').isPresented() + + self.page.Navigate('Windows Data Centers') + self.page = DataCentersPage(self.page.driver) + self.page.deploy_data_center('test08') + + status = self.page.get_datacenter_status('test08') + assert 'Deploy in progress' in status +""" if __name__ == '__main__': unittest2.main() From ccd0a3c904a09267e7065228561a628c0cdf35d2 Mon Sep 17 00:00:00 2001 From: Timur Nurlygayanov Date: Tue, 26 Mar 2013 13:29:50 +0400 Subject: [PATCH 23/37] Hot fix for WebUI tests. --- tests/selenium/test.py | 36 +++++++++++++++++------------------- 1 file changed, 17 insertions(+), 19 deletions(-) diff --git a/tests/selenium/test.py b/tests/selenium/test.py index 1af0699..b1c2b60 100644 --- a/tests/selenium/test.py +++ b/tests/selenium/test.py @@ -5,7 +5,21 @@ from login_page import LoginPage from datacenters_page import DataCentersPage from selenium import webdriver -""" + +class SanityTests(unittest2.TestCase): + + @classmethod + def setUpClass(self): + driver = webdriver.Firefox() + self.page = LoginPage(driver) + self.page.login() + self.page.Navigate('Project:Windows Data Centers') + self.page = DataCentersPage(driver) + + @classmethod + def tearDownClass(self): + self.page.driver.close() + def test_01_create_data_center(self): self.page.create_data_center('dc1') assert self.page.Link('dc1').isPresented() @@ -29,22 +43,6 @@ from selenium import webdriver for i in range(2, 9): name = 'datacenter' + str(i) assert self.page.Link(name).isPresented() -""" - - -class SanityTests(unittest2.TestCase): - - @classmethod - def setUpClass(self): - driver = webdriver.Firefox() - self.page = LoginPage(driver) - self.page.login() - self.page.Navigate('Project:Windows Data Centers') - self.page = DataCentersPage(driver) - - @classmethod - def tearDownClass(self): - self.page.driver.close() def test_05_create_service_ad(self): name = 'dc001.local' @@ -59,7 +57,7 @@ class SanityTests(unittest2.TestCase): self.page.create_service('Active Directory', ad_parameters) assert self.page.Link(name).isPresented() -""" + def test_06_create_service_ad_two_instances(self): name = 'dc002.local' self.page.Navigate('Windows Data Centers') @@ -151,7 +149,7 @@ class SanityTests(unittest2.TestCase): status = self.page.get_datacenter_status('test08') assert 'Deploy in progress' in status -""" + if __name__ == '__main__': unittest2.main() From a0069520804b0ef863dee4419ce42a48cfee898e Mon Sep 17 00:00:00 2001 From: Serg Melikyan Date: Tue, 26 Mar 2013 13:36:05 +0400 Subject: [PATCH 24/37] Send Env to Conductor for deletion --- portas/portas/api/v1/environments.py | 24 +++++++++++++++++++++ portas/portas/api/v1/sessions.py | 31 ++++++++++++---------------- portas/portas/common/service.py | 12 ++++++++--- 3 files changed, 46 insertions(+), 21 deletions(-) diff --git a/portas/portas/api/v1/environments.py b/portas/portas/api/v1/environments.py index 4129bbf..5e85fa7 100644 --- a/portas/portas/api/v1/environments.py +++ b/portas/portas/api/v1/environments.py @@ -1,10 +1,16 @@ +from amqplib.client_0_8 import Message +import anyjson +import eventlet from webob import exc +from portas.common import config from portas.api.v1 import get_env_status from portas.db.session import get_session from portas.db.models import Environment from portas.openstack.common import wsgi from portas.openstack.common import log as logging +amqp = eventlet.patcher.import_patched('amqplib.client_0_8') +rabbitmq = config.CONF.rabbitmq log = logging.getLogger(__name__) @@ -85,6 +91,24 @@ class Controller(object): with session.begin(): session.delete(environment) + #preparing data for removal from conductor + env = environment.description + env['services'] = [] + env['deleted'] = True + + connection = amqp.Connection('{0}:{1}'. + format(rabbitmq.host, rabbitmq.port), + virtual_host=rabbitmq.virtual_host, + userid=rabbitmq.userid, + password=rabbitmq.password, + ssl=rabbitmq.use_ssl, insist=True) + channel = connection.channel() + channel.exchange_declare('tasks', 'direct', durable=True, + auto_delete=False) + + channel.basic_publish(Message(body=anyjson.serialize(env)), 'tasks', + 'tasks') + return None diff --git a/portas/portas/api/v1/sessions.py b/portas/portas/api/v1/sessions.py index b1584d0..5fb30c0 100644 --- a/portas/portas/api/v1/sessions.py +++ b/portas/portas/api/v1/sessions.py @@ -1,7 +1,6 @@ from amqplib.client_0_8 import Message import anyjson import eventlet -from eventlet.semaphore import Semaphore from webob import exc from portas.common import config from portas.db.models import Session, Status, Environment @@ -15,18 +14,6 @@ log = logging.getLogger(__name__) class Controller(object): - def __init__(self): - self.write_lock = Semaphore(1) - connection = amqp.Connection('{0}:{1}'. - format(rabbitmq.host, rabbitmq.port), - virtual_host=rabbitmq.virtual_host, - userid=rabbitmq.userid, - password=rabbitmq.password, - ssl=rabbitmq.use_ssl, insist=True) - self.ch = connection.channel() - self.ch.exchange_declare('tasks', 'direct', durable=True, - auto_delete=False) - def index(self, request, environment_id): log.debug(_('Session:List '.format(environment_id))) @@ -121,14 +108,22 @@ class Controller(object): session.state = 'deploying' session.save(unit) - #Set X-Auth-Tokenconductor for conductor + #Set X-Auth-Token for conductor env = session.description env['token'] = request.context.auth_token - with self.write_lock: - self.ch.basic_publish(Message(body=anyjson. - serialize(env)), - 'tasks', 'tasks') + connection = amqp.Connection('{0}:{1}'. + format(rabbitmq.host, rabbitmq.port), + virtual_host=rabbitmq.virtual_host, + userid=rabbitmq.userid, + password=rabbitmq.password, + ssl=rabbitmq.use_ssl, insist=True) + channel = connection.channel() + channel.exchange_declare('tasks', 'direct', durable=True, + auto_delete=False) + + channel.basic_publish(Message(body=anyjson.serialize(env)), 'tasks', + 'tasks') def create_resource(): diff --git a/portas/portas/common/service.py b/portas/portas/common/service.py index 2d8f38e..238304e 100644 --- a/portas/portas/common/service.py +++ b/portas/portas/common/service.py @@ -55,7 +55,7 @@ class TaskResultHandlerService(service.Service): def handle_report(msg): log.debug(_('Got report message from orchestration engine:\n{0}'. - format(msg.body))) + format(msg.body))) params = anyjson.deserialize(msg.body) params['entity_id'] = params['id'] @@ -76,10 +76,16 @@ def handle_report(msg): def handle_result(msg): - log.debug(_('Got result message from orchestration engine:\n{0}'. - format(msg.body))) + log.debug(_('Got result message from ' + 'orchestration engine:\n{0}'.format(msg.body))) environment_result = anyjson.deserialize(msg.body) + if environment_result['deleted']: + log.debug(_('Result for environment {0} is dropped. ' + 'Environment is deleted'.format(environment_result['id']))) + + msg.channel.basic_ack(msg.delivery_tag) + return session = get_session() environment = session.query(Environment).get(environment_result['id']) From fe143e4675669ed4ef519df5ef7bf7045b1fce89 Mon Sep 17 00:00:00 2001 From: Serg Melikyan Date: Tue, 26 Mar 2013 13:46:29 +0400 Subject: [PATCH 25/37] #KEERO-222 Remove random part of unit name --- portas/portas/api/v1/active_directories.py | 3 +-- portas/portas/api/v1/webservers.py | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/portas/portas/api/v1/active_directories.py b/portas/portas/api/v1/active_directories.py index bacdb2a..ade7a6f 100644 --- a/portas/portas/api/v1/active_directories.py +++ b/portas/portas/api/v1/active_directories.py @@ -38,8 +38,7 @@ class Controller(object): for unit in active_directory['units']: unit_count += 1 unit['id'] = uuidutils.generate_uuid() - unit['name'] = 'dc{0}{1}'.format(unit_count, - active_directory['id'][:4]) + unit['name'] = 'dc{0}'.format(unit_count) draft = prepare_draft(draft) draft['services']['activeDirectories'].append(active_directory) diff --git a/portas/portas/api/v1/webservers.py b/portas/portas/api/v1/webservers.py index ef1344f..ddc73c3 100644 --- a/portas/portas/api/v1/webservers.py +++ b/portas/portas/api/v1/webservers.py @@ -36,7 +36,7 @@ class Controller(object): for unit in webServer['units']: unit_count += 1 unit['id'] = uuidutils.generate_uuid() - unit['name'] = 'iis{0}{1}'.format(unit_count, webServer['id'][:3]) + unit['name'] = 'iis{0}'.format(unit_count) draft = prepare_draft(draft) draft['services']['webServers'].append(webServer) From 9a63086bd57fc8cd511482573d33b91fdaeedb1c Mon Sep 17 00:00:00 2001 From: Serg Melikyan Date: Tue, 26 Mar 2013 14:36:44 +0400 Subject: [PATCH 26/37] Added support for setup.py Added localization and documentation skeletons PEP8 Fixes, optimized imports Change-Id: I4abf8a73975d31b11213b29e9e3e178fdcf9c86c --- conductor/.gitignore | 20 + conductor/babel.cfg | 1 + conductor/conductor/app.py | 8 +- conductor/doc/source/_static/basic.css | 416 ++++++++++++++++++ conductor/doc/source/_static/default.css | 230 ++++++++++ conductor/doc/source/_static/header-line.gif | Bin 0 -> 48 bytes conductor/doc/source/_static/header_bg.jpg | Bin 0 -> 3738 bytes conductor/doc/source/_static/jquery.tweet.js | 154 +++++++ conductor/doc/source/_static/nature.css | 245 +++++++++++ .../doc/source/_static/openstack_logo.png | Bin 0 -> 3670 bytes conductor/doc/source/_static/tweaks.css | 94 ++++ conductor/doc/source/_templates/.placeholder | 0 conductor/doc/source/_theme/layout.html | 83 ++++ conductor/doc/source/_theme/theme.conf | 4 + conductor/doc/source/conf.py | 242 ++++++++++ conductor/doc/source/index.rst | 20 + conductor/setup.cfg | 33 ++ conductor/setup.py | 49 +++ 18 files changed, 1594 insertions(+), 5 deletions(-) create mode 100644 conductor/.gitignore create mode 100644 conductor/babel.cfg create mode 100644 conductor/doc/source/_static/basic.css create mode 100644 conductor/doc/source/_static/default.css create mode 100644 conductor/doc/source/_static/header-line.gif create mode 100644 conductor/doc/source/_static/header_bg.jpg create mode 100644 conductor/doc/source/_static/jquery.tweet.js create mode 100644 conductor/doc/source/_static/nature.css create mode 100644 conductor/doc/source/_static/openstack_logo.png create mode 100644 conductor/doc/source/_static/tweaks.css create mode 100644 conductor/doc/source/_templates/.placeholder create mode 100644 conductor/doc/source/_theme/layout.html create mode 100644 conductor/doc/source/_theme/theme.conf create mode 100644 conductor/doc/source/conf.py create mode 100644 conductor/doc/source/index.rst create mode 100644 conductor/setup.cfg create mode 100644 conductor/setup.py diff --git a/conductor/.gitignore b/conductor/.gitignore new file mode 100644 index 0000000..6133130 --- /dev/null +++ b/conductor/.gitignore @@ -0,0 +1,20 @@ +#IntelJ Idea +.idea/ + +#virtualenv +.venv/ + +#Build results +build/ +dist/ +*.egg-info/ + +#Python +*.pyc + +#Translation build +*.mo +*.pot + +#SQLite Database files +*.sqlite \ No newline at end of file diff --git a/conductor/babel.cfg b/conductor/babel.cfg new file mode 100644 index 0000000..efceab8 --- /dev/null +++ b/conductor/babel.cfg @@ -0,0 +1 @@ +[python: **.py] diff --git a/conductor/conductor/app.py b/conductor/conductor/app.py index 52d8b2a..a69e8e8 100644 --- a/conductor/conductor/app.py +++ b/conductor/conductor/app.py @@ -1,12 +1,9 @@ import datetime import glob -import json import sys from conductor.openstack.common import service from workflow import Workflow -import cloud_formation -import windows_agent from commands.dispatcher import CommandDispatcher from config import Config import reporting @@ -14,6 +11,7 @@ import rabbitmq config = Config(sys.argv[1] if len(sys.argv) > 1 else None) + def task_received(task, message_id): with rabbitmq.RmqClient() as rmqclient: print 'Starting at', datetime.datetime.now() @@ -23,7 +21,8 @@ def task_received(task, message_id): workflows = [] for path in glob.glob("data/workflows/*.xml"): print "loading", path - workflow = Workflow(path, task, command_dispatcher, config, reporter) + workflow = Workflow(path, task, command_dispatcher, config, + reporter) workflows.append(workflow) while True: @@ -41,7 +40,6 @@ def task_received(task, message_id): print 'Finished at', datetime.datetime.now() - class ConductorWorkflowService(service.Service): def __init__(self): super(ConductorWorkflowService, self).__init__() diff --git a/conductor/doc/source/_static/basic.css b/conductor/doc/source/_static/basic.css new file mode 100644 index 0000000..d909ce3 --- /dev/null +++ b/conductor/doc/source/_static/basic.css @@ -0,0 +1,416 @@ +/** + * Sphinx stylesheet -- basic theme + * ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + */ + +/* -- main layout ----------------------------------------------------------- */ + +div.clearer { + clear: both; +} + +/* -- relbar ---------------------------------------------------------------- */ + +div.related { + width: 100%; + font-size: 90%; +} + +div.related h3 { + display: none; +} + +div.related ul { + margin: 0; + padding: 0 0 0 10px; + list-style: none; +} + +div.related li { + display: inline; +} + +div.related li.right { + float: right; + margin-right: 5px; +} + +/* -- sidebar --------------------------------------------------------------- */ + +div.sphinxsidebarwrapper { + padding: 10px 5px 0 10px; +} + +div.sphinxsidebar { + float: left; + width: 230px; + margin-left: -100%; + font-size: 90%; +} + +div.sphinxsidebar ul { + list-style: none; +} + +div.sphinxsidebar ul ul, +div.sphinxsidebar ul.want-points { + margin-left: 20px; + list-style: square; +} + +div.sphinxsidebar ul ul { + margin-top: 0; + margin-bottom: 0; +} + +div.sphinxsidebar form { + margin-top: 10px; +} + +div.sphinxsidebar input { + border: 1px solid #98dbcc; + font-family: sans-serif; + font-size: 1em; +} + +img { + border: 0; +} + +/* -- search page ----------------------------------------------------------- */ + +ul.search { + margin: 10px 0 0 20px; + padding: 0; +} + +ul.search li { + padding: 5px 0 5px 20px; + background-image: url(file.png); + background-repeat: no-repeat; + background-position: 0 7px; +} + +ul.search li a { + font-weight: bold; +} + +ul.search li div.context { + color: #888; + margin: 2px 0 0 30px; + text-align: left; +} + +ul.keywordmatches li.goodmatch a { + font-weight: bold; +} + +/* -- index page ------------------------------------------------------------ */ + +table.contentstable { + width: 90%; +} + +table.contentstable p.biglink { + line-height: 150%; +} + +a.biglink { + font-size: 1.3em; +} + +span.linkdescr { + font-style: italic; + padding-top: 5px; + font-size: 90%; +} + +/* -- general index --------------------------------------------------------- */ + +table.indextable td { + text-align: left; + vertical-align: top; +} + +table.indextable dl, table.indextable dd { + margin-top: 0; + margin-bottom: 0; +} + +table.indextable tr.pcap { + height: 10px; +} + +table.indextable tr.cap { + margin-top: 10px; + background-color: #f2f2f2; +} + +img.toggler { + margin-right: 3px; + margin-top: 3px; + cursor: pointer; +} + +/* -- general body styles --------------------------------------------------- */ + +a.headerlink { + visibility: hidden; +} + +h1:hover > a.headerlink, +h2:hover > a.headerlink, +h3:hover > a.headerlink, +h4:hover > a.headerlink, +h5:hover > a.headerlink, +h6:hover > a.headerlink, +dt:hover > a.headerlink { + visibility: visible; +} + +div.body p.caption { + text-align: inherit; +} + +div.body td { + text-align: left; +} + +.field-list ul { + padding-left: 1em; +} + +.first { +} + +p.rubric { + margin-top: 30px; + font-weight: bold; +} + +/* -- sidebars -------------------------------------------------------------- */ + +div.sidebar { + margin: 0 0 0.5em 1em; + border: 1px solid #ddb; + padding: 7px 7px 0 7px; + background-color: #ffe; + width: 40%; + float: right; +} + +p.sidebar-title { + font-weight: bold; +} + +/* -- topics ---------------------------------------------------------------- */ + +div.topic { + border: 1px solid #ccc; + padding: 7px 7px 0 7px; + margin: 10px 0 10px 0; +} + +p.topic-title { + font-size: 1.1em; + font-weight: bold; + margin-top: 10px; +} + +/* -- admonitions ----------------------------------------------------------- */ + +div.admonition { + margin-top: 10px; + margin-bottom: 10px; + padding: 7px; +} + +div.admonition dt { + font-weight: bold; +} + +div.admonition dl { + margin-bottom: 0; +} + +p.admonition-title { + margin: 0px 10px 5px 0px; + font-weight: bold; +} + +div.body p.centered { + text-align: center; + margin-top: 25px; +} + +/* -- tables ---------------------------------------------------------------- */ + +table.docutils { + border: 0; + border-collapse: collapse; +} + +table.docutils td, table.docutils th { + padding: 1px 8px 1px 0; + border-top: 0; + border-left: 0; + border-right: 0; + border-bottom: 1px solid #aaa; +} + +table.field-list td, table.field-list th { + border: 0 !important; +} + +table.footnote td, table.footnote th { + border: 0 !important; +} + +th { + text-align: left; + padding-right: 5px; +} + +/* -- other body styles ----------------------------------------------------- */ + +dl { + margin-bottom: 15px; +} + +dd p { + margin-top: 0px; +} + +dd ul, dd table { + margin-bottom: 10px; +} + +dd { + margin-top: 3px; + margin-bottom: 10px; + margin-left: 30px; +} + +dt:target, .highlight { + background-color: #fbe54e; +} + +dl.glossary dt { + font-weight: bold; + font-size: 1.1em; +} + +.field-list ul { + margin: 0; + padding-left: 1em; +} + +.field-list p { + margin: 0; +} + +.refcount { + color: #060; +} + +.optional { + font-size: 1.3em; +} + +.versionmodified { + font-style: italic; +} + +.system-message { + background-color: #fda; + padding: 5px; + border: 3px solid red; +} + +.footnote:target { + background-color: #ffa +} + +.line-block { + display: block; + margin-top: 1em; + margin-bottom: 1em; +} + +.line-block .line-block { + margin-top: 0; + margin-bottom: 0; + margin-left: 1.5em; +} + +/* -- code displays --------------------------------------------------------- */ + +pre { + overflow: auto; +} + +td.linenos pre { + padding: 5px 0px; + border: 0; + background-color: transparent; + color: #aaa; +} + +table.highlighttable { + margin-left: 0.5em; +} + +table.highlighttable td { + padding: 0 0.5em 0 0.5em; +} + +tt.descname { + background-color: transparent; + font-weight: bold; + font-size: 1.2em; +} + +tt.descclassname { + background-color: transparent; +} + +tt.xref, a tt { + background-color: transparent; + font-weight: bold; +} + +h1 tt, h2 tt, h3 tt, h4 tt, h5 tt, h6 tt { + background-color: transparent; +} + +/* -- math display ---------------------------------------------------------- */ + +img.math { + vertical-align: middle; +} + +div.body div.math p { + text-align: center; +} + +span.eqno { + float: right; +} + +/* -- printout stylesheet --------------------------------------------------- */ + +@media print { + div.document, + div.documentwrapper, + div.bodywrapper { + margin: 0 !important; + width: 100%; + } + + div.sphinxsidebar, + div.related, + div.footer, + #top-link { + display: none; + } +} diff --git a/conductor/doc/source/_static/default.css b/conductor/doc/source/_static/default.css new file mode 100644 index 0000000..c8091ec --- /dev/null +++ b/conductor/doc/source/_static/default.css @@ -0,0 +1,230 @@ +/** + * Sphinx stylesheet -- default theme + * ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + */ + +@import url("basic.css"); + +/* -- page layout ----------------------------------------------------------- */ + +body { + font-family: sans-serif; + font-size: 100%; + background-color: #11303d; + color: #000; + margin: 0; + padding: 0; +} + +div.document { + background-color: #1c4e63; +} + +div.documentwrapper { + float: left; + width: 100%; +} + +div.bodywrapper { + margin: 0 0 0 230px; +} + +div.body { + background-color: #ffffff; + color: #000000; + padding: 0 20px 30px 20px; +} + +div.footer { + color: #ffffff; + width: 100%; + padding: 9px 0 9px 0; + text-align: center; + font-size: 75%; +} + +div.footer a { + color: #ffffff; + text-decoration: underline; +} + +div.related { + background-color: #133f52; + line-height: 30px; + color: #ffffff; +} + +div.related a { + color: #ffffff; +} + +div.sphinxsidebar { +} + +div.sphinxsidebar h3 { + font-family: 'Trebuchet MS', sans-serif; + color: #ffffff; + font-size: 1.4em; + font-weight: normal; + margin: 0; + padding: 0; +} + +div.sphinxsidebar h3 a { + color: #ffffff; +} + +div.sphinxsidebar h4 { + font-family: 'Trebuchet MS', sans-serif; + color: #ffffff; + font-size: 1.3em; + font-weight: normal; + margin: 5px 0 0 0; + padding: 0; +} + +div.sphinxsidebar p { + color: #ffffff; +} + +div.sphinxsidebar p.topless { + margin: 5px 10px 10px 10px; +} + +div.sphinxsidebar ul { + margin: 10px; + padding: 0; + color: #ffffff; +} + +div.sphinxsidebar a { + color: #98dbcc; +} + +div.sphinxsidebar input { + border: 1px solid #98dbcc; + font-family: sans-serif; + font-size: 1em; +} + +/* -- body styles ----------------------------------------------------------- */ + +a { + color: #355f7c; + text-decoration: none; +} + +a:hover { + text-decoration: underline; +} + +div.body p, div.body dd, div.body li { + text-align: left; + line-height: 130%; +} + +div.body h1, +div.body h2, +div.body h3, +div.body h4, +div.body h5, +div.body h6 { + font-family: 'Trebuchet MS', sans-serif; + background-color: #f2f2f2; + font-weight: normal; + color: #20435c; + border-bottom: 1px solid #ccc; + margin: 20px -20px 10px -20px; + padding: 3px 0 3px 10px; +} + +div.body h1 { margin-top: 0; font-size: 200%; } +div.body h2 { font-size: 160%; } +div.body h3 { font-size: 140%; } +div.body h4 { font-size: 120%; } +div.body h5 { font-size: 110%; } +div.body h6 { font-size: 100%; } + +a.headerlink { + color: #c60f0f; + font-size: 0.8em; + padding: 0 4px 0 4px; + text-decoration: none; +} + +a.headerlink:hover { + background-color: #c60f0f; + color: white; +} + +div.body p, div.body dd, div.body li { + text-align: left; + line-height: 130%; +} + +div.admonition p.admonition-title + p { + display: inline; +} + +div.admonition p { + margin-bottom: 5px; +} + +div.admonition pre { + margin-bottom: 5px; +} + +div.admonition ul, div.admonition ol { + margin-bottom: 5px; +} + +div.note { + background-color: #eee; + border: 1px solid #ccc; +} + +div.seealso { + background-color: #ffc; + border: 1px solid #ff6; +} + +div.topic { + background-color: #eee; +} + +div.warning { + background-color: #ffe4e4; + border: 1px solid #f66; +} + +p.admonition-title { + display: inline; +} + +p.admonition-title:after { + content: ":"; +} + +pre { + padding: 5px; + background-color: #eeffcc; + color: #333333; + line-height: 120%; + border: 1px solid #ac9; + border-left: none; + border-right: none; +} + +tt { + background-color: #ecf0f3; + padding: 0 1px 0 1px; + font-size: 0.95em; +} + +.warning tt { + background: #efc2c2; +} + +.note tt { + background: #d6d6d6; +} diff --git a/conductor/doc/source/_static/header-line.gif b/conductor/doc/source/_static/header-line.gif new file mode 100644 index 0000000000000000000000000000000000000000..3601730e03488b7b5f92dc992d23ad753357c167 GIT binary patch literal 48 zcmZ?wbhEHbWMg1uXkcVG`smgF|Nj+#vM@3*Ff!;c00Bsbfr-7RpY8O^Kn4bD08FwB Aga7~l literal 0 HcmV?d00001 diff --git a/conductor/doc/source/_static/header_bg.jpg b/conductor/doc/source/_static/header_bg.jpg new file mode 100644 index 0000000000000000000000000000000000000000..f788c41c26481728fa4329c17c87bde36001adc1 GIT binary patch literal 3738 zcmd5-YdDna8vedHnM0NtYi6>>At7O=uyTsZup5R_40A9)aXQa}U(l^=gSg=J*&3mKp$aM0r>UIFDe9Zy(vs} zWf)kqO2Y_n0$>ZQ0D&hY4tWjpY?Ii5?V)h*kc0fz?%ZIj3|{;F8E5l%d0)&*Hx~ulvc_*73u8%R zsVMV~ne!JY);&pWott~QIZYJFTXliYc2};JEU{X7W6;ZPfz;)U;U4#mEuK@K*=SC3BR-m&x9(Nna@>b@%FS34|P^jtsXRb5>z9gtPp;_MI2F3o*k z>csA-?CX4b;~4P-*L$+Mmb|51F)eD*wCc`Jt(9}C${Zo=!Uin=u_yMC^;`X!x$##4 z+~}dkT`NF@Uhw0r+6g_)?e!h8IX+OE^C96>UOsv0GPMD6(kr#ljhXRnA=O>Qj@%iT zqBF7aQ*}BG)h@6r0%#azk!r9yrN6>9dq~>KadV$~cGG?Hjk>~it^5rd#zS4KE*p+4 z;;B)%oBK8PNTs=A)a-z`n?3zJ%+h{`=>ijk4sYKr*>`eN1H`~Lo|Tm!o6qN{S* zeNl=NcpGzD55)XnLC|>g)~w={=c#4*x^;mk4Zo_FOFlffP@!?1`c+TogTVR4kp9-q z`d5cMBzNxk6qjPRK9*WY3uHS=bnm_QJvSMBBS_A#3i=ywsg6^|9rfruW0MhdGwHDO z?1gJRMQVecKE^gV{%uo(b)zl^Hd&vmnwFh88h*-?FJ;y=Hdqvt!K|s<$>xlzR=G4{ zZgGOCF43IXS?62B)w*N&dXt%U8X^Bjx}^%Yf>VFpFoKSGP%k?ems;&&J)|Dx(qtQD zu2tS)<_Qz4#LhBKYkl@Og}G)^5+F4P($Fk>)}{uMVv|;Sz2i4$XJ_WTw*;n>3N805rnXhbC52SC={E3rXRlrs|I6f;o|Cn%eje59{axu9sivy4oYmg=j|fLt3<3 zFce84aNb8GbK;y>RbBu71YBcYKL3@M3N25yoE%BtG z^K!`WTQ|fb-Ysa7T)mEw&4_b)PWYgc!)3W)H+neR9o^f|AXdgY1`gN+pvgzbbk`M z*Ts6${7M`2)9XIPy^MoXTiiP2GTp_OtgWMshnH)M&ZSO0)cet!oWo_0_&hV(0?Qdb zdo(sw{I#{hI`SWPM`N=U^#+MgN-*rZ#J7Cm7Jj89`5ehd_{z&9->Jc7$F(X4)&|`K z5rEgd;@dhi-IzJnSVpMd!Gf_G-QW+ zjVMrIas1)g%)GJ;(=oaK};O^)NYdS1`XR?K_;I7qj zhii5}x^he{U3M+GF+WpYws#=Pt#S9xB_X5QE7W+_rQdwMhukJnQj}5cnCz_sIJ#r0 zJa5drkRPI$X(4YdpCswJe#5aN4Jjw3V3Nzt&`lcKBI~#;!>jq7j8y# zvHrFg_#P376A45^hp-KU*P=R;DVdPK*w7D@Gw+`XsSpm^L-VkCooZF61sPAnnjsT# zND4C{>G#P10F_&txEoE!rX%Iy*L}Kna=Q%fDLJ_rF*LujRITZ)$g!?UYLkCXOoz-S z_p`Hny*Rh--l)aYQC&-2dd%;%VKGC1<1DJm_n~`nk4^yS`}&P zM}5bOypW0hwtvrwnE>}g1Mq+B>09qPp1b$hn6kC_iqF`tX#G-t7D$n}Ky9t}sUqiI zOe@odQ?JueZ+sg`-zoQ}J4if6vv1c9x{BDme+F6z{8esU^Kio zK_oPy9}@nlGywSOZy9`^- zzBg>C9|rgWF{pcCogEV@;d}VHrgeBl=5Dr*th4V!1`Z9Zrz9le1zHC#sM3{j#G2R?WMhl6b_yyoEAxX>Zixl$16`+^d$ihNtuIBUafyiCEv#oksNL<4= z*oDXsc7-(ww^9-b-6_|bITySG1N2C-7p0L4+V@R%j=4@ygc=89bmSNy38$S=ZiDyP z0SrqrVA;zi8kYBZ2@Mx(2Lx~-*bc@d1#4R($RJv$9ZTfx_t7Kc|HIHnd&@I386P?& z?d6Vd(48n${cTNFFCoSIUj#O{mmt%M&xCIFmR9Y3f{2UnF4e9@uFZOaYiY|CLdbDa z%xS9x4SHi7Fr-1?CnDqRK?)n&$TTBW5J?O&o{TnNCnLw*{QmT7{c}flSbp9&xi*zF z1TdUn&_!$_WxQbMKGkgsl}B%+N5ZV%Hy6_zJ>dejD89yCBMw9(d}z2fWjYH_nV6!F zqe_rI2H5Pi0^~S6)jjnu%lqZN*eQq6!||a24+edpSH_{C8Ew^g8dw2qdrH!@*E7K* z)00Bb8uUsai%v6Oa^L@3E02r|EG%EdV>q;=#2Q9Wjv3l?dAur$4bzyOl3M6 z1hf%&o*#2R&xnS1z4&R`Uq%`Ut0_P{BOwt;FuDb$1")); + }); + return $(returning); + }, + linkUser: function() { + var returning = []; + var regexp = /[\@]+([A-Za-z0-9-_]+)/gi; + this.each(function() { + returning.push(this.replace(regexp,"@$1")); + }); + return $(returning); + }, + linkHash: function() { + var returning = []; + var regexp = / [\#]+([A-Za-z0-9-_]+)/gi; + this.each(function() { + returning.push(this.replace(regexp, ' #$1')); + }); + return $(returning); + }, + capAwesome: function() { + var returning = []; + this.each(function() { + returning.push(this.replace(/\b(awesome)\b/gi, '$1')); + }); + return $(returning); + }, + capEpic: function() { + var returning = []; + this.each(function() { + returning.push(this.replace(/\b(epic)\b/gi, '$1')); + }); + return $(returning); + }, + makeHeart: function() { + var returning = []; + this.each(function() { + returning.push(this.replace(/(<)+[3]/gi, "♥")); + }); + return $(returning); + } + }); + + function relative_time(time_value) { + var parsed_date = Date.parse(time_value); + var relative_to = (arguments.length > 1) ? arguments[1] : new Date(); + var delta = parseInt((relative_to.getTime() - parsed_date) / 1000); + var pluralize = function (singular, n) { + return '' + n + ' ' + singular + (n == 1 ? '' : 's'); + }; + if(delta < 60) { + return 'less than a minute ago'; + } else if(delta < (45*60)) { + return 'about ' + pluralize("minute", parseInt(delta / 60)) + ' ago'; + } else if(delta < (24*60*60)) { + return 'about ' + pluralize("hour", parseInt(delta / 3600)) + ' ago'; + } else { + return 'about ' + pluralize("day", parseInt(delta / 86400)) + ' ago'; + } + } + + function build_url() { + var proto = ('https:' == document.location.protocol ? 'https:' : 'http:'); + if (s.list) { + return proto+"//api.twitter.com/1/"+s.username[0]+"/lists/"+s.list+"/statuses.json?per_page="+s.count+"&callback=?"; + } else if (s.query == null && s.username.length == 1) { + return proto+'//twitter.com/status/user_timeline/'+s.username[0]+'.json?count='+s.count+'&callback=?'; + } else { + var query = (s.query || 'from:'+s.username.join('%20OR%20from:')); + return proto+'//search.twitter.com/search.json?&q='+query+'&rpp='+s.count+'&callback=?'; + } + } + + return this.each(function(){ + var list = $('
    ').appendTo(this); + var intro = '

    '+s.intro_text+'

    '; + var outro = '

    '+s.outro_text+'

    '; + var loading = $('

    '+s.loading_text+'

    '); + + if(typeof(s.username) == "string"){ + s.username = [s.username]; + } + + if (s.loading_text) $(this).append(loading); + $.getJSON(build_url(), function(data){ + if (s.loading_text) loading.remove(); + if (s.intro_text) list.before(intro); + $.each((data.results || data), function(i,item){ + // auto join text based on verb tense and content + if (s.join_text == "auto") { + if (item.text.match(/^(@([A-Za-z0-9-_]+)) .*/i)) { + var join_text = s.auto_join_text_reply; + } else if (item.text.match(/(^\w+:\/\/[A-Za-z0-9-_]+\.[A-Za-z0-9-_:%&\?\/.=]+) .*/i)) { + var join_text = s.auto_join_text_url; + } else if (item.text.match(/^((\w+ed)|just) .*/im)) { + var join_text = s.auto_join_text_ed; + } else if (item.text.match(/^(\w*ing) .*/i)) { + var join_text = s.auto_join_text_ing; + } else { + var join_text = s.auto_join_text_default; + } + } else { + var join_text = s.join_text; + }; + + var from_user = item.from_user || item.user.screen_name; + var profile_image_url = item.profile_image_url || item.user.profile_image_url; + var join_template = ' '+join_text+' '; + var join = ((s.join_text) ? join_template : ' '); + var avatar_template = ''+from_user+'\'s avatar'; + var avatar = (s.avatar_size ? avatar_template : ''); + var date = ''+relative_time(item.created_at)+''; + var text = '' +$([item.text]).linkUrl().linkUser().linkHash().makeHeart().capAwesome().capEpic()[0]+ ''; + + // until we create a template option, arrange the items below to alter a tweet's display. + list.append('
  • ' + avatar + date + join + text + '
    • '); + + list.children('li:first').addClass('tweet_first'); + list.children('li:odd').addClass('tweet_even'); + list.children('li:even').addClass('tweet_odd'); + }); + if (s.outro_text) list.after(outro); + }); + + }); + }; +})(jQuery); \ No newline at end of file diff --git a/conductor/doc/source/_static/nature.css b/conductor/doc/source/_static/nature.css new file mode 100644 index 0000000..a98bd42 --- /dev/null +++ b/conductor/doc/source/_static/nature.css @@ -0,0 +1,245 @@ +/* + * nature.css_t + * ~~~~~~~~~~~~ + * + * Sphinx stylesheet -- nature theme. + * + * :copyright: Copyright 2007-2011 by the Sphinx team, see AUTHORS. + * :license: BSD, see LICENSE for details. + * + */ + +@import url("basic.css"); + +/* -- page layout ----------------------------------------------------------- */ + +body { + font-family: Arial, sans-serif; + font-size: 100%; + background-color: #111; + color: #555; + margin: 0; + padding: 0; +} + +div.documentwrapper { + float: left; + width: 100%; +} + +div.bodywrapper { + margin: 0 0 0 {{ theme_sidebarwidth|toint }}px; +} + +hr { + border: 1px solid #B1B4B6; +} + +div.document { + background-color: #eee; +} + +div.body { + background-color: #ffffff; + color: #3E4349; + padding: 0 30px 30px 30px; + font-size: 0.9em; +} + +div.footer { + color: #555; + width: 100%; + padding: 13px 0; + text-align: center; + font-size: 75%; +} + +div.footer a { + color: #444; + text-decoration: underline; +} + +div.related { + background-color: #6BA81E; + line-height: 32px; + color: #fff; + text-shadow: 0px 1px 0 #444; + font-size: 0.9em; +} + +div.related a { + color: #E2F3CC; +} + +div.sphinxsidebar { + font-size: 0.75em; + line-height: 1.5em; +} + +div.sphinxsidebarwrapper{ + padding: 20px 0; +} + +div.sphinxsidebar h3, +div.sphinxsidebar h4 { + font-family: Arial, sans-serif; + color: #222; + font-size: 1.2em; + font-weight: normal; + margin: 0; + padding: 5px 10px; + background-color: #ddd; + text-shadow: 1px 1px 0 white +} + +div.sphinxsidebar h4{ + font-size: 1.1em; +} + +div.sphinxsidebar h3 a { + color: #444; +} + + +div.sphinxsidebar p { + color: #888; + padding: 5px 20px; +} + +div.sphinxsidebar p.topless { +} + +div.sphinxsidebar ul { + margin: 10px 20px; + padding: 0; + color: #000; +} + +div.sphinxsidebar a { + color: #444; +} + +div.sphinxsidebar input { + border: 1px solid #ccc; + font-family: sans-serif; + font-size: 1em; +} + +div.sphinxsidebar input[type=text]{ + margin-left: 20px; +} + +/* -- body styles ----------------------------------------------------------- */ + +a { + color: #005B81; + text-decoration: none; +} + +a:hover { + color: #E32E00; + text-decoration: underline; +} + +div.body h1, +div.body h2, +div.body h3, +div.body h4, +div.body h5, +div.body h6 { + font-family: Arial, sans-serif; + background-color: #BED4EB; + font-weight: normal; + color: #212224; + margin: 30px 0px 10px 0px; + padding: 5px 0 5px 10px; + text-shadow: 0px 1px 0 white +} + +div.body h1 { border-top: 20px solid white; margin-top: 0; font-size: 200%; } +div.body h2 { font-size: 150%; background-color: #C8D5E3; } +div.body h3 { font-size: 120%; background-color: #D8DEE3; } +div.body h4 { font-size: 110%; background-color: #D8DEE3; } +div.body h5 { font-size: 100%; background-color: #D8DEE3; } +div.body h6 { font-size: 100%; background-color: #D8DEE3; } + +a.headerlink { + color: #c60f0f; + font-size: 0.8em; + padding: 0 4px 0 4px; + text-decoration: none; +} + +a.headerlink:hover { + background-color: #c60f0f; + color: white; +} + +div.body p, div.body dd, div.body li { + line-height: 1.5em; +} + +div.admonition p.admonition-title + p { + display: inline; +} + +div.highlight{ + background-color: white; +} + +div.note { + background-color: #eee; + border: 1px solid #ccc; +} + +div.seealso { + background-color: #ffc; + border: 1px solid #ff6; +} + +div.topic { + background-color: #eee; +} + +div.warning { + background-color: #ffe4e4; + border: 1px solid #f66; +} + +p.admonition-title { + display: inline; +} + +p.admonition-title:after { + content: ":"; +} + +pre { + padding: 10px; + background-color: White; + color: #222; + line-height: 1.2em; + border: 1px solid #C6C9CB; + font-size: 1.1em; + margin: 1.5em 0 1.5em 0; + -webkit-box-shadow: 1px 1px 1px #d8d8d8; + -moz-box-shadow: 1px 1px 1px #d8d8d8; +} + +tt { + background-color: #ecf0f3; + color: #222; + /* padding: 1px 2px; */ + font-size: 1.1em; + font-family: monospace; +} + +.viewcode-back { + font-family: Arial, sans-serif; +} + +div.viewcode-block:target { + background-color: #f4debf; + border-top: 1px solid #ac9; + border-bottom: 1px solid #ac9; +} diff --git a/conductor/doc/source/_static/openstack_logo.png b/conductor/doc/source/_static/openstack_logo.png new file mode 100644 index 0000000000000000000000000000000000000000..146faec5cfe3773824f4caf39e4480e4974d10df GIT binary patch literal 3670 zcmV-c4yo~pP)CW75Qp#l)U;+N6jaIz6Nf$t6dNV>^>ETzcpQ=%tMaf0k|rg72+IW`z$FyfE+D{1@tt$t5DmX)*;QV?c;%+5Z&egAgfXTQJq-mZkC z>pFAHu}U=Axde_?s!99ZfDg_+9TYzDa6N1R3adhx&2Mb7>9w`KpMNz!>U5t2XQ8lZ zu+!+H7(PRwF@jAkwvI;|8|=Z_dfzV`Kpi;I!e=|Ql+HAdEag?VZ^Ilw9XJj9N1#1a z?UFC!)X62`CRIe^9YCLKbJ` z&O@f0zt{Z1YDF1utg2$F+rzvrncys+g37Xsd8)idSW(=}t#~qF#qBo29*@^ZCs<$W zpa144=o4g0z63h_ttPfIpH-FyG^MAH+6B~r$(4qw+Uv{2d#h`$lq+i+#Tf%CAzDFUh!pzX(6nW{EASJAQkhm!+}aGpHc z;(+N`S*@tYmump1T37E}J;!$0#F>^M*mT_X1x~bvnp&qP9IHI#bj-0z8FR+=p+e#*w3ugV#wX``sR-CI1!YiQsfc@Om<;1MBw zlfqH9z4Q|m*C?URU1OG(`UYn>Q8<|I!mby#FlN5MMFE8;Pyh$skbR?ngFLt?%nWSkS-#W5umy>@^DyAERP~{E&`M%0(qi&((^ahqL}u^jT<2dcf)p< z%Fxc9J$nh_`>_oNYC?oy`rIDY46Yrw4si3Qn~oXV%dJ}IlUD-40>QipyGa_dV0Z%J ztcEXm5yxR0gySJ04{nnbm#vP=Hq&GI<8VxcZ34pRjt6m%pE2H|!+HBJQrdBdyKHJR z2O_}hp!5bXuwniQYTF>yI|=cjT+2l`9T3|H+l4%ryPxWQm(ODW#8Ctj_CplcO=)qj zD#d~V6BahR9NY1kE5rF)_j<|!Cqnpq0uOKhL%w z>y8OyeTM1?REXc{0|3b=#WPZneh80PxL=Ljau1~+CgtMgg-vccMDX-L z9^7An_;!lFAi`#G_1F*OdM|Z$EVQs0m0$?mY}(baOZ%Zpd62#Pyg!3Jd4d zD^8+lSir&T6Y9-p9L#Wz6$5nXLjdOl?7Lv!TeMr}F14ranauW9=L>ubu*x>Bcrgwp zjrT@{rL*2Fc}Ilwn07QvdJfMOO2=(1Px)6&ih7lg839!Bx&}lQER~T`^7_x@fXo({ zCZMeZYt*!VgMTg>PR)PBaIwubzRY%jjE`-s zG;B}>2!lD=QLOTfQOEZKIEz*;yTJ9(Af0zNv;IDq7#Fr#W{Ap+7Sq1N3TL21X|h2t z=Dk>^bGSsRX-u+cZ23mMB_Ioc0yNIfcfLWB>$hVU3W3>d&a?IM+bGRGt+t}aiv(eh z(D6Z9N>U2|Qxle(!UVTeEKE6W))3WI5z48Rs8d5v0GwmyC8iQiUJO8KS?QwHl2abL zNW+hadDdPc8z%MSOG$l&WR@!!&M{WLmrnS=-0G#&`a)chX>mN9W1>|yqve@lL8a`f zXRmn$B8P=dLxE!2rIi}a*gh%FI4j?C;b@L=WgypiTRf==n6DKr9mUExo6a@{wLM-I z9%V9{!;5G!<8fMYikfEbrGXRQN-9*24}kIIpP&dEg@fiLqAY5|jjv}$P3x0avZODU zdX`c|G>h`1f=3uEu)L9C)H5%frni#HZXcX`TD{iQ-e2qXxj_f%|WW;byDMc%7+uBy}Y?KLC?jp%yyyeBNkqQ-*osw2ex&97Q{#C7%CdSDMNIV zTdC(LEm?&qPcNOjM)h9Grs|M(gsuhV8@96?m4WkQ>j{bJIs)m^neL%ua!i+N8>Lh+ zKu#7rF~VOH@hb{zGXYwys!Um4Vkf+H8Hj6?^eI%kT%j+HA0K=6qdQ@nfR57Q`Jm9T zc)Yg9-`e~BRE!xoKZ z=mP|0Kihr}V1$5sHw$QekmoL)lQ;~@H$S)}s3xuwypiubB?1%OyBpwC08TH!=?BrQ zhOp`PTu;%u0}Q=XKGb7d$g8*;de8c1UI|Re2R;;Radh_D!FIZg+JP`oJg>5 z;&B7eVAomZe>j~hOOIVRO_Q7eSGz37hxmnsG!n%HX`C6gSqFcg(RLmikn%EPR*wel zrsc;>!vQ<>2ZW`lk`MbNLopFd#_9mh8iKPH;KbjC@xJU${pdxuTF{uO(eG#9t*>XP z_4Seh`r_#q$^xeiuy(=eSouv66cpS!t3n`|j`6xnmSs1q@;0!I)m<6eYHHGMRdB87 ziruozT=gn@yp`B9oGxD-b7PqhZum|oJCfLB38&8v51ijj-Pb`qvCr3FtJ0aFms2h3(n0-}3jJ~J$ zCzep7-MIZFbo$(m8zWm?SoRl__blLE+!fFBVVk1&XLg+vmVNcTk9O2+q?x#F0LZUN zu6oM~C)(7^0|az4nM}@aZf<@RkH0CR8<-Yn-fZe+Dbr#iJWSt#tnR4^h<@ePXWmeHIO4q^X zCbiy(=k3R1o1}0E+7x*OOe-qnIXG{#N_rqK*1NH}Qz6aumTR`YTgo5K=q=61;5@b- zrgUA_Qz=)(TPN!tCZE|{?B0*r9ov5Fcip6xQ2;Yqs*2_o7TFKGp0|~bcP@6+a(rz^ zXXmmyBfT}ucw_t(6s+f^t_)nc>RKW<-q_&J35vN+RPLsR?VAsQeHLyCR7AWvxFOVc zAg-xl=j*RipzaKWx3lAf?ei`PoM;bbAL>svH?JqQwjSulb9bghytRt%*5x-no>xlf zh7qj0LYRXVDU})?Btsy7^71*ujsEP_ACyd)P)*ULWBCXox@PUfwmQ#)Vl&oeIqpQY zHMgU+xe0EhQ)RmjdB3JHGdrsvJ9?A=WwOrn)J?BH{+D&O_@SKdrj2|8Z{hS1T(k>&Zlt;p=tqw*mVY1aLt=u^eAHkW>8cb#@q& z4-SLa@ii zCt7NGrLv)1Scy9ew-sOwwLYn2a6T#KzJgnbacm7Z20q6tcs~C!0DI+r(=$l+x{=W0A}~0&W)ll4*&oF07*qoM6N<$f~n6U7ytkO literal 0 HcmV?d00001 diff --git a/conductor/doc/source/_static/tweaks.css b/conductor/doc/source/_static/tweaks.css new file mode 100644 index 0000000..3f3fb3f --- /dev/null +++ b/conductor/doc/source/_static/tweaks.css @@ -0,0 +1,94 @@ +body { + background: #fff url(../_static/header_bg.jpg) top left no-repeat; +} + +#header { + width: 950px; + margin: 0 auto; + height: 102px; +} + +#header h1#logo { + background: url(../_static/openstack_logo.png) top left no-repeat; + display: block; + float: left; + text-indent: -9999px; + width: 175px; + height: 55px; +} + +#navigation { + background: url(../_static/header-line.gif) repeat-x 0 bottom; + display: block; + float: left; + margin: 27px 0 0 25px; + padding: 0; +} + +#navigation li{ + float: left; + display: block; + margin-right: 25px; +} + +#navigation li a { + display: block; + font-weight: normal; + text-decoration: none; + background-position: 50% 0; + padding: 20px 0 5px; + color: #353535; + font-size: 14px; +} + +#navigation li a.current, #navigation li a.section { + border-bottom: 3px solid #cf2f19; + color: #cf2f19; +} + +div.related { + background-color: #cde2f8; + border: 1px solid #b0d3f8; +} + +div.related a { + color: #4078ba; + text-shadow: none; +} + +div.sphinxsidebarwrapper { + padding-top: 0; +} + +pre { + color: #555; +} + +div.documentwrapper h1, div.documentwrapper h2, div.documentwrapper h3, div.documentwrapper h4, div.documentwrapper h5, div.documentwrapper h6 { + font-family: 'PT Sans', sans-serif !important; + color: #264D69; + border-bottom: 1px dotted #C5E2EA; + padding: 0; + background: none; + padding-bottom: 5px; +} + +div.documentwrapper h3 { + color: #CF2F19; +} + +a.headerlink { + color: #fff !important; + margin-left: 5px; + background: #CF2F19 !important; +} + +div.body { + margin-top: -25px; + margin-left: 230px; +} + +div.document { + width: 960px; + margin: 0 auto; +} \ No newline at end of file diff --git a/conductor/doc/source/_templates/.placeholder b/conductor/doc/source/_templates/.placeholder new file mode 100644 index 0000000..e69de29 diff --git a/conductor/doc/source/_theme/layout.html b/conductor/doc/source/_theme/layout.html new file mode 100644 index 0000000..750b782 --- /dev/null +++ b/conductor/doc/source/_theme/layout.html @@ -0,0 +1,83 @@ +{% extends "basic/layout.html" %} +{% set css_files = css_files + ['_static/tweaks.css'] %} +{% set script_files = script_files + ['_static/jquery.tweet.js'] %} + +{%- macro sidebar() %} + {%- if not embedded %}{% if not theme_nosidebar|tobool %} +
    +
    + {%- block sidebarlogo %} + {%- if logo %} + + {%- endif %} + {%- endblock %} + {%- block sidebartoc %} + {%- if display_toc %} +

    {{ _('Table Of Contents') }}

    + {{ toc }} + {%- endif %} + {%- endblock %} + {%- block sidebarrel %} + {%- if prev %} +

    {{ _('Previous topic') }}

    +

    {{ prev.title }}

    + {%- endif %} + {%- if next %} +

    {{ _('Next topic') }}

    +

    {{ next.title }}

    + {%- endif %} + {%- endblock %} + {%- block sidebarsourcelink %} + {%- if show_source and has_source and sourcename %} +

    {{ _('This Page') }}

    + + {%- endif %} + {%- endblock %} + {%- if customsidebar %} + {% include customsidebar %} + {%- endif %} + {%- block sidebarsearch %} + {%- if pagename != "search" %} + + + {%- endif %} + {%- endblock %} +
    +
    + {%- endif %}{% endif %} +{%- endmacro %} + +{% block relbar1 %}{% endblock relbar1 %} + +{% block header %} + +{% endblock %} \ No newline at end of file diff --git a/conductor/doc/source/_theme/theme.conf b/conductor/doc/source/_theme/theme.conf new file mode 100644 index 0000000..1cc4004 --- /dev/null +++ b/conductor/doc/source/_theme/theme.conf @@ -0,0 +1,4 @@ +[theme] +inherit = basic +stylesheet = nature.css +pygments_style = tango diff --git a/conductor/doc/source/conf.py b/conductor/doc/source/conf.py new file mode 100644 index 0000000..e9b38f9 --- /dev/null +++ b/conductor/doc/source/conf.py @@ -0,0 +1,242 @@ + +# -*- coding: utf-8 -*- +# Copyright (c) 2010 OpenStack Foundation. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# +# Conductor documentation build configuration file, created by +# sphinx-quickstart on Tue February 28 13:50:15 2013. +# +# This file is execfile()'d with the current directory set to its containing +# dir. +# +# Note that not all possible configuration values are present in this +# autogenerated file. +# +# All configuration values have a default; values that are commented out +# serve to show the default. + +import os +import sys + +# If extensions (or modules to document with autodoc) are in another directory, +# add these directories to sys.path here. If the directory is relative to the +# documentation root, use os.path.abspath to make it absolute, like shown here. +sys.path = [os.path.abspath('../../conductor'), + os.path.abspath('../..'), + os.path.abspath('../../bin') + ] + sys.path + +# -- General configuration --------------------------------------------------- + +# Add any Sphinx extension module names here, as strings. They can be +# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom ones. +extensions = ['sphinx.ext.coverage', + 'sphinx.ext.ifconfig', + 'sphinx.ext.intersphinx', + 'sphinx.ext.pngmath', + 'sphinx.ext.graphviz'] + +# Add any paths that contain templates here, relative to this directory. +templates_path = [] +if os.getenv('HUDSON_PUBLISH_DOCS'): + templates_path = ['_ga', '_templates'] +else: + templates_path = ['_templates'] + +# The suffix of source filenames. +source_suffix = '.rst' + +# The encoding of source files. +#source_encoding = 'utf-8' + +# The master toctree document. +master_doc = 'index' + +# General information about the project. +project = u'Conductor' +copyright = u'2013, Mirantis, Inc.' + +# The version info for the project you're documenting, acts as replacement for +# |version| and |release|, also used in various other places throughout the +# built documents. +# +# The short X.Y version. +from conductor.version import version_info as conductor_version +# The full version, including alpha/beta/rc tags. +release = conductor_version.version_string_with_vcs() +# The short X.Y version. +version = conductor_version.canonical_version_string() + +# The language for content autogenerated by Sphinx. Refer to documentation +# for a list of supported languages. +#language = None + +# There are two options for replacing |today|: either, you set today to some +# non-false value, then it is used: +#today = '' +# Else, today_fmt is used as the format for a strftime call. +#today_fmt = '%B %d, %Y' + +# List of documents that shouldn't be included in the build. +#unused_docs = [] + +# List of directories, relative to source directory, that shouldn't be searched +# for source files. +exclude_trees = ['api'] + +# The reST default role (for this markup: `text`) to use for all documents. +#default_role = None + +# If true, '()' will be appended to :func: etc. cross-reference text. +#add_function_parentheses = True + +# If true, the current module name will be prepended to all description +# unit titles (such as .. function::). +#add_module_names = True + +# If true, sectionauthor and moduleauthor directives will be shown in the +# output. They are ignored by default. +show_authors = True + +# The name of the Pygments (syntax highlighting) style to use. +pygments_style = 'sphinx' + +# A list of ignored prefixes for module index sorting. +modindex_common_prefix = ['portas.'] + +# -- Options for man page output -------------------------------------------- + +# Grouping the document tree for man pages. +# List of tuples 'sourcefile', 'target', u'title', u'Authors name', 'manual' + +man_pages = [ + ('man/conductor', 'conductor', u'Conductor Orchestrator', + [u'Mirantis, Inc.'], 1) +] + + +# -- Options for HTML output ------------------------------------------------- + +# The theme to use for HTML and HTML Help pages. Major themes that come with +# Sphinx are currently 'default' and 'sphinxdoc'. +html_theme_path = ["."] +html_theme = '_theme' + +# Theme options are theme-specific and customize the look and feel of a theme +# further. For a list of options available for each theme, see the +# documentation. +#html_theme_options = {} + +# Add any paths that contain custom themes here, relative to this directory. +#html_theme_path = ['_theme'] + +# The name for this set of Sphinx documents. If None, it defaults to +# " v documentation". +#html_title = None + +# A shorter title for the navigation bar. Default is the same as html_title. +#html_short_title = None + +# The name of an image file (relative to this directory) to place at the top +# of the sidebar. +#html_logo = None + +# The name of an image file (within the static path) to use as favicon of the +# docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32 +# pixels large. +#html_favicon = None + +# Add any paths that contain custom static files (such as style sheets) here, +# relative to this directory. They are copied after the builtin static files, +# so a file named "default.css" will overwrite the builtin "default.css". +html_static_path = ['_static'] + +# If not '', a 'Last updated on:' timestamp is inserted at every page bottom, +# using the given strftime format. +#html_last_updated_fmt = '%b %d, %Y' +git_cmd = "git log --pretty=format:'%ad, commit %h' --date=local -n1" +html_last_updated_fmt = os.popen(git_cmd).read() + +# If true, SmartyPants will be used to convert quotes and dashes to +# typographically correct entities. +#html_use_smartypants = True + +# Custom sidebar templates, maps document names to template names. +#html_sidebars = {} + +# Additional templates that should be rendered to pages, maps page names to +# template names. +#html_additional_pages = {} + +# If false, no module index is generated. +html_use_modindex = False + +# If false, no index is generated. +html_use_index = False + +# If true, the index is split into individual pages for each letter. +#html_split_index = False + +# If true, links to the reST sources are added to the pages. +#html_show_sourcelink = True + +# If true, an OpenSearch description file will be output, and all pages will +# contain a tag referring to it. The value of this option must be the +# base URL from which the finished HTML is served. +#html_use_opensearch = '' + +# If nonempty, this is the file name suffix for HTML files (e.g. ".xhtml"). +#html_file_suffix = '' + +# Output file base name for HTML help builder. +htmlhelp_basename = 'conductordoc' + + +# -- Options for LaTeX output ------------------------------------------------ + +# The paper size ('letter' or 'a4'). +#latex_paper_size = 'letter' + +# The font size ('10pt', '11pt' or '12pt'). +#latex_font_size = '10pt' + +# Grouping the document tree into LaTeX files. List of tuples +# (source start file, target name, title, author, +# documentclass [howto/manual]). +latex_documents = [ + ('index', 'Conductor.tex', u'Conductor Documentation', + u'Keero Team', 'manual'), +] + +# The name of an image file (relative to this directory) to place at the top of +# the title page. +#latex_logo = None + +# For "manual" documents, if this is true, then toplevel headings are parts, +# not chapters. +#latex_use_parts = False + +# Additional stuff for the LaTeX preamble. +#latex_preamble = '' + +# Documents to append as an appendix to all manuals. +#latex_appendices = [] + +# If false, no module index is generated. +#latex_use_modindex = True + +# Example configuration for intersphinx: refer to the Python standard library. +intersphinx_mapping = {'python': ('http://docs.python.org/', None)} diff --git a/conductor/doc/source/index.rst b/conductor/doc/source/index.rst new file mode 100644 index 0000000..50ce142 --- /dev/null +++ b/conductor/doc/source/index.rst @@ -0,0 +1,20 @@ +.. + Copyright 2013, Mirantis Inc. + All Rights Reserved. + + Licensed under the Apache License, Version 2.0 (the "License"); you may + not use this file except in compliance with the License. You may obtain + a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + License for the specific language governing permissions and limitations + under the License. + +Welcome to Conductor's documentation! +================================== + +We rule the world! \ No newline at end of file diff --git a/conductor/setup.cfg b/conductor/setup.cfg new file mode 100644 index 0000000..6e6f655 --- /dev/null +++ b/conductor/setup.cfg @@ -0,0 +1,33 @@ +[build_sphinx] +all_files = 1 +build-dir = doc/build +source-dir = doc/source + +[egg_info] +tag_build = +tag_date = 0 +tag_svn_revision = 0 + +[compile_catalog] +directory = conductor/locale +domain = conductor + +[update_catalog] +domain = conductor +output_dir = conductor/locale +input_file = conductor/locale/conductor.pot + +[extract_messages] +keywords = _ gettext ngettext l_ lazy_gettext +mapping_file = babel.cfg +output_file = conductor/locale/conductor.pot + +[nosetests] +# NOTE(jkoelker) To run the test suite under nose install the following +# coverage http://pypi.python.org/pypi/coverage +# tissue http://pypi.python.org/pypi/tissue (pep8 checker) +# openstack-nose https://github.com/jkoelker/openstack-nose +verbosity=2 +cover-package = conductor +cover-html = true +cover-erase = true \ No newline at end of file diff --git a/conductor/setup.py b/conductor/setup.py new file mode 100644 index 0000000..fb9da8c --- /dev/null +++ b/conductor/setup.py @@ -0,0 +1,49 @@ +#!/usr/bin/python +# Copyright (c) 2010 OpenStack, LLC. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import setuptools + +from conductor.openstack.common import setup + +requires = setup.parse_requirements() +depend_links = setup.parse_dependency_links() +project = 'conductor' + +setuptools.setup( + name=project, + version=setup.get_version(project, '2013.1'), + description='The Conductor is orchestration engine server', + license='Apache License (2.0)', + author='Mirantis, Inc.', + author_email='openstack@lists.launchpad.net', + url='http://conductor.openstack.org/', + packages=setuptools.find_packages(exclude=['bin']), + test_suite='nose.collector', + cmdclass=setup.get_cmdclass(), + include_package_data=True, + install_requires=requires, + dependency_links=depend_links, + classifiers=[ + 'Development Status :: 4 - Beta', + 'License :: OSI Approved :: Apache Software License', + 'Operating System :: POSIX :: Linux', + 'Programming Language :: Python :: 2.7', + 'Environment :: No Input/Output (Daemon)', + 'Environment :: OpenStack', + ], + scripts=['bin/conductor'], + py_modules=[] +) From 2eb185428de93cf11f68e51a390d600f8b3457c7 Mon Sep 17 00:00:00 2001 From: Stan Lagun Date: Tue, 26 Mar 2013 14:27:37 +0400 Subject: [PATCH 27/37] Use Heat REST API instead of command-line --- conductor/conductor/app.py | 12 ++-- conductor/conductor/cloud_formation.py | 8 +-- .../conductor/commands/cloud_formation.py | 56 ++++++++++++++----- conductor/conductor/commands/dispatcher.py | 4 +- conductor/conductor/config.py | 5 ++ conductor/etc/conductor.conf | 4 +- conductor/test.json | 3 +- conductor/tools/pip-requires | 1 + 8 files changed, 68 insertions(+), 25 deletions(-) diff --git a/conductor/conductor/app.py b/conductor/conductor/app.py index a69e8e8..da17e9b 100644 --- a/conductor/conductor/app.py +++ b/conductor/conductor/app.py @@ -17,7 +17,8 @@ def task_received(task, message_id): print 'Starting at', datetime.datetime.now() reporter = reporting.Reporter(rmqclient, message_id, task['id']) - command_dispatcher = CommandDispatcher(task['name'], rmqclient) + command_dispatcher = CommandDispatcher( + task['name'], rmqclient, task['token']) workflows = [] for path in glob.glob("data/workflows/*.xml"): print "loading", path @@ -26,9 +27,12 @@ def task_received(task, message_id): workflows.append(workflow) while True: - for workflow in workflows: - workflow.execute() - if not command_dispatcher.execute_pending(): + try: + for workflow in workflows: + workflow.execute() + if not command_dispatcher.execute_pending(): + break + except Exception: break command_dispatcher.close() diff --git a/conductor/conductor/cloud_formation.py b/conductor/conductor/cloud_formation.py index 1642714..e7894bf 100644 --- a/conductor/conductor/cloud_formation.py +++ b/conductor/conductor/cloud_formation.py @@ -1,7 +1,7 @@ import base64 import xml_code_engine - +import config def update_cf_stack(engine, context, body, template, mappings, arguments, **kwargs): @@ -17,15 +17,15 @@ def update_cf_stack(engine, context, body, template, def prepare_user_data(context, template='Default', **kwargs): - config = context['/config'] + settings = config.CONF.rabbitmq + with open('data/init.ps1') as init_script_file: with open('data/templates/agent-config/%s.template' % template) as template_file: init_script = init_script_file.read() template_data = template_file.read() template_data = template_data.replace( - '%RABBITMQ_HOST%', - config.get_setting('rabbitmq', 'host') or 'localhost') + '%RABBITMQ_HOST%', settings.host) template_data = template_data.replace( '%RESULT_QUEUE%', '-execution-results-%s' % str(context['/dataSource']['name'])) diff --git a/conductor/conductor/commands/cloud_formation.py b/conductor/conductor/commands/cloud_formation.py index d0225ed..ab62ac5 100644 --- a/conductor/conductor/commands/cloud_formation.py +++ b/conductor/conductor/commands/cloud_formation.py @@ -1,16 +1,23 @@ import anyjson import os import uuid +import eventlet import conductor.helpers from command import CommandBase -from subprocess import call +import conductor.config +from heatclient.client import Client +import heatclient.exc + class HeatExecutor(CommandBase): - def __init__(self, stack): + def __init__(self, stack, token): self._pending_list = [] self._stack = stack + settings = conductor.config.CONF.heat + self._heat_client = Client('1', settings.url, + token_only=True, token=token) def execute(self, template, mappings, arguments, callback): with open('data/templates/cf/%s.template' % template) as template_file: @@ -43,15 +50,15 @@ class HeatExecutor(CommandBase): print 'Executing heat template', anyjson.dumps(template), \ 'with arguments', arguments, 'on stack', self._stack - if not os.path.exists("tmp"): - os.mkdir("tmp") - file_name = "tmp/" + str(uuid.uuid4()) - print "Saving template to", file_name - with open(file_name, "w") as f: - f.write(anyjson.dumps(template)) - - arguments_str = ';'.join(['%s=%s' % (key, value) - for (key, value) in arguments.items()]) + # if not os.path.exists("tmp"): + # os.mkdir("tmp") + # file_name = "tmp/" + str(uuid.uuid4()) + # print "Saving template to", file_name + # with open(file_name, "w") as f: + # f.write(anyjson.dumps(template)) + # + # arguments_str = ';'.join(['%s=%s' % (key, value) + # for (key, value) in arguments.items()]) # call([ # "./heat_run", "stack-create", # "-f" + file_name, @@ -59,12 +66,35 @@ class HeatExecutor(CommandBase): # self._stack # ]) + try: + self._heat_client.stacks.update( + stack_id=self._stack, + parameters=arguments, + template=template) + self._wait_state('UPDATE_COMPLETE') + except heatclient.exc.HTTPNotFound: + self._heat_client.stacks.create( + stack_name=self._stack, + parameters=arguments, + template=template) + self._wait_state('CREATE_COMPLETE') + pending_list = self._pending_list self._pending_list = [] for item in pending_list: item['callback'](True) - - return True + + def _wait_state(self, state): + while True: + status = self._heat_client.stacks.get( + stack_id=self._stack).stack_status + if 'IN_PROGRESS' in status: + eventlet.sleep(1) + continue + if status != state: + raise EnvironmentError() + return + diff --git a/conductor/conductor/commands/dispatcher.py b/conductor/conductor/commands/dispatcher.py index 606266e..624ce30 100644 --- a/conductor/conductor/commands/dispatcher.py +++ b/conductor/conductor/commands/dispatcher.py @@ -4,9 +4,9 @@ import windows_agent class CommandDispatcher(command.CommandBase): - def __init__(self, environment_id, rmqclient): + def __init__(self, environment_id, rmqclient, token): self._command_map = { - 'cf': cloud_formation.HeatExecutor(environment_id), + 'cf': cloud_formation.HeatExecutor(environment_id, token), 'agent': windows_agent.WindowsAgentExecutor( environment_id, rmqclient, environment_id) } diff --git a/conductor/conductor/config.py b/conductor/conductor/config.py index 1e42cad..5190918 100644 --- a/conductor/conductor/config.py +++ b/conductor/conductor/config.py @@ -45,9 +45,14 @@ rabbit_opts = [ cfg.StrOpt('virtual_host', default='/'), ] +heat_opts = [ + cfg.StrOpt('url') +] + CONF = cfg.CONF CONF.register_opts(paste_deploy_opts, group='paste_deploy') CONF.register_opts(rabbit_opts, group='rabbitmq') +CONF.register_opts(heat_opts, group='heat') CONF.import_opt('verbose', 'conductor.openstack.common.log') diff --git a/conductor/etc/conductor.conf b/conductor/etc/conductor.conf index 189eeed..24930b1 100644 --- a/conductor/etc/conductor.conf +++ b/conductor/etc/conductor.conf @@ -1,9 +1,11 @@ [DEFAULT] log_file = logs/conductor.log +[heat] +url = http://172.18.124.101:8004/v1/16eb78cbb688459c8308d89678bcef50 [rabbitmq] -host = localhost +host = 172.18.124.101 port = 5672 virtual_host = keero login = keero diff --git a/conductor/test.json b/conductor/test.json index 2427070..c2815d5 100644 --- a/conductor/test.json +++ b/conductor/test.json @@ -1,6 +1,7 @@ { - "name": "MyDataCenter", + "name": "MyDataCenterx", "id": "adc6d143f9584d10808c7ef4d07e4802", + "token": "MIINIQYJKoZIhvcNAQcCoIINEjCCDQ4CAQExCTAHBgUrDgMCGjCCC-oGCSqGSIb3DQEHAaCCC+sEggvneyJhY2Nlc3MiOiB7InRva2VuIjogeyJpc3N1ZWRfYXQiOiAiMjAxMy0wMy0yNlQwNjo0NTozNy4zOTI0MDAiLCAiZXhwaXJlcyI6ICIyMDEzLTAzLTI3VDA2OjQ1OjM3WiIsICJpZCI6ICJwbGFjZWhvbGRlciIsICJ0ZW5hbnQiOiB7ImRlc2NyaXB0aW9uIjogbnVsbCwgImVuYWJsZWQiOiB0cnVlLCAiaWQiOiAiMTZlYjc4Y2JiNjg4NDU5YzgzMDhkODk2NzhiY2VmNTAiLCAibmFtZSI6ICJhZG1pbiJ9fSwgInNlcnZpY2VDYXRhbG9nIjogW3siZW5kcG9pbnRzIjogW3siYWRtaW5VUkwiOiAiaHR0cDovLzE3Mi4xOC4xMjQuMTAxOjg3NzQvdjIvMTZlYjc4Y2JiNjg4NDU5YzgzMDhkODk2NzhiY2VmNTAiLCAicmVnaW9uIjogIlJlZ2lvbk9uZSIsICJpbnRlcm5hbFVSTCI6ICJodHRwOi8vMTcyLjE4LjEyNC4xMDE6ODc3NC92Mi8xNmViNzhjYmI2ODg0NTljODMwOGQ4OTY3OGJjZWY1MCIsICJpZCI6ICIwNGFlNjM2ZTdhYzc0NmJjYjExM2EwYzI5NDYzMzgzMCIsICJwdWJsaWNVUkwiOiAiaHR0cDovLzE3Mi4xOC4xMjQuMTAxOjg3NzQvdjIvMTZlYjc4Y2JiNjg4NDU5YzgzMDhkODk2NzhiY2VmNTAifV0sICJlbmRwb2ludHNfbGlua3MiOiBbXSwgInR5cGUiOiAiY29tcHV0ZSIsICJuYW1lIjogIm5vdmEifSwgeyJlbmRwb2ludHMiOiBbeyJhZG1pblVSTCI6ICJodHRwOi8vMTcyLjE4LjEyNC4xMDE6MzMzMyIsICJyZWdpb24iOiAiUmVnaW9uT25lIiwgImludGVybmFsVVJMIjogImh0dHA6Ly8xNzIuMTguMTI0LjEwMTozMzMzIiwgImlkIjogIjA5MmJkMjMyMGU5ZDRlYWY4ZDBlZjEzNDhjOGU3NTJjIiwgInB1YmxpY1VSTCI6ICJodHRwOi8vMTcyLjE4LjEyNC4xMDE6MzMzMyJ9XSwgImVuZHBvaW50c19saW5rcyI6IFtdLCAidHlwZSI6ICJzMyIsICJuYW1lIjogInMzIn0sIHsiZW5kcG9pbnRzIjogW3siYWRtaW5VUkwiOiAiaHR0cDovLzE3Mi4xOC4xMjQuMTAxOjkyOTIiLCAicmVnaW9uIjogIlJlZ2lvbk9uZSIsICJpbnRlcm5hbFVSTCI6ICJodHRwOi8vMTcyLjE4LjEyNC4xMDE6OTI5MiIsICJpZCI6ICI1ZWUzNjdjYzRhNjY0YmQzYTYyNmI2MjBkMzFhYzcwYyIsICJwdWJsaWNVUkwiOiAiaHR0cDovLzE3Mi4xOC4xMjQuMTAxOjkyOTIifV0sICJlbmRwb2ludHNfbGlua3MiOiBbXSwgInR5cGUiOiAiaW1hZ2UiLCAibmFtZSI6ICJnbGFuY2UifSwgeyJlbmRwb2ludHMiOiBbeyJhZG1pblVSTCI6ICJodHRwOi8vMTcyLjE4LjEyNC4xMDE6ODAwMC92MSIsICJyZWdpb24iOiAiUmVnaW9uT25lIiwgImludGVybmFsVVJMIjogImh0dHA6Ly8xNzIuMTguMTI0LjEwMTo4MDAwL3YxIiwgImlkIjogIjM3MzMzYmQwNDkxOTQzY2FiNWEyZGM5N2I5YWQzYjE2IiwgInB1YmxpY1VSTCI6ICJodHRwOi8vMTcyLjE4LjEyNC4xMDE6ODAwMC92MSJ9XSwgImVuZHBvaW50c19saW5rcyI6IFtdLCAidHlwZSI6ICJjbG91ZGZvcm1hdGlvbiIsICJuYW1lIjogImhlYXQtY2ZuIn0sIHsiZW5kcG9pbnRzIjogW3siYWRtaW5VUkwiOiAiaHR0cDovLzE3Mi4xOC4xMjQuMTAxOjg3NzYvdjEvMTZlYjc4Y2JiNjg4NDU5YzgzMDhkODk2NzhiY2VmNTAiLCAicmVnaW9uIjogIlJlZ2lvbk9uZSIsICJpbnRlcm5hbFVSTCI6ICJodHRwOi8vMTcyLjE4LjEyNC4xMDE6ODc3Ni92MS8xNmViNzhjYmI2ODg0NTljODMwOGQ4OTY3OGJjZWY1MCIsICJpZCI6ICI4NTgwYjMzOTAxZWU0YTUyOWI0OGMyMzU0ZjFiMWNhZSIsICJwdWJsaWNVUkwiOiAiaHR0cDovLzE3Mi4xOC4xMjQuMTAxOjg3NzYvdjEvMTZlYjc4Y2JiNjg4NDU5YzgzMDhkODk2NzhiY2VmNTAifV0sICJlbmRwb2ludHNfbGlua3MiOiBbXSwgInR5cGUiOiAidm9sdW1lIiwgIm5hbWUiOiAiY2luZGVyIn0sIHsiZW5kcG9pbnRzIjogW3siYWRtaW5VUkwiOiAiaHR0cDovLzE3Mi4xOC4xMjQuMTAxOjg3NzMvc2VydmljZXMvQWRtaW4iLCAicmVnaW9uIjogIlJlZ2lvbk9uZSIsICJpbnRlcm5hbFVSTCI6ICJodHRwOi8vMTcyLjE4LjEyNC4xMDE6ODc3My9zZXJ2aWNlcy9DbG91ZCIsICJpZCI6ICIwYTViOTIyNTNiZjg0NTAwYTA4OWY1N2VkMmYzZDY3NSIsICJwdWJsaWNVUkwiOiAiaHR0cDovLzE3Mi4xOC4xMjQuMTAxOjg3NzMvc2VydmljZXMvQ2xvdWQifV0sICJlbmRwb2ludHNfbGlua3MiOiBbXSwgInR5cGUiOiAiZWMyIiwgIm5hbWUiOiAiZWMyIn0sIHsiZW5kcG9pbnRzIjogW3siYWRtaW5VUkwiOiAiaHR0cDovLzE3Mi4xOC4xMjQuMTAxOjgwMDQvdjEvMTZlYjc4Y2JiNjg4NDU5YzgzMDhkODk2NzhiY2VmNTAiLCAicmVnaW9uIjogIlJlZ2lvbk9uZSIsICJpbnRlcm5hbFVSTCI6ICJodHRwOi8vMTcyLjE4LjEyNC4xMDE6ODAwNC92MS8xNmViNzhjYmI2ODg0NTljODMwOGQ4OTY3OGJjZWY1MCIsICJpZCI6ICJhMjRjMGY1ZmUzMmQ0ZDU5YWEwMTk1Mzg3OGFlMDQwNyIsICJwdWJsaWNVUkwiOiAiaHR0cDovLzE3Mi4xOC4xMjQuMTAxOjgwMDQvdjEvMTZlYjc4Y2JiNjg4NDU5YzgzMDhkODk2NzhiY2VmNTAifV0sICJlbmRwb2ludHNfbGlua3MiOiBbXSwgInR5cGUiOiAib3JjaGVzdHJhdGlvbiIsICJuYW1lIjogImhlYXQifSwgeyJlbmRwb2ludHMiOiBbeyJhZG1pblVSTCI6ICJodHRwOi8vMTcyLjE4LjEyNC4xMDE6MzUzNTcvdjIuMCIsICJyZWdpb24iOiAiUmVnaW9uT25lIiwgImludGVybmFsVVJMIjogImh0dHA6Ly8xNzIuMTguMTI0LjEwMTo1MDAwL3YyLjAiLCAiaWQiOiAiNGM4M2VlYjk3MDA5NDg3M2FiNjg3NjUzNWJlZjgxZWEiLCAicHVibGljVVJMIjogImh0dHA6Ly8xNzIuMTguMTI0LjEwMTo1MDAwL3YyLjAifV0sICJlbmRwb2ludHNfbGlua3MiOiBbXSwgInR5cGUiOiAiaWRlbnRpdHkiLCAibmFtZSI6ICJrZXlzdG9uZSJ9XSwgInVzZXIiOiB7InVzZXJuYW1lIjogImFkbWluIiwgInJvbGVzX2xpbmtzIjogW10sICJpZCI6ICJmMmNkZWM4NTQ2MmQ0N2UzODQ5ZTZmMzE3NGRhMTk4NSIsICJyb2xlcyI6IFt7Im5hbWUiOiAiYWRtaW4ifV0sICJuYW1lIjogImFkbWluIn0sICJtZXRhZGF0YSI6IHsiaXNfYWRtaW4iOiAwLCAicm9sZXMiOiBbIjc4N2JlODdjMGFkMjQ3ODJiNTQ4NWU5NjNhZjllNzllIl19fX0xgf8wgfwCAQEwXDBXMQswCQYDVQQGEwJVUzEOMAwGA1UECBMFVW5zZXQxDjAMBgNVBAcTBVVuc2V0MQ4wDAYDVQQKEwVVbnNldDEYMBYGA1UEAxMPd3d3LmV4YW1wbGUuY29tAgEBMAcGBSsOAwIaMA0GCSqGSIb3DQEBAQUABIGAURfgqd8iZ-UWZTta2pyKzXBXm9nmdzlOY-TN8526LWH4jrU1uuimAZKSjZUCwmnaSvoXHLlP6CSGvNUJWDDu6YFNmDfmatVqFrTij4EFGruExmtUxmhbQOnAyhKqIxHFg2t3VKEB2tVhLGSzoSH1dM2+j0-I0JgOLWIStVFEF5A=", "services": { "activeDirectories": [ { diff --git a/conductor/tools/pip-requires b/conductor/tools/pip-requires index 816ab98..ac910b0 100644 --- a/conductor/tools/pip-requires +++ b/conductor/tools/pip-requires @@ -5,5 +5,6 @@ puka Paste PasteDeploy iso8601>=0.1.4 +python-heatclient http://tarballs.openstack.org/oslo-config/oslo-config-2013.1b4.tar.gz#egg=oslo-config From 042b28f76d4f62e671a4a815450bf2dd2c58c4fc Mon Sep 17 00:00:00 2001 From: Serg Melikyan Date: Tue, 26 Mar 2013 14:44:10 +0400 Subject: [PATCH 28/37] Updated OpenStack Common (Fixed issue with notifier package) --- .../openstack/common/eventlet_backdoor.py | 174 +- .../conductor/openstack/common/exception.py | 284 +-- .../openstack/common/gettextutils.py | 66 +- .../conductor/openstack/common/importutils.py | 134 +- .../conductor/openstack/common/jsonutils.py | 282 +-- conductor/conductor/openstack/common/local.py | 96 +- conductor/conductor/openstack/common/log.py | 1065 +++++------ .../conductor/openstack/common/loopingcall.py | 190 +- .../openstack/common/notifier/__init__.py | 14 + .../openstack/common/notifier/api.py | 182 ++ .../openstack/common/notifier/log_notifier.py | 35 + .../common/notifier/no_op_notifier.py | 19 + .../openstack/common/notifier/rpc_notifier.py | 46 + .../common/notifier/rpc_notifier2.py | 52 + .../common/notifier/test_notifier.py | 22 + .../conductor/openstack/common/service.py | 664 +++---- conductor/conductor/openstack/common/setup.py | 726 ++++---- .../conductor/openstack/common/sslutils.py | 160 +- .../conductor/openstack/common/threadgroup.py | 228 +-- .../conductor/openstack/common/timeutils.py | 372 ++-- .../conductor/openstack/common/uuidutils.py | 78 +- .../conductor/openstack/common/version.py | 188 +- conductor/conductor/openstack/common/wsgi.py | 1594 ++++++++--------- .../conductor/openstack/common/xmlutils.py | 148 +- 24 files changed, 3609 insertions(+), 3210 deletions(-) create mode 100644 conductor/conductor/openstack/common/notifier/__init__.py create mode 100644 conductor/conductor/openstack/common/notifier/api.py create mode 100644 conductor/conductor/openstack/common/notifier/log_notifier.py create mode 100644 conductor/conductor/openstack/common/notifier/no_op_notifier.py create mode 100644 conductor/conductor/openstack/common/notifier/rpc_notifier.py create mode 100644 conductor/conductor/openstack/common/notifier/rpc_notifier2.py create mode 100644 conductor/conductor/openstack/common/notifier/test_notifier.py diff --git a/conductor/conductor/openstack/common/eventlet_backdoor.py b/conductor/conductor/openstack/common/eventlet_backdoor.py index 7605e26..c0ad460 100644 --- a/conductor/conductor/openstack/common/eventlet_backdoor.py +++ b/conductor/conductor/openstack/common/eventlet_backdoor.py @@ -1,87 +1,87 @@ -# vim: tabstop=4 shiftwidth=4 softtabstop=4 - -# Copyright (c) 2012 OpenStack Foundation. -# Administrator of the National Aeronautics and Space Administration. -# All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -import gc -import pprint -import sys -import traceback - -import eventlet -import eventlet.backdoor -import greenlet -from oslo.config import cfg - -eventlet_backdoor_opts = [ - cfg.IntOpt('backdoor_port', - default=None, - help='port for eventlet backdoor to listen') -] - -CONF = cfg.CONF -CONF.register_opts(eventlet_backdoor_opts) - - -def _dont_use_this(): - print "Don't use this, just disconnect instead" - - -def _find_objects(t): - return filter(lambda o: isinstance(o, t), gc.get_objects()) - - -def _print_greenthreads(): - for i, gt in enumerate(_find_objects(greenlet.greenlet)): - print i, gt - traceback.print_stack(gt.gr_frame) - print - - -def _print_nativethreads(): - for threadId, stack in sys._current_frames().items(): - print threadId - traceback.print_stack(stack) - print - - -def initialize_if_enabled(): - backdoor_locals = { - 'exit': _dont_use_this, # So we don't exit the entire process - 'quit': _dont_use_this, # So we don't exit the entire process - 'fo': _find_objects, - 'pgt': _print_greenthreads, - 'pnt': _print_nativethreads, - } - - if CONF.backdoor_port is None: - return None - - # NOTE(johannes): The standard sys.displayhook will print the value of - # the last expression and set it to __builtin__._, which overwrites - # the __builtin__._ that gettext sets. Let's switch to using pprint - # since it won't interact poorly with gettext, and it's easier to - # read the output too. - def displayhook(val): - if val is not None: - pprint.pprint(val) - sys.displayhook = displayhook - - sock = eventlet.listen(('localhost', CONF.backdoor_port)) - port = sock.getsockname()[1] - eventlet.spawn_n(eventlet.backdoor.backdoor_server, sock, - locals=backdoor_locals) - return port +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright (c) 2012 OpenStack Foundation. +# Administrator of the National Aeronautics and Space Administration. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import gc +import pprint +import sys +import traceback + +import eventlet +import eventlet.backdoor +import greenlet +from oslo.config import cfg + +eventlet_backdoor_opts = [ + cfg.IntOpt('backdoor_port', + default=None, + help='port for eventlet backdoor to listen') +] + +CONF = cfg.CONF +CONF.register_opts(eventlet_backdoor_opts) + + +def _dont_use_this(): + print "Don't use this, just disconnect instead" + + +def _find_objects(t): + return filter(lambda o: isinstance(o, t), gc.get_objects()) + + +def _print_greenthreads(): + for i, gt in enumerate(_find_objects(greenlet.greenlet)): + print i, gt + traceback.print_stack(gt.gr_frame) + print + + +def _print_nativethreads(): + for threadId, stack in sys._current_frames().items(): + print threadId + traceback.print_stack(stack) + print + + +def initialize_if_enabled(): + backdoor_locals = { + 'exit': _dont_use_this, # So we don't exit the entire process + 'quit': _dont_use_this, # So we don't exit the entire process + 'fo': _find_objects, + 'pgt': _print_greenthreads, + 'pnt': _print_nativethreads, + } + + if CONF.backdoor_port is None: + return None + + # NOTE(johannes): The standard sys.displayhook will print the value of + # the last expression and set it to __builtin__._, which overwrites + # the __builtin__._ that gettext sets. Let's switch to using pprint + # since it won't interact poorly with gettext, and it's easier to + # read the output too. + def displayhook(val): + if val is not None: + pprint.pprint(val) + sys.displayhook = displayhook + + sock = eventlet.listen(('localhost', CONF.backdoor_port)) + port = sock.getsockname()[1] + eventlet.spawn_n(eventlet.backdoor.backdoor_server, sock, + locals=backdoor_locals) + return port diff --git a/conductor/conductor/openstack/common/exception.py b/conductor/conductor/openstack/common/exception.py index beabfcd..5890c58 100644 --- a/conductor/conductor/openstack/common/exception.py +++ b/conductor/conductor/openstack/common/exception.py @@ -1,142 +1,142 @@ -# vim: tabstop=4 shiftwidth=4 softtabstop=4 - -# Copyright 2011 OpenStack Foundation. -# All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -""" -Exceptions common to OpenStack projects -""" - -import logging - -from conductor.openstack.common.gettextutils import _ - -_FATAL_EXCEPTION_FORMAT_ERRORS = False - - -class Error(Exception): - def __init__(self, message=None): - super(Error, self).__init__(message) - - -class ApiError(Error): - def __init__(self, message='Unknown', code='Unknown'): - self.message = message - self.code = code - super(ApiError, self).__init__('%s: %s' % (code, message)) - - -class NotFound(Error): - pass - - -class UnknownScheme(Error): - - msg = "Unknown scheme '%s' found in URI" - - def __init__(self, scheme): - msg = self.__class__.msg % scheme - super(UnknownScheme, self).__init__(msg) - - -class BadStoreUri(Error): - - msg = "The Store URI %s was malformed. Reason: %s" - - def __init__(self, uri, reason): - msg = self.__class__.msg % (uri, reason) - super(BadStoreUri, self).__init__(msg) - - -class Duplicate(Error): - pass - - -class NotAuthorized(Error): - pass - - -class NotEmpty(Error): - pass - - -class Invalid(Error): - pass - - -class BadInputError(Exception): - """Error resulting from a client sending bad input to a server""" - pass - - -class MissingArgumentError(Error): - pass - - -class DatabaseMigrationError(Error): - pass - - -class ClientConnectionError(Exception): - """Error resulting from a client connecting to a server""" - pass - - -def wrap_exception(f): - def _wrap(*args, **kw): - try: - return f(*args, **kw) - except Exception, e: - if not isinstance(e, Error): - #exc_type, exc_value, exc_traceback = sys.exc_info() - logging.exception(_('Uncaught exception')) - #logging.error(traceback.extract_stack(exc_traceback)) - raise Error(str(e)) - raise - _wrap.func_name = f.func_name - return _wrap - - -class OpenstackException(Exception): - """ - Base Exception - - To correctly use this class, inherit from it and define - a 'message' property. That message will get printf'd - with the keyword arguments provided to the constructor. - """ - message = "An unknown exception occurred" - - def __init__(self, **kwargs): - try: - self._error_string = self.message % kwargs - - except Exception as e: - if _FATAL_EXCEPTION_FORMAT_ERRORS: - raise e - else: - # at least get the core message out if something happened - self._error_string = self.message - - def __str__(self): - return self._error_string - - -class MalformedRequestBody(OpenstackException): - message = "Malformed message body: %(reason)s" - - -class InvalidContentType(OpenstackException): - message = "Invalid content type %(content_type)s" +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright 2011 OpenStack Foundation. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +""" +Exceptions common to OpenStack projects +""" + +import logging + +from conductor.openstack.common.gettextutils import _ + +_FATAL_EXCEPTION_FORMAT_ERRORS = False + + +class Error(Exception): + def __init__(self, message=None): + super(Error, self).__init__(message) + + +class ApiError(Error): + def __init__(self, message='Unknown', code='Unknown'): + self.message = message + self.code = code + super(ApiError, self).__init__('%s: %s' % (code, message)) + + +class NotFound(Error): + pass + + +class UnknownScheme(Error): + + msg = "Unknown scheme '%s' found in URI" + + def __init__(self, scheme): + msg = self.__class__.msg % scheme + super(UnknownScheme, self).__init__(msg) + + +class BadStoreUri(Error): + + msg = "The Store URI %s was malformed. Reason: %s" + + def __init__(self, uri, reason): + msg = self.__class__.msg % (uri, reason) + super(BadStoreUri, self).__init__(msg) + + +class Duplicate(Error): + pass + + +class NotAuthorized(Error): + pass + + +class NotEmpty(Error): + pass + + +class Invalid(Error): + pass + + +class BadInputError(Exception): + """Error resulting from a client sending bad input to a server""" + pass + + +class MissingArgumentError(Error): + pass + + +class DatabaseMigrationError(Error): + pass + + +class ClientConnectionError(Exception): + """Error resulting from a client connecting to a server""" + pass + + +def wrap_exception(f): + def _wrap(*args, **kw): + try: + return f(*args, **kw) + except Exception, e: + if not isinstance(e, Error): + #exc_type, exc_value, exc_traceback = sys.exc_info() + logging.exception(_('Uncaught exception')) + #logging.error(traceback.extract_stack(exc_traceback)) + raise Error(str(e)) + raise + _wrap.func_name = f.func_name + return _wrap + + +class OpenstackException(Exception): + """ + Base Exception + + To correctly use this class, inherit from it and define + a 'message' property. That message will get printf'd + with the keyword arguments provided to the constructor. + """ + message = "An unknown exception occurred" + + def __init__(self, **kwargs): + try: + self._error_string = self.message % kwargs + + except Exception as e: + if _FATAL_EXCEPTION_FORMAT_ERRORS: + raise e + else: + # at least get the core message out if something happened + self._error_string = self.message + + def __str__(self): + return self._error_string + + +class MalformedRequestBody(OpenstackException): + message = "Malformed message body: %(reason)s" + + +class InvalidContentType(OpenstackException): + message = "Invalid content type %(content_type)s" diff --git a/conductor/conductor/openstack/common/gettextutils.py b/conductor/conductor/openstack/common/gettextutils.py index 000d23b..3a81206 100644 --- a/conductor/conductor/openstack/common/gettextutils.py +++ b/conductor/conductor/openstack/common/gettextutils.py @@ -1,33 +1,33 @@ -# vim: tabstop=4 shiftwidth=4 softtabstop=4 - -# Copyright 2012 Red Hat, Inc. -# All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -""" -gettext for openstack-common modules. - -Usual usage in an openstack.common module: - - from conductor.openstack.common.gettextutils import _ -""" - -import gettext - - -t = gettext.translation('openstack-common', 'locale', fallback=True) - - -def _(msg): - return t.ugettext(msg) +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright 2012 Red Hat, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +""" +gettext for openstack-common modules. + +Usual usage in an openstack.common module: + + from conductor.openstack.common.gettextutils import _ +""" + +import gettext + + +t = gettext.translation('conductor', 'locale', fallback=True) + + +def _(msg): + return t.ugettext(msg) diff --git a/conductor/conductor/openstack/common/importutils.py b/conductor/conductor/openstack/common/importutils.py index 6862ca8..3bd277f 100644 --- a/conductor/conductor/openstack/common/importutils.py +++ b/conductor/conductor/openstack/common/importutils.py @@ -1,67 +1,67 @@ -# vim: tabstop=4 shiftwidth=4 softtabstop=4 - -# Copyright 2011 OpenStack Foundation. -# All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -""" -Import related utilities and helper functions. -""" - -import sys -import traceback - - -def import_class(import_str): - """Returns a class from a string including module and class""" - mod_str, _sep, class_str = import_str.rpartition('.') - try: - __import__(mod_str) - return getattr(sys.modules[mod_str], class_str) - except (ValueError, AttributeError): - raise ImportError('Class %s cannot be found (%s)' % - (class_str, - traceback.format_exception(*sys.exc_info()))) - - -def import_object(import_str, *args, **kwargs): - """Import a class and return an instance of it.""" - return import_class(import_str)(*args, **kwargs) - - -def import_object_ns(name_space, import_str, *args, **kwargs): - """ - Import a class and return an instance of it, first by trying - to find the class in a default namespace, then failing back to - a full path if not found in the default namespace. - """ - import_value = "%s.%s" % (name_space, import_str) - try: - return import_class(import_value)(*args, **kwargs) - except ImportError: - return import_class(import_str)(*args, **kwargs) - - -def import_module(import_str): - """Import a module.""" - __import__(import_str) - return sys.modules[import_str] - - -def try_import(import_str, default=None): - """Try to import a module and if it fails return default.""" - try: - return import_module(import_str) - except ImportError: - return default +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright 2011 OpenStack Foundation. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +""" +Import related utilities and helper functions. +""" + +import sys +import traceback + + +def import_class(import_str): + """Returns a class from a string including module and class""" + mod_str, _sep, class_str = import_str.rpartition('.') + try: + __import__(mod_str) + return getattr(sys.modules[mod_str], class_str) + except (ValueError, AttributeError): + raise ImportError('Class %s cannot be found (%s)' % + (class_str, + traceback.format_exception(*sys.exc_info()))) + + +def import_object(import_str, *args, **kwargs): + """Import a class and return an instance of it.""" + return import_class(import_str)(*args, **kwargs) + + +def import_object_ns(name_space, import_str, *args, **kwargs): + """ + Import a class and return an instance of it, first by trying + to find the class in a default namespace, then failing back to + a full path if not found in the default namespace. + """ + import_value = "%s.%s" % (name_space, import_str) + try: + return import_class(import_value)(*args, **kwargs) + except ImportError: + return import_class(import_str)(*args, **kwargs) + + +def import_module(import_str): + """Import a module.""" + __import__(import_str) + return sys.modules[import_str] + + +def try_import(import_str, default=None): + """Try to import a module and if it fails return default.""" + try: + return import_module(import_str) + except ImportError: + return default diff --git a/conductor/conductor/openstack/common/jsonutils.py b/conductor/conductor/openstack/common/jsonutils.py index c281b6a..4d3ddd0 100644 --- a/conductor/conductor/openstack/common/jsonutils.py +++ b/conductor/conductor/openstack/common/jsonutils.py @@ -1,141 +1,141 @@ -# vim: tabstop=4 shiftwidth=4 softtabstop=4 - -# Copyright 2010 United States Government as represented by the -# Administrator of the National Aeronautics and Space Administration. -# Copyright 2011 Justin Santa Barbara -# All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -''' -JSON related utilities. - -This module provides a few things: - - 1) A handy function for getting an object down to something that can be - JSON serialized. See to_primitive(). - - 2) Wrappers around loads() and dumps(). The dumps() wrapper will - automatically use to_primitive() for you if needed. - - 3) This sets up anyjson to use the loads() and dumps() wrappers if anyjson - is available. -''' - - -import datetime -import functools -import inspect -import itertools -import json -import xmlrpclib - -from conductor.openstack.common import timeutils - - -def to_primitive(value, convert_instances=False, convert_datetime=True, - level=0, max_depth=3): - """Convert a complex object into primitives. - - Handy for JSON serialization. We can optionally handle instances, - but since this is a recursive function, we could have cyclical - data structures. - - To handle cyclical data structures we could track the actual objects - visited in a set, but not all objects are hashable. Instead we just - track the depth of the object inspections and don't go too deep. - - Therefore, convert_instances=True is lossy ... be aware. - - """ - nasty = [inspect.ismodule, inspect.isclass, inspect.ismethod, - inspect.isfunction, inspect.isgeneratorfunction, - inspect.isgenerator, inspect.istraceback, inspect.isframe, - inspect.iscode, inspect.isbuiltin, inspect.isroutine, - inspect.isabstract] - for test in nasty: - if test(value): - return unicode(value) - - # value of itertools.count doesn't get caught by inspects - # above and results in infinite loop when list(value) is called. - if type(value) == itertools.count: - return unicode(value) - - # FIXME(vish): Workaround for LP bug 852095. Without this workaround, - # tests that raise an exception in a mocked method that - # has a @wrap_exception with a notifier will fail. If - # we up the dependency to 0.5.4 (when it is released) we - # can remove this workaround. - if getattr(value, '__module__', None) == 'mox': - return 'mock' - - if level > max_depth: - return '?' - - # The try block may not be necessary after the class check above, - # but just in case ... - try: - recursive = functools.partial(to_primitive, - convert_instances=convert_instances, - convert_datetime=convert_datetime, - level=level, - max_depth=max_depth) - # It's not clear why xmlrpclib created their own DateTime type, but - # for our purposes, make it a datetime type which is explicitly - # handled - if isinstance(value, xmlrpclib.DateTime): - value = datetime.datetime(*tuple(value.timetuple())[:6]) - - if isinstance(value, (list, tuple)): - return [recursive(v) for v in value] - elif isinstance(value, dict): - return dict((k, recursive(v)) for k, v in value.iteritems()) - elif convert_datetime and isinstance(value, datetime.datetime): - return timeutils.strtime(value) - elif hasattr(value, 'iteritems'): - return recursive(dict(value.iteritems()), level=level + 1) - elif hasattr(value, '__iter__'): - return recursive(list(value)) - elif convert_instances and hasattr(value, '__dict__'): - # Likely an instance of something. Watch for cycles. - # Ignore class member vars. - return recursive(value.__dict__, level=level + 1) - else: - return value - except TypeError: - # Class objects are tricky since they may define something like - # __iter__ defined but it isn't callable as list(). - return unicode(value) - - -def dumps(value, default=to_primitive, **kwargs): - return json.dumps(value, default=default, **kwargs) - - -def loads(s): - return json.loads(s) - - -def load(s): - return json.load(s) - - -try: - import anyjson -except ImportError: - pass -else: - anyjson._modules.append((__name__, 'dumps', TypeError, - 'loads', ValueError, 'load')) - anyjson.force_implementation(__name__) +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright 2010 United States Government as represented by the +# Administrator of the National Aeronautics and Space Administration. +# Copyright 2011 Justin Santa Barbara +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +''' +JSON related utilities. + +This module provides a few things: + + 1) A handy function for getting an object down to something that can be + JSON serialized. See to_primitive(). + + 2) Wrappers around loads() and dumps(). The dumps() wrapper will + automatically use to_primitive() for you if needed. + + 3) This sets up anyjson to use the loads() and dumps() wrappers if anyjson + is available. +''' + + +import datetime +import functools +import inspect +import itertools +import json +import xmlrpclib + +from conductor.openstack.common import timeutils + + +def to_primitive(value, convert_instances=False, convert_datetime=True, + level=0, max_depth=3): + """Convert a complex object into primitives. + + Handy for JSON serialization. We can optionally handle instances, + but since this is a recursive function, we could have cyclical + data structures. + + To handle cyclical data structures we could track the actual objects + visited in a set, but not all objects are hashable. Instead we just + track the depth of the object inspections and don't go too deep. + + Therefore, convert_instances=True is lossy ... be aware. + + """ + nasty = [inspect.ismodule, inspect.isclass, inspect.ismethod, + inspect.isfunction, inspect.isgeneratorfunction, + inspect.isgenerator, inspect.istraceback, inspect.isframe, + inspect.iscode, inspect.isbuiltin, inspect.isroutine, + inspect.isabstract] + for test in nasty: + if test(value): + return unicode(value) + + # value of itertools.count doesn't get caught by inspects + # above and results in infinite loop when list(value) is called. + if type(value) == itertools.count: + return unicode(value) + + # FIXME(vish): Workaround for LP bug 852095. Without this workaround, + # tests that raise an exception in a mocked method that + # has a @wrap_exception with a notifier will fail. If + # we up the dependency to 0.5.4 (when it is released) we + # can remove this workaround. + if getattr(value, '__module__', None) == 'mox': + return 'mock' + + if level > max_depth: + return '?' + + # The try block may not be necessary after the class check above, + # but just in case ... + try: + recursive = functools.partial(to_primitive, + convert_instances=convert_instances, + convert_datetime=convert_datetime, + level=level, + max_depth=max_depth) + # It's not clear why xmlrpclib created their own DateTime type, but + # for our purposes, make it a datetime type which is explicitly + # handled + if isinstance(value, xmlrpclib.DateTime): + value = datetime.datetime(*tuple(value.timetuple())[:6]) + + if isinstance(value, (list, tuple)): + return [recursive(v) for v in value] + elif isinstance(value, dict): + return dict((k, recursive(v)) for k, v in value.iteritems()) + elif convert_datetime and isinstance(value, datetime.datetime): + return timeutils.strtime(value) + elif hasattr(value, 'iteritems'): + return recursive(dict(value.iteritems()), level=level + 1) + elif hasattr(value, '__iter__'): + return recursive(list(value)) + elif convert_instances and hasattr(value, '__dict__'): + # Likely an instance of something. Watch for cycles. + # Ignore class member vars. + return recursive(value.__dict__, level=level + 1) + else: + return value + except TypeError: + # Class objects are tricky since they may define something like + # __iter__ defined but it isn't callable as list(). + return unicode(value) + + +def dumps(value, default=to_primitive, **kwargs): + return json.dumps(value, default=default, **kwargs) + + +def loads(s): + return json.loads(s) + + +def load(s): + return json.load(s) + + +try: + import anyjson +except ImportError: + pass +else: + anyjson._modules.append((__name__, 'dumps', TypeError, + 'loads', ValueError, 'load')) + anyjson.force_implementation(__name__) diff --git a/conductor/conductor/openstack/common/local.py b/conductor/conductor/openstack/common/local.py index 9b1d22a..f1bfc82 100644 --- a/conductor/conductor/openstack/common/local.py +++ b/conductor/conductor/openstack/common/local.py @@ -1,48 +1,48 @@ -# vim: tabstop=4 shiftwidth=4 softtabstop=4 - -# Copyright 2011 OpenStack Foundation. -# All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -"""Greenthread local storage of variables using weak references""" - -import weakref - -from eventlet import corolocal - - -class WeakLocal(corolocal.local): - def __getattribute__(self, attr): - rval = corolocal.local.__getattribute__(self, attr) - if rval: - # NOTE(mikal): this bit is confusing. What is stored is a weak - # reference, not the value itself. We therefore need to lookup - # the weak reference and return the inner value here. - rval = rval() - return rval - - def __setattr__(self, attr, value): - value = weakref.ref(value) - return corolocal.local.__setattr__(self, attr, value) - - -# NOTE(mikal): the name "store" should be deprecated in the future -store = WeakLocal() - -# A "weak" store uses weak references and allows an object to fall out of scope -# when it falls out of scope in the code that uses the thread local storage. A -# "strong" store will hold a reference to the object so that it never falls out -# of scope. -weak_store = WeakLocal() -strong_store = corolocal.local +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright 2011 OpenStack Foundation. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +"""Greenthread local storage of variables using weak references""" + +import weakref + +from eventlet import corolocal + + +class WeakLocal(corolocal.local): + def __getattribute__(self, attr): + rval = corolocal.local.__getattribute__(self, attr) + if rval: + # NOTE(mikal): this bit is confusing. What is stored is a weak + # reference, not the value itself. We therefore need to lookup + # the weak reference and return the inner value here. + rval = rval() + return rval + + def __setattr__(self, attr, value): + value = weakref.ref(value) + return corolocal.local.__setattr__(self, attr, value) + + +# NOTE(mikal): the name "store" should be deprecated in the future +store = WeakLocal() + +# A "weak" store uses weak references and allows an object to fall out of scope +# when it falls out of scope in the code that uses the thread local storage. A +# "strong" store will hold a reference to the object so that it never falls out +# of scope. +weak_store = WeakLocal() +strong_store = corolocal.local diff --git a/conductor/conductor/openstack/common/log.py b/conductor/conductor/openstack/common/log.py index 31ed37f..d8cd9fa 100644 --- a/conductor/conductor/openstack/common/log.py +++ b/conductor/conductor/openstack/common/log.py @@ -1,522 +1,543 @@ -# vim: tabstop=4 shiftwidth=4 softtabstop=4 - -# Copyright 2011 OpenStack Foundation. -# Copyright 2010 United States Government as represented by the -# Administrator of the National Aeronautics and Space Administration. -# All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -"""Openstack logging handler. - -This module adds to logging functionality by adding the option to specify -a context object when calling the various log methods. If the context object -is not specified, default formatting is used. Additionally, an instance uuid -may be passed as part of the log message, which is intended to make it easier -for admins to find messages related to a specific instance. - -It also allows setting of formatting information through conf. - -""" - -import cStringIO -import inspect -import itertools -import logging -import logging.config -import logging.handlers -import os -import stat -import sys -import traceback - -from oslo.config import cfg - -from conductor.openstack.common.gettextutils import _ -from conductor.openstack.common import jsonutils -from conductor.openstack.common import local -from conductor.openstack.common import notifier - - -_DEFAULT_LOG_FORMAT = "%(asctime)s %(levelname)8s [%(name)s] %(message)s" -_DEFAULT_LOG_DATE_FORMAT = "%Y-%m-%d %H:%M:%S" - -common_cli_opts = [ - cfg.BoolOpt('debug', - short='d', - default=False, - help='Print debugging output (set logging level to ' - 'DEBUG instead of default WARNING level).'), - cfg.BoolOpt('verbose', - short='v', - default=False, - help='Print more verbose output (set logging level to ' - 'INFO instead of default WARNING level).'), -] - -logging_cli_opts = [ - cfg.StrOpt('log-config', - metavar='PATH', - help='If this option is specified, the logging configuration ' - 'file specified is used and overrides any other logging ' - 'options specified. Please see the Python logging module ' - 'documentation for details on logging configuration ' - 'files.'), - cfg.StrOpt('log-format', - default=_DEFAULT_LOG_FORMAT, - metavar='FORMAT', - help='A logging.Formatter log message format string which may ' - 'use any of the available logging.LogRecord attributes. ' - 'Default: %(default)s'), - cfg.StrOpt('log-date-format', - default=_DEFAULT_LOG_DATE_FORMAT, - metavar='DATE_FORMAT', - help='Format string for %%(asctime)s in log records. ' - 'Default: %(default)s'), - cfg.StrOpt('log-file', - metavar='PATH', - deprecated_name='logfile', - help='(Optional) Name of log file to output to. ' - 'If not set, logging will go to stdout.'), - cfg.StrOpt('log-dir', - deprecated_name='logdir', - help='(Optional) The directory to keep log files in ' - '(will be prepended to --log-file)'), - cfg.BoolOpt('use-syslog', - default=False, - help='Use syslog for logging.'), - cfg.StrOpt('syslog-log-facility', - default='LOG_USER', - help='syslog facility to receive log lines') -] - -generic_log_opts = [ - cfg.BoolOpt('use_stderr', - default=True, - help='Log output to standard error'), - cfg.StrOpt('logfile_mode', - default='0644', - help='Default file mode used when creating log files'), -] - -log_opts = [ - cfg.StrOpt('logging_context_format_string', - default='%(asctime)s.%(msecs)03d %(levelname)s %(name)s ' - '[%(request_id)s %(user)s %(tenant)s] %(instance)s' - '%(message)s', - help='format string to use for log messages with context'), - cfg.StrOpt('logging_default_format_string', - default='%(asctime)s.%(msecs)03d %(process)d %(levelname)s ' - '%(name)s [-] %(instance)s%(message)s', - help='format string to use for log messages without context'), - cfg.StrOpt('logging_debug_format_suffix', - default='%(funcName)s %(pathname)s:%(lineno)d', - help='data to append to log format when level is DEBUG'), - cfg.StrOpt('logging_exception_prefix', - default='%(asctime)s.%(msecs)03d %(process)d TRACE %(name)s ' - '%(instance)s', - help='prefix each line of exception output with this format'), - cfg.ListOpt('default_log_levels', - default=[ - 'amqplib=WARN', - 'sqlalchemy=WARN', - 'boto=WARN', - 'suds=INFO', - 'keystone=INFO', - 'eventlet.wsgi.server=WARN' - ], - help='list of logger=LEVEL pairs'), - cfg.BoolOpt('publish_errors', - default=False, - help='publish error events'), - cfg.BoolOpt('fatal_deprecations', - default=False, - help='make deprecations fatal'), - - # NOTE(mikal): there are two options here because sometimes we are handed - # a full instance (and could include more information), and other times we - # are just handed a UUID for the instance. - cfg.StrOpt('instance_format', - default='[instance: %(uuid)s] ', - help='If an instance is passed with the log message, format ' - 'it like this'), - cfg.StrOpt('instance_uuid_format', - default='[instance: %(uuid)s] ', - help='If an instance UUID is passed with the log message, ' - 'format it like this'), -] - -CONF = cfg.CONF -CONF.register_cli_opts(common_cli_opts) -CONF.register_cli_opts(logging_cli_opts) -CONF.register_opts(generic_log_opts) -CONF.register_opts(log_opts) - -# our new audit level -# NOTE(jkoelker) Since we synthesized an audit level, make the logging -# module aware of it so it acts like other levels. -logging.AUDIT = logging.INFO + 1 -logging.addLevelName(logging.AUDIT, 'AUDIT') - - -try: - NullHandler = logging.NullHandler -except AttributeError: # NOTE(jkoelker) NullHandler added in Python 2.7 - class NullHandler(logging.Handler): - def handle(self, record): - pass - - def emit(self, record): - pass - - def createLock(self): - self.lock = None - - -def _dictify_context(context): - if context is None: - return None - if not isinstance(context, dict) and getattr(context, 'to_dict', None): - context = context.to_dict() - return context - - -def _get_binary_name(): - return os.path.basename(inspect.stack()[-1][1]) - - -def _get_log_file_path(binary=None): - logfile = CONF.log_file - logdir = CONF.log_dir - - if logfile and not logdir: - return logfile - - if logfile and logdir: - return os.path.join(logdir, logfile) - - if logdir: - binary = binary or _get_binary_name() - return '%s.log' % (os.path.join(logdir, binary),) - - -class ContextAdapter(logging.LoggerAdapter): - warn = logging.LoggerAdapter.warning - - def __init__(self, logger, project_name, version_string): - self.logger = logger - self.project = project_name - self.version = version_string - - def audit(self, msg, *args, **kwargs): - self.log(logging.AUDIT, msg, *args, **kwargs) - - def deprecated(self, msg, *args, **kwargs): - stdmsg = _("Deprecated: %s") % msg - if CONF.fatal_deprecations: - self.critical(stdmsg, *args, **kwargs) - raise DeprecatedConfig(msg=stdmsg) - else: - self.warn(stdmsg, *args, **kwargs) - - def process(self, msg, kwargs): - if 'extra' not in kwargs: - kwargs['extra'] = {} - extra = kwargs['extra'] - - context = kwargs.pop('context', None) - if not context: - context = getattr(local.store, 'context', None) - if context: - extra.update(_dictify_context(context)) - - instance = kwargs.pop('instance', None) - instance_extra = '' - if instance: - instance_extra = CONF.instance_format % instance - else: - instance_uuid = kwargs.pop('instance_uuid', None) - if instance_uuid: - instance_extra = (CONF.instance_uuid_format - % {'uuid': instance_uuid}) - extra.update({'instance': instance_extra}) - - extra.update({"project": self.project}) - extra.update({"version": self.version}) - extra['extra'] = extra.copy() - return msg, kwargs - - -class JSONFormatter(logging.Formatter): - def __init__(self, fmt=None, datefmt=None): - # NOTE(jkoelker) we ignore the fmt argument, but its still there - # since logging.config.fileConfig passes it. - self.datefmt = datefmt - - def formatException(self, ei, strip_newlines=True): - lines = traceback.format_exception(*ei) - if strip_newlines: - lines = [itertools.ifilter( - lambda x: x, - line.rstrip().splitlines()) for line in lines] - lines = list(itertools.chain(*lines)) - return lines - - def format(self, record): - message = {'message': record.getMessage(), - 'asctime': self.formatTime(record, self.datefmt), - 'name': record.name, - 'msg': record.msg, - 'args': record.args, - 'levelname': record.levelname, - 'levelno': record.levelno, - 'pathname': record.pathname, - 'filename': record.filename, - 'module': record.module, - 'lineno': record.lineno, - 'funcname': record.funcName, - 'created': record.created, - 'msecs': record.msecs, - 'relative_created': record.relativeCreated, - 'thread': record.thread, - 'thread_name': record.threadName, - 'process_name': record.processName, - 'process': record.process, - 'traceback': None} - - if hasattr(record, 'extra'): - message['extra'] = record.extra - - if record.exc_info: - message['traceback'] = self.formatException(record.exc_info) - - return jsonutils.dumps(message) - - -class PublishErrorsHandler(logging.Handler): - def emit(self, record): - if ('conductor.openstack.common.notifier.log_notifier' in - CONF.notification_driver): - return - notifier.api.notify(None, 'error.publisher', - 'error_notification', - notifier.api.ERROR, - dict(error=record.msg)) - - -def _create_logging_excepthook(product_name): - def logging_excepthook(type, value, tb): - extra = {} - if CONF.verbose: - extra['exc_info'] = (type, value, tb) - getLogger(product_name).critical(str(value), **extra) - return logging_excepthook - - -def setup(product_name): - """Setup logging.""" - if CONF.log_config: - logging.config.fileConfig(CONF.log_config) - else: - _setup_logging_from_conf() - sys.excepthook = _create_logging_excepthook(product_name) - - -def set_defaults(logging_context_format_string): - cfg.set_defaults(log_opts, - logging_context_format_string= - logging_context_format_string) - - -def _find_facility_from_conf(): - facility_names = logging.handlers.SysLogHandler.facility_names - facility = getattr(logging.handlers.SysLogHandler, - CONF.syslog_log_facility, - None) - - if facility is None and CONF.syslog_log_facility in facility_names: - facility = facility_names.get(CONF.syslog_log_facility) - - if facility is None: - valid_facilities = facility_names.keys() - consts = ['LOG_AUTH', 'LOG_AUTHPRIV', 'LOG_CRON', 'LOG_DAEMON', - 'LOG_FTP', 'LOG_KERN', 'LOG_LPR', 'LOG_MAIL', 'LOG_NEWS', - 'LOG_AUTH', 'LOG_SYSLOG', 'LOG_USER', 'LOG_UUCP', - 'LOG_LOCAL0', 'LOG_LOCAL1', 'LOG_LOCAL2', 'LOG_LOCAL3', - 'LOG_LOCAL4', 'LOG_LOCAL5', 'LOG_LOCAL6', 'LOG_LOCAL7'] - valid_facilities.extend(consts) - raise TypeError(_('syslog facility must be one of: %s') % - ', '.join("'%s'" % fac - for fac in valid_facilities)) - - return facility - - -def _setup_logging_from_conf(): - log_root = getLogger(None).logger - for handler in log_root.handlers: - log_root.removeHandler(handler) - - if CONF.use_syslog: - facility = _find_facility_from_conf() - syslog = logging.handlers.SysLogHandler(address='/dev/log', - facility=facility) - log_root.addHandler(syslog) - - logpath = _get_log_file_path() - if logpath: - filelog = logging.handlers.WatchedFileHandler(logpath) - log_root.addHandler(filelog) - - mode = int(CONF.logfile_mode, 8) - st = os.stat(logpath) - if st.st_mode != (stat.S_IFREG | mode): - os.chmod(logpath, mode) - - if CONF.use_stderr: - streamlog = ColorHandler() - log_root.addHandler(streamlog) - - elif not CONF.log_file: - # pass sys.stdout as a positional argument - # python2.6 calls the argument strm, in 2.7 it's stream - streamlog = logging.StreamHandler(sys.stdout) - log_root.addHandler(streamlog) - - if CONF.publish_errors: - log_root.addHandler(PublishErrorsHandler(logging.ERROR)) - - for handler in log_root.handlers: - datefmt = CONF.log_date_format - if CONF.log_format: - handler.setFormatter(logging.Formatter(fmt=CONF.log_format, - datefmt=datefmt)) - else: - handler.setFormatter(LegacyFormatter(datefmt=datefmt)) - - if CONF.debug: - log_root.setLevel(logging.DEBUG) - elif CONF.verbose: - log_root.setLevel(logging.INFO) - else: - log_root.setLevel(logging.WARNING) - - level = logging.NOTSET - for pair in CONF.default_log_levels: - mod, _sep, level_name = pair.partition('=') - level = logging.getLevelName(level_name) - logger = logging.getLogger(mod) - logger.setLevel(level) - for handler in log_root.handlers: - logger.addHandler(handler) - -_loggers = {} - - -def getLogger(name='unknown', version='unknown'): - if name not in _loggers: - _loggers[name] = ContextAdapter(logging.getLogger(name), - name, - version) - return _loggers[name] - - -class WritableLogger(object): - """A thin wrapper that responds to `write` and logs.""" - - def __init__(self, logger, level=logging.INFO): - self.logger = logger - self.level = level - - def write(self, msg): - self.logger.log(self.level, msg) - - -class LegacyFormatter(logging.Formatter): - """A context.RequestContext aware formatter configured through flags. - - The flags used to set format strings are: logging_context_format_string - and logging_default_format_string. You can also specify - logging_debug_format_suffix to append extra formatting if the log level is - debug. - - For information about what variables are available for the formatter see: - http://docs.python.org/library/logging.html#formatter - - """ - - def format(self, record): - """Uses contextstring if request_id is set, otherwise default.""" - # NOTE(sdague): default the fancier formating params - # to an empty string so we don't throw an exception if - # they get used - for key in ('instance', 'color'): - if key not in record.__dict__: - record.__dict__[key] = '' - - if record.__dict__.get('request_id', None): - self._fmt = CONF.logging_context_format_string - else: - self._fmt = CONF.logging_default_format_string - - if (record.levelno == logging.DEBUG and - CONF.logging_debug_format_suffix): - self._fmt += " " + CONF.logging_debug_format_suffix - - # Cache this on the record, Logger will respect our formated copy - if record.exc_info: - record.exc_text = self.formatException(record.exc_info, record) - return logging.Formatter.format(self, record) - - def formatException(self, exc_info, record=None): - """Format exception output with CONF.logging_exception_prefix.""" - if not record: - return logging.Formatter.formatException(self, exc_info) - - stringbuffer = cStringIO.StringIO() - traceback.print_exception(exc_info[0], exc_info[1], exc_info[2], - None, stringbuffer) - lines = stringbuffer.getvalue().split('\n') - stringbuffer.close() - - if CONF.logging_exception_prefix.find('%(asctime)') != -1: - record.asctime = self.formatTime(record, self.datefmt) - - formatted_lines = [] - for line in lines: - pl = CONF.logging_exception_prefix % record.__dict__ - fl = '%s%s' % (pl, line) - formatted_lines.append(fl) - return '\n'.join(formatted_lines) - - -class ColorHandler(logging.StreamHandler): - LEVEL_COLORS = { - logging.DEBUG: '\033[00;32m', # GREEN - logging.INFO: '\033[00;36m', # CYAN - logging.AUDIT: '\033[01;36m', # BOLD CYAN - logging.WARN: '\033[01;33m', # BOLD YELLOW - logging.ERROR: '\033[01;31m', # BOLD RED - logging.CRITICAL: '\033[01;31m', # BOLD RED - } - - def format(self, record): - record.color = self.LEVEL_COLORS[record.levelno] - return logging.StreamHandler.format(self, record) - - -class DeprecatedConfig(Exception): - message = _("Fatal call to deprecated config: %(msg)s") - - def __init__(self, msg): - super(Exception, self).__init__(self.message % dict(msg=msg)) +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright 2011 OpenStack Foundation. +# Copyright 2010 United States Government as represented by the +# Administrator of the National Aeronautics and Space Administration. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +"""Openstack logging handler. + +This module adds to logging functionality by adding the option to specify +a context object when calling the various log methods. If the context object +is not specified, default formatting is used. Additionally, an instance uuid +may be passed as part of the log message, which is intended to make it easier +for admins to find messages related to a specific instance. + +It also allows setting of formatting information through conf. + +""" + +import ConfigParser +import cStringIO +import inspect +import itertools +import logging +import logging.config +import logging.handlers +import os +import stat +import sys +import traceback + +from oslo.config import cfg + +from conductor.openstack.common.gettextutils import _ +from conductor.openstack.common import jsonutils +from conductor.openstack.common import local +from conductor.openstack.common import notifier + + +_DEFAULT_LOG_FORMAT = "%(asctime)s %(levelname)8s [%(name)s] %(message)s" +_DEFAULT_LOG_DATE_FORMAT = "%Y-%m-%d %H:%M:%S" + +common_cli_opts = [ + cfg.BoolOpt('debug', + short='d', + default=False, + help='Print debugging output (set logging level to ' + 'DEBUG instead of default WARNING level).'), + cfg.BoolOpt('verbose', + short='v', + default=False, + help='Print more verbose output (set logging level to ' + 'INFO instead of default WARNING level).'), +] + +logging_cli_opts = [ + cfg.StrOpt('log-config', + metavar='PATH', + help='If this option is specified, the logging configuration ' + 'file specified is used and overrides any other logging ' + 'options specified. Please see the Python logging module ' + 'documentation for details on logging configuration ' + 'files.'), + cfg.StrOpt('log-format', + default=_DEFAULT_LOG_FORMAT, + metavar='FORMAT', + help='A logging.Formatter log message format string which may ' + 'use any of the available logging.LogRecord attributes. ' + 'Default: %(default)s'), + cfg.StrOpt('log-date-format', + default=_DEFAULT_LOG_DATE_FORMAT, + metavar='DATE_FORMAT', + help='Format string for %%(asctime)s in log records. ' + 'Default: %(default)s'), + cfg.StrOpt('log-file', + metavar='PATH', + deprecated_name='logfile', + help='(Optional) Name of log file to output to. ' + 'If no default is set, logging will go to stdout.'), + cfg.StrOpt('log-dir', + deprecated_name='logdir', + help='(Optional) The base directory used for relative ' + '--log-file paths'), + cfg.BoolOpt('use-syslog', + default=False, + help='Use syslog for logging.'), + cfg.StrOpt('syslog-log-facility', + default='LOG_USER', + help='syslog facility to receive log lines') +] + +generic_log_opts = [ + cfg.BoolOpt('use_stderr', + default=True, + help='Log output to standard error'), + cfg.StrOpt('logfile_mode', + default='0644', + help='Default file mode used when creating log files'), +] + +log_opts = [ + cfg.StrOpt('logging_context_format_string', + default='%(asctime)s.%(msecs)03d %(levelname)s %(name)s ' + '[%(request_id)s %(user)s %(tenant)s] %(instance)s' + '%(message)s', + help='format string to use for log messages with context'), + cfg.StrOpt('logging_default_format_string', + default='%(asctime)s.%(msecs)03d %(process)d %(levelname)s ' + '%(name)s [-] %(instance)s%(message)s', + help='format string to use for log messages without context'), + cfg.StrOpt('logging_debug_format_suffix', + default='%(funcName)s %(pathname)s:%(lineno)d', + help='data to append to log format when level is DEBUG'), + cfg.StrOpt('logging_exception_prefix', + default='%(asctime)s.%(msecs)03d %(process)d TRACE %(name)s ' + '%(instance)s', + help='prefix each line of exception output with this format'), + cfg.ListOpt('default_log_levels', + default=[ + 'amqplib=WARN', + 'sqlalchemy=WARN', + 'boto=WARN', + 'suds=INFO', + 'keystone=INFO', + 'eventlet.wsgi.server=WARN' + ], + help='list of logger=LEVEL pairs'), + cfg.BoolOpt('publish_errors', + default=False, + help='publish error events'), + cfg.BoolOpt('fatal_deprecations', + default=False, + help='make deprecations fatal'), + + # NOTE(mikal): there are two options here because sometimes we are handed + # a full instance (and could include more information), and other times we + # are just handed a UUID for the instance. + cfg.StrOpt('instance_format', + default='[instance: %(uuid)s] ', + help='If an instance is passed with the log message, format ' + 'it like this'), + cfg.StrOpt('instance_uuid_format', + default='[instance: %(uuid)s] ', + help='If an instance UUID is passed with the log message, ' + 'format it like this'), +] + +CONF = cfg.CONF +CONF.register_cli_opts(common_cli_opts) +CONF.register_cli_opts(logging_cli_opts) +CONF.register_opts(generic_log_opts) +CONF.register_opts(log_opts) + +# our new audit level +# NOTE(jkoelker) Since we synthesized an audit level, make the logging +# module aware of it so it acts like other levels. +logging.AUDIT = logging.INFO + 1 +logging.addLevelName(logging.AUDIT, 'AUDIT') + + +try: + NullHandler = logging.NullHandler +except AttributeError: # NOTE(jkoelker) NullHandler added in Python 2.7 + class NullHandler(logging.Handler): + def handle(self, record): + pass + + def emit(self, record): + pass + + def createLock(self): + self.lock = None + + +def _dictify_context(context): + if context is None: + return None + if not isinstance(context, dict) and getattr(context, 'to_dict', None): + context = context.to_dict() + return context + + +def _get_binary_name(): + return os.path.basename(inspect.stack()[-1][1]) + + +def _get_log_file_path(binary=None): + logfile = CONF.log_file + logdir = CONF.log_dir + + if logfile and not logdir: + return logfile + + if logfile and logdir: + return os.path.join(logdir, logfile) + + if logdir: + binary = binary or _get_binary_name() + return '%s.log' % (os.path.join(logdir, binary),) + + +class ContextAdapter(logging.LoggerAdapter): + warn = logging.LoggerAdapter.warning + + def __init__(self, logger, project_name, version_string): + self.logger = logger + self.project = project_name + self.version = version_string + + def audit(self, msg, *args, **kwargs): + self.log(logging.AUDIT, msg, *args, **kwargs) + + def deprecated(self, msg, *args, **kwargs): + stdmsg = _("Deprecated: %s") % msg + if CONF.fatal_deprecations: + self.critical(stdmsg, *args, **kwargs) + raise DeprecatedConfig(msg=stdmsg) + else: + self.warn(stdmsg, *args, **kwargs) + + def process(self, msg, kwargs): + if 'extra' not in kwargs: + kwargs['extra'] = {} + extra = kwargs['extra'] + + context = kwargs.pop('context', None) + if not context: + context = getattr(local.store, 'context', None) + if context: + extra.update(_dictify_context(context)) + + instance = kwargs.pop('instance', None) + instance_extra = '' + if instance: + instance_extra = CONF.instance_format % instance + else: + instance_uuid = kwargs.pop('instance_uuid', None) + if instance_uuid: + instance_extra = (CONF.instance_uuid_format + % {'uuid': instance_uuid}) + extra.update({'instance': instance_extra}) + + extra.update({"project": self.project}) + extra.update({"version": self.version}) + extra['extra'] = extra.copy() + return msg, kwargs + + +class JSONFormatter(logging.Formatter): + def __init__(self, fmt=None, datefmt=None): + # NOTE(jkoelker) we ignore the fmt argument, but its still there + # since logging.config.fileConfig passes it. + self.datefmt = datefmt + + def formatException(self, ei, strip_newlines=True): + lines = traceback.format_exception(*ei) + if strip_newlines: + lines = [itertools.ifilter( + lambda x: x, + line.rstrip().splitlines()) for line in lines] + lines = list(itertools.chain(*lines)) + return lines + + def format(self, record): + message = {'message': record.getMessage(), + 'asctime': self.formatTime(record, self.datefmt), + 'name': record.name, + 'msg': record.msg, + 'args': record.args, + 'levelname': record.levelname, + 'levelno': record.levelno, + 'pathname': record.pathname, + 'filename': record.filename, + 'module': record.module, + 'lineno': record.lineno, + 'funcname': record.funcName, + 'created': record.created, + 'msecs': record.msecs, + 'relative_created': record.relativeCreated, + 'thread': record.thread, + 'thread_name': record.threadName, + 'process_name': record.processName, + 'process': record.process, + 'traceback': None} + + if hasattr(record, 'extra'): + message['extra'] = record.extra + + if record.exc_info: + message['traceback'] = self.formatException(record.exc_info) + + return jsonutils.dumps(message) + + +class PublishErrorsHandler(logging.Handler): + def emit(self, record): + if ('conductor.openstack.common.notifier.log_notifier' in + CONF.notification_driver): + return + notifier.api.notify(None, 'error.publisher', + 'error_notification', + notifier.api.ERROR, + dict(error=record.msg)) + + +def _create_logging_excepthook(product_name): + def logging_excepthook(type, value, tb): + extra = {} + if CONF.verbose: + extra['exc_info'] = (type, value, tb) + getLogger(product_name).critical(str(value), **extra) + return logging_excepthook + + +class LogConfigError(Exception): + + message = _('Error loading logging config %(log_config)s: %(err_msg)s') + + def __init__(self, log_config, err_msg): + self.log_config = log_config + self.err_msg = err_msg + + def __str__(self): + return self.message % dict(log_config=self.log_config, + err_msg=self.err_msg) + + +def _load_log_config(log_config): + try: + logging.config.fileConfig(log_config) + except ConfigParser.Error, exc: + raise LogConfigError(log_config, str(exc)) + + +def setup(product_name): + """Setup logging.""" + if CONF.log_config: + _load_log_config(CONF.log_config) + else: + _setup_logging_from_conf() + sys.excepthook = _create_logging_excepthook(product_name) + + +def set_defaults(logging_context_format_string): + cfg.set_defaults(log_opts, + logging_context_format_string= + logging_context_format_string) + + +def _find_facility_from_conf(): + facility_names = logging.handlers.SysLogHandler.facility_names + facility = getattr(logging.handlers.SysLogHandler, + CONF.syslog_log_facility, + None) + + if facility is None and CONF.syslog_log_facility in facility_names: + facility = facility_names.get(CONF.syslog_log_facility) + + if facility is None: + valid_facilities = facility_names.keys() + consts = ['LOG_AUTH', 'LOG_AUTHPRIV', 'LOG_CRON', 'LOG_DAEMON', + 'LOG_FTP', 'LOG_KERN', 'LOG_LPR', 'LOG_MAIL', 'LOG_NEWS', + 'LOG_AUTH', 'LOG_SYSLOG', 'LOG_USER', 'LOG_UUCP', + 'LOG_LOCAL0', 'LOG_LOCAL1', 'LOG_LOCAL2', 'LOG_LOCAL3', + 'LOG_LOCAL4', 'LOG_LOCAL5', 'LOG_LOCAL6', 'LOG_LOCAL7'] + valid_facilities.extend(consts) + raise TypeError(_('syslog facility must be one of: %s') % + ', '.join("'%s'" % fac + for fac in valid_facilities)) + + return facility + + +def _setup_logging_from_conf(): + log_root = getLogger(None).logger + for handler in log_root.handlers: + log_root.removeHandler(handler) + + if CONF.use_syslog: + facility = _find_facility_from_conf() + syslog = logging.handlers.SysLogHandler(address='/dev/log', + facility=facility) + log_root.addHandler(syslog) + + logpath = _get_log_file_path() + if logpath: + filelog = logging.handlers.WatchedFileHandler(logpath) + log_root.addHandler(filelog) + + mode = int(CONF.logfile_mode, 8) + st = os.stat(logpath) + if st.st_mode != (stat.S_IFREG | mode): + os.chmod(logpath, mode) + + if CONF.use_stderr: + streamlog = ColorHandler() + log_root.addHandler(streamlog) + + elif not CONF.log_file: + # pass sys.stdout as a positional argument + # python2.6 calls the argument strm, in 2.7 it's stream + streamlog = logging.StreamHandler(sys.stdout) + log_root.addHandler(streamlog) + + if CONF.publish_errors: + log_root.addHandler(PublishErrorsHandler(logging.ERROR)) + + for handler in log_root.handlers: + datefmt = CONF.log_date_format + if CONF.log_format: + handler.setFormatter(logging.Formatter(fmt=CONF.log_format, + datefmt=datefmt)) + else: + handler.setFormatter(LegacyFormatter(datefmt=datefmt)) + + if CONF.debug: + log_root.setLevel(logging.DEBUG) + elif CONF.verbose: + log_root.setLevel(logging.INFO) + else: + log_root.setLevel(logging.WARNING) + + level = logging.NOTSET + for pair in CONF.default_log_levels: + mod, _sep, level_name = pair.partition('=') + level = logging.getLevelName(level_name) + logger = logging.getLogger(mod) + logger.setLevel(level) + for handler in log_root.handlers: + logger.addHandler(handler) + +_loggers = {} + + +def getLogger(name='unknown', version='unknown'): + if name not in _loggers: + _loggers[name] = ContextAdapter(logging.getLogger(name), + name, + version) + return _loggers[name] + + +class WritableLogger(object): + """A thin wrapper that responds to `write` and logs.""" + + def __init__(self, logger, level=logging.INFO): + self.logger = logger + self.level = level + + def write(self, msg): + self.logger.log(self.level, msg) + + +class LegacyFormatter(logging.Formatter): + """A context.RequestContext aware formatter configured through flags. + + The flags used to set format strings are: logging_context_format_string + and logging_default_format_string. You can also specify + logging_debug_format_suffix to append extra formatting if the log level is + debug. + + For information about what variables are available for the formatter see: + http://docs.python.org/library/logging.html#formatter + + """ + + def format(self, record): + """Uses contextstring if request_id is set, otherwise default.""" + # NOTE(sdague): default the fancier formating params + # to an empty string so we don't throw an exception if + # they get used + for key in ('instance', 'color'): + if key not in record.__dict__: + record.__dict__[key] = '' + + if record.__dict__.get('request_id', None): + self._fmt = CONF.logging_context_format_string + else: + self._fmt = CONF.logging_default_format_string + + if (record.levelno == logging.DEBUG and + CONF.logging_debug_format_suffix): + self._fmt += " " + CONF.logging_debug_format_suffix + + # Cache this on the record, Logger will respect our formated copy + if record.exc_info: + record.exc_text = self.formatException(record.exc_info, record) + return logging.Formatter.format(self, record) + + def formatException(self, exc_info, record=None): + """Format exception output with CONF.logging_exception_prefix.""" + if not record: + return logging.Formatter.formatException(self, exc_info) + + stringbuffer = cStringIO.StringIO() + traceback.print_exception(exc_info[0], exc_info[1], exc_info[2], + None, stringbuffer) + lines = stringbuffer.getvalue().split('\n') + stringbuffer.close() + + if CONF.logging_exception_prefix.find('%(asctime)') != -1: + record.asctime = self.formatTime(record, self.datefmt) + + formatted_lines = [] + for line in lines: + pl = CONF.logging_exception_prefix % record.__dict__ + fl = '%s%s' % (pl, line) + formatted_lines.append(fl) + return '\n'.join(formatted_lines) + + +class ColorHandler(logging.StreamHandler): + LEVEL_COLORS = { + logging.DEBUG: '\033[00;32m', # GREEN + logging.INFO: '\033[00;36m', # CYAN + logging.AUDIT: '\033[01;36m', # BOLD CYAN + logging.WARN: '\033[01;33m', # BOLD YELLOW + logging.ERROR: '\033[01;31m', # BOLD RED + logging.CRITICAL: '\033[01;31m', # BOLD RED + } + + def format(self, record): + record.color = self.LEVEL_COLORS[record.levelno] + return logging.StreamHandler.format(self, record) + + +class DeprecatedConfig(Exception): + message = _("Fatal call to deprecated config: %(msg)s") + + def __init__(self, msg): + super(Exception, self).__init__(self.message % dict(msg=msg)) diff --git a/conductor/conductor/openstack/common/loopingcall.py b/conductor/conductor/openstack/common/loopingcall.py index 238fab9..08135f6 100644 --- a/conductor/conductor/openstack/common/loopingcall.py +++ b/conductor/conductor/openstack/common/loopingcall.py @@ -1,95 +1,95 @@ -# vim: tabstop=4 shiftwidth=4 softtabstop=4 - -# Copyright 2010 United States Government as represented by the -# Administrator of the National Aeronautics and Space Administration. -# Copyright 2011 Justin Santa Barbara -# All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -import sys - -from eventlet import event -from eventlet import greenthread - -from conductor.openstack.common.gettextutils import _ -from conductor.openstack.common import log as logging -from conductor.openstack.common import timeutils - -LOG = logging.getLogger(__name__) - - -class LoopingCallDone(Exception): - """Exception to break out and stop a LoopingCall. - - The poll-function passed to LoopingCall can raise this exception to - break out of the loop normally. This is somewhat analogous to - StopIteration. - - An optional return-value can be included as the argument to the exception; - this return-value will be returned by LoopingCall.wait() - - """ - - def __init__(self, retvalue=True): - """:param retvalue: Value that LoopingCall.wait() should return.""" - self.retvalue = retvalue - - -class LoopingCall(object): - def __init__(self, f=None, *args, **kw): - self.args = args - self.kw = kw - self.f = f - self._running = False - - def start(self, interval, initial_delay=None): - self._running = True - done = event.Event() - - def _inner(): - if initial_delay: - greenthread.sleep(initial_delay) - - try: - while self._running: - start = timeutils.utcnow() - self.f(*self.args, **self.kw) - end = timeutils.utcnow() - if not self._running: - break - delay = interval - timeutils.delta_seconds(start, end) - if delay <= 0: - LOG.warn(_('task run outlasted interval by %s sec') % - -delay) - greenthread.sleep(delay if delay > 0 else 0) - except LoopingCallDone, e: - self.stop() - done.send(e.retvalue) - except Exception: - LOG.exception(_('in looping call')) - done.send_exception(*sys.exc_info()) - return - else: - done.send(True) - - self.done = done - - greenthread.spawn_n(_inner) - return self.done - - def stop(self): - self._running = False - - def wait(self): - return self.done.wait() +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright 2010 United States Government as represented by the +# Administrator of the National Aeronautics and Space Administration. +# Copyright 2011 Justin Santa Barbara +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import sys + +from eventlet import event +from eventlet import greenthread + +from conductor.openstack.common.gettextutils import _ +from conductor.openstack.common import log as logging +from conductor.openstack.common import timeutils + +LOG = logging.getLogger(__name__) + + +class LoopingCallDone(Exception): + """Exception to break out and stop a LoopingCall. + + The poll-function passed to LoopingCall can raise this exception to + break out of the loop normally. This is somewhat analogous to + StopIteration. + + An optional return-value can be included as the argument to the exception; + this return-value will be returned by LoopingCall.wait() + + """ + + def __init__(self, retvalue=True): + """:param retvalue: Value that LoopingCall.wait() should return.""" + self.retvalue = retvalue + + +class LoopingCall(object): + def __init__(self, f=None, *args, **kw): + self.args = args + self.kw = kw + self.f = f + self._running = False + + def start(self, interval, initial_delay=None): + self._running = True + done = event.Event() + + def _inner(): + if initial_delay: + greenthread.sleep(initial_delay) + + try: + while self._running: + start = timeutils.utcnow() + self.f(*self.args, **self.kw) + end = timeutils.utcnow() + if not self._running: + break + delay = interval - timeutils.delta_seconds(start, end) + if delay <= 0: + LOG.warn(_('task run outlasted interval by %s sec') % + -delay) + greenthread.sleep(delay if delay > 0 else 0) + except LoopingCallDone, e: + self.stop() + done.send(e.retvalue) + except Exception: + LOG.exception(_('in looping call')) + done.send_exception(*sys.exc_info()) + return + else: + done.send(True) + + self.done = done + + greenthread.spawn_n(_inner) + return self.done + + def stop(self): + self._running = False + + def wait(self): + return self.done.wait() diff --git a/conductor/conductor/openstack/common/notifier/__init__.py b/conductor/conductor/openstack/common/notifier/__init__.py new file mode 100644 index 0000000..45c3b46 --- /dev/null +++ b/conductor/conductor/openstack/common/notifier/__init__.py @@ -0,0 +1,14 @@ +# Copyright 2011 OpenStack Foundation. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. diff --git a/conductor/conductor/openstack/common/notifier/api.py b/conductor/conductor/openstack/common/notifier/api.py new file mode 100644 index 0000000..d5629e8 --- /dev/null +++ b/conductor/conductor/openstack/common/notifier/api.py @@ -0,0 +1,182 @@ +# Copyright 2011 OpenStack Foundation. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import uuid + +from oslo.config import cfg + +from conductor.openstack.common import context +from conductor.openstack.common.gettextutils import _ +from conductor.openstack.common import importutils +from conductor.openstack.common import jsonutils +from conductor.openstack.common import log as logging +from conductor.openstack.common import timeutils + + +LOG = logging.getLogger(__name__) + +notifier_opts = [ + cfg.MultiStrOpt('notification_driver', + default=[], + help='Driver or drivers to handle sending notifications'), + cfg.StrOpt('default_notification_level', + default='INFO', + help='Default notification level for outgoing notifications'), + cfg.StrOpt('default_publisher_id', + default='$host', + help='Default publisher_id for outgoing notifications'), +] + +CONF = cfg.CONF +CONF.register_opts(notifier_opts) + +WARN = 'WARN' +INFO = 'INFO' +ERROR = 'ERROR' +CRITICAL = 'CRITICAL' +DEBUG = 'DEBUG' + +log_levels = (DEBUG, WARN, INFO, ERROR, CRITICAL) + + +class BadPriorityException(Exception): + pass + + +def notify_decorator(name, fn): + """ decorator for notify which is used from utils.monkey_patch() + + :param name: name of the function + :param function: - object of the function + :returns: function -- decorated function + + """ + def wrapped_func(*args, **kwarg): + body = {} + body['args'] = [] + body['kwarg'] = {} + for arg in args: + body['args'].append(arg) + for key in kwarg: + body['kwarg'][key] = kwarg[key] + + ctxt = context.get_context_from_function_and_args(fn, args, kwarg) + notify(ctxt, + CONF.default_publisher_id, + name, + CONF.default_notification_level, + body) + return fn(*args, **kwarg) + return wrapped_func + + +def publisher_id(service, host=None): + if not host: + host = CONF.host + return "%s.%s" % (service, host) + + +def notify(context, publisher_id, event_type, priority, payload): + """Sends a notification using the specified driver + + :param publisher_id: the source worker_type.host of the message + :param event_type: the literal type of event (ex. Instance Creation) + :param priority: patterned after the enumeration of Python logging + levels in the set (DEBUG, WARN, INFO, ERROR, CRITICAL) + :param payload: A python dictionary of attributes + + Outgoing message format includes the above parameters, and appends the + following: + + message_id + a UUID representing the id for this notification + + timestamp + the GMT timestamp the notification was sent at + + The composite message will be constructed as a dictionary of the above + attributes, which will then be sent via the transport mechanism defined + by the driver. + + Message example:: + + {'message_id': str(uuid.uuid4()), + 'publisher_id': 'compute.host1', + 'timestamp': timeutils.utcnow(), + 'priority': 'WARN', + 'event_type': 'compute.create_instance', + 'payload': {'instance_id': 12, ... }} + + """ + if priority not in log_levels: + raise BadPriorityException( + _('%s not in valid priorities') % priority) + + # Ensure everything is JSON serializable. + payload = jsonutils.to_primitive(payload, convert_instances=True) + + msg = dict(message_id=str(uuid.uuid4()), + publisher_id=publisher_id, + event_type=event_type, + priority=priority, + payload=payload, + timestamp=str(timeutils.utcnow())) + + for driver in _get_drivers(): + try: + driver.notify(context, msg) + except Exception as e: + LOG.exception(_("Problem '%(e)s' attempting to " + "send to notification system. " + "Payload=%(payload)s") + % dict(e=e, payload=payload)) + + +_drivers = None + + +def _get_drivers(): + """Instantiate, cache, and return drivers based on the CONF.""" + global _drivers + if _drivers is None: + _drivers = {} + for notification_driver in CONF.notification_driver: + add_driver(notification_driver) + + return _drivers.values() + + +def add_driver(notification_driver): + """Add a notification driver at runtime.""" + # Make sure the driver list is initialized. + _get_drivers() + if isinstance(notification_driver, basestring): + # Load and add + try: + driver = importutils.import_module(notification_driver) + _drivers[notification_driver] = driver + except ImportError: + LOG.exception(_("Failed to load notifier %s. " + "These notifications will not be sent.") % + notification_driver) + else: + # Driver is already loaded; just add the object. + _drivers[notification_driver] = notification_driver + + +def _reset_drivers(): + """Used by unit tests to reset the drivers.""" + global _drivers + _drivers = None diff --git a/conductor/conductor/openstack/common/notifier/log_notifier.py b/conductor/conductor/openstack/common/notifier/log_notifier.py new file mode 100644 index 0000000..9f159fa --- /dev/null +++ b/conductor/conductor/openstack/common/notifier/log_notifier.py @@ -0,0 +1,35 @@ +# Copyright 2011 OpenStack Foundation. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from oslo.config import cfg + +from conductor.openstack.common import jsonutils +from conductor.openstack.common import log as logging + + +CONF = cfg.CONF + + +def notify(_context, message): + """Notifies the recipient of the desired event given the model. + Log notifications using openstack's default logging system""" + + priority = message.get('priority', + CONF.default_notification_level) + priority = priority.lower() + logger = logging.getLogger( + 'conductor.openstack.common.notification.%s' % + message['event_type']) + getattr(logger, priority)(jsonutils.dumps(message)) diff --git a/conductor/conductor/openstack/common/notifier/no_op_notifier.py b/conductor/conductor/openstack/common/notifier/no_op_notifier.py new file mode 100644 index 0000000..bc7a56c --- /dev/null +++ b/conductor/conductor/openstack/common/notifier/no_op_notifier.py @@ -0,0 +1,19 @@ +# Copyright 2011 OpenStack Foundation. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + + +def notify(_context, message): + """Notifies the recipient of the desired event given the model""" + pass diff --git a/conductor/conductor/openstack/common/notifier/rpc_notifier.py b/conductor/conductor/openstack/common/notifier/rpc_notifier.py new file mode 100644 index 0000000..67d615d --- /dev/null +++ b/conductor/conductor/openstack/common/notifier/rpc_notifier.py @@ -0,0 +1,46 @@ +# Copyright 2011 OpenStack Foundation. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from oslo.config import cfg + +from conductor.openstack.common import context as req_context +from conductor.openstack.common.gettextutils import _ +from conductor.openstack.common import log as logging +from conductor.openstack.common import rpc + +LOG = logging.getLogger(__name__) + +notification_topic_opt = cfg.ListOpt( + 'notification_topics', default=['notifications', ], + help='AMQP topic used for openstack notifications') + +CONF = cfg.CONF +CONF.register_opt(notification_topic_opt) + + +def notify(context, message): + """Sends a notification via RPC""" + if not context: + context = req_context.get_admin_context() + priority = message.get('priority', + CONF.default_notification_level) + priority = priority.lower() + for topic in CONF.notification_topics: + topic = '%s.%s' % (topic, priority) + try: + rpc.notify(context, topic, message) + except Exception: + LOG.exception(_("Could not send notification to %(topic)s. " + "Payload=%(message)s"), locals()) diff --git a/conductor/conductor/openstack/common/notifier/rpc_notifier2.py b/conductor/conductor/openstack/common/notifier/rpc_notifier2.py new file mode 100644 index 0000000..3585e7e --- /dev/null +++ b/conductor/conductor/openstack/common/notifier/rpc_notifier2.py @@ -0,0 +1,52 @@ +# Copyright 2011 OpenStack Foundation. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +'''messaging based notification driver, with message envelopes''' + +from oslo.config import cfg + +from conductor.openstack.common import context as req_context +from conductor.openstack.common.gettextutils import _ +from conductor.openstack.common import log as logging +from conductor.openstack.common import rpc + +LOG = logging.getLogger(__name__) + +notification_topic_opt = cfg.ListOpt( + 'topics', default=['notifications', ], + help='AMQP topic(s) used for openstack notifications') + +opt_group = cfg.OptGroup(name='rpc_notifier2', + title='Options for rpc_notifier2') + +CONF = cfg.CONF +CONF.register_group(opt_group) +CONF.register_opt(notification_topic_opt, opt_group) + + +def notify(context, message): + """Sends a notification via RPC""" + if not context: + context = req_context.get_admin_context() + priority = message.get('priority', + CONF.default_notification_level) + priority = priority.lower() + for topic in CONF.rpc_notifier2.topics: + topic = '%s.%s' % (topic, priority) + try: + rpc.notify(context, topic, message, envelope=True) + except Exception: + LOG.exception(_("Could not send notification to %(topic)s. " + "Payload=%(message)s"), locals()) diff --git a/conductor/conductor/openstack/common/notifier/test_notifier.py b/conductor/conductor/openstack/common/notifier/test_notifier.py new file mode 100644 index 0000000..96c1746 --- /dev/null +++ b/conductor/conductor/openstack/common/notifier/test_notifier.py @@ -0,0 +1,22 @@ +# Copyright 2011 OpenStack Foundation. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + + +NOTIFICATIONS = [] + + +def notify(_context, message): + """Test notifier, stores notifications in memory for unittests.""" + NOTIFICATIONS.append(message) diff --git a/conductor/conductor/openstack/common/service.py b/conductor/conductor/openstack/common/service.py index df48769..a31b41a 100644 --- a/conductor/conductor/openstack/common/service.py +++ b/conductor/conductor/openstack/common/service.py @@ -1,332 +1,332 @@ -# vim: tabstop=4 shiftwidth=4 softtabstop=4 - -# Copyright 2010 United States Government as represented by the -# Administrator of the National Aeronautics and Space Administration. -# Copyright 2011 Justin Santa Barbara -# All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -"""Generic Node base class for all workers that run on hosts.""" - -import errno -import os -import random -import signal -import sys -import time - -import eventlet -import logging as std_logging -from oslo.config import cfg - -from conductor.openstack.common import eventlet_backdoor -from conductor.openstack.common.gettextutils import _ -from conductor.openstack.common import importutils -from conductor.openstack.common import log as logging -from conductor.openstack.common import threadgroup - - -rpc = importutils.try_import('conductor.openstack.common.rpc') -CONF = cfg.CONF -LOG = logging.getLogger(__name__) - - -class Launcher(object): - """Launch one or more services and wait for them to complete.""" - - def __init__(self): - """Initialize the service launcher. - - :returns: None - - """ - self._services = threadgroup.ThreadGroup() - eventlet_backdoor.initialize_if_enabled() - - @staticmethod - def run_service(service): - """Start and wait for a service to finish. - - :param service: service to run and wait for. - :returns: None - - """ - service.start() - service.wait() - - def launch_service(self, service): - """Load and start the given service. - - :param service: The service you would like to start. - :returns: None - - """ - self._services.add_thread(self.run_service, service) - - def stop(self): - """Stop all services which are currently running. - - :returns: None - - """ - self._services.stop() - - def wait(self): - """Waits until all services have been stopped, and then returns. - - :returns: None - - """ - self._services.wait() - - -class SignalExit(SystemExit): - def __init__(self, signo, exccode=1): - super(SignalExit, self).__init__(exccode) - self.signo = signo - - -class ServiceLauncher(Launcher): - def _handle_signal(self, signo, frame): - # Allow the process to be killed again and die from natural causes - signal.signal(signal.SIGTERM, signal.SIG_DFL) - signal.signal(signal.SIGINT, signal.SIG_DFL) - - raise SignalExit(signo) - - def wait(self): - signal.signal(signal.SIGTERM, self._handle_signal) - signal.signal(signal.SIGINT, self._handle_signal) - - LOG.debug(_('Full set of CONF:')) - CONF.log_opt_values(LOG, std_logging.DEBUG) - - status = None - try: - super(ServiceLauncher, self).wait() - except SignalExit as exc: - signame = {signal.SIGTERM: 'SIGTERM', - signal.SIGINT: 'SIGINT'}[exc.signo] - LOG.info(_('Caught %s, exiting'), signame) - status = exc.code - except SystemExit as exc: - status = exc.code - finally: - if rpc: - rpc.cleanup() - self.stop() - return status - - -class ServiceWrapper(object): - def __init__(self, service, workers): - self.service = service - self.workers = workers - self.children = set() - self.forktimes = [] - - -class ProcessLauncher(object): - def __init__(self): - self.children = {} - self.sigcaught = None - self.running = True - rfd, self.writepipe = os.pipe() - self.readpipe = eventlet.greenio.GreenPipe(rfd, 'r') - - signal.signal(signal.SIGTERM, self._handle_signal) - signal.signal(signal.SIGINT, self._handle_signal) - - def _handle_signal(self, signo, frame): - self.sigcaught = signo - self.running = False - - # Allow the process to be killed again and die from natural causes - signal.signal(signal.SIGTERM, signal.SIG_DFL) - signal.signal(signal.SIGINT, signal.SIG_DFL) - - def _pipe_watcher(self): - # This will block until the write end is closed when the parent - # dies unexpectedly - self.readpipe.read() - - LOG.info(_('Parent process has died unexpectedly, exiting')) - - sys.exit(1) - - def _child_process(self, service): - # Setup child signal handlers differently - def _sigterm(*args): - signal.signal(signal.SIGTERM, signal.SIG_DFL) - raise SignalExit(signal.SIGTERM) - - signal.signal(signal.SIGTERM, _sigterm) - # Block SIGINT and let the parent send us a SIGTERM - signal.signal(signal.SIGINT, signal.SIG_IGN) - - # Reopen the eventlet hub to make sure we don't share an epoll - # fd with parent and/or siblings, which would be bad - eventlet.hubs.use_hub() - - # Close write to ensure only parent has it open - os.close(self.writepipe) - # Create greenthread to watch for parent to close pipe - eventlet.spawn_n(self._pipe_watcher) - - # Reseed random number generator - random.seed() - - launcher = Launcher() - launcher.run_service(service) - - def _start_child(self, wrap): - if len(wrap.forktimes) > wrap.workers: - # Limit ourselves to one process a second (over the period of - # number of workers * 1 second). This will allow workers to - # start up quickly but ensure we don't fork off children that - # die instantly too quickly. - if time.time() - wrap.forktimes[0] < wrap.workers: - LOG.info(_('Forking too fast, sleeping')) - time.sleep(1) - - wrap.forktimes.pop(0) - - wrap.forktimes.append(time.time()) - - pid = os.fork() - if pid == 0: - # NOTE(johannes): All exceptions are caught to ensure this - # doesn't fallback into the loop spawning children. It would - # be bad for a child to spawn more children. - status = 0 - try: - self._child_process(wrap.service) - except SignalExit as exc: - signame = {signal.SIGTERM: 'SIGTERM', - signal.SIGINT: 'SIGINT'}[exc.signo] - LOG.info(_('Caught %s, exiting'), signame) - status = exc.code - except SystemExit as exc: - status = exc.code - except BaseException: - LOG.exception(_('Unhandled exception')) - status = 2 - finally: - wrap.service.stop() - - os._exit(status) - - LOG.info(_('Started child %d'), pid) - - wrap.children.add(pid) - self.children[pid] = wrap - - return pid - - def launch_service(self, service, workers=1): - wrap = ServiceWrapper(service, workers) - - LOG.info(_('Starting %d workers'), wrap.workers) - while self.running and len(wrap.children) < wrap.workers: - self._start_child(wrap) - - def _wait_child(self): - try: - # Don't block if no child processes have exited - pid, status = os.waitpid(0, os.WNOHANG) - if not pid: - return None - except OSError as exc: - if exc.errno not in (errno.EINTR, errno.ECHILD): - raise - return None - - if os.WIFSIGNALED(status): - sig = os.WTERMSIG(status) - LOG.info(_('Child %(pid)d killed by signal %(sig)d'), - dict(pid=pid, sig=sig)) - else: - code = os.WEXITSTATUS(status) - LOG.info(_('Child %(pid)s exited with status %(code)d'), - dict(pid=pid, code=code)) - - if pid not in self.children: - LOG.warning(_('pid %d not in child list'), pid) - return None - - wrap = self.children.pop(pid) - wrap.children.remove(pid) - return wrap - - def wait(self): - """Loop waiting on children to die and respawning as necessary""" - - LOG.debug(_('Full set of CONF:')) - CONF.log_opt_values(LOG, std_logging.DEBUG) - - while self.running: - wrap = self._wait_child() - if not wrap: - # Yield to other threads if no children have exited - # Sleep for a short time to avoid excessive CPU usage - # (see bug #1095346) - eventlet.greenthread.sleep(.01) - continue - - while self.running and len(wrap.children) < wrap.workers: - self._start_child(wrap) - - if self.sigcaught: - signame = {signal.SIGTERM: 'SIGTERM', - signal.SIGINT: 'SIGINT'}[self.sigcaught] - LOG.info(_('Caught %s, stopping children'), signame) - - for pid in self.children: - try: - os.kill(pid, signal.SIGTERM) - except OSError as exc: - if exc.errno != errno.ESRCH: - raise - - # Wait for children to die - if self.children: - LOG.info(_('Waiting on %d children to exit'), len(self.children)) - while self.children: - self._wait_child() - - -class Service(object): - """Service object for binaries running on hosts.""" - - def __init__(self, threads=1000): - self.tg = threadgroup.ThreadGroup(threads) - - def start(self): - pass - - def stop(self): - self.tg.stop() - - def wait(self): - self.tg.wait() - - -def launch(service, workers=None): - if workers: - launcher = ProcessLauncher() - launcher.launch_service(service, workers=workers) - else: - launcher = ServiceLauncher() - launcher.launch_service(service) - return launcher +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright 2010 United States Government as represented by the +# Administrator of the National Aeronautics and Space Administration. +# Copyright 2011 Justin Santa Barbara +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +"""Generic Node base class for all workers that run on hosts.""" + +import errno +import os +import random +import signal +import sys +import time + +import eventlet +import logging as std_logging +from oslo.config import cfg + +from conductor.openstack.common import eventlet_backdoor +from conductor.openstack.common.gettextutils import _ +from conductor.openstack.common import importutils +from conductor.openstack.common import log as logging +from conductor.openstack.common import threadgroup + + +rpc = importutils.try_import('conductor.openstack.common.rpc') +CONF = cfg.CONF +LOG = logging.getLogger(__name__) + + +class Launcher(object): + """Launch one or more services and wait for them to complete.""" + + def __init__(self): + """Initialize the service launcher. + + :returns: None + + """ + self._services = threadgroup.ThreadGroup() + eventlet_backdoor.initialize_if_enabled() + + @staticmethod + def run_service(service): + """Start and wait for a service to finish. + + :param service: service to run and wait for. + :returns: None + + """ + service.start() + service.wait() + + def launch_service(self, service): + """Load and start the given service. + + :param service: The service you would like to start. + :returns: None + + """ + self._services.add_thread(self.run_service, service) + + def stop(self): + """Stop all services which are currently running. + + :returns: None + + """ + self._services.stop() + + def wait(self): + """Waits until all services have been stopped, and then returns. + + :returns: None + + """ + self._services.wait() + + +class SignalExit(SystemExit): + def __init__(self, signo, exccode=1): + super(SignalExit, self).__init__(exccode) + self.signo = signo + + +class ServiceLauncher(Launcher): + def _handle_signal(self, signo, frame): + # Allow the process to be killed again and die from natural causes + signal.signal(signal.SIGTERM, signal.SIG_DFL) + signal.signal(signal.SIGINT, signal.SIG_DFL) + + raise SignalExit(signo) + + def wait(self): + signal.signal(signal.SIGTERM, self._handle_signal) + signal.signal(signal.SIGINT, self._handle_signal) + + LOG.debug(_('Full set of CONF:')) + CONF.log_opt_values(LOG, std_logging.DEBUG) + + status = None + try: + super(ServiceLauncher, self).wait() + except SignalExit as exc: + signame = {signal.SIGTERM: 'SIGTERM', + signal.SIGINT: 'SIGINT'}[exc.signo] + LOG.info(_('Caught %s, exiting'), signame) + status = exc.code + except SystemExit as exc: + status = exc.code + finally: + if rpc: + rpc.cleanup() + self.stop() + return status + + +class ServiceWrapper(object): + def __init__(self, service, workers): + self.service = service + self.workers = workers + self.children = set() + self.forktimes = [] + + +class ProcessLauncher(object): + def __init__(self): + self.children = {} + self.sigcaught = None + self.running = True + rfd, self.writepipe = os.pipe() + self.readpipe = eventlet.greenio.GreenPipe(rfd, 'r') + + signal.signal(signal.SIGTERM, self._handle_signal) + signal.signal(signal.SIGINT, self._handle_signal) + + def _handle_signal(self, signo, frame): + self.sigcaught = signo + self.running = False + + # Allow the process to be killed again and die from natural causes + signal.signal(signal.SIGTERM, signal.SIG_DFL) + signal.signal(signal.SIGINT, signal.SIG_DFL) + + def _pipe_watcher(self): + # This will block until the write end is closed when the parent + # dies unexpectedly + self.readpipe.read() + + LOG.info(_('Parent process has died unexpectedly, exiting')) + + sys.exit(1) + + def _child_process(self, service): + # Setup child signal handlers differently + def _sigterm(*args): + signal.signal(signal.SIGTERM, signal.SIG_DFL) + raise SignalExit(signal.SIGTERM) + + signal.signal(signal.SIGTERM, _sigterm) + # Block SIGINT and let the parent send us a SIGTERM + signal.signal(signal.SIGINT, signal.SIG_IGN) + + # Reopen the eventlet hub to make sure we don't share an epoll + # fd with parent and/or siblings, which would be bad + eventlet.hubs.use_hub() + + # Close write to ensure only parent has it open + os.close(self.writepipe) + # Create greenthread to watch for parent to close pipe + eventlet.spawn_n(self._pipe_watcher) + + # Reseed random number generator + random.seed() + + launcher = Launcher() + launcher.run_service(service) + + def _start_child(self, wrap): + if len(wrap.forktimes) > wrap.workers: + # Limit ourselves to one process a second (over the period of + # number of workers * 1 second). This will allow workers to + # start up quickly but ensure we don't fork off children that + # die instantly too quickly. + if time.time() - wrap.forktimes[0] < wrap.workers: + LOG.info(_('Forking too fast, sleeping')) + time.sleep(1) + + wrap.forktimes.pop(0) + + wrap.forktimes.append(time.time()) + + pid = os.fork() + if pid == 0: + # NOTE(johannes): All exceptions are caught to ensure this + # doesn't fallback into the loop spawning children. It would + # be bad for a child to spawn more children. + status = 0 + try: + self._child_process(wrap.service) + except SignalExit as exc: + signame = {signal.SIGTERM: 'SIGTERM', + signal.SIGINT: 'SIGINT'}[exc.signo] + LOG.info(_('Caught %s, exiting'), signame) + status = exc.code + except SystemExit as exc: + status = exc.code + except BaseException: + LOG.exception(_('Unhandled exception')) + status = 2 + finally: + wrap.service.stop() + + os._exit(status) + + LOG.info(_('Started child %d'), pid) + + wrap.children.add(pid) + self.children[pid] = wrap + + return pid + + def launch_service(self, service, workers=1): + wrap = ServiceWrapper(service, workers) + + LOG.info(_('Starting %d workers'), wrap.workers) + while self.running and len(wrap.children) < wrap.workers: + self._start_child(wrap) + + def _wait_child(self): + try: + # Don't block if no child processes have exited + pid, status = os.waitpid(0, os.WNOHANG) + if not pid: + return None + except OSError as exc: + if exc.errno not in (errno.EINTR, errno.ECHILD): + raise + return None + + if os.WIFSIGNALED(status): + sig = os.WTERMSIG(status) + LOG.info(_('Child %(pid)d killed by signal %(sig)d'), + dict(pid=pid, sig=sig)) + else: + code = os.WEXITSTATUS(status) + LOG.info(_('Child %(pid)s exited with status %(code)d'), + dict(pid=pid, code=code)) + + if pid not in self.children: + LOG.warning(_('pid %d not in child list'), pid) + return None + + wrap = self.children.pop(pid) + wrap.children.remove(pid) + return wrap + + def wait(self): + """Loop waiting on children to die and respawning as necessary""" + + LOG.debug(_('Full set of CONF:')) + CONF.log_opt_values(LOG, std_logging.DEBUG) + + while self.running: + wrap = self._wait_child() + if not wrap: + # Yield to other threads if no children have exited + # Sleep for a short time to avoid excessive CPU usage + # (see bug #1095346) + eventlet.greenthread.sleep(.01) + continue + + while self.running and len(wrap.children) < wrap.workers: + self._start_child(wrap) + + if self.sigcaught: + signame = {signal.SIGTERM: 'SIGTERM', + signal.SIGINT: 'SIGINT'}[self.sigcaught] + LOG.info(_('Caught %s, stopping children'), signame) + + for pid in self.children: + try: + os.kill(pid, signal.SIGTERM) + except OSError as exc: + if exc.errno != errno.ESRCH: + raise + + # Wait for children to die + if self.children: + LOG.info(_('Waiting on %d children to exit'), len(self.children)) + while self.children: + self._wait_child() + + +class Service(object): + """Service object for binaries running on hosts.""" + + def __init__(self, threads=1000): + self.tg = threadgroup.ThreadGroup(threads) + + def start(self): + pass + + def stop(self): + self.tg.stop() + + def wait(self): + self.tg.wait() + + +def launch(service, workers=None): + if workers: + launcher = ProcessLauncher() + launcher.launch_service(service, workers=workers) + else: + launcher = ServiceLauncher() + launcher.launch_service(service) + return launcher diff --git a/conductor/conductor/openstack/common/setup.py b/conductor/conductor/openstack/common/setup.py index 0e9ff88..dec74fd 100644 --- a/conductor/conductor/openstack/common/setup.py +++ b/conductor/conductor/openstack/common/setup.py @@ -1,359 +1,367 @@ -# vim: tabstop=4 shiftwidth=4 softtabstop=4 - -# Copyright 2011 OpenStack Foundation. -# Copyright 2012-2013 Hewlett-Packard Development Company, L.P. -# All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -""" -Utilities with minimum-depends for use in setup.py -""" - -import email -import os -import re -import subprocess -import sys - -from setuptools.command import sdist - - -def parse_mailmap(mailmap='.mailmap'): - mapping = {} - if os.path.exists(mailmap): - with open(mailmap, 'r') as fp: - for l in fp: - try: - canonical_email, alias = re.match( - r'[^#]*?(<.+>).*(<.+>).*', l).groups() - except AttributeError: - continue - mapping[alias] = canonical_email - return mapping - - -def _parse_git_mailmap(git_dir, mailmap='.mailmap'): - mailmap = os.path.join(os.path.dirname(git_dir), mailmap) - return parse_mailmap(mailmap) - - -def canonicalize_emails(changelog, mapping): - """Takes in a string and an email alias mapping and replaces all - instances of the aliases in the string with their real email. - """ - for alias, email_address in mapping.iteritems(): - changelog = changelog.replace(alias, email_address) - return changelog - - -# Get requirements from the first file that exists -def get_reqs_from_files(requirements_files): - for requirements_file in requirements_files: - if os.path.exists(requirements_file): - with open(requirements_file, 'r') as fil: - return fil.read().split('\n') - return [] - - -def parse_requirements(requirements_files=['requirements.txt', - 'tools/pip-requires']): - requirements = [] - for line in get_reqs_from_files(requirements_files): - # For the requirements list, we need to inject only the portion - # after egg= so that distutils knows the package it's looking for - # such as: - # -e git://github.com/openstack/nova/master#egg=nova - if re.match(r'\s*-e\s+', line): - requirements.append(re.sub(r'\s*-e\s+.*#egg=(.*)$', r'\1', - line)) - # such as: - # http://github.com/openstack/nova/zipball/master#egg=nova - elif re.match(r'\s*https?:', line): - requirements.append(re.sub(r'\s*https?:.*#egg=(.*)$', r'\1', - line)) - # -f lines are for index locations, and don't get used here - elif re.match(r'\s*-f\s+', line): - pass - # argparse is part of the standard library starting with 2.7 - # adding it to the requirements list screws distro installs - elif line == 'argparse' and sys.version_info >= (2, 7): - pass - else: - requirements.append(line) - - return requirements - - -def parse_dependency_links(requirements_files=['requirements.txt', - 'tools/pip-requires']): - dependency_links = [] - # dependency_links inject alternate locations to find packages listed - # in requirements - for line in get_reqs_from_files(requirements_files): - # skip comments and blank lines - if re.match(r'(\s*#)|(\s*$)', line): - continue - # lines with -e or -f need the whole line, minus the flag - if re.match(r'\s*-[ef]\s+', line): - dependency_links.append(re.sub(r'\s*-[ef]\s+', '', line)) - # lines that are only urls can go in unmolested - elif re.match(r'\s*https?:', line): - dependency_links.append(line) - return dependency_links - - -def _run_shell_command(cmd, throw_on_error=False): - if os.name == 'nt': - output = subprocess.Popen(["cmd.exe", "/C", cmd], - stdout=subprocess.PIPE, - stderr=subprocess.PIPE) - else: - output = subprocess.Popen(["/bin/sh", "-c", cmd], - stdout=subprocess.PIPE, - stderr=subprocess.PIPE) - out = output.communicate() - if output.returncode and throw_on_error: - raise Exception("%s returned %d" % cmd, output.returncode) - if len(out) == 0: - return None - if len(out[0].strip()) == 0: - return None - return out[0].strip() - - -def _get_git_directory(): - parent_dir = os.path.dirname(__file__) - while True: - git_dir = os.path.join(parent_dir, '.git') - if os.path.exists(git_dir): - return git_dir - parent_dir, child = os.path.split(parent_dir) - if not child: # reached to root dir - return None - - -def write_git_changelog(): - """Write a changelog based on the git changelog.""" - new_changelog = 'ChangeLog' - git_dir = _get_git_directory() - if not os.getenv('SKIP_WRITE_GIT_CHANGELOG'): - if git_dir: - git_log_cmd = 'git --git-dir=%s log' % git_dir - changelog = _run_shell_command(git_log_cmd) - mailmap = _parse_git_mailmap(git_dir) - with open(new_changelog, "w") as changelog_file: - changelog_file.write(canonicalize_emails(changelog, mailmap)) - else: - open(new_changelog, 'w').close() - - -def generate_authors(): - """Create AUTHORS file using git commits.""" - jenkins_email = 'jenkins@review.(openstack|stackforge).org' - old_authors = 'AUTHORS.in' - new_authors = 'AUTHORS' - git_dir = _get_git_directory() - if not os.getenv('SKIP_GENERATE_AUTHORS'): - if git_dir: - # don't include jenkins email address in AUTHORS file - git_log_cmd = ("git --git-dir=" + git_dir + - " log --format='%aN <%aE>' | sort -u | " - "egrep -v '" + jenkins_email + "'") - changelog = _run_shell_command(git_log_cmd) - mailmap = _parse_git_mailmap(git_dir) - with open(new_authors, 'w') as new_authors_fh: - new_authors_fh.write(canonicalize_emails(changelog, mailmap)) - if os.path.exists(old_authors): - with open(old_authors, "r") as old_authors_fh: - new_authors_fh.write('\n' + old_authors_fh.read()) - else: - open(new_authors, 'w').close() - - -_rst_template = """%(heading)s -%(underline)s - -.. automodule:: %(module)s - :members: - :undoc-members: - :show-inheritance: -""" - - -def get_cmdclass(): - """Return dict of commands to run from setup.py.""" - - cmdclass = dict() - - def _find_modules(arg, dirname, files): - for filename in files: - if filename.endswith('.py') and filename != '__init__.py': - arg["%s.%s" % (dirname.replace('/', '.'), - filename[:-3])] = True - - class LocalSDist(sdist.sdist): - """Builds the ChangeLog and Authors files from VC first.""" - - def run(self): - write_git_changelog() - generate_authors() - # sdist.sdist is an old style class, can't use super() - sdist.sdist.run(self) - - cmdclass['sdist'] = LocalSDist - - # If Sphinx is installed on the box running setup.py, - # enable setup.py to build the documentation, otherwise, - # just ignore it - try: - from sphinx.setup_command import BuildDoc - - class LocalBuildDoc(BuildDoc): - - builders = ['html', 'man'] - - def generate_autoindex(self): - print "**Autodocumenting from %s" % os.path.abspath(os.curdir) - modules = {} - option_dict = self.distribution.get_option_dict('build_sphinx') - source_dir = os.path.join(option_dict['source_dir'][1], 'api') - if not os.path.exists(source_dir): - os.makedirs(source_dir) - for pkg in self.distribution.packages: - if '.' not in pkg: - os.path.walk(pkg, _find_modules, modules) - module_list = modules.keys() - module_list.sort() - autoindex_filename = os.path.join(source_dir, 'autoindex.rst') - with open(autoindex_filename, 'w') as autoindex: - autoindex.write(""".. toctree:: - :maxdepth: 1 - -""") - for module in module_list: - output_filename = os.path.join(source_dir, - "%s.rst" % module) - heading = "The :mod:`%s` Module" % module - underline = "=" * len(heading) - values = dict(module=module, heading=heading, - underline=underline) - - print "Generating %s" % output_filename - with open(output_filename, 'w') as output_file: - output_file.write(_rst_template % values) - autoindex.write(" %s.rst\n" % module) - - def run(self): - if not os.getenv('SPHINX_DEBUG'): - self.generate_autoindex() - - for builder in self.builders: - self.builder = builder - self.finalize_options() - self.project = self.distribution.get_name() - self.version = self.distribution.get_version() - self.release = self.distribution.get_version() - BuildDoc.run(self) - - class LocalBuildLatex(LocalBuildDoc): - builders = ['latex'] - - cmdclass['build_sphinx'] = LocalBuildDoc - cmdclass['build_sphinx_latex'] = LocalBuildLatex - except ImportError: - pass - - return cmdclass - - -def _get_revno(git_dir): - """Return the number of commits since the most recent tag. - - We use git-describe to find this out, but if there are no - tags then we fall back to counting commits since the beginning - of time. - """ - describe = _run_shell_command( - "git --git-dir=%s describe --always" % git_dir) - if "-" in describe: - return describe.rsplit("-", 2)[-2] - - # no tags found - revlist = _run_shell_command( - "git --git-dir=%s rev-list --abbrev-commit HEAD" % git_dir) - return len(revlist.splitlines()) - - -def _get_version_from_git(pre_version): - """Return a version which is equal to the tag that's on the current - revision if there is one, or tag plus number of additional revisions - if the current revision has no tag.""" - - git_dir = _get_git_directory() - if git_dir: - if pre_version: - try: - return _run_shell_command( - "git --git-dir=" + git_dir + " describe --exact-match", - throw_on_error=True).replace('-', '.') - except Exception: - sha = _run_shell_command( - "git --git-dir=" + git_dir + " log -n1 --pretty=format:%h") - return "%s.a%s.g%s" % (pre_version, _get_revno(git_dir), sha) - else: - return _run_shell_command( - "git --git-dir=" + git_dir + " describe --always").replace( - '-', '.') - return None - - -def _get_version_from_pkg_info(package_name): - """Get the version from PKG-INFO file if we can.""" - try: - pkg_info_file = open('PKG-INFO', 'r') - except (IOError, OSError): - return None - try: - pkg_info = email.message_from_file(pkg_info_file) - except email.MessageError: - return None - # Check to make sure we're in our own dir - if pkg_info.get('Name', None) != package_name: - return None - return pkg_info.get('Version', None) - - -def get_version(package_name, pre_version=None): - """Get the version of the project. First, try getting it from PKG-INFO, if - it exists. If it does, that means we're in a distribution tarball or that - install has happened. Otherwise, if there is no PKG-INFO file, pull the - version from git. - - We do not support setup.py version sanity in git archive tarballs, nor do - we support packagers directly sucking our git repo into theirs. We expect - that a source tarball be made from our git repo - or that if someone wants - to make a source tarball from a fork of our repo with additional tags in it - that they understand and desire the results of doing that. - """ - version = os.environ.get("OSLO_PACKAGE_VERSION", None) - if version: - return version - version = _get_version_from_pkg_info(package_name) - if version: - return version - version = _get_version_from_git(pre_version) - if version: - return version - raise Exception("Versioning for this project requires either an sdist" - " tarball, or access to an upstream git repository.") +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright 2011 OpenStack Foundation. +# Copyright 2012-2013 Hewlett-Packard Development Company, L.P. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +""" +Utilities with minimum-depends for use in setup.py +""" + +import email +import os +import re +import subprocess +import sys + +from setuptools.command import sdist + + +def parse_mailmap(mailmap='.mailmap'): + mapping = {} + if os.path.exists(mailmap): + with open(mailmap, 'r') as fp: + for l in fp: + try: + canonical_email, alias = re.match( + r'[^#]*?(<.+>).*(<.+>).*', l).groups() + except AttributeError: + continue + mapping[alias] = canonical_email + return mapping + + +def _parse_git_mailmap(git_dir, mailmap='.mailmap'): + mailmap = os.path.join(os.path.dirname(git_dir), mailmap) + return parse_mailmap(mailmap) + + +def canonicalize_emails(changelog, mapping): + """Takes in a string and an email alias mapping and replaces all + instances of the aliases in the string with their real email. + """ + for alias, email_address in mapping.iteritems(): + changelog = changelog.replace(alias, email_address) + return changelog + + +# Get requirements from the first file that exists +def get_reqs_from_files(requirements_files): + for requirements_file in requirements_files: + if os.path.exists(requirements_file): + with open(requirements_file, 'r') as fil: + return fil.read().split('\n') + return [] + + +def parse_requirements(requirements_files=['requirements.txt', + 'tools/pip-requires']): + requirements = [] + for line in get_reqs_from_files(requirements_files): + # For the requirements list, we need to inject only the portion + # after egg= so that distutils knows the package it's looking for + # such as: + # -e git://github.com/openstack/nova/master#egg=nova + if re.match(r'\s*-e\s+', line): + requirements.append(re.sub(r'\s*-e\s+.*#egg=(.*)$', r'\1', + line)) + # such as: + # http://github.com/openstack/nova/zipball/master#egg=nova + elif re.match(r'\s*https?:', line): + requirements.append(re.sub(r'\s*https?:.*#egg=(.*)$', r'\1', + line)) + # -f lines are for index locations, and don't get used here + elif re.match(r'\s*-f\s+', line): + pass + # argparse is part of the standard library starting with 2.7 + # adding it to the requirements list screws distro installs + elif line == 'argparse' and sys.version_info >= (2, 7): + pass + else: + requirements.append(line) + + return requirements + + +def parse_dependency_links(requirements_files=['requirements.txt', + 'tools/pip-requires']): + dependency_links = [] + # dependency_links inject alternate locations to find packages listed + # in requirements + for line in get_reqs_from_files(requirements_files): + # skip comments and blank lines + if re.match(r'(\s*#)|(\s*$)', line): + continue + # lines with -e or -f need the whole line, minus the flag + if re.match(r'\s*-[ef]\s+', line): + dependency_links.append(re.sub(r'\s*-[ef]\s+', '', line)) + # lines that are only urls can go in unmolested + elif re.match(r'\s*https?:', line): + dependency_links.append(line) + return dependency_links + + +def _run_shell_command(cmd, throw_on_error=False): + if os.name == 'nt': + output = subprocess.Popen(["cmd.exe", "/C", cmd], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + else: + output = subprocess.Popen(["/bin/sh", "-c", cmd], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + out = output.communicate() + if output.returncode and throw_on_error: + raise Exception("%s returned %d" % cmd, output.returncode) + if len(out) == 0: + return None + if len(out[0].strip()) == 0: + return None + return out[0].strip() + + +def _get_git_directory(): + parent_dir = os.path.dirname(__file__) + while True: + git_dir = os.path.join(parent_dir, '.git') + if os.path.exists(git_dir): + return git_dir + parent_dir, child = os.path.split(parent_dir) + if not child: # reached to root dir + return None + + +def write_git_changelog(): + """Write a changelog based on the git changelog.""" + new_changelog = 'ChangeLog' + git_dir = _get_git_directory() + if not os.getenv('SKIP_WRITE_GIT_CHANGELOG'): + if git_dir: + git_log_cmd = 'git --git-dir=%s log' % git_dir + changelog = _run_shell_command(git_log_cmd) + mailmap = _parse_git_mailmap(git_dir) + with open(new_changelog, "w") as changelog_file: + changelog_file.write(canonicalize_emails(changelog, mailmap)) + else: + open(new_changelog, 'w').close() + + +def generate_authors(): + """Create AUTHORS file using git commits.""" + jenkins_email = 'jenkins@review.(openstack|stackforge).org' + old_authors = 'AUTHORS.in' + new_authors = 'AUTHORS' + git_dir = _get_git_directory() + if not os.getenv('SKIP_GENERATE_AUTHORS'): + if git_dir: + # don't include jenkins email address in AUTHORS file + git_log_cmd = ("git --git-dir=" + git_dir + + " log --format='%aN <%aE>' | sort -u | " + "egrep -v '" + jenkins_email + "'") + changelog = _run_shell_command(git_log_cmd) + signed_cmd = ("git log --git-dir=" + git_dir + + " | grep -i Co-authored-by: | sort -u") + signed_entries = _run_shell_command(signed_cmd) + if signed_entries: + new_entries = "\n".join( + [signed.split(":", 1)[1].strip() + for signed in signed_entries.split("\n") if signed]) + changelog = "\n".join((changelog, new_entries)) + mailmap = _parse_git_mailmap(git_dir) + with open(new_authors, 'w') as new_authors_fh: + new_authors_fh.write(canonicalize_emails(changelog, mailmap)) + if os.path.exists(old_authors): + with open(old_authors, "r") as old_authors_fh: + new_authors_fh.write('\n' + old_authors_fh.read()) + else: + open(new_authors, 'w').close() + + +_rst_template = """%(heading)s +%(underline)s + +.. automodule:: %(module)s + :members: + :undoc-members: + :show-inheritance: +""" + + +def get_cmdclass(): + """Return dict of commands to run from setup.py.""" + + cmdclass = dict() + + def _find_modules(arg, dirname, files): + for filename in files: + if filename.endswith('.py') and filename != '__init__.py': + arg["%s.%s" % (dirname.replace('/', '.'), + filename[:-3])] = True + + class LocalSDist(sdist.sdist): + """Builds the ChangeLog and Authors files from VC first.""" + + def run(self): + write_git_changelog() + generate_authors() + # sdist.sdist is an old style class, can't use super() + sdist.sdist.run(self) + + cmdclass['sdist'] = LocalSDist + + # If Sphinx is installed on the box running setup.py, + # enable setup.py to build the documentation, otherwise, + # just ignore it + try: + from sphinx.setup_command import BuildDoc + + class LocalBuildDoc(BuildDoc): + + builders = ['html', 'man'] + + def generate_autoindex(self): + print "**Autodocumenting from %s" % os.path.abspath(os.curdir) + modules = {} + option_dict = self.distribution.get_option_dict('build_sphinx') + source_dir = os.path.join(option_dict['source_dir'][1], 'api') + if not os.path.exists(source_dir): + os.makedirs(source_dir) + for pkg in self.distribution.packages: + if '.' not in pkg: + os.path.walk(pkg, _find_modules, modules) + module_list = modules.keys() + module_list.sort() + autoindex_filename = os.path.join(source_dir, 'autoindex.rst') + with open(autoindex_filename, 'w') as autoindex: + autoindex.write(""".. toctree:: + :maxdepth: 1 + +""") + for module in module_list: + output_filename = os.path.join(source_dir, + "%s.rst" % module) + heading = "The :mod:`%s` Module" % module + underline = "=" * len(heading) + values = dict(module=module, heading=heading, + underline=underline) + + print "Generating %s" % output_filename + with open(output_filename, 'w') as output_file: + output_file.write(_rst_template % values) + autoindex.write(" %s.rst\n" % module) + + def run(self): + if not os.getenv('SPHINX_DEBUG'): + self.generate_autoindex() + + for builder in self.builders: + self.builder = builder + self.finalize_options() + self.project = self.distribution.get_name() + self.version = self.distribution.get_version() + self.release = self.distribution.get_version() + BuildDoc.run(self) + + class LocalBuildLatex(LocalBuildDoc): + builders = ['latex'] + + cmdclass['build_sphinx'] = LocalBuildDoc + cmdclass['build_sphinx_latex'] = LocalBuildLatex + except ImportError: + pass + + return cmdclass + + +def _get_revno(git_dir): + """Return the number of commits since the most recent tag. + + We use git-describe to find this out, but if there are no + tags then we fall back to counting commits since the beginning + of time. + """ + describe = _run_shell_command( + "git --git-dir=%s describe --always" % git_dir) + if "-" in describe: + return describe.rsplit("-", 2)[-2] + + # no tags found + revlist = _run_shell_command( + "git --git-dir=%s rev-list --abbrev-commit HEAD" % git_dir) + return len(revlist.splitlines()) + + +def _get_version_from_git(pre_version): + """Return a version which is equal to the tag that's on the current + revision if there is one, or tag plus number of additional revisions + if the current revision has no tag.""" + + git_dir = _get_git_directory() + if git_dir: + if pre_version: + try: + return _run_shell_command( + "git --git-dir=" + git_dir + " describe --exact-match", + throw_on_error=True).replace('-', '.') + except Exception: + sha = _run_shell_command( + "git --git-dir=" + git_dir + " log -n1 --pretty=format:%h") + return "%s.a%s.g%s" % (pre_version, _get_revno(git_dir), sha) + else: + return _run_shell_command( + "git --git-dir=" + git_dir + " describe --always").replace( + '-', '.') + return None + + +def _get_version_from_pkg_info(package_name): + """Get the version from PKG-INFO file if we can.""" + try: + pkg_info_file = open('PKG-INFO', 'r') + except (IOError, OSError): + return None + try: + pkg_info = email.message_from_file(pkg_info_file) + except email.MessageError: + return None + # Check to make sure we're in our own dir + if pkg_info.get('Name', None) != package_name: + return None + return pkg_info.get('Version', None) + + +def get_version(package_name, pre_version=None): + """Get the version of the project. First, try getting it from PKG-INFO, if + it exists. If it does, that means we're in a distribution tarball or that + install has happened. Otherwise, if there is no PKG-INFO file, pull the + version from git. + + We do not support setup.py version sanity in git archive tarballs, nor do + we support packagers directly sucking our git repo into theirs. We expect + that a source tarball be made from our git repo - or that if someone wants + to make a source tarball from a fork of our repo with additional tags in it + that they understand and desire the results of doing that. + """ + version = os.environ.get("OSLO_PACKAGE_VERSION", None) + if version: + return version + version = _get_version_from_pkg_info(package_name) + if version: + return version + version = _get_version_from_git(pre_version) + if version: + return version + raise Exception("Versioning for this project requires either an sdist" + " tarball, or access to an upstream git repository.") diff --git a/conductor/conductor/openstack/common/sslutils.py b/conductor/conductor/openstack/common/sslutils.py index 4168e87..6ccbac8 100644 --- a/conductor/conductor/openstack/common/sslutils.py +++ b/conductor/conductor/openstack/common/sslutils.py @@ -1,80 +1,80 @@ -# vim: tabstop=4 shiftwidth=4 softtabstop=4 - -# Copyright 2013 IBM -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -import os -import ssl - -from oslo.config import cfg - -from conductor.openstack.common.gettextutils import _ - - -ssl_opts = [ - cfg.StrOpt('ca_file', - default=None, - help="CA certificate file to use to verify " - "connecting clients"), - cfg.StrOpt('cert_file', - default=None, - help="Certificate file to use when starting " - "the server securely"), - cfg.StrOpt('key_file', - default=None, - help="Private key file to use when starting " - "the server securely"), -] - - -CONF = cfg.CONF -CONF.register_opts(ssl_opts, "ssl") - - -def is_enabled(): - cert_file = CONF.ssl.cert_file - key_file = CONF.ssl.key_file - ca_file = CONF.ssl.ca_file - use_ssl = cert_file or key_file - - if cert_file and not os.path.exists(cert_file): - raise RuntimeError(_("Unable to find cert_file : %s") % cert_file) - - if ca_file and not os.path.exists(ca_file): - raise RuntimeError(_("Unable to find ca_file : %s") % ca_file) - - if key_file and not os.path.exists(key_file): - raise RuntimeError(_("Unable to find key_file : %s") % key_file) - - if use_ssl and (not cert_file or not key_file): - raise RuntimeError(_("When running server in SSL mode, you must " - "specify both a cert_file and key_file " - "option value in your configuration file")) - - return use_ssl - - -def wrap(sock): - ssl_kwargs = { - 'server_side': True, - 'certfile': CONF.ssl.cert_file, - 'keyfile': CONF.ssl.key_file, - 'cert_reqs': ssl.CERT_NONE, - } - - if CONF.ssl.ca_file: - ssl_kwargs['ca_certs'] = CONF.ssl.ca_file - ssl_kwargs['cert_reqs'] = ssl.CERT_REQUIRED - - return ssl.wrap_socket(sock, **ssl_kwargs) +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright 2013 IBM +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import os +import ssl + +from oslo.config import cfg + +from conductor.openstack.common.gettextutils import _ + + +ssl_opts = [ + cfg.StrOpt('ca_file', + default=None, + help="CA certificate file to use to verify " + "connecting clients"), + cfg.StrOpt('cert_file', + default=None, + help="Certificate file to use when starting " + "the server securely"), + cfg.StrOpt('key_file', + default=None, + help="Private key file to use when starting " + "the server securely"), +] + + +CONF = cfg.CONF +CONF.register_opts(ssl_opts, "ssl") + + +def is_enabled(): + cert_file = CONF.ssl.cert_file + key_file = CONF.ssl.key_file + ca_file = CONF.ssl.ca_file + use_ssl = cert_file or key_file + + if cert_file and not os.path.exists(cert_file): + raise RuntimeError(_("Unable to find cert_file : %s") % cert_file) + + if ca_file and not os.path.exists(ca_file): + raise RuntimeError(_("Unable to find ca_file : %s") % ca_file) + + if key_file and not os.path.exists(key_file): + raise RuntimeError(_("Unable to find key_file : %s") % key_file) + + if use_ssl and (not cert_file or not key_file): + raise RuntimeError(_("When running server in SSL mode, you must " + "specify both a cert_file and key_file " + "option value in your configuration file")) + + return use_ssl + + +def wrap(sock): + ssl_kwargs = { + 'server_side': True, + 'certfile': CONF.ssl.cert_file, + 'keyfile': CONF.ssl.key_file, + 'cert_reqs': ssl.CERT_NONE, + } + + if CONF.ssl.ca_file: + ssl_kwargs['ca_certs'] = CONF.ssl.ca_file + ssl_kwargs['cert_reqs'] = ssl.CERT_REQUIRED + + return ssl.wrap_socket(sock, **ssl_kwargs) diff --git a/conductor/conductor/openstack/common/threadgroup.py b/conductor/conductor/openstack/common/threadgroup.py index c8d0d9e..5c986aa 100644 --- a/conductor/conductor/openstack/common/threadgroup.py +++ b/conductor/conductor/openstack/common/threadgroup.py @@ -1,114 +1,114 @@ -# vim: tabstop=4 shiftwidth=4 softtabstop=4 - -# Copyright 2012 Red Hat, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -from eventlet import greenlet -from eventlet import greenpool -from eventlet import greenthread - -from conductor.openstack.common import log as logging -from conductor.openstack.common import loopingcall - - -LOG = logging.getLogger(__name__) - - -def _thread_done(gt, *args, **kwargs): - """ Callback function to be passed to GreenThread.link() when we spawn() - Calls the :class:`ThreadGroup` to notify if. - - """ - kwargs['group'].thread_done(kwargs['thread']) - - -class Thread(object): - """ Wrapper around a greenthread, that holds a reference to the - :class:`ThreadGroup`. The Thread will notify the :class:`ThreadGroup` when - it has done so it can be removed from the threads list. - """ - def __init__(self, thread, group): - self.thread = thread - self.thread.link(_thread_done, group=group, thread=self) - - def stop(self): - self.thread.kill() - - def wait(self): - return self.thread.wait() - - -class ThreadGroup(object): - """ The point of the ThreadGroup classis to: - - * keep track of timers and greenthreads (making it easier to stop them - when need be). - * provide an easy API to add timers. - """ - def __init__(self, thread_pool_size=10): - self.pool = greenpool.GreenPool(thread_pool_size) - self.threads = [] - self.timers = [] - - def add_timer(self, interval, callback, initial_delay=None, - *args, **kwargs): - pulse = loopingcall.LoopingCall(callback, *args, **kwargs) - pulse.start(interval=interval, - initial_delay=initial_delay) - self.timers.append(pulse) - - def add_thread(self, callback, *args, **kwargs): - gt = self.pool.spawn(callback, *args, **kwargs) - th = Thread(gt, self) - self.threads.append(th) - - def thread_done(self, thread): - self.threads.remove(thread) - - def stop(self): - current = greenthread.getcurrent() - for x in self.threads: - if x is current: - # don't kill the current thread. - continue - try: - x.stop() - except Exception as ex: - LOG.exception(ex) - - for x in self.timers: - try: - x.stop() - except Exception as ex: - LOG.exception(ex) - self.timers = [] - - def wait(self): - for x in self.timers: - try: - x.wait() - except greenlet.GreenletExit: - pass - except Exception as ex: - LOG.exception(ex) - current = greenthread.getcurrent() - for x in self.threads: - if x is current: - continue - try: - x.wait() - except greenlet.GreenletExit: - pass - except Exception as ex: - LOG.exception(ex) +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright 2012 Red Hat, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from eventlet import greenlet +from eventlet import greenpool +from eventlet import greenthread + +from conductor.openstack.common import log as logging +from conductor.openstack.common import loopingcall + + +LOG = logging.getLogger(__name__) + + +def _thread_done(gt, *args, **kwargs): + """ Callback function to be passed to GreenThread.link() when we spawn() + Calls the :class:`ThreadGroup` to notify if. + + """ + kwargs['group'].thread_done(kwargs['thread']) + + +class Thread(object): + """ Wrapper around a greenthread, that holds a reference to the + :class:`ThreadGroup`. The Thread will notify the :class:`ThreadGroup` when + it has done so it can be removed from the threads list. + """ + def __init__(self, thread, group): + self.thread = thread + self.thread.link(_thread_done, group=group, thread=self) + + def stop(self): + self.thread.kill() + + def wait(self): + return self.thread.wait() + + +class ThreadGroup(object): + """ The point of the ThreadGroup classis to: + + * keep track of timers and greenthreads (making it easier to stop them + when need be). + * provide an easy API to add timers. + """ + def __init__(self, thread_pool_size=10): + self.pool = greenpool.GreenPool(thread_pool_size) + self.threads = [] + self.timers = [] + + def add_timer(self, interval, callback, initial_delay=None, + *args, **kwargs): + pulse = loopingcall.LoopingCall(callback, *args, **kwargs) + pulse.start(interval=interval, + initial_delay=initial_delay) + self.timers.append(pulse) + + def add_thread(self, callback, *args, **kwargs): + gt = self.pool.spawn(callback, *args, **kwargs) + th = Thread(gt, self) + self.threads.append(th) + + def thread_done(self, thread): + self.threads.remove(thread) + + def stop(self): + current = greenthread.getcurrent() + for x in self.threads: + if x is current: + # don't kill the current thread. + continue + try: + x.stop() + except Exception as ex: + LOG.exception(ex) + + for x in self.timers: + try: + x.stop() + except Exception as ex: + LOG.exception(ex) + self.timers = [] + + def wait(self): + for x in self.timers: + try: + x.wait() + except greenlet.GreenletExit: + pass + except Exception as ex: + LOG.exception(ex) + current = greenthread.getcurrent() + for x in self.threads: + if x is current: + continue + try: + x.wait() + except greenlet.GreenletExit: + pass + except Exception as ex: + LOG.exception(ex) diff --git a/conductor/conductor/openstack/common/timeutils.py b/conductor/conductor/openstack/common/timeutils.py index 35cdf8c..6094365 100644 --- a/conductor/conductor/openstack/common/timeutils.py +++ b/conductor/conductor/openstack/common/timeutils.py @@ -1,186 +1,186 @@ -# vim: tabstop=4 shiftwidth=4 softtabstop=4 - -# Copyright 2011 OpenStack Foundation. -# All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -""" -Time related utilities and helper functions. -""" - -import calendar -import datetime - -import iso8601 - - -# ISO 8601 extended time format with microseconds -_ISO8601_TIME_FORMAT_SUBSECOND = '%Y-%m-%dT%H:%M:%S.%f' -_ISO8601_TIME_FORMAT = '%Y-%m-%dT%H:%M:%S' -PERFECT_TIME_FORMAT = _ISO8601_TIME_FORMAT_SUBSECOND - - -def isotime(at=None, subsecond=False): - """Stringify time in ISO 8601 format""" - if not at: - at = utcnow() - st = at.strftime(_ISO8601_TIME_FORMAT - if not subsecond - else _ISO8601_TIME_FORMAT_SUBSECOND) - tz = at.tzinfo.tzname(None) if at.tzinfo else 'UTC' - st += ('Z' if tz == 'UTC' else tz) - return st - - -def parse_isotime(timestr): - """Parse time from ISO 8601 format""" - try: - return iso8601.parse_date(timestr) - except iso8601.ParseError as e: - raise ValueError(e.message) - except TypeError as e: - raise ValueError(e.message) - - -def strtime(at=None, fmt=PERFECT_TIME_FORMAT): - """Returns formatted utcnow.""" - if not at: - at = utcnow() - return at.strftime(fmt) - - -def parse_strtime(timestr, fmt=PERFECT_TIME_FORMAT): - """Turn a formatted time back into a datetime.""" - return datetime.datetime.strptime(timestr, fmt) - - -def normalize_time(timestamp): - """Normalize time in arbitrary timezone to UTC naive object""" - offset = timestamp.utcoffset() - if offset is None: - return timestamp - return timestamp.replace(tzinfo=None) - offset - - -def is_older_than(before, seconds): - """Return True if before is older than seconds.""" - if isinstance(before, basestring): - before = parse_strtime(before).replace(tzinfo=None) - return utcnow() - before > datetime.timedelta(seconds=seconds) - - -def is_newer_than(after, seconds): - """Return True if after is newer than seconds.""" - if isinstance(after, basestring): - after = parse_strtime(after).replace(tzinfo=None) - return after - utcnow() > datetime.timedelta(seconds=seconds) - - -def utcnow_ts(): - """Timestamp version of our utcnow function.""" - return calendar.timegm(utcnow().timetuple()) - - -def utcnow(): - """Overridable version of utils.utcnow.""" - if utcnow.override_time: - try: - return utcnow.override_time.pop(0) - except AttributeError: - return utcnow.override_time - return datetime.datetime.utcnow() - - -def iso8601_from_timestamp(timestamp): - """Returns a iso8601 formated date from timestamp""" - return isotime(datetime.datetime.utcfromtimestamp(timestamp)) - - -utcnow.override_time = None - - -def set_time_override(override_time=datetime.datetime.utcnow()): - """ - Override utils.utcnow to return a constant time or a list thereof, - one at a time. - """ - utcnow.override_time = override_time - - -def advance_time_delta(timedelta): - """Advance overridden time using a datetime.timedelta.""" - assert(not utcnow.override_time is None) - try: - for dt in utcnow.override_time: - dt += timedelta - except TypeError: - utcnow.override_time += timedelta - - -def advance_time_seconds(seconds): - """Advance overridden time by seconds.""" - advance_time_delta(datetime.timedelta(0, seconds)) - - -def clear_time_override(): - """Remove the overridden time.""" - utcnow.override_time = None - - -def marshall_now(now=None): - """Make an rpc-safe datetime with microseconds. - - Note: tzinfo is stripped, but not required for relative times.""" - if not now: - now = utcnow() - return dict(day=now.day, month=now.month, year=now.year, hour=now.hour, - minute=now.minute, second=now.second, - microsecond=now.microsecond) - - -def unmarshall_time(tyme): - """Unmarshall a datetime dict.""" - return datetime.datetime(day=tyme['day'], - month=tyme['month'], - year=tyme['year'], - hour=tyme['hour'], - minute=tyme['minute'], - second=tyme['second'], - microsecond=tyme['microsecond']) - - -def delta_seconds(before, after): - """ - Compute the difference in seconds between two date, time, or - datetime objects (as a float, to microsecond resolution). - """ - delta = after - before - try: - return delta.total_seconds() - except AttributeError: - return ((delta.days * 24 * 3600) + delta.seconds + - float(delta.microseconds) / (10 ** 6)) - - -def is_soon(dt, window): - """ - Determines if time is going to happen in the next window seconds. - - :params dt: the time - :params window: minimum seconds to remain to consider the time not soon - - :return: True if expiration is within the given duration - """ - soon = (utcnow() + datetime.timedelta(seconds=window)) - return normalize_time(dt) <= soon +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright 2011 OpenStack Foundation. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +""" +Time related utilities and helper functions. +""" + +import calendar +import datetime + +import iso8601 + + +# ISO 8601 extended time format with microseconds +_ISO8601_TIME_FORMAT_SUBSECOND = '%Y-%m-%dT%H:%M:%S.%f' +_ISO8601_TIME_FORMAT = '%Y-%m-%dT%H:%M:%S' +PERFECT_TIME_FORMAT = _ISO8601_TIME_FORMAT_SUBSECOND + + +def isotime(at=None, subsecond=False): + """Stringify time in ISO 8601 format""" + if not at: + at = utcnow() + st = at.strftime(_ISO8601_TIME_FORMAT + if not subsecond + else _ISO8601_TIME_FORMAT_SUBSECOND) + tz = at.tzinfo.tzname(None) if at.tzinfo else 'UTC' + st += ('Z' if tz == 'UTC' else tz) + return st + + +def parse_isotime(timestr): + """Parse time from ISO 8601 format""" + try: + return iso8601.parse_date(timestr) + except iso8601.ParseError as e: + raise ValueError(e.message) + except TypeError as e: + raise ValueError(e.message) + + +def strtime(at=None, fmt=PERFECT_TIME_FORMAT): + """Returns formatted utcnow.""" + if not at: + at = utcnow() + return at.strftime(fmt) + + +def parse_strtime(timestr, fmt=PERFECT_TIME_FORMAT): + """Turn a formatted time back into a datetime.""" + return datetime.datetime.strptime(timestr, fmt) + + +def normalize_time(timestamp): + """Normalize time in arbitrary timezone to UTC naive object""" + offset = timestamp.utcoffset() + if offset is None: + return timestamp + return timestamp.replace(tzinfo=None) - offset + + +def is_older_than(before, seconds): + """Return True if before is older than seconds.""" + if isinstance(before, basestring): + before = parse_strtime(before).replace(tzinfo=None) + return utcnow() - before > datetime.timedelta(seconds=seconds) + + +def is_newer_than(after, seconds): + """Return True if after is newer than seconds.""" + if isinstance(after, basestring): + after = parse_strtime(after).replace(tzinfo=None) + return after - utcnow() > datetime.timedelta(seconds=seconds) + + +def utcnow_ts(): + """Timestamp version of our utcnow function.""" + return calendar.timegm(utcnow().timetuple()) + + +def utcnow(): + """Overridable version of utils.utcnow.""" + if utcnow.override_time: + try: + return utcnow.override_time.pop(0) + except AttributeError: + return utcnow.override_time + return datetime.datetime.utcnow() + + +def iso8601_from_timestamp(timestamp): + """Returns a iso8601 formated date from timestamp""" + return isotime(datetime.datetime.utcfromtimestamp(timestamp)) + + +utcnow.override_time = None + + +def set_time_override(override_time=datetime.datetime.utcnow()): + """ + Override utils.utcnow to return a constant time or a list thereof, + one at a time. + """ + utcnow.override_time = override_time + + +def advance_time_delta(timedelta): + """Advance overridden time using a datetime.timedelta.""" + assert(not utcnow.override_time is None) + try: + for dt in utcnow.override_time: + dt += timedelta + except TypeError: + utcnow.override_time += timedelta + + +def advance_time_seconds(seconds): + """Advance overridden time by seconds.""" + advance_time_delta(datetime.timedelta(0, seconds)) + + +def clear_time_override(): + """Remove the overridden time.""" + utcnow.override_time = None + + +def marshall_now(now=None): + """Make an rpc-safe datetime with microseconds. + + Note: tzinfo is stripped, but not required for relative times.""" + if not now: + now = utcnow() + return dict(day=now.day, month=now.month, year=now.year, hour=now.hour, + minute=now.minute, second=now.second, + microsecond=now.microsecond) + + +def unmarshall_time(tyme): + """Unmarshall a datetime dict.""" + return datetime.datetime(day=tyme['day'], + month=tyme['month'], + year=tyme['year'], + hour=tyme['hour'], + minute=tyme['minute'], + second=tyme['second'], + microsecond=tyme['microsecond']) + + +def delta_seconds(before, after): + """ + Compute the difference in seconds between two date, time, or + datetime objects (as a float, to microsecond resolution). + """ + delta = after - before + try: + return delta.total_seconds() + except AttributeError: + return ((delta.days * 24 * 3600) + delta.seconds + + float(delta.microseconds) / (10 ** 6)) + + +def is_soon(dt, window): + """ + Determines if time is going to happen in the next window seconds. + + :params dt: the time + :params window: minimum seconds to remain to consider the time not soon + + :return: True if expiration is within the given duration + """ + soon = (utcnow() + datetime.timedelta(seconds=window)) + return normalize_time(dt) <= soon diff --git a/conductor/conductor/openstack/common/uuidutils.py b/conductor/conductor/openstack/common/uuidutils.py index fff6309..7608acb 100644 --- a/conductor/conductor/openstack/common/uuidutils.py +++ b/conductor/conductor/openstack/common/uuidutils.py @@ -1,39 +1,39 @@ -# vim: tabstop=4 shiftwidth=4 softtabstop=4 - -# Copyright (c) 2012 Intel Corporation. -# All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -""" -UUID related utilities and helper functions. -""" - -import uuid - - -def generate_uuid(): - return str(uuid.uuid4()) - - -def is_uuid_like(val): - """Returns validation of a value as a UUID. - - For our purposes, a UUID is a canonical form string: - aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa - - """ - try: - return str(uuid.UUID(val)) == val - except (TypeError, ValueError, AttributeError): - return False +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright (c) 2012 Intel Corporation. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +""" +UUID related utilities and helper functions. +""" + +import uuid + + +def generate_uuid(): + return str(uuid.uuid4()) + + +def is_uuid_like(val): + """Returns validation of a value as a UUID. + + For our purposes, a UUID is a canonical form string: + aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa + + """ + try: + return str(uuid.UUID(val)) == val + except (TypeError, ValueError, AttributeError): + return False diff --git a/conductor/conductor/openstack/common/version.py b/conductor/conductor/openstack/common/version.py index 0b80a60..080a89e 100644 --- a/conductor/conductor/openstack/common/version.py +++ b/conductor/conductor/openstack/common/version.py @@ -1,94 +1,94 @@ - -# Copyright 2012 OpenStack Foundation -# Copyright 2012-2013 Hewlett-Packard Development Company, L.P. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -""" -Utilities for consuming the version from pkg_resources. -""" - -import pkg_resources - - -class VersionInfo(object): - - def __init__(self, package): - """Object that understands versioning for a package - :param package: name of the python package, such as glance, or - python-glanceclient - """ - self.package = package - self.release = None - self.version = None - self._cached_version = None - - def __str__(self): - """Make the VersionInfo object behave like a string.""" - return self.version_string() - - def __repr__(self): - """Include the name.""" - return "VersionInfo(%s:%s)" % (self.package, self.version_string()) - - def _get_version_from_pkg_resources(self): - """Get the version of the package from the pkg_resources record - associated with the package.""" - try: - requirement = pkg_resources.Requirement.parse(self.package) - provider = pkg_resources.get_provider(requirement) - return provider.version - except pkg_resources.DistributionNotFound: - # The most likely cause for this is running tests in a tree - # produced from a tarball where the package itself has not been - # installed into anything. Revert to setup-time logic. - from conductor.openstack.common import setup - return setup.get_version(self.package) - - def release_string(self): - """Return the full version of the package including suffixes indicating - VCS status. - """ - if self.release is None: - self.release = self._get_version_from_pkg_resources() - - return self.release - - def version_string(self): - """Return the short version minus any alpha/beta tags.""" - if self.version is None: - parts = [] - for part in self.release_string().split('.'): - if part[0].isdigit(): - parts.append(part) - else: - break - self.version = ".".join(parts) - - return self.version - - # Compatibility functions - canonical_version_string = version_string - version_string_with_vcs = release_string - - def cached_version_string(self, prefix=""): - """Generate an object which will expand in a string context to - the results of version_string(). We do this so that don't - call into pkg_resources every time we start up a program when - passing version information into the CONF constructor, but - rather only do the calculation when and if a version is requested - """ - if not self._cached_version: - self._cached_version = "%s%s" % (prefix, - self.version_string()) - return self._cached_version + +# Copyright 2012 OpenStack Foundation +# Copyright 2012-2013 Hewlett-Packard Development Company, L.P. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +""" +Utilities for consuming the version from pkg_resources. +""" + +import pkg_resources + + +class VersionInfo(object): + + def __init__(self, package): + """Object that understands versioning for a package + :param package: name of the python package, such as glance, or + python-glanceclient + """ + self.package = package + self.release = None + self.version = None + self._cached_version = None + + def __str__(self): + """Make the VersionInfo object behave like a string.""" + return self.version_string() + + def __repr__(self): + """Include the name.""" + return "VersionInfo(%s:%s)" % (self.package, self.version_string()) + + def _get_version_from_pkg_resources(self): + """Get the version of the package from the pkg_resources record + associated with the package.""" + try: + requirement = pkg_resources.Requirement.parse(self.package) + provider = pkg_resources.get_provider(requirement) + return provider.version + except pkg_resources.DistributionNotFound: + # The most likely cause for this is running tests in a tree + # produced from a tarball where the package itself has not been + # installed into anything. Revert to setup-time logic. + from conductor.openstack.common import setup + return setup.get_version(self.package) + + def release_string(self): + """Return the full version of the package including suffixes indicating + VCS status. + """ + if self.release is None: + self.release = self._get_version_from_pkg_resources() + + return self.release + + def version_string(self): + """Return the short version minus any alpha/beta tags.""" + if self.version is None: + parts = [] + for part in self.release_string().split('.'): + if part[0].isdigit(): + parts.append(part) + else: + break + self.version = ".".join(parts) + + return self.version + + # Compatibility functions + canonical_version_string = version_string + version_string_with_vcs = release_string + + def cached_version_string(self, prefix=""): + """Generate an object which will expand in a string context to + the results of version_string(). We do this so that don't + call into pkg_resources every time we start up a program when + passing version information into the CONF constructor, but + rather only do the calculation when and if a version is requested + """ + if not self._cached_version: + self._cached_version = "%s%s" % (prefix, + self.version_string()) + return self._cached_version diff --git a/conductor/conductor/openstack/common/wsgi.py b/conductor/conductor/openstack/common/wsgi.py index c367224..9df3188 100644 --- a/conductor/conductor/openstack/common/wsgi.py +++ b/conductor/conductor/openstack/common/wsgi.py @@ -1,797 +1,797 @@ -# vim: tabstop=4 shiftwidth=4 softtabstop=4 - -# Copyright 2011 OpenStack Foundation. -# All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -"""Utility methods for working with WSGI servers.""" - -import eventlet -eventlet.patcher.monkey_patch(all=False, socket=True) - -import datetime -import errno -import socket -import sys -import time - -import eventlet.wsgi -from oslo.config import cfg -import routes -import routes.middleware -import webob.dec -import webob.exc -from xml.dom import minidom -from xml.parsers import expat - -from conductor.openstack.common import exception -from conductor.openstack.common.gettextutils import _ -from conductor.openstack.common import jsonutils -from conductor.openstack.common import log as logging -from conductor.openstack.common import service -from conductor.openstack.common import sslutils -from conductor.openstack.common import xmlutils - -socket_opts = [ - cfg.IntOpt('backlog', - default=4096, - help="Number of backlog requests to configure the socket with"), - cfg.IntOpt('tcp_keepidle', - default=600, - help="Sets the value of TCP_KEEPIDLE in seconds for each " - "server socket. Not supported on OS X."), -] - -CONF = cfg.CONF -CONF.register_opts(socket_opts) - -LOG = logging.getLogger(__name__) - - -def run_server(application, port): - """Run a WSGI server with the given application.""" - sock = eventlet.listen(('0.0.0.0', port)) - eventlet.wsgi.server(sock, application) - - -class Service(service.Service): - """ - Provides a Service API for wsgi servers. - - This gives us the ability to launch wsgi servers with the - Launcher classes in service.py. - """ - - def __init__(self, application, port, - host='0.0.0.0', backlog=4096, threads=1000): - self.application = application - self._port = port - self._host = host - self._backlog = backlog if backlog else CONF.backlog - super(Service, self).__init__(threads) - - def _get_socket(self, host, port, backlog): - # TODO(dims): eventlet's green dns/socket module does not actually - # support IPv6 in getaddrinfo(). We need to get around this in the - # future or monitor upstream for a fix - info = socket.getaddrinfo(host, - port, - socket.AF_UNSPEC, - socket.SOCK_STREAM)[0] - family = info[0] - bind_addr = info[-1] - - sock = None - retry_until = time.time() + 30 - while not sock and time.time() < retry_until: - try: - sock = eventlet.listen(bind_addr, - backlog=backlog, - family=family) - if sslutils.is_enabled(): - sock = sslutils.wrap(sock) - - except socket.error, err: - if err.args[0] != errno.EADDRINUSE: - raise - eventlet.sleep(0.1) - if not sock: - raise RuntimeError(_("Could not bind to %(host)s:%(port)s " - "after trying for 30 seconds") % - {'host': host, 'port': port}) - sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) - # sockets can hang around forever without keepalive - sock.setsockopt(socket.SOL_SOCKET, socket.SO_KEEPALIVE, 1) - - # This option isn't available in the OS X version of eventlet - if hasattr(socket, 'TCP_KEEPIDLE'): - sock.setsockopt(socket.IPPROTO_TCP, - socket.TCP_KEEPIDLE, - CONF.tcp_keepidle) - - return sock - - def start(self): - """Start serving this service using the provided server instance. - - :returns: None - - """ - super(Service, self).start() - self._socket = self._get_socket(self._host, self._port, self._backlog) - self.tg.add_thread(self._run, self.application, self._socket) - - @property - def backlog(self): - return self._backlog - - @property - def host(self): - return self._socket.getsockname()[0] if self._socket else self._host - - @property - def port(self): - return self._socket.getsockname()[1] if self._socket else self._port - - def stop(self): - """Stop serving this API. - - :returns: None - - """ - super(Service, self).stop() - - def _run(self, application, socket): - """Start a WSGI server in a new green thread.""" - logger = logging.getLogger('eventlet.wsgi') - eventlet.wsgi.server(socket, - application, - custom_pool=self.tg.pool, - log=logging.WritableLogger(logger)) - - -class Middleware(object): - """ - Base WSGI middleware wrapper. These classes require an application to be - initialized that will be called next. By default the middleware will - simply call its wrapped app, or you can override __call__ to customize its - behavior. - """ - - def __init__(self, application): - self.application = application - - def process_request(self, req): - """ - Called on each request. - - If this returns None, the next application down the stack will be - executed. If it returns a response then that response will be returned - and execution will stop here. - """ - return None - - def process_response(self, response): - """Do whatever you'd like to the response.""" - return response - - @webob.dec.wsgify - def __call__(self, req): - response = self.process_request(req) - if response: - return response - response = req.get_response(self.application) - return self.process_response(response) - - -class Debug(Middleware): - """ - Helper class that can be inserted into any WSGI application chain - to get information about the request and response. - """ - - @webob.dec.wsgify - def __call__(self, req): - print ("*" * 40) + " REQUEST ENVIRON" - for key, value in req.environ.items(): - print key, "=", value - print - resp = req.get_response(self.application) - - print ("*" * 40) + " RESPONSE HEADERS" - for (key, value) in resp.headers.iteritems(): - print key, "=", value - print - - resp.app_iter = self.print_generator(resp.app_iter) - - return resp - - @staticmethod - def print_generator(app_iter): - """ - Iterator that prints the contents of a wrapper string iterator - when iterated. - """ - print ("*" * 40) + " BODY" - for part in app_iter: - sys.stdout.write(part) - sys.stdout.flush() - yield part - print - - -class Router(object): - - """ - WSGI middleware that maps incoming requests to WSGI apps. - """ - - def __init__(self, mapper): - """ - Create a router for the given routes.Mapper. - - Each route in `mapper` must specify a 'controller', which is a - WSGI app to call. You'll probably want to specify an 'action' as - well and have your controller be a wsgi.Controller, who will route - the request to the action method. - - Examples: - mapper = routes.Mapper() - sc = ServerController() - - # Explicit mapping of one route to a controller+action - mapper.connect(None, "/svrlist", controller=sc, action="list") - - # Actions are all implicitly defined - mapper.resource("server", "servers", controller=sc) - - # Pointing to an arbitrary WSGI app. You can specify the - # {path_info:.*} parameter so the target app can be handed just that - # section of the URL. - mapper.connect(None, "/v1.0/{path_info:.*}", controller=BlogApp()) - """ - self.map = mapper - self._router = routes.middleware.RoutesMiddleware(self._dispatch, - self.map) - - @webob.dec.wsgify - def __call__(self, req): - """ - Route the incoming request to a controller based on self.map. - If no match, return a 404. - """ - return self._router - - @staticmethod - @webob.dec.wsgify - def _dispatch(req): - """ - Called by self._router after matching the incoming request to a route - and putting the information into req.environ. Either returns 404 - or the routed WSGI app's response. - """ - match = req.environ['wsgiorg.routing_args'][1] - if not match: - return webob.exc.HTTPNotFound() - app = match['controller'] - return app - - -class Request(webob.Request): - """Add some Openstack API-specific logic to the base webob.Request.""" - - default_request_content_types = ('application/json', 'application/xml') - default_accept_types = ('application/json', 'application/xml') - default_accept_type = 'application/json' - - def best_match_content_type(self, supported_content_types=None): - """Determine the requested response content-type. - - Based on the query extension then the Accept header. - Defaults to default_accept_type if we don't find a preference - - """ - supported_content_types = (supported_content_types or - self.default_accept_types) - - parts = self.path.rsplit('.', 1) - if len(parts) > 1: - ctype = 'application/{0}'.format(parts[1]) - if ctype in supported_content_types: - return ctype - - bm = self.accept.best_match(supported_content_types) - return bm or self.default_accept_type - - def get_content_type(self, allowed_content_types=None): - """Determine content type of the request body. - - Does not do any body introspection, only checks header - - """ - if "Content-Type" not in self.headers: - return None - - content_type = self.content_type - allowed_content_types = (allowed_content_types or - self.default_request_content_types) - - if content_type not in allowed_content_types: - raise exception.InvalidContentType(content_type=content_type) - return content_type - - -class Resource(object): - """ - WSGI app that handles (de)serialization and controller dispatch. - - Reads routing information supplied by RoutesMiddleware and calls - the requested action method upon its deserializer, controller, - and serializer. Those three objects may implement any of the basic - controller action methods (create, update, show, index, delete) - along with any that may be specified in the api router. A 'default' - method may also be implemented to be used in place of any - non-implemented actions. Deserializer methods must accept a request - argument and return a dictionary. Controller methods must accept a - request argument. Additionally, they must also accept keyword - arguments that represent the keys returned by the Deserializer. They - may raise a webob.exc exception or return a dict, which will be - serialized by requested content type. - """ - def __init__(self, controller, deserializer=None, serializer=None): - """ - :param controller: object that implement methods created by routes lib - :param deserializer: object that supports webob request deserialization - through controller-like actions - :param serializer: object that supports webob response serialization - through controller-like actions - """ - self.controller = controller - self.serializer = serializer or ResponseSerializer() - self.deserializer = deserializer or RequestDeserializer() - - @webob.dec.wsgify(RequestClass=Request) - def __call__(self, request): - """WSGI method that controls (de)serialization and method dispatch.""" - - try: - action, action_args, accept = self.deserialize_request(request) - except exception.InvalidContentType: - msg = _("Unsupported Content-Type") - return webob.exc.HTTPUnsupportedMediaType(explanation=msg) - except exception.MalformedRequestBody: - msg = _("Malformed request body") - return webob.exc.HTTPBadRequest(explanation=msg) - - action_result = self.execute_action(action, request, **action_args) - try: - return self.serialize_response(action, action_result, accept) - # return unserializable result (typically a webob exc) - except Exception: - return action_result - - def deserialize_request(self, request): - return self.deserializer.deserialize(request) - - def serialize_response(self, action, action_result, accept): - return self.serializer.serialize(action_result, accept, action) - - def execute_action(self, action, request, **action_args): - return self.dispatch(self.controller, action, request, **action_args) - - def dispatch(self, obj, action, *args, **kwargs): - """Find action-specific method on self and call it.""" - try: - method = getattr(obj, action) - except AttributeError: - method = getattr(obj, 'default') - - return method(*args, **kwargs) - - def get_action_args(self, request_environment): - """Parse dictionary created by routes library.""" - try: - args = request_environment['wsgiorg.routing_args'][1].copy() - except Exception: - return {} - - try: - del args['controller'] - except KeyError: - pass - - try: - del args['format'] - except KeyError: - pass - - return args - - -class ActionDispatcher(object): - """Maps method name to local methods through action name.""" - - def dispatch(self, *args, **kwargs): - """Find and call local method.""" - action = kwargs.pop('action', 'default') - action_method = getattr(self, str(action), self.default) - return action_method(*args, **kwargs) - - def default(self, data): - raise NotImplementedError() - - -class DictSerializer(ActionDispatcher): - """Default request body serialization""" - - def serialize(self, data, action='default'): - return self.dispatch(data, action=action) - - def default(self, data): - return "" - - -class JSONDictSerializer(DictSerializer): - """Default JSON request body serialization""" - - def default(self, data): - def sanitizer(obj): - if isinstance(obj, datetime.datetime): - _dtime = obj - datetime.timedelta(microseconds=obj.microsecond) - return _dtime.isoformat() - return unicode(obj) - return jsonutils.dumps(data, default=sanitizer) - - -class XMLDictSerializer(DictSerializer): - - def __init__(self, metadata=None, xmlns=None): - """ - :param metadata: information needed to deserialize xml into - a dictionary. - :param xmlns: XML namespace to include with serialized xml - """ - super(XMLDictSerializer, self).__init__() - self.metadata = metadata or {} - self.xmlns = xmlns - - def default(self, data): - # We expect data to contain a single key which is the XML root. - root_key = data.keys()[0] - doc = minidom.Document() - node = self._to_xml_node(doc, self.metadata, root_key, data[root_key]) - - return self.to_xml_string(node) - - def to_xml_string(self, node, has_atom=False): - self._add_xmlns(node, has_atom) - return node.toprettyxml(indent=' ', encoding='UTF-8') - - #NOTE (ameade): the has_atom should be removed after all of the - # xml serializers and view builders have been updated to the current - # spec that required all responses include the xmlns:atom, the has_atom - # flag is to prevent current tests from breaking - def _add_xmlns(self, node, has_atom=False): - if self.xmlns is not None: - node.setAttribute('xmlns', self.xmlns) - if has_atom: - node.setAttribute('xmlns:atom', "http://www.w3.org/2005/Atom") - - def _to_xml_node(self, doc, metadata, nodename, data): - """Recursive method to convert data members to XML nodes.""" - result = doc.createElement(nodename) - - # Set the xml namespace if one is specified - # TODO(justinsb): We could also use prefixes on the keys - xmlns = metadata.get('xmlns', None) - if xmlns: - result.setAttribute('xmlns', xmlns) - - #TODO(bcwaldon): accomplish this without a type-check - if type(data) is list: - collections = metadata.get('list_collections', {}) - if nodename in collections: - metadata = collections[nodename] - for item in data: - node = doc.createElement(metadata['item_name']) - node.setAttribute(metadata['item_key'], str(item)) - result.appendChild(node) - return result - singular = metadata.get('plurals', {}).get(nodename, None) - if singular is None: - if nodename.endswith('s'): - singular = nodename[:-1] - else: - singular = 'item' - for item in data: - node = self._to_xml_node(doc, metadata, singular, item) - result.appendChild(node) - #TODO(bcwaldon): accomplish this without a type-check - elif type(data) is dict: - collections = metadata.get('dict_collections', {}) - if nodename in collections: - metadata = collections[nodename] - for k, v in data.items(): - node = doc.createElement(metadata['item_name']) - node.setAttribute(metadata['item_key'], str(k)) - text = doc.createTextNode(str(v)) - node.appendChild(text) - result.appendChild(node) - return result - attrs = metadata.get('attributes', {}).get(nodename, {}) - for k, v in data.items(): - if k in attrs: - result.setAttribute(k, str(v)) - else: - node = self._to_xml_node(doc, metadata, k, v) - result.appendChild(node) - else: - # Type is atom - node = doc.createTextNode(str(data)) - result.appendChild(node) - return result - - def _create_link_nodes(self, xml_doc, links): - link_nodes = [] - for link in links: - link_node = xml_doc.createElement('atom:link') - link_node.setAttribute('rel', link['rel']) - link_node.setAttribute('href', link['href']) - if 'type' in link: - link_node.setAttribute('type', link['type']) - link_nodes.append(link_node) - return link_nodes - - -class ResponseHeadersSerializer(ActionDispatcher): - """Default response headers serialization""" - - def serialize(self, response, data, action): - self.dispatch(response, data, action=action) - - def default(self, response, data): - response.status_int = 200 - - -class ResponseSerializer(object): - """Encode the necessary pieces into a response object""" - - def __init__(self, body_serializers=None, headers_serializer=None): - self.body_serializers = { - 'application/xml': XMLDictSerializer(), - 'application/json': JSONDictSerializer(), - } - self.body_serializers.update(body_serializers or {}) - - self.headers_serializer = (headers_serializer or - ResponseHeadersSerializer()) - - def serialize(self, response_data, content_type, action='default'): - """Serialize a dict into a string and wrap in a wsgi.Request object. - - :param response_data: dict produced by the Controller - :param content_type: expected mimetype of serialized response body - - """ - response = webob.Response() - self.serialize_headers(response, response_data, action) - self.serialize_body(response, response_data, content_type, action) - return response - - def serialize_headers(self, response, data, action): - self.headers_serializer.serialize(response, data, action) - - def serialize_body(self, response, data, content_type, action): - response.headers['Content-Type'] = content_type - if data is not None: - serializer = self.get_body_serializer(content_type) - response.body = serializer.serialize(data, action) - - def get_body_serializer(self, content_type): - try: - return self.body_serializers[content_type] - except (KeyError, TypeError): - raise exception.InvalidContentType(content_type=content_type) - - -class RequestHeadersDeserializer(ActionDispatcher): - """Default request headers deserializer""" - - def deserialize(self, request, action): - return self.dispatch(request, action=action) - - def default(self, request): - return {} - - -class RequestDeserializer(object): - """Break up a Request object into more useful pieces.""" - - def __init__(self, body_deserializers=None, headers_deserializer=None, - supported_content_types=None): - - self.supported_content_types = supported_content_types - - self.body_deserializers = { - 'application/xml': XMLDeserializer(), - 'application/json': JSONDeserializer(), - } - self.body_deserializers.update(body_deserializers or {}) - - self.headers_deserializer = (headers_deserializer or - RequestHeadersDeserializer()) - - def deserialize(self, request): - """Extract necessary pieces of the request. - - :param request: Request object - :returns: tuple of (expected controller action name, dictionary of - keyword arguments to pass to the controller, the expected - content type of the response) - - """ - action_args = self.get_action_args(request.environ) - action = action_args.pop('action', None) - - action_args.update(self.deserialize_headers(request, action)) - action_args.update(self.deserialize_body(request, action)) - - accept = self.get_expected_content_type(request) - - return (action, action_args, accept) - - def deserialize_headers(self, request, action): - return self.headers_deserializer.deserialize(request, action) - - def deserialize_body(self, request, action): - if not len(request.body) > 0: - LOG.debug(_("Empty body provided in request")) - return {} - - try: - content_type = request.get_content_type() - except exception.InvalidContentType: - LOG.debug(_("Unrecognized Content-Type provided in request")) - raise - - if content_type is None: - LOG.debug(_("No Content-Type provided in request")) - return {} - - try: - deserializer = self.get_body_deserializer(content_type) - except exception.InvalidContentType: - LOG.debug(_("Unable to deserialize body as provided Content-Type")) - raise - - return deserializer.deserialize(request.body, action) - - def get_body_deserializer(self, content_type): - try: - return self.body_deserializers[content_type] - except (KeyError, TypeError): - raise exception.InvalidContentType(content_type=content_type) - - def get_expected_content_type(self, request): - return request.best_match_content_type(self.supported_content_types) - - def get_action_args(self, request_environment): - """Parse dictionary created by routes library.""" - try: - args = request_environment['wsgiorg.routing_args'][1].copy() - except Exception: - return {} - - try: - del args['controller'] - except KeyError: - pass - - try: - del args['format'] - except KeyError: - pass - - return args - - -class TextDeserializer(ActionDispatcher): - """Default request body deserialization""" - - def deserialize(self, datastring, action='default'): - return self.dispatch(datastring, action=action) - - def default(self, datastring): - return {} - - -class JSONDeserializer(TextDeserializer): - - def _from_json(self, datastring): - try: - return jsonutils.loads(datastring) - except ValueError: - msg = _("cannot understand JSON") - raise exception.MalformedRequestBody(reason=msg) - - def default(self, datastring): - return {'body': self._from_json(datastring)} - - -class XMLDeserializer(TextDeserializer): - - def __init__(self, metadata=None): - """ - :param metadata: information needed to deserialize xml into - a dictionary. - """ - super(XMLDeserializer, self).__init__() - self.metadata = metadata or {} - - def _from_xml(self, datastring): - plurals = set(self.metadata.get('plurals', {})) - - try: - node = xmlutils.safe_minidom_parse_string(datastring).childNodes[0] - return {node.nodeName: self._from_xml_node(node, plurals)} - except expat.ExpatError: - msg = _("cannot understand XML") - raise exception.MalformedRequestBody(reason=msg) - - def _from_xml_node(self, node, listnames): - """Convert a minidom node to a simple Python type. - - :param listnames: list of XML node names whose subnodes should - be considered list items. - - """ - - if len(node.childNodes) == 1 and node.childNodes[0].nodeType == 3: - return node.childNodes[0].nodeValue - elif node.nodeName in listnames: - return [self._from_xml_node(n, listnames) for n in node.childNodes] - else: - result = dict() - for attr in node.attributes.keys(): - result[attr] = node.attributes[attr].nodeValue - for child in node.childNodes: - if child.nodeType != node.TEXT_NODE: - result[child.nodeName] = self._from_xml_node(child, - listnames) - return result - - def find_first_child_named(self, parent, name): - """Search a nodes children for the first child with a given name""" - for node in parent.childNodes: - if node.nodeName == name: - return node - return None - - def find_children_named(self, parent, name): - """Return all of a nodes children who have the given name""" - for node in parent.childNodes: - if node.nodeName == name: - yield node - - def extract_text(self, node): - """Get the text field contained by the given node""" - if len(node.childNodes) == 1: - child = node.childNodes[0] - if child.nodeType == child.TEXT_NODE: - return child.nodeValue - return "" - - def default(self, datastring): - return {'body': self._from_xml(datastring)} +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright 2011 OpenStack Foundation. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +"""Utility methods for working with WSGI servers.""" + +import eventlet +eventlet.patcher.monkey_patch(all=False, socket=True) + +import datetime +import errno +import socket +import sys +import time + +import eventlet.wsgi +from oslo.config import cfg +import routes +import routes.middleware +import webob.dec +import webob.exc +from xml.dom import minidom +from xml.parsers import expat + +from conductor.openstack.common import exception +from conductor.openstack.common.gettextutils import _ +from conductor.openstack.common import jsonutils +from conductor.openstack.common import log as logging +from conductor.openstack.common import service +from conductor.openstack.common import sslutils +from conductor.openstack.common import xmlutils + +socket_opts = [ + cfg.IntOpt('backlog', + default=4096, + help="Number of backlog requests to configure the socket with"), + cfg.IntOpt('tcp_keepidle', + default=600, + help="Sets the value of TCP_KEEPIDLE in seconds for each " + "server socket. Not supported on OS X."), +] + +CONF = cfg.CONF +CONF.register_opts(socket_opts) + +LOG = logging.getLogger(__name__) + + +def run_server(application, port, **kwargs): + """Run a WSGI server with the given application.""" + sock = eventlet.listen(('0.0.0.0', port)) + eventlet.wsgi.server(sock, application, **kwargs) + + +class Service(service.Service): + """ + Provides a Service API for wsgi servers. + + This gives us the ability to launch wsgi servers with the + Launcher classes in service.py. + """ + + def __init__(self, application, port, + host='0.0.0.0', backlog=4096, threads=1000): + self.application = application + self._port = port + self._host = host + self._backlog = backlog if backlog else CONF.backlog + super(Service, self).__init__(threads) + + def _get_socket(self, host, port, backlog): + # TODO(dims): eventlet's green dns/socket module does not actually + # support IPv6 in getaddrinfo(). We need to get around this in the + # future or monitor upstream for a fix + info = socket.getaddrinfo(host, + port, + socket.AF_UNSPEC, + socket.SOCK_STREAM)[0] + family = info[0] + bind_addr = info[-1] + + sock = None + retry_until = time.time() + 30 + while not sock and time.time() < retry_until: + try: + sock = eventlet.listen(bind_addr, + backlog=backlog, + family=family) + if sslutils.is_enabled(): + sock = sslutils.wrap(sock) + + except socket.error, err: + if err.args[0] != errno.EADDRINUSE: + raise + eventlet.sleep(0.1) + if not sock: + raise RuntimeError(_("Could not bind to %(host)s:%(port)s " + "after trying for 30 seconds") % + {'host': host, 'port': port}) + sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + # sockets can hang around forever without keepalive + sock.setsockopt(socket.SOL_SOCKET, socket.SO_KEEPALIVE, 1) + + # This option isn't available in the OS X version of eventlet + if hasattr(socket, 'TCP_KEEPIDLE'): + sock.setsockopt(socket.IPPROTO_TCP, + socket.TCP_KEEPIDLE, + CONF.tcp_keepidle) + + return sock + + def start(self): + """Start serving this service using the provided server instance. + + :returns: None + + """ + super(Service, self).start() + self._socket = self._get_socket(self._host, self._port, self._backlog) + self.tg.add_thread(self._run, self.application, self._socket) + + @property + def backlog(self): + return self._backlog + + @property + def host(self): + return self._socket.getsockname()[0] if self._socket else self._host + + @property + def port(self): + return self._socket.getsockname()[1] if self._socket else self._port + + def stop(self): + """Stop serving this API. + + :returns: None + + """ + super(Service, self).stop() + + def _run(self, application, socket): + """Start a WSGI server in a new green thread.""" + logger = logging.getLogger('eventlet.wsgi') + eventlet.wsgi.server(socket, + application, + custom_pool=self.tg.pool, + log=logging.WritableLogger(logger)) + + +class Middleware(object): + """ + Base WSGI middleware wrapper. These classes require an application to be + initialized that will be called next. By default the middleware will + simply call its wrapped app, or you can override __call__ to customize its + behavior. + """ + + def __init__(self, application): + self.application = application + + def process_request(self, req): + """ + Called on each request. + + If this returns None, the next application down the stack will be + executed. If it returns a response then that response will be returned + and execution will stop here. + """ + return None + + def process_response(self, response): + """Do whatever you'd like to the response.""" + return response + + @webob.dec.wsgify + def __call__(self, req): + response = self.process_request(req) + if response: + return response + response = req.get_response(self.application) + return self.process_response(response) + + +class Debug(Middleware): + """ + Helper class that can be inserted into any WSGI application chain + to get information about the request and response. + """ + + @webob.dec.wsgify + def __call__(self, req): + print ("*" * 40) + " REQUEST ENVIRON" + for key, value in req.environ.items(): + print key, "=", value + print + resp = req.get_response(self.application) + + print ("*" * 40) + " RESPONSE HEADERS" + for (key, value) in resp.headers.iteritems(): + print key, "=", value + print + + resp.app_iter = self.print_generator(resp.app_iter) + + return resp + + @staticmethod + def print_generator(app_iter): + """ + Iterator that prints the contents of a wrapper string iterator + when iterated. + """ + print ("*" * 40) + " BODY" + for part in app_iter: + sys.stdout.write(part) + sys.stdout.flush() + yield part + print + + +class Router(object): + + """ + WSGI middleware that maps incoming requests to WSGI apps. + """ + + def __init__(self, mapper): + """ + Create a router for the given routes.Mapper. + + Each route in `mapper` must specify a 'controller', which is a + WSGI app to call. You'll probably want to specify an 'action' as + well and have your controller be a wsgi.Controller, who will route + the request to the action method. + + Examples: + mapper = routes.Mapper() + sc = ServerController() + + # Explicit mapping of one route to a controller+action + mapper.connect(None, "/svrlist", controller=sc, action="list") + + # Actions are all implicitly defined + mapper.resource("server", "servers", controller=sc) + + # Pointing to an arbitrary WSGI app. You can specify the + # {path_info:.*} parameter so the target app can be handed just that + # section of the URL. + mapper.connect(None, "/v1.0/{path_info:.*}", controller=BlogApp()) + """ + self.map = mapper + self._router = routes.middleware.RoutesMiddleware(self._dispatch, + self.map) + + @webob.dec.wsgify + def __call__(self, req): + """ + Route the incoming request to a controller based on self.map. + If no match, return a 404. + """ + return self._router + + @staticmethod + @webob.dec.wsgify + def _dispatch(req): + """ + Called by self._router after matching the incoming request to a route + and putting the information into req.environ. Either returns 404 + or the routed WSGI app's response. + """ + match = req.environ['wsgiorg.routing_args'][1] + if not match: + return webob.exc.HTTPNotFound() + app = match['controller'] + return app + + +class Request(webob.Request): + """Add some Openstack API-specific logic to the base webob.Request.""" + + default_request_content_types = ('application/json', 'application/xml') + default_accept_types = ('application/json', 'application/xml') + default_accept_type = 'application/json' + + def best_match_content_type(self, supported_content_types=None): + """Determine the requested response content-type. + + Based on the query extension then the Accept header. + Defaults to default_accept_type if we don't find a preference + + """ + supported_content_types = (supported_content_types or + self.default_accept_types) + + parts = self.path.rsplit('.', 1) + if len(parts) > 1: + ctype = 'application/{0}'.format(parts[1]) + if ctype in supported_content_types: + return ctype + + bm = self.accept.best_match(supported_content_types) + return bm or self.default_accept_type + + def get_content_type(self, allowed_content_types=None): + """Determine content type of the request body. + + Does not do any body introspection, only checks header + + """ + if "Content-Type" not in self.headers: + return None + + content_type = self.content_type + allowed_content_types = (allowed_content_types or + self.default_request_content_types) + + if content_type not in allowed_content_types: + raise exception.InvalidContentType(content_type=content_type) + return content_type + + +class Resource(object): + """ + WSGI app that handles (de)serialization and controller dispatch. + + Reads routing information supplied by RoutesMiddleware and calls + the requested action method upon its deserializer, controller, + and serializer. Those three objects may implement any of the basic + controller action methods (create, update, show, index, delete) + along with any that may be specified in the api router. A 'default' + method may also be implemented to be used in place of any + non-implemented actions. Deserializer methods must accept a request + argument and return a dictionary. Controller methods must accept a + request argument. Additionally, they must also accept keyword + arguments that represent the keys returned by the Deserializer. They + may raise a webob.exc exception or return a dict, which will be + serialized by requested content type. + """ + def __init__(self, controller, deserializer=None, serializer=None): + """ + :param controller: object that implement methods created by routes lib + :param deserializer: object that supports webob request deserialization + through controller-like actions + :param serializer: object that supports webob response serialization + through controller-like actions + """ + self.controller = controller + self.serializer = serializer or ResponseSerializer() + self.deserializer = deserializer or RequestDeserializer() + + @webob.dec.wsgify(RequestClass=Request) + def __call__(self, request): + """WSGI method that controls (de)serialization and method dispatch.""" + + try: + action, action_args, accept = self.deserialize_request(request) + except exception.InvalidContentType: + msg = _("Unsupported Content-Type") + return webob.exc.HTTPUnsupportedMediaType(explanation=msg) + except exception.MalformedRequestBody: + msg = _("Malformed request body") + return webob.exc.HTTPBadRequest(explanation=msg) + + action_result = self.execute_action(action, request, **action_args) + try: + return self.serialize_response(action, action_result, accept) + # return unserializable result (typically a webob exc) + except Exception: + return action_result + + def deserialize_request(self, request): + return self.deserializer.deserialize(request) + + def serialize_response(self, action, action_result, accept): + return self.serializer.serialize(action_result, accept, action) + + def execute_action(self, action, request, **action_args): + return self.dispatch(self.controller, action, request, **action_args) + + def dispatch(self, obj, action, *args, **kwargs): + """Find action-specific method on self and call it.""" + try: + method = getattr(obj, action) + except AttributeError: + method = getattr(obj, 'default') + + return method(*args, **kwargs) + + def get_action_args(self, request_environment): + """Parse dictionary created by routes library.""" + try: + args = request_environment['wsgiorg.routing_args'][1].copy() + except Exception: + return {} + + try: + del args['controller'] + except KeyError: + pass + + try: + del args['format'] + except KeyError: + pass + + return args + + +class ActionDispatcher(object): + """Maps method name to local methods through action name.""" + + def dispatch(self, *args, **kwargs): + """Find and call local method.""" + action = kwargs.pop('action', 'default') + action_method = getattr(self, str(action), self.default) + return action_method(*args, **kwargs) + + def default(self, data): + raise NotImplementedError() + + +class DictSerializer(ActionDispatcher): + """Default request body serialization""" + + def serialize(self, data, action='default'): + return self.dispatch(data, action=action) + + def default(self, data): + return "" + + +class JSONDictSerializer(DictSerializer): + """Default JSON request body serialization""" + + def default(self, data): + def sanitizer(obj): + if isinstance(obj, datetime.datetime): + _dtime = obj - datetime.timedelta(microseconds=obj.microsecond) + return _dtime.isoformat() + return unicode(obj) + return jsonutils.dumps(data, default=sanitizer) + + +class XMLDictSerializer(DictSerializer): + + def __init__(self, metadata=None, xmlns=None): + """ + :param metadata: information needed to deserialize xml into + a dictionary. + :param xmlns: XML namespace to include with serialized xml + """ + super(XMLDictSerializer, self).__init__() + self.metadata = metadata or {} + self.xmlns = xmlns + + def default(self, data): + # We expect data to contain a single key which is the XML root. + root_key = data.keys()[0] + doc = minidom.Document() + node = self._to_xml_node(doc, self.metadata, root_key, data[root_key]) + + return self.to_xml_string(node) + + def to_xml_string(self, node, has_atom=False): + self._add_xmlns(node, has_atom) + return node.toprettyxml(indent=' ', encoding='UTF-8') + + #NOTE (ameade): the has_atom should be removed after all of the + # xml serializers and view builders have been updated to the current + # spec that required all responses include the xmlns:atom, the has_atom + # flag is to prevent current tests from breaking + def _add_xmlns(self, node, has_atom=False): + if self.xmlns is not None: + node.setAttribute('xmlns', self.xmlns) + if has_atom: + node.setAttribute('xmlns:atom', "http://www.w3.org/2005/Atom") + + def _to_xml_node(self, doc, metadata, nodename, data): + """Recursive method to convert data members to XML nodes.""" + result = doc.createElement(nodename) + + # Set the xml namespace if one is specified + # TODO(justinsb): We could also use prefixes on the keys + xmlns = metadata.get('xmlns', None) + if xmlns: + result.setAttribute('xmlns', xmlns) + + #TODO(bcwaldon): accomplish this without a type-check + if type(data) is list: + collections = metadata.get('list_collections', {}) + if nodename in collections: + metadata = collections[nodename] + for item in data: + node = doc.createElement(metadata['item_name']) + node.setAttribute(metadata['item_key'], str(item)) + result.appendChild(node) + return result + singular = metadata.get('plurals', {}).get(nodename, None) + if singular is None: + if nodename.endswith('s'): + singular = nodename[:-1] + else: + singular = 'item' + for item in data: + node = self._to_xml_node(doc, metadata, singular, item) + result.appendChild(node) + #TODO(bcwaldon): accomplish this without a type-check + elif type(data) is dict: + collections = metadata.get('dict_collections', {}) + if nodename in collections: + metadata = collections[nodename] + for k, v in data.items(): + node = doc.createElement(metadata['item_name']) + node.setAttribute(metadata['item_key'], str(k)) + text = doc.createTextNode(str(v)) + node.appendChild(text) + result.appendChild(node) + return result + attrs = metadata.get('attributes', {}).get(nodename, {}) + for k, v in data.items(): + if k in attrs: + result.setAttribute(k, str(v)) + else: + node = self._to_xml_node(doc, metadata, k, v) + result.appendChild(node) + else: + # Type is atom + node = doc.createTextNode(str(data)) + result.appendChild(node) + return result + + def _create_link_nodes(self, xml_doc, links): + link_nodes = [] + for link in links: + link_node = xml_doc.createElement('atom:link') + link_node.setAttribute('rel', link['rel']) + link_node.setAttribute('href', link['href']) + if 'type' in link: + link_node.setAttribute('type', link['type']) + link_nodes.append(link_node) + return link_nodes + + +class ResponseHeadersSerializer(ActionDispatcher): + """Default response headers serialization""" + + def serialize(self, response, data, action): + self.dispatch(response, data, action=action) + + def default(self, response, data): + response.status_int = 200 + + +class ResponseSerializer(object): + """Encode the necessary pieces into a response object""" + + def __init__(self, body_serializers=None, headers_serializer=None): + self.body_serializers = { + 'application/xml': XMLDictSerializer(), + 'application/json': JSONDictSerializer(), + } + self.body_serializers.update(body_serializers or {}) + + self.headers_serializer = (headers_serializer or + ResponseHeadersSerializer()) + + def serialize(self, response_data, content_type, action='default'): + """Serialize a dict into a string and wrap in a wsgi.Request object. + + :param response_data: dict produced by the Controller + :param content_type: expected mimetype of serialized response body + + """ + response = webob.Response() + self.serialize_headers(response, response_data, action) + self.serialize_body(response, response_data, content_type, action) + return response + + def serialize_headers(self, response, data, action): + self.headers_serializer.serialize(response, data, action) + + def serialize_body(self, response, data, content_type, action): + response.headers['Content-Type'] = content_type + if data is not None: + serializer = self.get_body_serializer(content_type) + response.body = serializer.serialize(data, action) + + def get_body_serializer(self, content_type): + try: + return self.body_serializers[content_type] + except (KeyError, TypeError): + raise exception.InvalidContentType(content_type=content_type) + + +class RequestHeadersDeserializer(ActionDispatcher): + """Default request headers deserializer""" + + def deserialize(self, request, action): + return self.dispatch(request, action=action) + + def default(self, request): + return {} + + +class RequestDeserializer(object): + """Break up a Request object into more useful pieces.""" + + def __init__(self, body_deserializers=None, headers_deserializer=None, + supported_content_types=None): + + self.supported_content_types = supported_content_types + + self.body_deserializers = { + 'application/xml': XMLDeserializer(), + 'application/json': JSONDeserializer(), + } + self.body_deserializers.update(body_deserializers or {}) + + self.headers_deserializer = (headers_deserializer or + RequestHeadersDeserializer()) + + def deserialize(self, request): + """Extract necessary pieces of the request. + + :param request: Request object + :returns: tuple of (expected controller action name, dictionary of + keyword arguments to pass to the controller, the expected + content type of the response) + + """ + action_args = self.get_action_args(request.environ) + action = action_args.pop('action', None) + + action_args.update(self.deserialize_headers(request, action)) + action_args.update(self.deserialize_body(request, action)) + + accept = self.get_expected_content_type(request) + + return (action, action_args, accept) + + def deserialize_headers(self, request, action): + return self.headers_deserializer.deserialize(request, action) + + def deserialize_body(self, request, action): + if not len(request.body) > 0: + LOG.debug(_("Empty body provided in request")) + return {} + + try: + content_type = request.get_content_type() + except exception.InvalidContentType: + LOG.debug(_("Unrecognized Content-Type provided in request")) + raise + + if content_type is None: + LOG.debug(_("No Content-Type provided in request")) + return {} + + try: + deserializer = self.get_body_deserializer(content_type) + except exception.InvalidContentType: + LOG.debug(_("Unable to deserialize body as provided Content-Type")) + raise + + return deserializer.deserialize(request.body, action) + + def get_body_deserializer(self, content_type): + try: + return self.body_deserializers[content_type] + except (KeyError, TypeError): + raise exception.InvalidContentType(content_type=content_type) + + def get_expected_content_type(self, request): + return request.best_match_content_type(self.supported_content_types) + + def get_action_args(self, request_environment): + """Parse dictionary created by routes library.""" + try: + args = request_environment['wsgiorg.routing_args'][1].copy() + except Exception: + return {} + + try: + del args['controller'] + except KeyError: + pass + + try: + del args['format'] + except KeyError: + pass + + return args + + +class TextDeserializer(ActionDispatcher): + """Default request body deserialization""" + + def deserialize(self, datastring, action='default'): + return self.dispatch(datastring, action=action) + + def default(self, datastring): + return {} + + +class JSONDeserializer(TextDeserializer): + + def _from_json(self, datastring): + try: + return jsonutils.loads(datastring) + except ValueError: + msg = _("cannot understand JSON") + raise exception.MalformedRequestBody(reason=msg) + + def default(self, datastring): + return {'body': self._from_json(datastring)} + + +class XMLDeserializer(TextDeserializer): + + def __init__(self, metadata=None): + """ + :param metadata: information needed to deserialize xml into + a dictionary. + """ + super(XMLDeserializer, self).__init__() + self.metadata = metadata or {} + + def _from_xml(self, datastring): + plurals = set(self.metadata.get('plurals', {})) + + try: + node = xmlutils.safe_minidom_parse_string(datastring).childNodes[0] + return {node.nodeName: self._from_xml_node(node, plurals)} + except expat.ExpatError: + msg = _("cannot understand XML") + raise exception.MalformedRequestBody(reason=msg) + + def _from_xml_node(self, node, listnames): + """Convert a minidom node to a simple Python type. + + :param listnames: list of XML node names whose subnodes should + be considered list items. + + """ + + if len(node.childNodes) == 1 and node.childNodes[0].nodeType == 3: + return node.childNodes[0].nodeValue + elif node.nodeName in listnames: + return [self._from_xml_node(n, listnames) for n in node.childNodes] + else: + result = dict() + for attr in node.attributes.keys(): + result[attr] = node.attributes[attr].nodeValue + for child in node.childNodes: + if child.nodeType != node.TEXT_NODE: + result[child.nodeName] = self._from_xml_node(child, + listnames) + return result + + def find_first_child_named(self, parent, name): + """Search a nodes children for the first child with a given name""" + for node in parent.childNodes: + if node.nodeName == name: + return node + return None + + def find_children_named(self, parent, name): + """Return all of a nodes children who have the given name""" + for node in parent.childNodes: + if node.nodeName == name: + yield node + + def extract_text(self, node): + """Get the text field contained by the given node""" + if len(node.childNodes) == 1: + child = node.childNodes[0] + if child.nodeType == child.TEXT_NODE: + return child.nodeValue + return "" + + def default(self, datastring): + return {'body': self._from_xml(datastring)} diff --git a/conductor/conductor/openstack/common/xmlutils.py b/conductor/conductor/openstack/common/xmlutils.py index ae7c077..3370048 100644 --- a/conductor/conductor/openstack/common/xmlutils.py +++ b/conductor/conductor/openstack/common/xmlutils.py @@ -1,74 +1,74 @@ -# vim: tabstop=4 shiftwidth=4 softtabstop=4 - -# Copyright 2013 IBM -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -from xml.dom import minidom -from xml.parsers import expat -from xml import sax -from xml.sax import expatreader - - -class ProtectedExpatParser(expatreader.ExpatParser): - """An expat parser which disables DTD's and entities by default.""" - - def __init__(self, forbid_dtd=True, forbid_entities=True, - *args, **kwargs): - # Python 2.x old style class - expatreader.ExpatParser.__init__(self, *args, **kwargs) - self.forbid_dtd = forbid_dtd - self.forbid_entities = forbid_entities - - def start_doctype_decl(self, name, sysid, pubid, has_internal_subset): - raise ValueError("Inline DTD forbidden") - - def entity_decl(self, entityName, is_parameter_entity, value, base, - systemId, publicId, notationName): - raise ValueError(" entity declaration forbidden") - - def unparsed_entity_decl(self, name, base, sysid, pubid, notation_name): - # expat 1.2 - raise ValueError(" unparsed entity forbidden") - - def external_entity_ref(self, context, base, systemId, publicId): - raise ValueError(" external entity forbidden") - - def notation_decl(self, name, base, sysid, pubid): - raise ValueError(" notation forbidden") - - def reset(self): - expatreader.ExpatParser.reset(self) - if self.forbid_dtd: - self._parser.StartDoctypeDeclHandler = self.start_doctype_decl - self._parser.EndDoctypeDeclHandler = None - if self.forbid_entities: - self._parser.EntityDeclHandler = self.entity_decl - self._parser.UnparsedEntityDeclHandler = self.unparsed_entity_decl - self._parser.ExternalEntityRefHandler = self.external_entity_ref - self._parser.NotationDeclHandler = self.notation_decl - try: - self._parser.SkippedEntityHandler = None - except AttributeError: - # some pyexpat versions do not support SkippedEntity - pass - - -def safe_minidom_parse_string(xml_string): - """Parse an XML string using minidom safely. - - """ - try: - return minidom.parseString(xml_string, parser=ProtectedExpatParser()) - except sax.SAXParseException: - raise expat.ExpatError() +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright 2013 IBM +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from xml.dom import minidom +from xml.parsers import expat +from xml import sax +from xml.sax import expatreader + + +class ProtectedExpatParser(expatreader.ExpatParser): + """An expat parser which disables DTD's and entities by default.""" + + def __init__(self, forbid_dtd=True, forbid_entities=True, + *args, **kwargs): + # Python 2.x old style class + expatreader.ExpatParser.__init__(self, *args, **kwargs) + self.forbid_dtd = forbid_dtd + self.forbid_entities = forbid_entities + + def start_doctype_decl(self, name, sysid, pubid, has_internal_subset): + raise ValueError("Inline DTD forbidden") + + def entity_decl(self, entityName, is_parameter_entity, value, base, + systemId, publicId, notationName): + raise ValueError(" entity declaration forbidden") + + def unparsed_entity_decl(self, name, base, sysid, pubid, notation_name): + # expat 1.2 + raise ValueError(" unparsed entity forbidden") + + def external_entity_ref(self, context, base, systemId, publicId): + raise ValueError(" external entity forbidden") + + def notation_decl(self, name, base, sysid, pubid): + raise ValueError(" notation forbidden") + + def reset(self): + expatreader.ExpatParser.reset(self) + if self.forbid_dtd: + self._parser.StartDoctypeDeclHandler = self.start_doctype_decl + self._parser.EndDoctypeDeclHandler = None + if self.forbid_entities: + self._parser.EntityDeclHandler = self.entity_decl + self._parser.UnparsedEntityDeclHandler = self.unparsed_entity_decl + self._parser.ExternalEntityRefHandler = self.external_entity_ref + self._parser.NotationDeclHandler = self.notation_decl + try: + self._parser.SkippedEntityHandler = None + except AttributeError: + # some pyexpat versions do not support SkippedEntity + pass + + +def safe_minidom_parse_string(xml_string): + """Parse an XML string using minidom safely. + + """ + try: + return minidom.parseString(xml_string, parser=ProtectedExpatParser()) + except sax.SAXParseException: + raise expat.ExpatError() From 7be1f27f0bd6b04d83a675c92db5cd8674488c2b Mon Sep 17 00:00:00 2001 From: Stan Lagun Date: Tue, 26 Mar 2013 19:06:56 +0400 Subject: [PATCH 29/37] Naming conventions changed --- conductor/conductor/app.py | 22 ++++++++++---- conductor/conductor/cloud_formation.py | 29 +++++++++++++++--- .../conductor/commands/cloud_formation.py | 6 ++++ conductor/conductor/commands/dispatcher.py | 2 +- conductor/conductor/commands/windows_agent.py | 8 ++--- conductor/conductor/helpers.py | 5 +++- conductor/conductor/windows_agent.py | 4 +-- conductor/conductor/workflow.py | 14 ++++----- .../templates/agent-config/Default.template | 1 + conductor/data/workflows/AD.xml | 30 +++++++++++++++---- conductor/data/workflows/Common.xml | 7 +++++ conductor/data/workflows/IIS.xml | 15 ++++++---- 12 files changed, 107 insertions(+), 36 deletions(-) create mode 100644 conductor/data/workflows/Common.xml diff --git a/conductor/conductor/app.py b/conductor/conductor/app.py index da17e9b..fe44f2d 100644 --- a/conductor/conductor/app.py +++ b/conductor/conductor/app.py @@ -9,6 +9,9 @@ from config import Config import reporting import rabbitmq +import windows_agent +import cloud_formation + config = Config(sys.argv[1] if len(sys.argv) > 1 else None) @@ -18,7 +21,7 @@ def task_received(task, message_id): reporter = reporting.Reporter(rmqclient, message_id, task['id']) command_dispatcher = CommandDispatcher( - task['name'], rmqclient, task['token']) + task['id'], rmqclient, task['token']) workflows = [] for path in glob.glob("data/workflows/*.xml"): print "loading", path @@ -28,14 +31,21 @@ def task_received(task, message_id): while True: try: - for workflow in workflows: - workflow.execute() + while True: + result = False + for workflow in workflows: + if workflow.execute(): + result = True + if not result: + break if not command_dispatcher.execute_pending(): break except Exception: break command_dispatcher.close() + + del task['token'] result_msg = rabbitmq.Message() result_msg.body = task result_msg.id = message_id @@ -59,9 +69,9 @@ class ConductorWorkflowService(service.Service): while True: try: with rabbitmq.RmqClient() as rmq: - rmq.declare('tasks', 'tasks') - rmq.declare('task-results', 'tasks') - with rmq.open('tasks') as subscription: + rmq.declare('tasks2', 'tasks2') + rmq.declare('task-results', 'tasks2') + with rmq.open('tasks2') as subscription: while True: msg = subscription.get_message() self.tg.add_thread( diff --git a/conductor/conductor/cloud_formation.py b/conductor/conductor/cloud_formation.py index 7303801..9798832 100644 --- a/conductor/conductor/cloud_formation.py +++ b/conductor/conductor/cloud_formation.py @@ -2,6 +2,8 @@ import base64 import xml_code_engine import config +from random import choice +import time def update_cf_stack(engine, context, body, template, mappings, arguments, **kwargs): @@ -16,7 +18,7 @@ def update_cf_stack(engine, context, body, template, arguments=arguments, callback=callback) -def prepare_user_data(context, template='Default', **kwargs): +def prepare_user_data(context, hostname, service, unit, template='Default', **kwargs): settings = config.CONF.rabbitmq with open('data/init.ps1') as init_script_file: @@ -26,17 +28,36 @@ def prepare_user_data(context, template='Default', **kwargs): template_data = template_file.read() template_data = template_data.replace( '%RABBITMQ_HOST%', settings.host) + template_data = template_data.replace( + '%RABBITMQ_INPUT_QUEUE%', + '-'.join([str(context['/dataSource']['id']), + str(service), str(unit)]) + ) template_data = template_data.replace( '%RESULT_QUEUE%', - '-execution-results-%s' % str(context['/dataSource']['name'])) + '-execution-results-%s' % str(context['/dataSource']['id'])) - return init_script.replace( + init_script = init_script.replace( '%WINDOWS_AGENT_CONFIG_BASE64%', base64.b64encode(template_data)) + init_script = init_script.replace('%INTERNAL_HOSTNAME%', hostname) + + return init_script + +def generate_hostname(**kwargs): + chars = 'abcdefghijklmnopqrstuvwxyz' + prefix = ''.join(choice(chars) for _ in range(4)) + return prefix + str(int(time.time() * 10)) + + + xml_code_engine.XmlCodeEngine.register_function( update_cf_stack, "update-cf-stack") xml_code_engine.XmlCodeEngine.register_function( - prepare_user_data, "prepare_user_data") + prepare_user_data, "prepare-user-data") + +xml_code_engine.XmlCodeEngine.register_function( + generate_hostname, "generate-hostname") diff --git a/conductor/conductor/commands/cloud_formation.py b/conductor/conductor/commands/cloud_formation.py index ab62ac5..44f4e08 100644 --- a/conductor/conductor/commands/cloud_formation.py +++ b/conductor/conductor/commands/cloud_formation.py @@ -67,18 +67,24 @@ class HeatExecutor(CommandBase): # ]) try: + print 'try update' self._heat_client.stacks.update( stack_id=self._stack, parameters=arguments, template=template) + print 'wait update' self._wait_state('UPDATE_COMPLETE') except heatclient.exc.HTTPNotFound: + print 'try create' + self._heat_client.stacks.create( stack_name=self._stack, parameters=arguments, template=template) + print 'wait create' self._wait_state('CREATE_COMPLETE') + pending_list = self._pending_list self._pending_list = [] diff --git a/conductor/conductor/commands/dispatcher.py b/conductor/conductor/commands/dispatcher.py index 8208856..d44f224 100644 --- a/conductor/conductor/commands/dispatcher.py +++ b/conductor/conductor/commands/dispatcher.py @@ -8,7 +8,7 @@ class CommandDispatcher(command.CommandBase): self._command_map = { 'cf': cloud_formation.HeatExecutor(environment_id, token), 'agent': windows_agent.WindowsAgentExecutor( - environment_id, rmqclient, environment_id) + environment_id, rmqclient) } def execute(self, name, **kwargs): diff --git a/conductor/conductor/commands/windows_agent.py b/conductor/conductor/commands/windows_agent.py index 978ddd2..8573489 100644 --- a/conductor/conductor/commands/windows_agent.py +++ b/conductor/conductor/commands/windows_agent.py @@ -7,14 +7,14 @@ from command import CommandBase class WindowsAgentExecutor(CommandBase): - def __init__(self, stack, rmqclient, environment): + def __init__(self, stack, rmqclient): self._stack = stack self._rmqclient = rmqclient self._pending_list = [] - self._results_queue = '-execution-results-%s' % str(environment) + self._results_queue = '-execution-results-%s' % str(stack) rmqclient.declare(self._results_queue) - def execute(self, template, mappings, host, callback): + def execute(self, template, mappings, host, service, callback): with open('data/templates/agent/%s.template' % template) as template_file: template_data = template_file.read() @@ -23,7 +23,7 @@ class WindowsAgentExecutor(CommandBase): json.loads(template_data), mappings) id = str(uuid.uuid4()).lower() - host = ('%s-%s' % (self._stack, host)).lower().replace(' ', '-') + host = ('%s-%s-%s' % (self._stack, service, host)).lower() self._pending_list.append({ 'id': id, 'callback': callback diff --git a/conductor/conductor/helpers.py b/conductor/conductor/helpers.py index 435a35b..b133836 100644 --- a/conductor/conductor/helpers.py +++ b/conductor/conductor/helpers.py @@ -1,6 +1,5 @@ import types - def transform_json(json, mappings): if isinstance(json, types.ListType): return [transform_json(t, mappings) for t in json] @@ -47,3 +46,7 @@ def find(f, seq): return item, index index += 1 return None, -1 + + +def generate(length): + return ''.join(choice(chars) for _ in range(length)) diff --git a/conductor/conductor/windows_agent.py b/conductor/conductor/windows_agent.py index 2e7761c..3a4fda7 100644 --- a/conductor/conductor/windows_agent.py +++ b/conductor/conductor/windows_agent.py @@ -1,7 +1,7 @@ import xml_code_engine -def send_command(engine, context, body, template, host, mappings=None, +def send_command(engine, context, body, template, service, host, mappings=None, result=None, **kwargs): if not mappings: mappings = {} command_dispatcher = context['/commandDispatcher'] @@ -18,7 +18,7 @@ def send_command(engine, context, body, template, host, mappings=None, command_dispatcher.execute(name='agent', template=template, mappings=mappings, - host=host, callback=callback) + host=host, service=service, callback=callback) xml_code_engine.XmlCodeEngine.register_function(send_command, "send-command") \ No newline at end of file diff --git a/conductor/conductor/workflow.py b/conductor/conductor/workflow.py index 46d1d3d..94d357e 100644 --- a/conductor/conductor/workflow.py +++ b/conductor/conductor/workflow.py @@ -17,14 +17,12 @@ class Workflow(object): self._reporter = reporter def execute(self): - while True: - context = function_context.Context() - context['/dataSource'] = self._data - context['/commandDispatcher'] = self._command_dispatcher - context['/config'] = self._config - context['/reporter'] = self._reporter - if not self._engine.execute(context): - break + context = function_context.Context() + context['/dataSource'] = self._data + context['/commandDispatcher'] = self._command_dispatcher + context['/config'] = self._config + context['/reporter'] = self._reporter + return self._engine.execute(context) @staticmethod def _get_path(obj, path, create_non_existing=False): diff --git a/conductor/data/templates/agent-config/Default.template b/conductor/data/templates/agent-config/Default.template index 179bfb8..ff5c3c4 100644 --- a/conductor/data/templates/agent-config/Default.template +++ b/conductor/data/templates/agent-config/Default.template @@ -22,6 +22,7 @@ + diff --git a/conductor/data/workflows/AD.xml b/conductor/data/workflows/AD.xml index 929069f..c394fbd 100644 --- a/conductor/data/workflows/AD.xml +++ b/conductor/data/workflows/AD.xml @@ -5,7 +5,7 @@ - + @@ -13,11 +13,13 @@ - - - + + @@ -44,6 +46,9 @@ + @@ -64,6 +69,9 @@ + @@ -88,6 +96,9 @@ + @@ -113,6 +124,9 @@ + + + + + Creating instance - + WS- - + + + + + + + ' and @.state.primaryDcIp)] @@ -151,7 +154,7 @@ +