[Puppet] Puppet module refactored

We can now put to Hiera MuranoPL objects (dicts and arrays) as a values. They will be serialized into YAML and put into Hiera. The same object can be obtained inside Puppet manifest from Hiera. * setHieraValue is deprecated now. putHieraData introduced instead. Since now it is possible to pass a dictionary object instead of key/value pair * shell files renamed to lowcase * yaml files processed by python and PyYAML instead of puppet module * gerrit configuration refactored as an example of the new api usage Change-Id: I61cff31d42fcc2ba46de4faa7f0fbc0cb117c08b
2016-05-17 02:13:18 +03:00 · 2016-05-17 02:13:18 +03:00 · 4543a4913b
commit 4543a4913b
parent 38ba7778e5
20 changed files with 261 additions and 144 deletions
--- a/murano-apps/Gerrit/package/Classes/Gerrit.yaml
+++ b/murano-apps/Gerrit/package/Classes/Gerrit.yaml
@ -118,44 +118,51 @@ Methods:
      - If: $this.ldap != null
        Then:
          - $.createLdapConfiguration()
-      - $.instance.setHieraValue( 'gerrit_war_url', $this.warUrl)
-      - $.instance.setHieraValue( 'gerrit_github_token', '')
-      - $.instance.setHieraValue( 'github_project_username', 'mirademo')
-      - $.instance.setHieraValue( 'github_project_password', 'github_project_password')
-      - $.instance.setHieraValue( 'gerrit_mysql_host', 'localhost')
-      - $.instance.setHieraValue( 'gerrit_email_private_key', 'gerrit_email_private_key')
-      - $.instance.setHieraValue( 'gerrit_rest_token_private_key', 'gerrit_rest_token_private_key45620')
-      - $.instance.setHieraValue( 'gerritbot_password', 'P@ssw0rd')
-      - $.instance.setHieraValue( 'gerritbot_ssh_rsa_key_contents', 'gerritbot_ssh_rsa_key_contents')
-      - $.instance.setHieraValue( 'gerritbot_ssh_rsa_pubkey_contents', 'gerritbot_ssh_rsa_pubkey_contents')
-      # - $.instance.setHieraValue( 'gerrit_ssl_cert_file_contents', 'gerrit_ssl_cert_file_contents')
-      # - $.instance.setHieraValue( 'gerrit_ssl_key_file_contents', 'gerrit_ssl_key_file_contents')
-      # - $.instance.setHieraValue( 'gerrit_ssl_chain_file_contents', 'gerrit_ssl_chain_file_contents')
-      - $.instance.setHieraValue( 'gerrit_ssh_dsa_key_contents', 'gerrit_ssh_dsa_key_contents')
-      - $.instance.setHieraValue( 'gerrit_ssh_dsa_pubkey_contents', 'gerrit_ssh_dsa_pubkey_contents')
-      - $.instance.setHieraValue( 'gerrit_ssh_rsa_key_contents', 'gerrit_ssh_rsa_key_contents')
-      - $.instance.setHieraValue( 'gerrit_ssh_rsa_pubkey_contents', 'gerrit_ssh_rsa_pubkey_contents')
-      - $.instance.setHieraValue( 'gerrit_ssh_project_rsa_key_contents', 'gerrit_ssh_project_rsa_key_contents')
-      - $.instance.setHieraValue( 'gerrit_ssh_project_rsa_pubkey_contents', 'gerrit_project_ssh_rsa_pubkey_contents')
-      - $.instance.setHieraValue( 'welcome_message_gerrit_ssh_private_key', 'welcome_message_gerrit_ssh_private_key')
-      - $.instance.setHieraValue( 'welcome_message_gerrit_ssh_public_key', 'welcome_message_gerrit_ssh_public_key')
-      - $.instance.setHieraValue( 'ssh_replication_rsa_key_contents', '')
-      - $.instance.setHieraValue( 'gerrit_replication_ssh_rsa_pubkey_contents', '')
-      - $.instance.setHieraValue( 'gerrit_lp_consumer_key', 'gerrit_lp_consumer_key')
-      - $.instance.setHieraValue( 'gerrit_lp_access_token', 'gerrit_lp_access_token')
-      - $.instance.setHieraValue( 'gerrit_lp_access_secret', 'gerrit_lp_access_secret')
-      - $.instance.setHieraValue( 'gerrit_notify_impact_file', '/etc/project-config/gerrit/notify_impact.yaml')
-      - $.instance.setHieraValue( 'gerrit_contactstore_appsec', 'gerrit_contactstore_appsec')
-      - $.instance.setHieraValue( 'gerrit_contactstore_pubkey', 'gerrit_contactstore_pubkey')
-      - $.instance.setHieraValue( 'gerrit_acls_dir', '/etc/project-config/gerrit/acls')
-      - $.instance.setHieraValue( 'gerrit_db_user', 'gerrit2')
-      - $.instance.setHieraValue( 'gerrit_db_password', '12345')
-      - $.instance.setHieraValue( 'gerrit_db_root_password', 'P@ssw0rd')

-      - $.instance.setHieraValue( 'swift_store_user', 'swift_store_user')
-      - $.instance.setHieraValue( 'swift_store_key', 'swift_store_key')
-      - $.instance.setHieraValue( 'jeepyb_project_file', '/etc/project-config/gerrit/projects.yaml')
-      - $.instance.setHieraValue( 'project_config_config_dir', '/etc/project-config/')
+      - $data:
+          gerrit_war_url: $this.warUrl
+          gerrit_github_token: ''
+          github_project_username: 'mirademo'
+          gerrit_mysql_host: 'localhost'
+          gerritbot_password: 'P@ssw0rd'
+          gerrit_db_user: 'gerrit2'
+          gerrit_db_password: '12345'
+          gerrit_db_root_password: 'P@ssw0rd'
+          gerrit_acls_dir: '/etc/project-config/gerrit/acls'
+          gerrit_notify_impact_file: '/etc/project-config/gerrit/notify_impact.yaml'
+          jeepyb_project_file: '/etc/project-config/gerrit/projects.yaml'
+          project_config_config_dir: '/etc/project-config/'
+          github_project_password: 'dummy content. to be done'
+          gerrit_email_private_key: 'dummy content. to be done'
+          gerrit_rest_token_private_key: 'dummy content. to be done'
+          welcome_message_gerrit_ssh_private_key: 'dummy content. to be done'
+          welcome_message_gerrit_ssh_public_key: 'dummy content. to be done'
+          ssh_replication_rsa_key_contents: 'dummy content. to be done'
+          gerrit_replication_ssh_rsa_pubkey_contents: 'dummy content. to be done'
+          gerrit_lp_consumer_key: 'dummy content. to be done'
+          gerrit_lp_access_token: 'dummy content. to be done'
+          gerrit_lp_access_secret: 'dummy content. to be done'
+          gerrit_contactstore_appsec: 'dummy content. to be done'
+          gerrit_contactstore_pubkey: 'dummy content. to be done'
+          swift_store_user: 'dummy content. to be done'
+          swift_store_key: 'dummy content. to be done'
+#          This fields autogenerated on vm
+#          TODO: These values should be stored in Murano too
+#          gerritbot_ssh_rsa_key_contents: 'dummy content. the value should be autogenerated'
+#          gerritbot_ssh_rsa_pubkey_contents: 'dummy content. the value should be autogenerated'
+#          gerrit_ssh_dsa_key_contents: 'dummy content. the value should be autogenerated'
+#          gerrit_ssh_dsa_pubkey_contents: 'dummy content. the value should be autogenerated'
+#          gerrit_ssh_rsa_key_contents: 'dummy content. the value should be autogenerated'
+#          gerrit_ssh_rsa_pubkey_contents: 'dummy content. the value should be autogenerated'
+#          gerrit_ssh_project_rsa_key_contents: 'dummy content. the value should be autogenerated'
+#          gerrit_ssh_project_rsa_pubkey_contents: 'dummy content. the value should be autogenerated'
+#          gerrit_ssl_cert_file_contents: 'dummy content. the value should be autogenerated'
+#          gerrit_ssl_key_file_contents: 'dummy content. the value should be autogenerated'
+#          gerrit_ssl_chain_file_contents: 'dummy content. the value should be autogenerated'
+
+      - $this.instance.putHieraData($data)
+      - $this.instance.syncHieraData()
+

  #NOTE: this method doesn't work without ldap
  createAccount:
--- a/murano-apps/Puppet/package/Classes/Hiera.yaml
+++ b/murano-apps/Puppet/package/Classes/Hiera.yaml
@ -0,0 +1,65 @@
+#  Licensed under the Apache License, Version 2.0 (the "License"); you may
+#  not use this file except in compliance with the License. You may obtain
+#  a copy of the License at
+#
+#       http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#  WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#  License for the specific language governing permissions and limitations
+#  under the License.
+
+Namespaces:
+    =: org.openstack.ci_cd_pipeline_murano_app.puppet
+    res: io.murano.resources
+    std: io.murano
+    sys: io.murano.system
+
+Name: Hiera
+
+Properties:
+  dataFilePath:
+    Contract: $.string()
+    Default: '/etc/puppet/hieradata/murano.yaml'
+    Usage: Const
+  configFilePath:
+    Contract: $.string()
+    Default: '/etc/puppet/hiera.yaml'
+    Usage: Const
+  configFile:
+    Contract: $.class(YamlFile)
+  dataFile:
+    Contract: $.class(YamlFile)
+
+Methods:
+  .init:
+    Body:
+      - $this.configFile: new(YamlFile, path => $this.configFilePath)
+      - $this.dataFile: new(YamlFile, path => $this.dataFilePath)
+      # add new data file to hiera
+      # this file is the only file to store all of data the passed throw Hiera class
+      - $config: !yaql "dict( ':hierarchy' => 'murano')"
+      - $this.configFile.putData($config)
+
+  putData:
+    Arguments:
+      - data:
+          Contract: {}
+    Body:
+      - $this.dataFile.putData($data)
+
+  syncronize:
+    Arguments:
+      instance:
+        Contract: $.class(PuppetInstance).notNull()
+    Body:
+      - $tool: new(YamlTool)
+      - $tool.update(
+          $instance,
+          $this.configFile.path,
+          $this.configFile.data)
+      - $tool.update(
+          $instance,
+          $this.dataFile.path,
+          $this.dataFile.data)
--- a/murano-apps/Puppet/package/Classes/PuppetInstance.yaml
+++ b/murano-apps/Puppet/package/Classes/PuppetInstance.yaml
@ -20,10 +20,15 @@ Name: PuppetInstance

 Extends: res:LinuxMuranoInstance

+Properties:
+  hiera:
+   Contract: $.class(Hiera)
+
 Methods:
-  initialize:
+  .init:
    Body:
      - $._environment: $.find(std:Environment).require()
+      - $.hiera: new(Hiera)

  deploy:
    Body:
@ -36,8 +41,7 @@ Methods:

      - $.installPuppet()
      - $.installHiera()
-      - $.installPuppetModule('reidmv/yamlfile')
-      - !yaql "$.setHieraConfigValue(':hierarchy', 'murano')"
+      - $.hiera.syncronize($this)
      - $.setAttr(hostDeployed, true)

  installPuppet:
@ -74,44 +78,8 @@ Methods:
      - $._environment.reporter.report($this,
        'Puppet module {0} installed on {1}'.format( $module, $.openstackId))

-  setHieraConfigValue:
-    Arguments:
-      - key:
-          Contract: $.string().notNull()
-      - value:
-          Contract: $.string().notNull()
-    Body:
-      - $._environment.reporter.report($this,
-        'setHieraConfigValue {0}={1}'.format( $key, $value))
-      - $.setYamlValue('/etc/puppet/hiera.yaml', $key, $value)
-
-  setYamlValue:
-    Arguments:
-      - path:
-          Contract: $.string().notNull()
-      - key:
-          Contract: $.string().notNull()
-      - value:
-          Contract: $.string().notNull()
-    Body:
-      - $resources: new(sys:Resources)
-      - $template: $resources.yaml('setYamlValue.template').bind(dict(
-          path => $path,
-          key => $key,
-          value => $value))
-      - $.agent.call($template, $resources)
-
-  getHieraValue:
-    Arguments:
-      - key:
-          Contract: $.string().notNull()
-    Body:
-      - $resources: new(sys:Resources)
-      - $template: $resources.yaml('getHieraData.template').bind(dict(
-          key => $key))
-      - $value: $.agent.call($template, $resources)
-      - Return: $value
-
+  # Deprecated
+  # use PuppetInstance.putHieraData()
  setHieraValue:
    Arguments:
      - key:
@ -120,5 +88,39 @@ Methods:
          Contract: $.string().notNull()
    Body:
      - $._environment.reporter.report($this,
-        'Put value {0} = {1}'.format( $key, $value))
-      - $.setYamlValue('/etc/puppet/hieradata/murano.yaml', $key, $value)
+        'Put value {0} = {1}'.format($key, $value))
+      - $data: dict($key => $value)
+      - $.hiera.putData($data)
+      - $.hiera.syncronize($this)
+
+  getHieraValue:
+    Arguments:
+      - key:
+          Contract: $.string().notNull()
+    Body:
+      - $resources: new(sys:Resources)
+      - $template: $resources.yaml('GetHieraData.template').bind(dict(
+          key => $key))
+      - $value: $.agent.call($template, $resources)
+      - Return: $value
+
+  #
+  # Put the data contained in the buffer to the instance
+  #
+  syncHieraData:
+    Body:
+      - $.hiera.syncronize($this)
+
+  #
+  # This method just adds the data to the inner buffer.
+  # The values related to the keys which are already exists will be updated
+  # method syncHieraData should be called to push all of the data to the
+  # instance
+  #
+  putHieraData:
+    Arguments:
+      data:
+        Contract: {}
+    Body:
+      - $.hiera.putData($data)
+
--- a/murano-apps/Puppet/package/Classes/YamlFile.yaml
+++ b/murano-apps/Puppet/package/Classes/YamlFile.yaml
@ -0,0 +1,40 @@
+#  Licensed under the Apache License, Version 2.0 (the "License"); you may
+#  not use this file except in compliance with the License. You may obtain
+#  a copy of the License at
+#
+#       http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#  WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#  License for the specific language governing permissions and limitations
+#  under the License.
+
+Namespaces:
+    =: org.openstack.ci_cd_pipeline_murano_app.puppet
+    res: io.murano.resources
+    std: io.murano
+    sys: io.murano.system
+
+Name: YamlFile
+
+Properties:
+  path:
+    Contract: $.string().notNull()
+    Usage: Const
+  data:
+    Contract: {}
+    Default: dict()
+    Usage: Out
+
+Methods:
+  .init:
+    Body:
+      $this.tool: new(YamlTool)
+
+  putData:
+    Arguments:
+      data:
+        Contract: {}
+    Body:
+      $this.data: $this.data.set($data)
--- a/murano-apps/Puppet/package/Classes/YamlTool.yaml
+++ b/murano-apps/Puppet/package/Classes/YamlTool.yaml
@ -0,0 +1,37 @@
+#  Licensed under the Apache License, Version 2.0 (the "License"); you may
+#  not use this file except in compliance with the License. You may obtain
+#  a copy of the License at
+#
+#       http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#  WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#  License for the specific language governing permissions and limitations
+#  under the License.
+
+# TODO: Make every method of this class static when moving to Mitaka
+Namespaces:
+    =: org.openstack.ci_cd_pipeline_murano_app.puppet
+    res: io.murano.resources
+    std: io.murano
+    sys: io.murano.system
+
+Name: YamlTool
+
+Methods:
+  update:
+    Arguments:
+      - instance:
+          Contract: $.class(res:LinuxMuranoInstance).notNull()
+          Usage: Const
+      - path:
+          Contract: $.string().notNull()
+      - data:
+          Contract: {}
+    Body:
+      - $resources: new(sys:Resources)
+      - $template: $resources.yaml('PutYamlData.template').bind(dict(
+          path => $path,
+          data => $data))
+      - $instance.agent.call($template, $resources)
--- a/murano-apps/Puppet/package/Resources/GetHieraData.template
+++ b/murano-apps/Puppet/package/Resources/GetHieraData.template
@ -12,7 +12,7 @@ Scripts:
  get_value:
    Type: Application
    Version: 1.0.0
-    EntryPoint: getHieraData.sh
+    EntryPoint: get_hiera_data.sh
    Files: []
    Options:
      captureStdout: true
--- a/murano-apps/Puppet/package/Resources/InstallHiera.template
+++ b/murano-apps/Puppet/package/Resources/InstallHiera.template
@ -11,7 +11,7 @@ Scripts:
  installHiera:
    Type: Application
    Version: 1.0.0
-    EntryPoint: installHiera.sh
+    EntryPoint: install_hiera.sh
    Files: ['hiera.yaml', 'murano.yaml']
    Options:
      captureStdout: true
--- a/murano-apps/Puppet/package/Resources/InstallPuppet.template
+++ b/murano-apps/Puppet/package/Resources/InstallPuppet.template
@ -11,7 +11,7 @@ Scripts:
  installPuppet:
    Type: Application
    Version: 1.0.0
-    EntryPoint: installPuppet.sh
+    EntryPoint: install_puppet.sh
    Files: []
    Options:
      captureStdout: true
--- a/murano-apps/Puppet/package/Resources/InstallPuppetModule.template
+++ b/murano-apps/Puppet/package/Resources/InstallPuppetModule.template
@ -12,7 +12,7 @@ Scripts:
  installPuppetModule:
    Type: Application
    Version: 1.0.0
-    EntryPoint: installPuppetModule.sh
+    EntryPoint: install_puppet_module.sh
    Files: []
    Options:
      captureStdout: true
--- a/murano-apps/Puppet/package/Resources/PutYamlData.template
+++ b/murano-apps/Puppet/package/Resources/PutYamlData.template
@ -0,0 +1,21 @@
+FormatVersion: 2.0.0
+Version: 1.0.0
+Name: Hiera Data
+
+Parameters:
+  path: $path
+  data: $data
+
+Body: |
+  import yaml
+
+  # I guees murano-agent does not execute requests in parallel
+  with open(args.path) as f:
+      dct = yaml.load(f)
+
+  dct.update(args.data)
+
+  with open(args.path, "w") as f:
+      yaml.safe_dump(dct, f, default_flow_style=False)
+
+Scripts: {}
--- a/murano-apps/Puppet/package/Resources/putHieraData.template
+++ b/murano-apps/Puppet/package/Resources/putHieraData.template
@ -1,21 +0,0 @@
-FormatVersion: 2.1.0
-Version: 1.0.0
-Name: Hiera Data
-
-Parameters:
-  key: $key
-  value: $value
-
-Body: |
-  return putHieraData('{0} {1} test'.format(args.key, args.value)).stdout
-
-Scripts:
-  putHieraData:
-    Type: Application
-    Version: 1.0.0
-    EntryPoint: putHieraData.sh
-    Files: []
-    Options:
-      captureStdout: true
-      captureStderr: true
-      verifyExitcode: true
--- a/murano-apps/Puppet/package/Resources/scripts/get_hiera_data.sh
+++ b/murano-apps/Puppet/package/Resources/scripts/get_hiera_data.sh
--- a/murano-apps/Puppet/package/Resources/scripts/install_hiera.sh
+++ b/murano-apps/Puppet/package/Resources/scripts/install_hiera.sh
--- a/murano-apps/Puppet/package/Resources/scripts/install_puppet.sh
+++ b/murano-apps/Puppet/package/Resources/scripts/install_puppet.sh
--- a/murano-apps/Puppet/package/Resources/scripts/install_puppet_module.sh
+++ b/murano-apps/Puppet/package/Resources/scripts/install_puppet_module.sh
--- a/murano-apps/Puppet/package/Resources/scripts/putHieraData.sh
+++ b/murano-apps/Puppet/package/Resources/scripts/putHieraData.sh
@ -1,7 +0,0 @@
-#!/usr/bin/env bash
-
-key=$1
-value=$2
-logger Put value to Hiera $key=$value
-
-puppet apply --execute "yaml_setting { 'example': target=>'/etc/puppet/hieradata/murano.yaml', key=>'$key', value=>'$value', }"
--- a/murano-apps/Puppet/package/Resources/scripts/setYamlValue.sh
+++ b/murano-apps/Puppet/package/Resources/scripts/setYamlValue.sh
@ -1,9 +0,0 @@
-#!/usr/bin/env bash
-
-path=$1
-key=$2
-value=$3
-
-logger Put value to $path $key=$value
-
-puppet apply --execute "yaml_setting { 'example': target=>'$path', key=>'$key', value=>'$value', }"
--- a/murano-apps/Puppet/package/Resources/setYamlValue.template
+++ b/murano-apps/Puppet/package/Resources/setYamlValue.template
@ -1,22 +0,0 @@
-FormatVersion: 2.1.0
-Version: 1.0.0
-Name: Hiera Data
-
-Parameters:
-  path: $path
-  key: $key
-  value: $value
-
-Body: |
-  return putHieraData('"{0}" "{1}" "{2}"'.format(args.path, args.key, args.value)).stdout
-
-Scripts:
-  putHieraData:
-    Type: Application
-    Version: 1.0.0
-    EntryPoint: setYamlValue.sh
-    Files: []
-    Options:
-      captureStdout: true
-      captureStderr: true
-      verifyExitcode: true
--- a/murano-apps/Puppet/package/manifest.yaml
+++ b/murano-apps/Puppet/package/manifest.yaml
@ -10,3 +10,6 @@ Author: 'Mirantis, Inc'
 Tags: [Server, Puppet]
 Classes:
 org.openstack.ci_cd_pipeline_murano_app.puppet.PuppetInstance: PuppetInstance.yaml
+ org.openstack.ci_cd_pipeline_murano_app.puppet.Hiera: Hiera.yaml
+ org.openstack.ci_cd_pipeline_murano_app.puppet.YamlTool: YamlTool.yaml
+ org.openstack.ci_cd_pipeline_murano_app.puppet.YamlFile: YamlFile.yaml
--- a/murano-apps/SystemConfig/package/Classes/SystemConfig.yaml
+++ b/murano-apps/SystemConfig/package/Classes/SystemConfig.yaml
@ -19,4 +19,5 @@ Methods:
      - $resources: new(sys:Resources)
      - $template: $resources.yaml('SystemConfig.template')
      - $instance.agent.call($template, $resources)
-      - $instance.setHieraValue( 'project_config_repo', $this.projectConfigRepo)
+      - $instance.setHieraValue( 'project_config_repo', $this.projectConfigRepo)
+      - $instance.installPuppetModule('reidmv/yamlfile')