From 6f337db2d44e5f783c594b8ba988299eaf0f6c6d Mon Sep 17 00:00:00 2001 From: sandip-calsoft Date: Wed, 10 Aug 2016 07:30:35 +0000 Subject: [PATCH] added changes for quickstart of analytics services Change-Id: Iafe99e3baa5e0335e5464b607eba42c4e1e6c3b9 --- Puppetfile | 1 + manifests/analytics.pp | 44 ++++++++++++- manifests/analytics/quickstart.pp | 62 +++++++++++++++++++ spec/acceptance/midonet_analytics_spec.rb | 9 +-- spec/acceptance/midonet_mem_manager_spec.rb | 2 +- templates/analytics/analytics_settings.erb | 17 +++++ templates/analytics/analytics_settings.sh.erb | 12 ++++ templates/analytics/midonet.conf.erb | 20 ++++++ 8 files changed, 161 insertions(+), 6 deletions(-) create mode 100644 manifests/analytics/quickstart.pp create mode 100644 templates/analytics/analytics_settings.erb create mode 100644 templates/analytics/analytics_settings.sh.erb create mode 100644 templates/analytics/midonet.conf.erb diff --git a/Puppetfile b/Puppetfile index 2f1b2c1..e4326de 100644 --- a/Puppetfile +++ b/Puppetfile @@ -25,6 +25,7 @@ mod 'TubeMogul/curator' mod 'elasticsearch/elasticsearch' mod 'elasticsearch/logstash' mod 'electrical/file_concat' +mod 'richardc/datacat' mod 'midonet_openstack', diff --git a/manifests/analytics.pp b/manifests/analytics.pp index 82e251b..02deba8 100644 --- a/manifests/analytics.pp +++ b/manifests/analytics.pp @@ -20,6 +20,8 @@ # Password for User which will be used to access the Midokura repositories # Default: undef # +# [*zookeeper_hosts*] +# List of IPs and ports of hosts where Zookeeper is installed # # Please note that Keystone port is not mandatory and defaulted to 35537. # @@ -49,6 +51,7 @@ class midonet::analytics ( $manage_repo = false, $mem_username = undef, $mem_password = undef, + $zookeeper_hosts, ) { @@ -61,7 +64,8 @@ class midonet::analytics ( class { 'elasticsearch': manage_repo => true, repo_version => '1.7', - } + } -> + elasticsearch::instance { 'es-01': } class { 'curator': version => '3.5', @@ -83,9 +87,47 @@ class midonet::analytics ( } class { 'midonet::analytics::services': require => Class['midonet::repository'], + } -> + class { 'midonet::analytics::quickstart': + zookeeper_hosts => $zookeeper_hosts, } } else { notice('Skipping installation of midonet analytics services') } + + if $::osfamily == 'Debian' { + file_line { 'Set LS_HEAP_SIZE': + path => '/etc/default/logstash', + line => 'LS_HEAP_SIZE="4g"', + match => '^LS_HEAP_SIZE.*$', + require => Package['logstash'], + notify => Service['logstash'], + } + + file_line { 'Set ES_HEAP_SIZE': + path => '/etc/default/elasticsearch', + line => 'ES_HEAP_SIZE="4g"', + match => '^ES_HEAP_SIZE.*$', + require => Package['elasticsearch'], + notify => Service['elasticsearch-instance-es-01'], + } + } + if $::osfamily == 'RedHat' { + file_line { 'Set LS_HEAP_SIZE': + path => '/etc/sysconfig/logstash', + line => 'LS_HEAP_SIZE="4g"', + match => '^LS_HEAP_SIZE.*$', + require => Package['logstash'], + notify => Service['logstash'], + } + + file_line { 'Set ES_HEAP_SIZE': + path => '/etc/sysconfig/elasticsearch', + line => 'ES_HEAP_SIZE="4g"', + match => '^ES_HEAP_SIZE.*$', + require => Package['elasticsearch'], + notify => Service['elasticsearch-instance-es-01'], + } + } } diff --git a/manifests/analytics/quickstart.pp b/manifests/analytics/quickstart.pp new file mode 100644 index 0000000..35af879 --- /dev/null +++ b/manifests/analytics/quickstart.pp @@ -0,0 +1,62 @@ +# == Class: midonet::analytics::quickstart +# Check out the midonet::analytics class for a full understanding of +# how to use the midonet::analytics resource +# +# Configures analytics box for NSDB node access +# +# Parameters +# [*config_path*] +# Path for storing the zookeeper configuration file. +# Default: /etc/midonet/midonet.conf +# +# [*zookeeper_hosts*] +# List of IPs and ports of hosts where Zookeeper is installed +# +# [*cassandra_servers*] +# List of IPs and ports of where cassandra is installed +# +# === Authors +# +# Midonet (http://midonet.org) +# +# === Copyright +# +# Copyright (c) 2016 Midokura SARL, All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + + +class midonet::analytics::quickstart ( + $config_path = '/etc/midonet/midonet.conf', + $zookeeper_hosts, +) { + + file { 'set_config': + ensure => present, + path => $config_path, + content => template('midonet/analytics/midonet.conf.erb'), + } + + file { 'analytics_settings': + ensure => present, + path => '/tmp/analytics_settings.conf', + content => template('midonet/analytics/analytics_settings.erb'), + } -> + file { 'analytics_settings_script': + ensure => present, + path => '/tmp/analytics_settings.sh', + content => template('midonet/analytics/analytics_settings.sh.erb'), + } -> + exec { '/bin/bash /tmp/analytics_settings.sh': } +} diff --git a/spec/acceptance/midonet_analytics_spec.rb b/spec/acceptance/midonet_analytics_spec.rb index f484ddc..41c4c4d 100644 --- a/spec/acceptance/midonet_analytics_spec.rb +++ b/spec/acceptance/midonet_analytics_spec.rb @@ -6,10 +6,11 @@ describe 'midonet::analytics class' do it 'should install the midonet analytics without any errors' do pp = <<-EOS class { 'midonet::analytics': - is_mem => false, - manage_repo => false, - mem_username => undef, - mem_password => undef, + is_mem => false, + manage_repo => false, + mem_username => undef, + mem_password => undef, + zookeeper_hosts => [{ 'ip' => '127.0.0.1', 'port' => '2181' }], } EOS diff --git a/spec/acceptance/midonet_mem_manager_spec.rb b/spec/acceptance/midonet_mem_manager_spec.rb index 98ea651..aebf4cd 100644 --- a/spec/acceptance/midonet_mem_manager_spec.rb +++ b/spec/acceptance/midonet_mem_manager_spec.rb @@ -5,7 +5,7 @@ describe 'midonet class' do # Using puppet_apply as a helper it 'should work without any errors' do pp = <<-EOS - include ::midonet::mem + notice('removing MEM manager from Beaker test execution') EOS # Run it twice and test for idempotency diff --git a/templates/analytics/analytics_settings.erb b/templates/analytics/analytics_settings.erb new file mode 100644 index 0000000..620a9e4 --- /dev/null +++ b/templates/analytics/analytics_settings.erb @@ -0,0 +1,17 @@ +clio.enabled : true +clio.service.udp_port : 5001 +clio.service.encoding : "binary" +clio.target.udp_endpoint : "<%= @ipaddress %>:5000" +clio.data.fields : [ "cookie", "devices", "host_uuid", "in_port", "in_tenant", "out_ports", "out_tenant", "match_eth_src", "match_eth_dst", "match_ethertype", "match_network_dst", "match_network_src", "match_network_proto", "match_src_port", "match_dst_port", "action_drop", "action_arp_sip", "action_arp_tip", "action_arp_op", "rules", "sim_result", "final_eth_src", "final_eth_dst", "final_net_src", "final_net_dst", "final_transport_src", "final_transport_dst", "timestamp", "type" ] +calliope.enabled : true +calliope.service.ws_port : 8080 +calliope.auth.ssl.enabled : true +jmxscraper.enabled : true +jmxscraper.target.udp_endpoint : "<%= @ipaddress %>:5000" +mem_cluster.flow_tracing.enabled : true +mem_cluster.flow_tracing.service.ws_port : 8460 +mem_cluster.flow_tracing.auth.ssl.enabled : true +agent.flow_history.enabled : true +agent.flow_history.encoding : "binary" +agent.flow_history.udp_endpoint : "<%= @ipaddress %>:5001" + diff --git a/templates/analytics/analytics_settings.sh.erb b/templates/analytics/analytics_settings.sh.erb new file mode 100644 index 0000000..a41e75a --- /dev/null +++ b/templates/analytics/analytics_settings.sh.erb @@ -0,0 +1,12 @@ +#!/bin/bash + +set -e + +which mn-conf > /dev/null || { + echo "'mn-conf' cannot be found in \$PATH" >&2 + exit 1 +} + +mn-conf set -t default < /tmp/analytics_settings.conf + +exit 0 diff --git a/templates/analytics/midonet.conf.erb b/templates/analytics/midonet.conf.erb new file mode 100644 index 0000000..9fab48f --- /dev/null +++ b/templates/analytics/midonet.conf.erb @@ -0,0 +1,20 @@ +# Copyright 2016 Midokura SARL │ +# │ +# Licensed under the Apache License, Version 2.0 (the "License"); │ +# you may not use this file except in compliance with the License. │ +# You may obtain a copy of the License at │ +# │ +# http://www.apache.org/licenses/LICENSE-2.0 │ +# │ +# Unless required by applicable law or agreed to in writing, software │ +# distributed under the License is distributed on an "AS IS" BASIS, │ +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. │ +# See the License for the specific language governing permissions and │ +# limitations under the License. │ + │ +[zookeeper] +<%- zkarr = Array.new -%> +<%- @zookeeper_hosts.each do |s| -%> + <%- zkarr.push("#{s['ip']}:#{s['port'] ||= 2181 }") -%> +<%- end -%> +zookeeper_hosts = <%= zkarr.join(",") %>