From c30e8b950ccc073b13a03b9923d91f0462e262bf Mon Sep 17 00:00:00 2001 From: akrzos Date: Thu, 1 Jun 2017 16:25:48 -0400 Subject: [PATCH] Expose the carbon port (2003) when installing Carbon/Graphite and docs * Expose carbon_cache_port to the config file and via the firewall * Small cleanup job around the Graphite/Grafana playbook for consistency * Docs for building a monitor host Change-Id: I9e15eac3ce749c341ebf464f165f73c0a87212e5 --- ansible/install/grafana.yml | 6 +- ansible/install/graphite.yml | 6 +- ansible/install/group_vars/all.yml | 3 +- .../install/roles/graphite/handlers/main.yml | 16 +- ansible/install/roles/graphite/tasks/main.yml | 137 +++++++++++------- .../carbon.conf => templates/carbon.conf.j2} | 73 +++++++--- doc/source/installation.rst | 130 +++++++++++++++++ doc/source/introduction.rst | 6 +- 8 files changed, 294 insertions(+), 83 deletions(-) rename ansible/install/roles/graphite/{files/carbon.conf => templates/carbon.conf.j2} (82%) diff --git a/ansible/install/grafana.yml b/ansible/install/grafana.yml index cab4196f2..84febe0bd 100644 --- a/ansible/install/grafana.yml +++ b/ansible/install/grafana.yml @@ -1,10 +1,10 @@ --- # -# Playbook to install grafana +# Playbook to install Grafana # - hosts: grafana remote_user: root roles: - - { role: epel } - - { role: grafana } + - epel + - grafana diff --git a/ansible/install/graphite.yml b/ansible/install/graphite.yml index 1f8080a0a..2f08507b9 100644 --- a/ansible/install/graphite.yml +++ b/ansible/install/graphite.yml @@ -1,10 +1,10 @@ --- # -# Playbook to install graphite-web +# Playbook to install Carbon and Graphite # - hosts: graphite remote_user: root roles: - - { role: epel } - - { role: graphite } + - epel + - graphite diff --git a/ansible/install/group_vars/all.yml b/ansible/install/group_vars/all.yml index 6029ec447..7803c54b1 100644 --- a/ansible/install/group_vars/all.yml +++ b/ansible/install/group_vars/all.yml @@ -220,12 +220,13 @@ graphite_web_docker_image: kambiz/graphite-web:0.9.15 grafana_docker_image: grafana/grafana:2.6.0 ######################################## -# Graphite Configuration +# Carbon/Graphite Configuration ######################################## # Graphite Server ip address (Collectd -> Graphite server) # you must fill out graphite_host prior to playbook execution graphite_host: graphite_port: 80 +carbon_cache_port: 2003 # Graphite prefix / Cloud name used both with graphite and grafana dashboards graphite_prefix: openstack # Graphite username and password for login on the dashboard diff --git a/ansible/install/roles/graphite/handlers/main.yml b/ansible/install/roles/graphite/handlers/main.yml index 3987061b4..982da4803 100644 --- a/ansible/install/roles/graphite/handlers/main.yml +++ b/ansible/install/roles/graphite/handlers/main.yml @@ -1,6 +1,18 @@ +--- +# +# Carbon and Graphite Handlers +# + - name: restart apache - service: name=httpd state=restarted enabled=true + service: + name: httpd + state: restarted + enabled: true become: true + - name: restart carbon-cache - service: name=carbon-cache state=restarted enabled=true + service: + name: carbon-cache + state: restarted + enabled: true become: true diff --git a/ansible/install/roles/graphite/tasks/main.yml b/ansible/install/roles/graphite/tasks/main.yml index 105f85135..e096f3b6e 100644 --- a/ansible/install/roles/graphite/tasks/main.yml +++ b/ansible/install/roles/graphite/tasks/main.yml @@ -4,7 +4,9 @@ # - name: Install graphite rpms - yum: name={{ item }} state=present + yum: + name: "{{ item }}" + state: present become: true with_items: - graphite-web @@ -18,11 +20,11 @@ - name: Copy setup-graphite-db.exp copy: - src=setup-graphite-db.exp - dest=/root/setup-graphite-db.exp - owner=root - group=root - mode=0755 + src: setup-graphite-db.exp + dest: /root/setup-graphite-db.exp + owner: root + group: root + mode: 0755 become: true - name: Create initial graphite db @@ -30,18 +32,18 @@ become: true when: graphite_db_installed.rc != 0 notify: - - restart apache + - restart apache - name: Setup httpd graphite-web config template: - src=graphite-web.conf.j2 - dest=/etc/httpd/conf.d/graphite-web.conf - owner=root - group=root - mode=0644 + src: graphite-web.conf.j2 + dest: /etc/httpd/conf.d/graphite-web.conf + owner: root + group: root + mode: 0644 become: true notify: - - restart apache + - restart apache ### begin firewall ### # we need TCP/80 open @@ -68,6 +70,12 @@ register: firewalld_graphite_port_exists no_log: true +- name: (carbon) Determine if TCP/{{carbon_cache_port}} is already active + shell: firewall-cmd --list-ports | egrep -q "^{{carbon_cache_port}}/tcp" + ignore_errors: true + register: firewalld_carbon_cache_port_exists + no_log: true + # add firewall rule via firewall-cmd - name: (graphite-web) Add firewall rule for TCP/{{graphite_port}} (firewalld) command: "{{ item }}" @@ -78,6 +86,16 @@ become: true when: firewalld_in_use.rc == 0 and firewalld_is_active.rc == 0 and firewalld_graphite_port_exists.rc != 0 +# add firewall rule via firewall-cmd +- name: (carbon) Add firewall rule for TCP/{{carbon_cache_port}} (firewalld) + command: "{{ item }}" + with_items: + - firewall-cmd --zone=public --add-port={{carbon_cache_port}}/tcp --permanent + - firewall-cmd --reload + ignore_errors: true + become: true + when: firewalld_in_use.rc == 0 and firewalld_is_active.rc == 0 and firewalld_carbon_cache_port_exists.rc != 0 + # iptables-services - name: (graphite-web) check firewall rules for TCP/{{graphite_port}} (iptables-services) shell: grep "dport {{graphite_port}} \-j ACCEPT" /etc/sysconfig/iptables | wc -l @@ -86,6 +104,13 @@ failed_when: iptables_graphite_port_exists == 127 no_log: true +- name: (carbon) check firewall rules for TCP/{{carbon_cache_port}} (iptables-services) + shell: grep "dport {{carbon_cache_port}} \-j ACCEPT" /etc/sysconfig/iptables | wc -l + ignore_errors: true + register: iptables_carbon_cache_port_exists + failed_when: iptables_carbon_cache_port_exists == 127 + no_log: true + - name: (graphite-web) Add firewall rule for TCP/{{graphite_port}} (iptables-services) lineinfile: dest: /etc/sysconfig/iptables @@ -96,6 +121,16 @@ when: firewalld_in_use.rc != 0 and firewalld_is_active.rc != 0 and iptables_graphite_port_exists.stdout|int == 0 register: iptables_needs_restart +- name: (carbon) Add firewall rule for TCP/{{carbon_cache_port}} (iptables-services) + lineinfile: + dest: /etc/sysconfig/iptables + line: '-A INPUT -p tcp -m tcp --dport {{carbon_cache_port}} -j ACCEPT' + regexp: '^INPUT -i lo -j ACCEPT' + insertbefore: '-A INPUT -i lo -j ACCEPT' + backup: yes + when: firewalld_in_use.rc != 0 and firewalld_is_active.rc != 0 and iptables_carbon_cache_port_exists.stdout|int == 0 + register: iptables_needs_restart + - name: (graphite-web) Restart iptables-services for TCP/{{graphite_port}} (iptables-services) shell: systemctl restart iptables.service ignore_errors: true @@ -103,57 +138,51 @@ ### end firewall ### -# Start graphite-web service - name: Setup httpd service - service: name=httpd state=started enabled=true + service: + name: httpd + state: started + enabled: true become: true -# remove silly welcome from apache (if it exists) - name: Remove httpd welcome config become: true - file: path=/etc/httpd/conf.d/welcome.conf state=absent + file: + path: /etc/httpd/conf.d/welcome.conf + state: absent notify: - - restart apache - -# -# setup the python-carbon service -# + - restart apache - name: Setup carbon-cache service - service: name=carbon-cache state=started enabled=true + service: + name: carbon-cache + state: started + enabled: true become: true -- name: copy carbon storage schema config +- name: Copy Carbon storage scheme and aggregation config files copy: - src=storage-schemas.conf - dest=/etc/carbon/storage-schemas.conf - owner=root - group=root - mode=0644 + src: "{{item.src}}" + dest: "{{item.dest}}" + owner: root + group: root + mode: 0644 + become: true + with_items: + - src: storage-schemas.conf + dest: /etc/carbon/storage-schemas.conf + - src: storage-aggregation.conf + dest: /etc/carbon/storage-aggregation.conf + notify: + - restart carbon-cache + +- name: Configure carbon.conf + template: + src: carbon.conf.j2 + dest: /etc/carbon/carbon.conf + owner: root + group: root + mode: 0644 become: true notify: - - restart carbon-cache - -- name: copy carbon storage aggregation config - copy: - src=storage-aggregation.conf - dest=/etc/carbon/storage-aggregation.conf - owner=root - group=root - mode=0644 - become: true - notify: - - restart carbon-cache - -- name: copy carbon config - copy: - src=carbon.conf - dest=/etc/carbon/carbon.conf - owner=root - group=root - mode=0644 - become: true - notify: - - restart carbon-cache - - + - restart carbon-cache diff --git a/ansible/install/roles/graphite/files/carbon.conf b/ansible/install/roles/graphite/templates/carbon.conf.j2 similarity index 82% rename from ansible/install/roles/graphite/files/carbon.conf rename to ansible/install/roles/graphite/templates/carbon.conf.j2 index 576969847..ac600981b 100644 --- a/ansible/install/roles/graphite/files/carbon.conf +++ b/ansible/install/roles/graphite/templates/carbon.conf.j2 @@ -8,7 +8,7 @@ # Defaults to ../ # GRAPHITE_CONF_DIR - Configuration directory (where this file lives). # Defaults to $GRAPHITE_ROOT/conf/ -# GRAPHITE_STORAGE_DIR - Storage directory for whipser/rrd/log/pid files. +# GRAPHITE_STORAGE_DIR - Storage directory for whisper/rrd/log/pid files. # Defaults to $GRAPHITE_ROOT/storage/ # # To change other directory paths, add settings to this file. The following @@ -37,15 +37,14 @@ CONF_DIR = /etc/carbon/ LOG_DIR = /var/log/carbon/ PID_DIR = /var/run/ -# Enable daily log rotation. If disabled, a kill -HUP can be used after a manual rotate - +# Enable daily log rotation. If disabled, carbon will automatically re-open +# the file if it's rotated out of place (e.g. by logrotate daemon) ENABLE_LOGROTATION = True # Specify the user to drop privileges to # If this is blank carbon runs as the user that invokes it # This user must have write access to the local data directory USER = carbon - # # NOTE: The above settings must be set under [relay] and [aggregator] # to take effect for those daemons as well @@ -69,16 +68,24 @@ MAX_UPDATES_PER_SECOND = 500 # MAX_UPDATES_PER_SECOND_ON_SHUTDOWN = 1000 # Softly limits the number of whisper files that get created each minute. -# Setting this value low (like at 50) is a good way to ensure your graphite +# Setting this value low (e.g. 50) is a good way to ensure that your carbon # system will not be adversely impacted when a bunch of new metrics are -# sent to it. The trade off is that it will take much longer for those metrics' -# database files to all get created and thus longer until the data becomes usable. -# Setting this value high (like "inf" for infinity) will cause graphite to create -# the files quickly but at the risk of slowing I/O down considerably for a while. +# sent to it. The trade off is that any metrics received in excess of this +# value will be silently dropped, and the whisper file will not be created +# until such point as a subsequent metric is received and fits within the +# defined rate limit. Setting this value high (like "inf" for infinity) will +# cause carbon to create the files quickly but at the risk of increased I/O. MAX_CREATES_PER_MINUTE = 2000 +# Set the interface and port for the line (plain text) listener. Setting the +# interface to 0.0.0.0 listens on all interfaces. Port can be set to 0 to +# disable this listener if it is not required. LINE_RECEIVER_INTERFACE = 0.0.0.0 -LINE_RECEIVER_PORT = 2003 +LINE_RECEIVER_PORT = {{carbon_cache_port}} + +# Set the TCP backlog for the listen socket created by the line receiver. You +# shouldn't change this unless you know what you're doing. +# LINE_RECEIVER_BACKLOG = 1024 # Set this to True to enable the UDP listener. By default this is off # because it is very common to run multiple carbon daemons and managing @@ -87,9 +94,16 @@ ENABLE_UDP_LISTENER = False UDP_RECEIVER_INTERFACE = 0.0.0.0 UDP_RECEIVER_PORT = 2003 +# Set the interface and port for the pickle listener. Setting the interface to +# 0.0.0.0 listens on all interfaces. Port can be set to 0 to disable this +# listener if it is not required. PICKLE_RECEIVER_INTERFACE = 0.0.0.0 PICKLE_RECEIVER_PORT = 2004 +# Set the TCP backlog for the listen socket created by the pickle receiver. You +# shouldn't change this unless you know what you're doing. +# PICKLE_RECEIVER_BACKLOG = 1024 + # Set to false to disable logging of successful connections LOG_LISTENER_CONNECTIONS = True @@ -101,6 +115,10 @@ USE_INSECURE_UNPICKLER = False CACHE_QUERY_INTERFACE = 0.0.0.0 CACHE_QUERY_PORT = 7002 +# Set the TCP backlog for the listen socket created by the cache query +# listener. You shouldn't change this unless you know what you're doing. +# CACHE_QUERY_BACKLOG = 1024 + # Set this to False to drop datapoints received after the cache # reaches MAX_CACHE_SIZE. If this is True (the default) then sockets # over which metrics are received will temporarily stop accepting @@ -224,7 +242,6 @@ WHISPER_FALLOCATE_CREATE = True [relay] -USER = carbon LINE_RECEIVER_INTERFACE = 0.0.0.0 LINE_RECEIVER_PORT = 2013 PICKLE_RECEIVER_INTERFACE = 0.0.0.0 @@ -253,6 +270,14 @@ RELAY_METHOD = rules # datapoint to more than one machine. REPLICATION_FACTOR = 1 +# For REPLICATION_FACTOR >=2, set DIVERSE_REPLICAS to True to guarantee replicas +# across distributed hosts. With this setting disabled, it's possible that replicas +# may be sent to different caches on the same host. This has been the default +# behavior since introduction of 'consistent-hashing' relay method. +# Note that enabling this on an existing pre-0.9.14 cluster will require rebalancing +# your metrics across the cluster nodes using a tool like Carbonate. +#DIVERSE_REPLICAS = False + # This is a list of carbon daemons we will send any relayed or # generated metrics to. The default provided would send to a single # carbon-cache instance on the default port. However if you @@ -273,13 +298,22 @@ DESTINATIONS = 127.0.0.1:2004 # This defines the maximum "message size" between carbon daemons. # You shouldn't need to tune this unless you really know what you're doing. -MAX_DATAPOINTS_PER_MESSAGE = 2000 -MAX_QUEUE_SIZE = 400000 +MAX_DATAPOINTS_PER_MESSAGE = 500 +MAX_QUEUE_SIZE = 10000 + +# This is the percentage that the queue must be empty before it will accept +# more messages. For a larger site, if the queue is very large it makes sense +# to tune this to allow for incoming stats. So if you have an average +# flow of 100k stats/minute, and a MAX_QUEUE_SIZE of 3,000,000, it makes sense +# to allow stats to start flowing when you've cleared the queue to 95% since +# you should have space to accommodate the next minute's worth of stats +# even before the relay incrementally clears more of the queue +QUEUE_LOW_WATERMARK_PCT = 0.8 # Set this to False to drop datapoints when any send queue (sending datapoints # to a downstream carbon daemon) hits MAX_QUEUE_SIZE. If this is True (the # default) then sockets over which metrics are received will temporarily stop accepting -# data until the send queues fall below 80% MAX_QUEUE_SIZE. +# data until the send queues fall below QUEUE_LOW_WATERMARK_PCT * MAX_QUEUE_SIZE. USE_FLOW_CONTROL = True # Set this to True to enable whitelisting and blacklisting of metrics in @@ -295,7 +329,6 @@ USE_FLOW_CONTROL = True [aggregator] -USER = carbon LINE_RECEIVER_INTERFACE = 0.0.0.0 LINE_RECEIVER_PORT = 2023 @@ -307,8 +340,14 @@ LOG_LISTENER_CONNECTIONS = True # If set true, metric received will be forwarded to DESTINATIONS in addition to # the output of the aggregation rules. If set false the carbon-aggregator will -# only ever send the output of aggregation. -FORWARD_ALL = True +# only ever send the output of aggregation. Default value is set to false and will not forward +FORWARD_ALL = False + +# Filenames of the configuration files to use for this instance of aggregator. +# Filenames are relative to CONF_DIR. +# +# AGGREGATION_RULES = aggregation-rules.conf +# REWRITE_RULES = rewrite-rules.conf # This is a list of carbon daemons we will send any relayed or # generated metrics to. The default provided would send to a single diff --git a/doc/source/installation.rst b/doc/source/installation.rst index 134749ba9..07a360135 100644 --- a/doc/source/installation.rst +++ b/doc/source/installation.rst @@ -373,3 +373,133 @@ will be available via http on different ports. from multiple Browbeat users at the same time will introduce variation into resulting performance data if the machine on which Browbeat is installed is resource constrained. + +================================== +Additional Components Installation +================================== + +Install Monitoring Host (Carbon/Graphite/Grafana) +------------------------------------------------- + +A monitoring host exposes System and Application performance metrics to the +Browbeat user via Grafana. It helps expose what may be causing your bottleneck +when you encounter a performance issue. + +Prerequisites +~~~~~~~~~~~~~ + +Hardware + +* Baremetal or Virtual Machine +* SSD storage + +Operating System + +* RHEL 7 +* CentOS 7 + +Repos + +* Red Hat Enterprise Linux 7Server - x86_64 - Server +* Red Hat Enterprise Linux 7Server - x86_64 - Server Optional + +RPM + +* epel-release +* ansible +* git + +Installation +~~~~~~~~~~~~ + +1. Deploy machine (RHEL7 is used in this example) +2. Install RPMS + +:: + + [root@dhcp23-93 ~]# yum install -y https://dl.fedoraproject.org/pub/epel/epel-release-latest-7.noarch.rpm + ... + [root@dhcp23-93 ~]# yum install -y ansible git + +3. Clone Browbeat + +:: + + [root@dhcp23-93 ~]# git clone https://github.com/openstack/browbeat.git + Cloning into 'browbeat'... + remote: Counting objects: 7533, done. + remote: Compressing objects: 100% (38/38), done. + remote: Total 7533 (delta 30), reused 36 (delta 23), pack-reused 7469 + Receiving objects: 100% (7533/7533), 5.26 MiB | 5.79 MiB/s, done. + Resolving deltas: 100% (4330/4330), done. + +4. Add a hosts file into ansible directory + +:: + + [root@dhcp23-93 ~]# cd browbeat/ansible/ + [root@dhcp23-93 ansible]# vi hosts + +Content of hosts file should be following + +:: + + [graphite] + localhost + + [grafana] + localhost + +5. Setup SSH config, SSH key and exchange for Ansible + +:: + + [root@dhcp23-93 ansible]# touch ssh-config + [root@dhcp23-93 ansible]# ssh-keygen + Generating public/private rsa key pair. + ... + [root@dhcp23-93 ansible]# ssh-copy-id root@localhost + ... + +6. Edit install variables + +:: + + [root@dhcp23-93 ansible]# vi install/group_vars/all.yml + +Depending on the environment you may need to edit more than just the following +variables - graphite_host and grafana_host + + +7. Install Carbon and Graphite via Ansible playbook + +:: + + [root@dhcp23-93 ansible]# ansible-playbook -i hosts install/graphite.yml + ... + +8. Install Grafana via Ansible playbook + +:: + + [root@dhcp23-93 ansible]# ansible-playbook -i hosts install/grafana.yml + ... + +9. Install Grafana dashboards via Ansible playbook + +:: + + [root@dhcp23-93 ansible]# ansible-playbook -i hosts install/grafana-dashboards.yml -e 'cloud_dashboards=false' + ... + +Now navigate to http://monitoring-host-address:3000 to verify Grafana is +installed, the Graphite data source exists and custom dashboards are uploaded. + +You can now point other clouds at this host in order to view System and +Application performance metrics. Depending on the number of clouds and +machines pointed at your monitoring server, you may need to add more disk IO +capacity, disk storage or carbon-cache+carbon-relay processes depending +entirely on the number of metrics and your environments capacity. There is a +Graphite dashboard included and it is recommended to install collectd on your +monitoring host such that you can see if you hit resource issues with your +monitoring host. diff --git a/doc/source/introduction.rst b/doc/source/introduction.rst index 459bf9321..b91e52b47 100644 --- a/doc/source/introduction.rst +++ b/doc/source/introduction.rst @@ -1,6 +1,6 @@ -===================== -Browbeat Introduction -===================== +============ +Introduction +============ This started as a project to help determine the number of database connections a given OpenStack deployment uses via stress tests. It has