From 55cd9984f22dcb6be014e4e67858df6b4be1d16b Mon Sep 17 00:00:00 2001
From: Christian Brandstetter <christian.brandstetter@est.fujitsu.com>
Date: Mon, 23 Jul 2018 12:47:16 +0200
Subject: [PATCH] Add tooling for building Docker image
Story: 2001694
Task: 23063
Change-Id: I5192adba0b815bc07db0498f170a0916009c5c40
---
docker/Dockerfile | 47 ++++++++++++
docker/README.rst | 76 +++++++++++++++++++
docker/build_image.sh | 126 +++++++++++++++++++++++++++++++
docker/health_check.py | 26 +++++++
docker/monasca_persister.conf.j2 | 64 ++++++++++++++++
docker/start.sh | 41 ++++++++++
6 files changed, 380 insertions(+)
create mode 100644 docker/Dockerfile
create mode 100644 docker/README.rst
create mode 100755 docker/build_image.sh
create mode 100644 docker/health_check.py
create mode 100644 docker/monasca_persister.conf.j2
create mode 100644 docker/start.sh
diff --git a/docker/Dockerfile b/docker/Dockerfile
new file mode 100644
index 00000000..06226b1e
--- /dev/null
+++ b/docker/Dockerfile
@@ -0,0 +1,47 @@
+ARG DOCKER_IMAGE=monasca-persister
+ARG APP_REPO=https://git.openstack.org/openstack/monasca-persister
+
+# Branch, tag or git hash to build from.
+ARG REPO_VERSION=master
+ARG CONSTRAINTS_BRANCH=master
+
+# Extra Python3 dependencies.
+ARG EXTRA_DEPS="influxdb"
+
+# Always start from `monasca-base` image and use specific tag of it.
+ARG BASE_TAG=master
+FROM monasca/base:$BASE_TAG
+
+# Environment variables used for our service or wait scripts.
+ENV \
+ DEBUG=false \
+ VERBOSE=true \
+ ZOOKEEPER_URI=zookeeper:2181 \
+ KAFKA_URI=kafka:9092 \
+ KAFKA_ALARM_HISTORY_BATCH_SIZE=1000 \
+ KAFKA_ALARM_HISTORY_WAIT_TIME=15 \
+ KAFKA_METRICS_BATCH_SIZE=1000 \
+ KAFKA_METRICS_WAIT_TIME=15 \
+ KAFKA_WAIT_FOR_TOPICS=alarm-state-transitions,metrics \
+ INFLUX_HOST=influxdb \
+ INFLUX_PORT=8086 \
+ INFLUX_USER=mon_persister \
+ INFLUX_PASSWORD=password \
+ INFLUX_DB=mon \
+ STAY_ALIVE_ON_FAILURE="false"
+
+# Copy all necessary files to proper locations.
+COPY monasca_persister.conf.j2 /etc/monasca/
+
+# Run here all additional steps your service need post installation.
+# Stay with only one `RUN` and use `&& \` for next steps to don't create
+# unnecessary image layers. Clean at the end to conserve space.
+#RUN \
+# echo "Some steps to do after main installation." && \
+# echo "Hello when building."
+
+# Expose port for specific service.
+#EXPOSE 1234
+
+# Implement start script in `start.sh` file.
+CMD ["/start.sh"]
diff --git a/docker/README.rst b/docker/README.rst
new file mode 100644
index 00000000..846c9f79
--- /dev/null
+++ b/docker/README.rst
@@ -0,0 +1,76 @@
+==================================
+Docker image for Monasca persister
+==================================
+The Monasca persister image is based on the monasca-base image.
+
+
+Building monasca-base image
+===========================
+See https://github.com/openstack/monasca-common/tree/master/docker/README.rst
+
+
+Building Monasca persister image (child)
+========================================
+
+
+Requirements from monasca-base image
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+health_check.py
+ This file will be used for checking the status of the Monasca persister
+ application.
+
+
+Scripts for child image
+~~~~~~~~~~~~~~~~~~~~~~~
+start.sh
+ In this starting script provide all steps that lead to the proper service
+ start. Including usage of wait scripts and templating of configuration
+ files. You also could provide the ability to allow running container after
+ service died for easier debugging.
+
+build_image.sh
+ Please read detailed build description inside the script.
+
+
+Build arguments (child)
+~~~~~~~~~~~~~~~~~~~~~~~
+====================== =========================
+Arguments Occurrence
+====================== =========================
+BASE_TAG Dockerfile
+====================== =========================
+
+
+Environment variables (child)
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+============================== =============================== ================================================
+Variable Default Description
+============================== =============================== ================================================
+DEBUG false If true, enable debug logging
+VERBOSE true If true, enable info logging
+ZOOKEEPER_URI zookeeper:2181 The host and port for zookeeper
+KAFKA_URI kafka:9092 The host and port for kafka
+KAFKA_WAIT_FOR_TOPICS alarm-state-transitions,metrics Topics to wait on at startup
+KAFKA_WAIT_RETRIES 24 Number of kafka connect attempts
+KAFKA_WAIT_DELAY 5 Seconds to wait between attempts
+KAFKA_ALARM_HISTORY_BATCH_SIZE 1000 Kafka consumer takes messages in a batch
+KAFKA_ALARM_HISTORY_WAIT_TIME 15 Seconds to wait if the batch size is not reached
+KAFKA_METRICS_BATCH_SIZE 1000 Kafka consumer takes messages in a batch
+KAFKA_METRICS_WAIT_TIME 15 Seconds to wait if the batch size is not reached
+INFLUX_HOST influxdb The host for influxdb
+INFLUX_PORT 8086 The port for influxdb
+INFLUX_USER mon_persister The influx username
+INFLUX_PASSWORD password The influx password
+INFLUX_DB mon The influx database name
+STAY_ALIVE_ON_FAILURE false If true, container runs 2 hours even start fails
+============================== =============================== ================================================
+
+
+Provide Configuration templates
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+* persister.conf.j2
+
+
+Links
+~~~~~
+https://github.com/openstack/monasca-persister/tree/master/monasca_persister
diff --git a/docker/build_image.sh b/docker/build_image.sh
new file mode 100755
index 00000000..b601dfa7
--- /dev/null
+++ b/docker/build_image.sh
@@ -0,0 +1,126 @@
+#!/bin/bash
+
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+# TODO(Dobroslaw): move this script to monasca-common/docker folder
+# and leave here small script to download it and execute using env variables
+# to minimize code duplication.
+
+set -x # Print each script step.
+set -eo pipefail # Exit the script if any statement returns error.
+
+# This script is used for building Docker image with proper labels.
+#
+# Example usage:
+# $ ./build_image.sh <repository_version> <upper_constains_branch>
+#
+# To build from master branch (default):
+# $ ./build_image.sh
+# To build specific version run this script in the following way:
+# $ ./build_image.sh stable/queens
+# Building from specific commit:
+# $ ./build_image.sh cb7f226
+# When building from a tag monasca-common will be used in version available
+# in upper constraint file:
+# $ ./build_image.sh 2.5.0
+# To build image from Gerrit patch sets that is targeting branch stable/queens:
+# $ ./build_image.sh refs/changes/51/558751/1 stable/queens
+
+[ -z "$DOCKER_IMAGE" ] && \
+ DOCKER_IMAGE=$(\grep DOCKER_IMAGE Dockerfile | cut -f2 -d"=")
+
+: "${REPO_VERSION:=$1}"
+[ -z "$REPO_VERSION" ] && \
+ REPO_VERSION=$(\grep REPO_VERSION Dockerfile | cut -f2 -d"=")
+# Let's stick to more readable version and disable SC2001 here.
+# shellcheck disable=SC2001
+REPO_VERSION_CLEAN=$(echo "$REPO_VERSION" | sed 's|/|-|g')
+
+[ -z "$APP_REPO" ] && APP_REPO=$(\grep APP_REPO Dockerfile | cut -f2 -d"=")
+GITHUB_REPO=$(echo "$APP_REPO" | sed 's/git.openstack.org/github.com/' | \
+ sed 's/ssh:/https:/')
+
+if [ -z "$CONSTRAINTS_FILE" ]; then
+ CONSTRAINTS_FILE=$(\grep CONSTRAINTS_FILE Dockerfile | cut -f2 -d"=") || true
+ : "${CONSTRAINTS_FILE:=http://git.openstack.org/cgit/openstack/requirements/plain/upper-constraints.txt}"
+fi
+
+: "${CONSTRAINTS_BRANCH:=$2}"
+[ -z "$CONSTRAINTS_BRANCH" ] && \
+ CONSTRAINTS_BRANCH=$(\grep CONSTRAINTS_BRANCH Dockerfile | cut -f2 -d"=")
+
+# When using stable version of repository use same stable constraints file.
+case "$REPO_VERSION" in
+ *stable*)
+ CONSTRAINTS_BRANCH_CLEAN="$REPO_VERSION"
+ ;;
+ *)
+ CONSTRAINTS_BRANCH_CLEAN="$CONSTRAINTS_BRANCH"
+ ;;
+esac
+
+# Monasca-common variables.
+if [ -z "$COMMON_REPO" ]; then
+ COMMON_REPO=$(\grep COMMON_REPO Dockerfile | cut -f2 -d"=") || true
+ : "${COMMON_REPO:=https://git.openstack.org/openstack/monasca-common}"
+fi
+if [ -z "$COMMON_VERSION" ]; then
+ COMMON_VERSION=$(\grep COMMON_VERSION Dockerfile | cut -f2 -d"=") || true
+ : "${COMMON_VERSION:=master}"
+fi
+
+# Clone project to temporary directory for getting proper commit number from
+# branches and tags. We need this for setting proper image labels.
+# Docker does not allow to get any data from inside of system when building
+# image.
+TMP_DIR=$(mktemp -d)
+(
+ cd "$TMP_DIR"
+ # This many steps are needed to support gerrit patch sets.
+ git init
+ git remote add origin "$APP_REPO"
+ git fetch origin "$REPO_VERSION"
+ git reset --hard FETCH_HEAD
+)
+GIT_COMMIT=$(git -C "$TMP_DIR" rev-parse FETCH_HEAD)
+[ -z "${GIT_COMMIT}" ] && echo "No git commit hash found" && exit 1
+rm -rf "$TMP_DIR"
+
+# Do the same for monasca-common.
+COMMON_TMP_DIR=$(mktemp -d)
+(
+ cd "$COMMON_TMP_DIR"
+ # This many steps are needed to support gerrit patch sets.
+ git init
+ git remote add origin "$COMMON_REPO"
+ git fetch origin "$COMMON_VERSION"
+ git reset --hard FETCH_HEAD
+)
+COMMON_GIT_COMMIT=$(git -C "$COMMON_TMP_DIR" rev-parse FETCH_HEAD)
+[ -z "${COMMON_GIT_COMMIT}" ] && echo "No git commit hash found" && exit 1
+rm -rf "$COMMON_TMP_DIR"
+
+CREATION_TIME=$(date -u +"%Y-%m-%dT%H:%M:%SZ")
+
+docker build --no-cache \
+ --build-arg CREATION_TIME="$CREATION_TIME" \
+ --build-arg GITHUB_REPO="$GITHUB_REPO" \
+ --build-arg APP_REPO="$APP_REPO" \
+ --build-arg REPO_VERSION="$REPO_VERSION" \
+ --build-arg GIT_COMMIT="$GIT_COMMIT" \
+ --build-arg CONSTRAINTS_FILE="$CONSTRAINTS_FILE" \
+ --build-arg CONSTRAINTS_BRANCH="$CONSTRAINTS_BRANCH_CLEAN" \
+ --build-arg COMMON_REPO="$COMMON_REPO" \
+ --build-arg COMMON_VERSION="$COMMON_VERSION" \
+ --build-arg COMMON_GIT_COMMIT="$COMMON_GIT_COMMIT" \
+ --tag "$DOCKER_IMAGE":"$REPO_VERSION_CLEAN" .
diff --git a/docker/health_check.py b/docker/health_check.py
new file mode 100644
index 00000000..7d38b641
--- /dev/null
+++ b/docker/health_check.py
@@ -0,0 +1,26 @@
+#!/usr/bin/env python
+# coding=utf-8
+
+# (C) Copyright 2018 FUJITSU LIMITED
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+"""Health check will returns 0 when service is working properly."""
+
+def main():
+ """health check for Monasca-persister"""
+ #TODO wait for health check endpoint ...
+ return 0
+
+if __name__ == '__main__':
+ main()
diff --git a/docker/monasca_persister.conf.j2 b/docker/monasca_persister.conf.j2
new file mode 100644
index 00000000..bc9f4f64
--- /dev/null
+++ b/docker/monasca_persister.conf.j2
@@ -0,0 +1,64 @@
+[DEFAULT]
+#log_file = monasca_persister.log
+#log_dir = /var/log/monasca_persister
+
+# Default log level is WARNING
+# Show debugging output in logs (sets DEBUG log level output)
+debug = {{ DEBUG }}
+# Show more verbose log output (sets INFO log level output) if debug is False
+verbose = {{ VERBOSE }}
+
+[repositories]
+# The driver to use for the metrics repository
+metrics_driver = monasca_persister.repositories.influxdb.metrics_repository:MetricInfluxdbRepository
+
+# The driver to use for the alarm state history repository
+alarm_state_history_driver = monasca_persister.repositories.influxdb.alarm_state_history_repository:AlarmStateHistInfluxdbRepository
+
+[zookeeper]
+# Comma separated list of host:port
+uri = {{ ZOOKEEPER_URI }}
+partition_interval_recheck_seconds = 15
+
+[kafka_alarm_history]
+# Comma separated list of Kafka broker host:port.
+uri = {{ KAFKA_URI }}
+group_id = 1_alarm-state-transitions
+topic = alarm-state-transitions
+consumer_id = 1
+client_id = 1
+database_batch_size = {{ KAFKA_ALARM_HISTORY_BATCH_SIZE }}
+max_wait_time_seconds = {{ KAFKA_ALARM_HISTORY_WAIT_TIME }}
+# The following 3 values are set to the kakfa-python defaults
+fetch_size_bytes = 4096
+buffer_size = 4096
+# 8 times buffer size
+max_buffer_size = 32768
+# Path in zookeeper for kafka consumer group partitioning algo
+zookeeper_path = /persister_partitions/alarm-state-transitions
+num_processors = 1
+
+[kafka_metrics]
+# Comma separated list of Kafka broker host:port
+uri = {{ KAFKA_URI }}
+group_id = 1_metrics
+topic = metrics
+consumer_id = 1
+client_id = 1
+database_batch_size = {{ KAFKA_METRICS_BATCH_SIZE }}
+max_wait_time_seconds = {{ KAFKA_METRICS_WAIT_TIME }}
+# The following 3 values are set to the kakfa-python defaults
+fetch_size_bytes = 4096
+buffer_size = 4096
+# 8 times buffer size
+max_buffer_size = 32768
+# Path in zookeeper for kafka consumer group partitioning algo
+zookeeper_path = /persister_partitions/metrics
+num_processors = 1
+
+[influxdb]
+database_name = {{ INFLUX_DB }}
+ip_address = {{ INFLUX_HOST }}
+port = {{ INFLUX_PORT }}
+user = {{ INFLUX_USER }}
+password = {{ INFLUX_PASSWORD }}
diff --git a/docker/start.sh b/docker/start.sh
new file mode 100644
index 00000000..a43fc3b4
--- /dev/null
+++ b/docker/start.sh
@@ -0,0 +1,41 @@
+#!/bin/sh
+
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+# Starting script.
+# All checks and configuration templating you need to do before service
+# could be safely started should be added in this file.
+
+set -eo pipefail # Exit the script if any statement returns error.
+
+# Test services we need before starting our service.
+echo "Start script: waiting for needed services"
+python3 /kafka_wait_for_topics.py
+
+# Template all config files before start, it will use env variables.
+# Read usage examples: https://pypi.org/project/Templer/
+echo "Start script: creating config files from templates"
+templer -v -f /etc/monasca/monasca_persister.conf.j2 /etc/monasca/monasca_persister.conf
+
+# Start our service.
+# gunicorn --args
+echo "Start script: starting container"
+monasca-persister --config-file /etc/monasca/monasca_persister.conf
+
+# Allow server to stay alive in case of failure for 2 hours for debugging.
+RESULT=$?
+if [ $RESULT != 0 ] && [ "$STAY_ALIVE_ON_FAILURE" = "true" ]; then
+ echo "Service died, waiting 120 min before exiting"
+ sleep 7200
+fi
+exit $RESULT