Add base Dockerfile and supporting scripts
Story: 2001694 Task: 12491 Change-Id: I81e0d0ecbb431ed7e26fcbcb4d347ac164c66736
This commit is contained in:
parent
f1f1ba90fd
commit
2ce968d052
120
docker/Dockerfile
Normal file
120
docker/Dockerfile
Normal file
@ -0,0 +1,120 @@
|
|||||||
|
FROM python:3.5.5-alpine3.7
|
||||||
|
|
||||||
|
COPY wait_for.sh kafka_wait_for_topics.py /
|
||||||
|
COPY ashrc /root/.ashrc
|
||||||
|
|
||||||
|
ENV \
|
||||||
|
ENV="/root/.ashrc" \
|
||||||
|
PIP_NO_CACHE_DIR="no" \
|
||||||
|
PIP_NO_COMPILE="no" \
|
||||||
|
PYTHONIOENCODING="utf-8"
|
||||||
|
|
||||||
|
RUN \
|
||||||
|
chmod +x /wait_for.sh /kafka_wait_for_topics.py && \
|
||||||
|
apk add --no-cache \
|
||||||
|
su-exec=0.2-r0 \
|
||||||
|
tini=0.16.1-r0 \
|
||||||
|
# We need this to allow users choose different time zone.
|
||||||
|
tzdata=2017c-r0 && \
|
||||||
|
# Cleaning.
|
||||||
|
rm -rf /var/cache/apk/* && \
|
||||||
|
rm -rf /var/log/* && \
|
||||||
|
rm -rf /tmp/*
|
||||||
|
|
||||||
|
# Get values from child images
|
||||||
|
ONBUILD ARG CREATION_TIME
|
||||||
|
ONBUILD ARG DOCKER_IMAGE
|
||||||
|
ONBUILD ARG APP_REPO
|
||||||
|
ONBUILD ARG GITHUB_REPO
|
||||||
|
ONBUILD ARG REPO_VERSION
|
||||||
|
ONBUILD ARG GIT_COMMIT
|
||||||
|
ONBUILD ARG CONSTRAINTS_BRANCH
|
||||||
|
ONBUILD ARG CONSTRAINTS_FILE=http://git.openstack.org/cgit/openstack/requirements/plain/upper-constraints.txt
|
||||||
|
ONBUILD ARG EXTRA_DEPS
|
||||||
|
ONBUILD ARG COMMON_REPO=https://git.openstack.org/openstack/monasca-common
|
||||||
|
|
||||||
|
# Build-time metadata as defined at
|
||||||
|
# https://github.com/opencontainers/image-spec/blob/master/annotations.md
|
||||||
|
ONBUILD LABEL org.opencontainers.image.created="$CREATION_TIME"
|
||||||
|
ONBUILD LABEL org.opencontainers.image.title="$DOCKER_IMAGE"
|
||||||
|
ONBUILD LABEL org.opencontainers.image.source="$APP_REPO"
|
||||||
|
ONBUILD LABEL org.opencontainers.image.url="$GITHUB_REPO"
|
||||||
|
ONBUILD LABEL org.opencontainers.image.version="$REPO_VERSION"
|
||||||
|
ONBUILD LABEL org.opencontainers.image.revision="$GIT_COMMIT"
|
||||||
|
ONBUILD LABEL org.opencontainers.image.licenses="Apache-2.0"
|
||||||
|
ONBUILD LABEL org.openstack.constraints_uri="$CONSTRAINTS_FILE?h=$CONSTRAINTS_BRANCH"
|
||||||
|
ONBUILD LABEL org.openstack.monasca.python.extra_deps="$EXTRA_DEPS"
|
||||||
|
|
||||||
|
# Every child image need to provide starting and health check script.
|
||||||
|
# If they're not provided build will fail. We want that for uniformity.
|
||||||
|
ONBUILD COPY start.sh health_check.py /
|
||||||
|
|
||||||
|
ONBUILD WORKDIR /
|
||||||
|
|
||||||
|
ONBUILD SHELL ["/bin/ash", "-eo", "pipefail", "-c"]
|
||||||
|
ONBUILD RUN \
|
||||||
|
chmod +x /start.sh && \
|
||||||
|
apk add --no-cache --virtual .build-deps \
|
||||||
|
g++=6.4.0-r5 \
|
||||||
|
git=2.15.2-r0 \
|
||||||
|
libffi-dev=3.2.1-r4 \
|
||||||
|
libressl-dev=2.6.5-r0 \
|
||||||
|
linux-headers=4.4.6-r2 \
|
||||||
|
make=4.2.1-r0 && \
|
||||||
|
# Clone repository and checkout requested version.
|
||||||
|
# This many steps are needed to support gerrit patch sets.
|
||||||
|
mkdir -p /app && \
|
||||||
|
git -C /app init && \
|
||||||
|
git -C /app remote add origin $APP_REPO && \
|
||||||
|
git -C /app fetch origin $REPO_VERSION && \
|
||||||
|
git -C /app reset --hard FETCH_HEAD && \
|
||||||
|
wget --output-document /app/upper-constraints.txt \
|
||||||
|
$CONSTRAINTS_FILE?h=$CONSTRAINTS_BRANCH && \
|
||||||
|
# When creating image from master, stable branch or commit use
|
||||||
|
# monasca-common from git repository.
|
||||||
|
[ ! $(git -C /app tag -l "${REPO_VERSION}") ] && \
|
||||||
|
sed -i "s|monasca-common.*|-e git+$COMMON_REPO@$CONSTRAINTS_BRANCH#egg=monasca-common|" \
|
||||||
|
/app/upper-constraints.txt || true && \
|
||||||
|
# Install packages needed by wait scripts and used for templating.
|
||||||
|
pip3 install \
|
||||||
|
pykafka \
|
||||||
|
PyMySQL \
|
||||||
|
Templer==1.1.4 \
|
||||||
|
--constraint /app/upper-constraints.txt && \
|
||||||
|
# Install our application with extra dependencies if provided.
|
||||||
|
pip3 install \
|
||||||
|
/app/. $EXTRA_DEPS \
|
||||||
|
--requirement /app/requirements.txt \
|
||||||
|
--constraint /app/upper-constraints.txt && \
|
||||||
|
# Save info about build to `/VERSIONS` file.
|
||||||
|
printf "App: %s\n" $DOCKER_IMAGE >> /VERSIONS && \
|
||||||
|
printf "Repository: %s\n" $APP_REPO >> /VERSIONS && \
|
||||||
|
printf "Version: %s\n" $REPO_VERSION >> /VERSIONS && \
|
||||||
|
printf "Build date: %s\n" $CREATION_TIME >> /VERSIONS && \
|
||||||
|
printf "Revision: %s\n" \
|
||||||
|
"$(git -C /app rev-parse FETCH_HEAD)" >> /VERSIONS && \
|
||||||
|
printf "Monasca-common version: %s\n" \
|
||||||
|
"$(pip3 freeze 2>1 | grep 'monasca-common')" >> /VERSIONS && \
|
||||||
|
printf "Constraints file: %s\n" \
|
||||||
|
"$CONSTRAINTS_FILE"?h="$CONSTRAINTS_BRANCH" >> /VERSIONS && \
|
||||||
|
# Clean after instalation.
|
||||||
|
apk del .build-deps && \
|
||||||
|
rm -rf \
|
||||||
|
/app \
|
||||||
|
/root/.cache/ \
|
||||||
|
# Pip is leaving monasca-common repo in /src so remove it.
|
||||||
|
/src/ \
|
||||||
|
/tmp/* \
|
||||||
|
/var/cache/apk/* \
|
||||||
|
/var/log/* && \
|
||||||
|
# Remove all Python pyc and pyo files.
|
||||||
|
find /usr/local -depth \
|
||||||
|
\( \
|
||||||
|
\( -type d -a \( -name test -o -name tests \) \) \
|
||||||
|
-o \( -type f -a \( -name '*.pyc' -o -name '*.pyo' \) \) \
|
||||||
|
\) -exec rm -rf '{}' +
|
||||||
|
|
||||||
|
ONBUILD HEALTHCHECK --interval=5m --timeout=3s \
|
||||||
|
CMD python3 health_check.py || exit 1
|
||||||
|
|
||||||
|
ENTRYPOINT ["/sbin/tini", "-s", "--"]
|
77
docker/README.rst
Normal file
77
docker/README.rst
Normal file
@ -0,0 +1,77 @@
|
|||||||
|
======================================
|
||||||
|
Docker base image for Monasca services
|
||||||
|
======================================
|
||||||
|
|
||||||
|
This image is used as a starting point for images of all Monasca services.
|
||||||
|
|
||||||
|
|
||||||
|
Building monasca-base
|
||||||
|
=====================
|
||||||
|
|
||||||
|
You need to have Docker installed (minimum supported version is ``17.09``).
|
||||||
|
Then you could build image inside of this folder:
|
||||||
|
|
||||||
|
``docker build --no-cache -t monasca-base:1.0.0 .``
|
||||||
|
|
||||||
|
|
||||||
|
Building child image
|
||||||
|
--------------------
|
||||||
|
|
||||||
|
In the ``example`` folder you could file sample of how to start building
|
||||||
|
new child image using ``monasca-base`` as start.
|
||||||
|
|
||||||
|
Requirements
|
||||||
|
~~~~~~~~~~~~
|
||||||
|
|
||||||
|
Every child image need to provide two files:
|
||||||
|
|
||||||
|
start.sh
|
||||||
|
In this starting script provide all steps that direct to proper service
|
||||||
|
start. Including usage of wait scripts and templating of configuration files.
|
||||||
|
You also could provide ability to allow running container after service died
|
||||||
|
for easier debugging.
|
||||||
|
|
||||||
|
health_check.py
|
||||||
|
This file will be used for checking status of application running in the
|
||||||
|
container. It will be useful for programs like Kubernetes or Docker Swarm
|
||||||
|
to properly handle services that are still running but stopped being
|
||||||
|
responsive. Avoid using `curl` directly and instead use `health_check.py`
|
||||||
|
written with specific service in mind. It will provide more flexibility
|
||||||
|
like when creating JSON request body.
|
||||||
|
|
||||||
|
|
||||||
|
Wait scripts
|
||||||
|
------------
|
||||||
|
|
||||||
|
Some Python libraries are already preinstalled: `pykafka` and `PyMySQL`.
|
||||||
|
They are used by wait scripts and in the process of creating child image
|
||||||
|
`pip3` will reinstall them to proper versions confronting to upper constraints
|
||||||
|
file.
|
||||||
|
|
||||||
|
This wait scripts will be available in every child image and could be used in
|
||||||
|
`start.sh` to avoid unnecessary errors and restarts of containers when they
|
||||||
|
are started.
|
||||||
|
|
||||||
|
::
|
||||||
|
|
||||||
|
python3 /kafka_wait_for_topics.py || exit 1
|
||||||
|
python3 /mysql_check.py || exit 1
|
||||||
|
/wait_for.sh 192.168.10.6:5000 || exit 1
|
||||||
|
|
||||||
|
Please, check content of every of this files for documentation of what
|
||||||
|
environment variables are used and more usage examples.
|
||||||
|
|
||||||
|
|
||||||
|
Useful commands
|
||||||
|
---------------
|
||||||
|
|
||||||
|
List all labels on image (you need to have ``jq`` installed):
|
||||||
|
|
||||||
|
``docker inspect monasca-api:master | jq .[].Config.Labels``
|
||||||
|
|
||||||
|
Get all steps from what Docker image was build:
|
||||||
|
|
||||||
|
::
|
||||||
|
|
||||||
|
docker history --no-trunc <IMAGE_ID>
|
||||||
|
docker history --no-trunc monasca-base:1.0.0
|
5
docker/ashrc
Normal file
5
docker/ashrc
Normal file
@ -0,0 +1,5 @@
|
|||||||
|
alias ll="ls -alp"
|
||||||
|
|
||||||
|
# Print versions on login to the container.
|
||||||
|
cat /VERSIONS
|
||||||
|
echo
|
41
docker/example/Dockerfile
Normal file
41
docker/example/Dockerfile
Normal file
@ -0,0 +1,41 @@
|
|||||||
|
# Example Dockerfile for creating Docker image.
|
||||||
|
ARG DOCKER_IMAGE=monasca-api
|
||||||
|
ARG APP_REPO=https://git.openstack.org/openstack/monasca-api
|
||||||
|
|
||||||
|
# Branch, tag or git hash to build from.
|
||||||
|
ARG REPO_VERSION=master
|
||||||
|
ARG CONSTRAINTS_BRANCH=master
|
||||||
|
|
||||||
|
# Extra Python3 dependencies.
|
||||||
|
ARG EXTRA_DEPS="gunicorn influxdb python-memcached"
|
||||||
|
|
||||||
|
# Always start from `monasca-base` image and use specific tag of it.
|
||||||
|
ARG BASE_TAG=1.0.0
|
||||||
|
FROM monasca-base:$BASE_TAG
|
||||||
|
|
||||||
|
# Environment variables used for our service or wait scripts.
|
||||||
|
ENV \
|
||||||
|
KAFKA_URI=kafka:9092 \
|
||||||
|
KAFKA_WAIT_FOR_TOPICS=alarm-state-transitions,metrics \
|
||||||
|
MYSQL_HOST=mysql \
|
||||||
|
MYSQL_USER=monapi \
|
||||||
|
MYSQL_PASSWORD=password \
|
||||||
|
MYSQL_DB=mon \
|
||||||
|
LOG_LEVEL=INFO \
|
||||||
|
STAY_ALIVE_ON_FAILURE="false"
|
||||||
|
|
||||||
|
# Copy all neccessary files to proper locations.
|
||||||
|
COPY config_1.yml.j2 config_2.yml.j2 /
|
||||||
|
|
||||||
|
# Run here all additionals steps your service need post installation.
|
||||||
|
# Stay with only one `RUN` and use `&& \` for next steps to don't create
|
||||||
|
# unnecessary image layers. Clean at the end to conserve space.
|
||||||
|
RUN \
|
||||||
|
echo "Some steps to do after main installation." && \
|
||||||
|
echo "Hello when building."
|
||||||
|
|
||||||
|
# Expose port for specific service.
|
||||||
|
EXPOSE 1234
|
||||||
|
|
||||||
|
# Implement start script in `start.sh` file.
|
||||||
|
CMD ["/start.sh"]
|
10
docker/example/README.rst
Normal file
10
docker/example/README.rst
Normal file
@ -0,0 +1,10 @@
|
|||||||
|
====================
|
||||||
|
Docker example image
|
||||||
|
====================
|
||||||
|
|
||||||
|
Example image to show how to build child containers from `monasca-base` image.
|
||||||
|
|
||||||
|
|
||||||
|
| Variable | Default | Description |
|
||||||
|
|-------------------------- |------------------|----------------------------------------------------|
|
||||||
|
| `STAY_ALIVE_ON_FAILURE` | `false` | If true, container runs 2 hours after service fail |
|
86
docker/example/build_image.sh
Executable file
86
docker/example/build_image.sh
Executable file
@ -0,0 +1,86 @@
|
|||||||
|
#!/bin/bash
|
||||||
|
|
||||||
|
# TODO(Dobroslaw): move this script to monasca-common/docker folder
|
||||||
|
# and leave here small script to download it and execute using env variables
|
||||||
|
# to minimize code duplication.
|
||||||
|
|
||||||
|
set -x # Print each script step.
|
||||||
|
set -eo pipefail # Exit the script if any statement returns error.
|
||||||
|
|
||||||
|
# This script is used for building Docker image with proper labels.
|
||||||
|
#
|
||||||
|
# Example usage:
|
||||||
|
# $ ./build_image.sh <repository_version> <upper_constains_branch>
|
||||||
|
#
|
||||||
|
# To build from master branch (default):
|
||||||
|
# $ ./build_image.sh
|
||||||
|
# To build specific version run this script in the following way:
|
||||||
|
# $ ./build_image.sh stable/queens
|
||||||
|
# Building from specific commit:
|
||||||
|
# $ ./build_image.sh cb7f226
|
||||||
|
# When building from a tag monasca-common will be used in version available
|
||||||
|
# in upper constraint file:
|
||||||
|
# $ ./build_image.sh 2.5.0
|
||||||
|
# To build image from Gerrit patch sets that is targeting branch stable/queens:
|
||||||
|
# $ ./build_image.sh refs/changes/51/558751/1 stable/queens
|
||||||
|
|
||||||
|
[ -z "$DOCKER_IMAGE" ] && \
|
||||||
|
DOCKER_IMAGE=$(\grep DOCKER_IMAGE Dockerfile | cut -f2 -d"=")
|
||||||
|
|
||||||
|
: "${REPO_VERSION:=$1}"
|
||||||
|
[ -z "$REPO_VERSION" ] && \
|
||||||
|
REPO_VERSION=$(\grep REPO_VERSION Dockerfile | cut -f2 -d"=")
|
||||||
|
# Let's stick to more readable version and disable SC2001 here.
|
||||||
|
# shellcheck disable=SC2001
|
||||||
|
REPO_VERSION_CLEAN=$(echo "$REPO_VERSION" | sed 's|/|-|g')
|
||||||
|
|
||||||
|
[ -z "$APP_REPO" ] && APP_REPO=$(\grep APP_REPO Dockerfile | cut -f2 -d"=")
|
||||||
|
GITHUB_REPO=$(echo "$APP_REPO" | sed 's/git.openstack.org/github.com/' | \
|
||||||
|
sed 's/ssh:/https:/')
|
||||||
|
|
||||||
|
: "${CONSTRAINTS_BRANCH:=$2}"
|
||||||
|
[ -z "$CONSTRAINTS_BRANCH" ] && \
|
||||||
|
CONSTRAINTS_BRANCH=$(\grep CONSTRAINTS_BRANCH Dockerfile | cut -f2 -d"=")
|
||||||
|
# When using stable version of repository use same stable constraints file.
|
||||||
|
case "$REPO_VERSION" in
|
||||||
|
*stable*)
|
||||||
|
CONSTRAINTS_BRANCH_CLEAN="$REPO_VERSION"
|
||||||
|
;;
|
||||||
|
*)
|
||||||
|
CONSTRAINTS_BRANCH_CLEAN="$CONSTRAINTS_BRANCH"
|
||||||
|
;;
|
||||||
|
esac
|
||||||
|
|
||||||
|
# Clone project to temporary directory for getting proper commit number from
|
||||||
|
# branches and tags. We need this for setting proper image labels.
|
||||||
|
# Docker does not allow to get any data from inside of system when building
|
||||||
|
# image.
|
||||||
|
TMP_DIR=$(mktemp -d)
|
||||||
|
(
|
||||||
|
cd "$TMP_DIR"
|
||||||
|
# This many steps are needed to support gerrit patch sets.
|
||||||
|
git init
|
||||||
|
git remote add origin "$APP_REPO"
|
||||||
|
git fetch origin "$REPO_VERSION"
|
||||||
|
git reset --hard FETCH_HEAD
|
||||||
|
)
|
||||||
|
GIT_COMMIT=$(git -C "$TMP_DIR" rev-parse FETCH_HEAD)
|
||||||
|
[ -z "${GIT_COMMIT}" ] && echo "No git commit hash found" && exit 1
|
||||||
|
rm -rf "$TMP_DIR"
|
||||||
|
|
||||||
|
# TODO(Dobroslaw): find a way to set label monasca-common with version
|
||||||
|
# we will be using with app.
|
||||||
|
|
||||||
|
CREATION_TIME=$(date -u +"%Y-%m-%dT%H:%M:%SZ")
|
||||||
|
# Docker tags don't like colons so use shorter version of ISO 8601 for them.
|
||||||
|
CREATION_TIME_SHORT=$(date -d "$CREATION_TIME" -u +"%Y%m%dT%H%M%SZ")
|
||||||
|
|
||||||
|
docker build --no-cache \
|
||||||
|
--build-arg CREATION_TIME="$CREATION_TIME" \
|
||||||
|
--build-arg GITHUB_REPO="$GITHUB_REPO" \
|
||||||
|
--build-arg APP_REPO="$APP_REPO" \
|
||||||
|
--build-arg REPO_VERSION="$REPO_VERSION" \
|
||||||
|
--build-arg GIT_COMMIT="$GIT_COMMIT" \
|
||||||
|
--build-arg CONSTRAINTS_BRANCH="$CONSTRAINTS_BRANCH_CLEAN" \
|
||||||
|
--tag "$DOCKER_IMAGE":"$REPO_VERSION_CLEAN" \
|
||||||
|
--tag "$DOCKER_IMAGE":"$REPO_VERSION_CLEAN"-"$CREATION_TIME_SHORT" .
|
1
docker/example/config_1.yml.j2
Normal file
1
docker/example/config_1.yml.j2
Normal file
@ -0,0 +1 @@
|
|||||||
|
kafka_uri: {{ KAFKA_URI }}
|
2
docker/example/config_2.yml.j2
Normal file
2
docker/example/config_2.yml.j2
Normal file
@ -0,0 +1,2 @@
|
|||||||
|
connection_string:
|
||||||
|
"mysql+pymysql://{{ MYSQL_USER }}:{{ MYSQL_PASSWORD }}@{{ MYSQL_HOST }}/{{ MYSQL_DB }}"
|
20
docker/example/health_check.py
Normal file
20
docker/example/health_check.py
Normal file
@ -0,0 +1,20 @@
|
|||||||
|
#!/usr/bin/env python
|
||||||
|
# coding=utf-8
|
||||||
|
|
||||||
|
# (C) Copyright 2018 FUJITSU LIMITED
|
||||||
|
#
|
||||||
|
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||||
|
# not use this file except in compliance with the License. You may obtain
|
||||||
|
# a copy of the License at
|
||||||
|
#
|
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
#
|
||||||
|
# Unless required by applicable law or agreed to in writing, software
|
||||||
|
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||||
|
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||||
|
# License for the specific language governing permissions and limitations
|
||||||
|
# under the License.
|
||||||
|
|
||||||
|
"""Health check will returns 0 when service is working properly."""
|
||||||
|
|
||||||
|
# TODO(Dobroslaw): Fill me with health check magic.
|
28
docker/example/start.sh
Normal file
28
docker/example/start.sh
Normal file
@ -0,0 +1,28 @@
|
|||||||
|
#!/bin/sh
|
||||||
|
# Starting script.
|
||||||
|
# All checks you need to do before service could be safely started should
|
||||||
|
# be added in this file.
|
||||||
|
|
||||||
|
set -e # Exit the script if any statement returns a non-true return value.
|
||||||
|
|
||||||
|
# Test services we need before starting our service.
|
||||||
|
echo "Start script: waiting for needed services"
|
||||||
|
python3 /kafka_wait_for_topics.py
|
||||||
|
python3 /mysql_check.py
|
||||||
|
|
||||||
|
# Template all config files before start, it will use env variables.
|
||||||
|
# Read usage examples: https://pypi.org/project/Templer/
|
||||||
|
echo "Start script: creating config files from templates"
|
||||||
|
templer /*.j2 /
|
||||||
|
|
||||||
|
# Start our service.
|
||||||
|
# gunicorn --args
|
||||||
|
echo "Start script: starting container"
|
||||||
|
|
||||||
|
# Allow server to stay alive in case of failure for 2 hours for debugging.
|
||||||
|
RESULT=$?
|
||||||
|
if [ $RESULT != 0 ] && [ "$STAY_ALIVE_ON_FAILURE" = "true" ]; then
|
||||||
|
echo "Service died, waiting 120 min before exiting"
|
||||||
|
sleep 7200
|
||||||
|
fi
|
||||||
|
exit $RESULT
|
145
docker/kafka_wait_for_topics.py
Normal file
145
docker/kafka_wait_for_topics.py
Normal file
@ -0,0 +1,145 @@
|
|||||||
|
#!/usr/bin/env python
|
||||||
|
# coding=utf-8
|
||||||
|
|
||||||
|
# (C) Copyright 2017 Hewlett Packard Enterprise Development LP
|
||||||
|
# (C) Copyright 2018 FUJITSU LIMITED
|
||||||
|
#
|
||||||
|
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||||
|
# not use this file except in compliance with the License. You may obtain
|
||||||
|
# a copy of the License at
|
||||||
|
#
|
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
#
|
||||||
|
# Unless required by applicable law or agreed to in writing, software
|
||||||
|
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||||
|
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||||
|
# License for the specific language governing permissions and limitations
|
||||||
|
# under the License.
|
||||||
|
|
||||||
|
"""Wait for specific Kafka topics.
|
||||||
|
|
||||||
|
For using this script you need to set two environment variables:
|
||||||
|
* `KAFKA_URI` for connection string to Kafka together with port.
|
||||||
|
Example: `kafka:9092`, `192.168.10.6:9092`.
|
||||||
|
* `KAFKA_WAIT_FOR_TOPICS` that contain topics that should exist in Kafka
|
||||||
|
to consider it's working. Many topics should be separated with comma.
|
||||||
|
Example: `retry-notifications,alarm-state-transitions`.
|
||||||
|
|
||||||
|
After making sure that this environment variables are set you can simply
|
||||||
|
execute this script in the following way:
|
||||||
|
`python3 kafka_wait_for_topics.py && ./start_service.sh`
|
||||||
|
`python3 kafka_wait_for_topics.py || exit 1`
|
||||||
|
|
||||||
|
Additional environment variables available are:
|
||||||
|
* `LOG_LEVEL` - default to `INFO`
|
||||||
|
* `KAFKA_WAIT_RETRIES` - number of retries, default to `24`
|
||||||
|
* `KAFKA_WAIT_INTERVAL` - in seconds, default to `5`
|
||||||
|
"""
|
||||||
|
|
||||||
|
import logging
|
||||||
|
import os
|
||||||
|
import sys
|
||||||
|
import time
|
||||||
|
|
||||||
|
from pykafka import KafkaClient
|
||||||
|
from pykafka.exceptions import NoBrokersAvailableError
|
||||||
|
|
||||||
|
# Run this script only with Python 3
|
||||||
|
if sys.version_info.major != 3:
|
||||||
|
sys.stdout.write("Sorry, requires Python 3.x\n")
|
||||||
|
sys.exit(1)
|
||||||
|
|
||||||
|
LOG_LEVEL = logging.getLevelName(os.environ.get('LOG_LEVEL', 'INFO'))
|
||||||
|
logging.basicConfig(level=LOG_LEVEL)
|
||||||
|
|
||||||
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
KAFKA_HOSTS = os.environ.get('KAFKA_URI', 'kafka:9092')
|
||||||
|
|
||||||
|
REQUIRED_TOPICS = os.environ.get('KAFKA_WAIT_FOR_TOPICS', '') \
|
||||||
|
.encode('utf-8').split(b',')
|
||||||
|
|
||||||
|
KAFKA_WAIT_RETRIES = int(os.environ.get('KAFKA_WAIT_RETRIES', '24'))
|
||||||
|
KAFKA_WAIT_INTERVAL = int(os.environ.get('KAFKA_WAIT_INTERVAL', '5'))
|
||||||
|
|
||||||
|
|
||||||
|
class TopicNoPartition(Exception):
|
||||||
|
"""Raise when topic has no partitions."""
|
||||||
|
|
||||||
|
|
||||||
|
class TopicNotFound(Exception):
|
||||||
|
"""Raise when topic was not found."""
|
||||||
|
|
||||||
|
|
||||||
|
def retry(retries=KAFKA_WAIT_RETRIES, delay=KAFKA_WAIT_INTERVAL,
|
||||||
|
check_exceptions=()):
|
||||||
|
"""Retry decorator."""
|
||||||
|
def decorator(func):
|
||||||
|
"""Decorator."""
|
||||||
|
def f_retry(*args, **kwargs):
|
||||||
|
"""Retry running function on exception after delay."""
|
||||||
|
for i in range(1, retries + 1):
|
||||||
|
try:
|
||||||
|
return func(*args, **kwargs)
|
||||||
|
# pylint: disable=W0703
|
||||||
|
# We want to catch all exceptions here to retry.
|
||||||
|
except check_exceptions + (Exception,) as exc:
|
||||||
|
if i < retries:
|
||||||
|
logger.info('Connection attempt %d of %d failed',
|
||||||
|
i, retries)
|
||||||
|
if isinstance(exc, check_exceptions):
|
||||||
|
logger.debug('Caught known exception, retrying...',
|
||||||
|
exc_info=True)
|
||||||
|
else:
|
||||||
|
logger.warn(
|
||||||
|
'Caught unknown exception, retrying...',
|
||||||
|
exc_info=True)
|
||||||
|
else:
|
||||||
|
logger.exception('Failed after %d attempts', retries)
|
||||||
|
|
||||||
|
raise
|
||||||
|
|
||||||
|
# No exception so wait before retrying
|
||||||
|
time.sleep(delay)
|
||||||
|
|
||||||
|
return f_retry
|
||||||
|
return decorator
|
||||||
|
|
||||||
|
|
||||||
|
@retry(check_exceptions=(TopicNoPartition, TopicNotFound))
|
||||||
|
def check_topics(client, req_topics):
|
||||||
|
"""Check for existence of provided topics in Kafka."""
|
||||||
|
client.update_cluster()
|
||||||
|
logger.debug('Found topics: %r', client.topics.keys())
|
||||||
|
|
||||||
|
for req_topic in req_topics:
|
||||||
|
if req_topic not in client.topics.keys():
|
||||||
|
err_topic_not_found = 'Topic not found: {}'.format(req_topic)
|
||||||
|
logger.warning(err_topic_not_found)
|
||||||
|
raise TopicNotFound(err_topic_not_found)
|
||||||
|
|
||||||
|
topic = client.topics[req_topic]
|
||||||
|
if not topic.partitions:
|
||||||
|
err_topic_no_part = 'Topic has no partitions: {}'.format(req_topic)
|
||||||
|
logger.warning(err_topic_no_part)
|
||||||
|
raise TopicNoPartition(err_topic_no_part)
|
||||||
|
|
||||||
|
logger.info('Topic is ready: %s', req_topic)
|
||||||
|
|
||||||
|
|
||||||
|
@retry(check_exceptions=(NoBrokersAvailableError,))
|
||||||
|
def connect_kafka(hosts):
|
||||||
|
"""Connect to Kafka with retries."""
|
||||||
|
return KafkaClient(hosts=hosts)
|
||||||
|
|
||||||
|
|
||||||
|
def main():
|
||||||
|
"""Start main part of the wait script."""
|
||||||
|
logger.info('Checking for available topics: %r', repr(REQUIRED_TOPICS))
|
||||||
|
|
||||||
|
client = connect_kafka(hosts=KAFKA_HOSTS)
|
||||||
|
check_topics(client, REQUIRED_TOPICS)
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == '__main__':
|
||||||
|
main()
|
58
docker/mysql_check.py
Normal file
58
docker/mysql_check.py
Normal file
@ -0,0 +1,58 @@
|
|||||||
|
#!/usr/bin/env python
|
||||||
|
# coding=utf-8
|
||||||
|
|
||||||
|
# (C) Copyright 2018 FUJITSU LIMITED
|
||||||
|
#
|
||||||
|
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||||
|
# not use this file except in compliance with the License. You may obtain
|
||||||
|
# a copy of the License at
|
||||||
|
#
|
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
#
|
||||||
|
# Unless required by applicable law or agreed to in writing, software
|
||||||
|
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||||
|
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||||
|
# License for the specific language governing permissions and limitations
|
||||||
|
# under the License.
|
||||||
|
|
||||||
|
"""Health check for MySQL returns 0 when all checks works properly.
|
||||||
|
|
||||||
|
It's checking if requested database already exists.
|
||||||
|
|
||||||
|
After making sure that this environment variables are set you can simply
|
||||||
|
execute this script in the following way:
|
||||||
|
`python3 mysql_check.py && ./start_service.sh`
|
||||||
|
`python3 mysql_check.py || exit 1`
|
||||||
|
"""
|
||||||
|
|
||||||
|
import logging
|
||||||
|
import os
|
||||||
|
import sys
|
||||||
|
|
||||||
|
import pymysql
|
||||||
|
|
||||||
|
# Run this script only with Python 3
|
||||||
|
if sys.version_info.major != 3:
|
||||||
|
sys.stdout.write("Sorry, requires Python 3.x\n")
|
||||||
|
sys.exit(1)
|
||||||
|
|
||||||
|
LOG_LEVEL = logging.getLevelName(os.environ.get('LOG_LEVEL', 'INFO'))
|
||||||
|
logging.basicConfig(level=LOG_LEVEL)
|
||||||
|
|
||||||
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
MYSQL_HOST = os.environ.get('MYSQL_HOST', 'mysql')
|
||||||
|
MYSQL_PORT = os.environ.get('MYSQL_HOST', 3306)
|
||||||
|
MYSQL_USER = os.environ.get('MYSQL_USER', 'monapi')
|
||||||
|
MYSQL_PASSWORD = os.environ.get('MYSQL_PASSWORD', 'password')
|
||||||
|
MYSQL_DB = os.environ.get('MYSQL_DB', 'mon')
|
||||||
|
|
||||||
|
MYSQL_WAIT_RETRIES = int(os.environ.get('MYSQL_WAIT_RETRIES', '24'))
|
||||||
|
MYSQL_WAIT_INTERVAL = int(os.environ.get('MYSQL_WAIT_INTERVAL', '5'))
|
||||||
|
|
||||||
|
# TODO(Dobroslaw): All checks and retry.
|
||||||
|
db = pymysql.connect(
|
||||||
|
host=MYSQL_HOST, port=MYSQL_PORT,
|
||||||
|
user=MYSQL_USER, passwd=MYSQL_PASSWORD,
|
||||||
|
db=MYSQL_DB
|
||||||
|
)
|
36
docker/wait_for.sh
Normal file
36
docker/wait_for.sh
Normal file
@ -0,0 +1,36 @@
|
|||||||
|
#!/bin/sh
|
||||||
|
|
||||||
|
# This script will return 0 when on specific address (like 192.168.10.6:5000)
|
||||||
|
# scanning will reveal that port is responding.
|
||||||
|
#
|
||||||
|
# Example usage:
|
||||||
|
# ./wait_for.sh 192.168.10.6:5000 && ./start_service.sh
|
||||||
|
# ./wait_for.sh 192.168.10.6:5000 || exit 1
|
||||||
|
#
|
||||||
|
# By default this script will check up to 24 times every 5 seconds.
|
||||||
|
# You can overwrite this values with environment variables:
|
||||||
|
# `WAIT_RETRIES`
|
||||||
|
# `WAIT_INTERVAL`
|
||||||
|
|
||||||
|
: "${WAIT_RETRIES:=24}"
|
||||||
|
: "${WAIT_INTERVAL:=5}"
|
||||||
|
|
||||||
|
wait_for() {
|
||||||
|
echo "Waiting for $1 to listen on $2..."
|
||||||
|
|
||||||
|
for i in $(seq $WAIT_RETRIES)
|
||||||
|
do
|
||||||
|
nc -z "$1" "$2" && return
|
||||||
|
echo "$1 not yet ready (attempt $i of $WAIT_RETRIES)"
|
||||||
|
sleep "$WAIT_INTERVAL"
|
||||||
|
done
|
||||||
|
echo "$1 failed to become ready, exiting..."
|
||||||
|
exit 1
|
||||||
|
}
|
||||||
|
|
||||||
|
for var in "$@"
|
||||||
|
do
|
||||||
|
host=${var%:*}
|
||||||
|
port=${var#*:}
|
||||||
|
wait_for "$host" "$port"
|
||||||
|
done
|
Loading…
x
Reference in New Issue
Block a user