Add base Dockerfile and supporting scripts
Story: 2001694 Task: 12491 Change-Id: I81e0d0ecbb431ed7e26fcbcb4d347ac164c66736
This commit is contained in:
parent
f1f1ba90fd
commit
2ce968d052
120
docker/Dockerfile
Normal file
120
docker/Dockerfile
Normal file
@ -0,0 +1,120 @@
|
||||
FROM python:3.5.5-alpine3.7
|
||||
|
||||
COPY wait_for.sh kafka_wait_for_topics.py /
|
||||
COPY ashrc /root/.ashrc
|
||||
|
||||
ENV \
|
||||
ENV="/root/.ashrc" \
|
||||
PIP_NO_CACHE_DIR="no" \
|
||||
PIP_NO_COMPILE="no" \
|
||||
PYTHONIOENCODING="utf-8"
|
||||
|
||||
RUN \
|
||||
chmod +x /wait_for.sh /kafka_wait_for_topics.py && \
|
||||
apk add --no-cache \
|
||||
su-exec=0.2-r0 \
|
||||
tini=0.16.1-r0 \
|
||||
# We need this to allow users choose different time zone.
|
||||
tzdata=2017c-r0 && \
|
||||
# Cleaning.
|
||||
rm -rf /var/cache/apk/* && \
|
||||
rm -rf /var/log/* && \
|
||||
rm -rf /tmp/*
|
||||
|
||||
# Get values from child images
|
||||
ONBUILD ARG CREATION_TIME
|
||||
ONBUILD ARG DOCKER_IMAGE
|
||||
ONBUILD ARG APP_REPO
|
||||
ONBUILD ARG GITHUB_REPO
|
||||
ONBUILD ARG REPO_VERSION
|
||||
ONBUILD ARG GIT_COMMIT
|
||||
ONBUILD ARG CONSTRAINTS_BRANCH
|
||||
ONBUILD ARG CONSTRAINTS_FILE=http://git.openstack.org/cgit/openstack/requirements/plain/upper-constraints.txt
|
||||
ONBUILD ARG EXTRA_DEPS
|
||||
ONBUILD ARG COMMON_REPO=https://git.openstack.org/openstack/monasca-common
|
||||
|
||||
# Build-time metadata as defined at
|
||||
# https://github.com/opencontainers/image-spec/blob/master/annotations.md
|
||||
ONBUILD LABEL org.opencontainers.image.created="$CREATION_TIME"
|
||||
ONBUILD LABEL org.opencontainers.image.title="$DOCKER_IMAGE"
|
||||
ONBUILD LABEL org.opencontainers.image.source="$APP_REPO"
|
||||
ONBUILD LABEL org.opencontainers.image.url="$GITHUB_REPO"
|
||||
ONBUILD LABEL org.opencontainers.image.version="$REPO_VERSION"
|
||||
ONBUILD LABEL org.opencontainers.image.revision="$GIT_COMMIT"
|
||||
ONBUILD LABEL org.opencontainers.image.licenses="Apache-2.0"
|
||||
ONBUILD LABEL org.openstack.constraints_uri="$CONSTRAINTS_FILE?h=$CONSTRAINTS_BRANCH"
|
||||
ONBUILD LABEL org.openstack.monasca.python.extra_deps="$EXTRA_DEPS"
|
||||
|
||||
# Every child image need to provide starting and health check script.
|
||||
# If they're not provided build will fail. We want that for uniformity.
|
||||
ONBUILD COPY start.sh health_check.py /
|
||||
|
||||
ONBUILD WORKDIR /
|
||||
|
||||
ONBUILD SHELL ["/bin/ash", "-eo", "pipefail", "-c"]
|
||||
ONBUILD RUN \
|
||||
chmod +x /start.sh && \
|
||||
apk add --no-cache --virtual .build-deps \
|
||||
g++=6.4.0-r5 \
|
||||
git=2.15.2-r0 \
|
||||
libffi-dev=3.2.1-r4 \
|
||||
libressl-dev=2.6.5-r0 \
|
||||
linux-headers=4.4.6-r2 \
|
||||
make=4.2.1-r0 && \
|
||||
# Clone repository and checkout requested version.
|
||||
# This many steps are needed to support gerrit patch sets.
|
||||
mkdir -p /app && \
|
||||
git -C /app init && \
|
||||
git -C /app remote add origin $APP_REPO && \
|
||||
git -C /app fetch origin $REPO_VERSION && \
|
||||
git -C /app reset --hard FETCH_HEAD && \
|
||||
wget --output-document /app/upper-constraints.txt \
|
||||
$CONSTRAINTS_FILE?h=$CONSTRAINTS_BRANCH && \
|
||||
# When creating image from master, stable branch or commit use
|
||||
# monasca-common from git repository.
|
||||
[ ! $(git -C /app tag -l "${REPO_VERSION}") ] && \
|
||||
sed -i "s|monasca-common.*|-e git+$COMMON_REPO@$CONSTRAINTS_BRANCH#egg=monasca-common|" \
|
||||
/app/upper-constraints.txt || true && \
|
||||
# Install packages needed by wait scripts and used for templating.
|
||||
pip3 install \
|
||||
pykafka \
|
||||
PyMySQL \
|
||||
Templer==1.1.4 \
|
||||
--constraint /app/upper-constraints.txt && \
|
||||
# Install our application with extra dependencies if provided.
|
||||
pip3 install \
|
||||
/app/. $EXTRA_DEPS \
|
||||
--requirement /app/requirements.txt \
|
||||
--constraint /app/upper-constraints.txt && \
|
||||
# Save info about build to `/VERSIONS` file.
|
||||
printf "App: %s\n" $DOCKER_IMAGE >> /VERSIONS && \
|
||||
printf "Repository: %s\n" $APP_REPO >> /VERSIONS && \
|
||||
printf "Version: %s\n" $REPO_VERSION >> /VERSIONS && \
|
||||
printf "Build date: %s\n" $CREATION_TIME >> /VERSIONS && \
|
||||
printf "Revision: %s\n" \
|
||||
"$(git -C /app rev-parse FETCH_HEAD)" >> /VERSIONS && \
|
||||
printf "Monasca-common version: %s\n" \
|
||||
"$(pip3 freeze 2>1 | grep 'monasca-common')" >> /VERSIONS && \
|
||||
printf "Constraints file: %s\n" \
|
||||
"$CONSTRAINTS_FILE"?h="$CONSTRAINTS_BRANCH" >> /VERSIONS && \
|
||||
# Clean after instalation.
|
||||
apk del .build-deps && \
|
||||
rm -rf \
|
||||
/app \
|
||||
/root/.cache/ \
|
||||
# Pip is leaving monasca-common repo in /src so remove it.
|
||||
/src/ \
|
||||
/tmp/* \
|
||||
/var/cache/apk/* \
|
||||
/var/log/* && \
|
||||
# Remove all Python pyc and pyo files.
|
||||
find /usr/local -depth \
|
||||
\( \
|
||||
\( -type d -a \( -name test -o -name tests \) \) \
|
||||
-o \( -type f -a \( -name '*.pyc' -o -name '*.pyo' \) \) \
|
||||
\) -exec rm -rf '{}' +
|
||||
|
||||
ONBUILD HEALTHCHECK --interval=5m --timeout=3s \
|
||||
CMD python3 health_check.py || exit 1
|
||||
|
||||
ENTRYPOINT ["/sbin/tini", "-s", "--"]
|
77
docker/README.rst
Normal file
77
docker/README.rst
Normal file
@ -0,0 +1,77 @@
|
||||
======================================
|
||||
Docker base image for Monasca services
|
||||
======================================
|
||||
|
||||
This image is used as a starting point for images of all Monasca services.
|
||||
|
||||
|
||||
Building monasca-base
|
||||
=====================
|
||||
|
||||
You need to have Docker installed (minimum supported version is ``17.09``).
|
||||
Then you could build image inside of this folder:
|
||||
|
||||
``docker build --no-cache -t monasca-base:1.0.0 .``
|
||||
|
||||
|
||||
Building child image
|
||||
--------------------
|
||||
|
||||
In the ``example`` folder you could file sample of how to start building
|
||||
new child image using ``monasca-base`` as start.
|
||||
|
||||
Requirements
|
||||
~~~~~~~~~~~~
|
||||
|
||||
Every child image need to provide two files:
|
||||
|
||||
start.sh
|
||||
In this starting script provide all steps that direct to proper service
|
||||
start. Including usage of wait scripts and templating of configuration files.
|
||||
You also could provide ability to allow running container after service died
|
||||
for easier debugging.
|
||||
|
||||
health_check.py
|
||||
This file will be used for checking status of application running in the
|
||||
container. It will be useful for programs like Kubernetes or Docker Swarm
|
||||
to properly handle services that are still running but stopped being
|
||||
responsive. Avoid using `curl` directly and instead use `health_check.py`
|
||||
written with specific service in mind. It will provide more flexibility
|
||||
like when creating JSON request body.
|
||||
|
||||
|
||||
Wait scripts
|
||||
------------
|
||||
|
||||
Some Python libraries are already preinstalled: `pykafka` and `PyMySQL`.
|
||||
They are used by wait scripts and in the process of creating child image
|
||||
`pip3` will reinstall them to proper versions confronting to upper constraints
|
||||
file.
|
||||
|
||||
This wait scripts will be available in every child image and could be used in
|
||||
`start.sh` to avoid unnecessary errors and restarts of containers when they
|
||||
are started.
|
||||
|
||||
::
|
||||
|
||||
python3 /kafka_wait_for_topics.py || exit 1
|
||||
python3 /mysql_check.py || exit 1
|
||||
/wait_for.sh 192.168.10.6:5000 || exit 1
|
||||
|
||||
Please, check content of every of this files for documentation of what
|
||||
environment variables are used and more usage examples.
|
||||
|
||||
|
||||
Useful commands
|
||||
---------------
|
||||
|
||||
List all labels on image (you need to have ``jq`` installed):
|
||||
|
||||
``docker inspect monasca-api:master | jq .[].Config.Labels``
|
||||
|
||||
Get all steps from what Docker image was build:
|
||||
|
||||
::
|
||||
|
||||
docker history --no-trunc <IMAGE_ID>
|
||||
docker history --no-trunc monasca-base:1.0.0
|
5
docker/ashrc
Normal file
5
docker/ashrc
Normal file
@ -0,0 +1,5 @@
|
||||
alias ll="ls -alp"
|
||||
|
||||
# Print versions on login to the container.
|
||||
cat /VERSIONS
|
||||
echo
|
41
docker/example/Dockerfile
Normal file
41
docker/example/Dockerfile
Normal file
@ -0,0 +1,41 @@
|
||||
# Example Dockerfile for creating Docker image.
|
||||
ARG DOCKER_IMAGE=monasca-api
|
||||
ARG APP_REPO=https://git.openstack.org/openstack/monasca-api
|
||||
|
||||
# Branch, tag or git hash to build from.
|
||||
ARG REPO_VERSION=master
|
||||
ARG CONSTRAINTS_BRANCH=master
|
||||
|
||||
# Extra Python3 dependencies.
|
||||
ARG EXTRA_DEPS="gunicorn influxdb python-memcached"
|
||||
|
||||
# Always start from `monasca-base` image and use specific tag of it.
|
||||
ARG BASE_TAG=1.0.0
|
||||
FROM monasca-base:$BASE_TAG
|
||||
|
||||
# Environment variables used for our service or wait scripts.
|
||||
ENV \
|
||||
KAFKA_URI=kafka:9092 \
|
||||
KAFKA_WAIT_FOR_TOPICS=alarm-state-transitions,metrics \
|
||||
MYSQL_HOST=mysql \
|
||||
MYSQL_USER=monapi \
|
||||
MYSQL_PASSWORD=password \
|
||||
MYSQL_DB=mon \
|
||||
LOG_LEVEL=INFO \
|
||||
STAY_ALIVE_ON_FAILURE="false"
|
||||
|
||||
# Copy all neccessary files to proper locations.
|
||||
COPY config_1.yml.j2 config_2.yml.j2 /
|
||||
|
||||
# Run here all additionals steps your service need post installation.
|
||||
# Stay with only one `RUN` and use `&& \` for next steps to don't create
|
||||
# unnecessary image layers. Clean at the end to conserve space.
|
||||
RUN \
|
||||
echo "Some steps to do after main installation." && \
|
||||
echo "Hello when building."
|
||||
|
||||
# Expose port for specific service.
|
||||
EXPOSE 1234
|
||||
|
||||
# Implement start script in `start.sh` file.
|
||||
CMD ["/start.sh"]
|
10
docker/example/README.rst
Normal file
10
docker/example/README.rst
Normal file
@ -0,0 +1,10 @@
|
||||
====================
|
||||
Docker example image
|
||||
====================
|
||||
|
||||
Example image to show how to build child containers from `monasca-base` image.
|
||||
|
||||
|
||||
| Variable | Default | Description |
|
||||
|-------------------------- |------------------|----------------------------------------------------|
|
||||
| `STAY_ALIVE_ON_FAILURE` | `false` | If true, container runs 2 hours after service fail |
|
86
docker/example/build_image.sh
Executable file
86
docker/example/build_image.sh
Executable file
@ -0,0 +1,86 @@
|
||||
#!/bin/bash
|
||||
|
||||
# TODO(Dobroslaw): move this script to monasca-common/docker folder
|
||||
# and leave here small script to download it and execute using env variables
|
||||
# to minimize code duplication.
|
||||
|
||||
set -x # Print each script step.
|
||||
set -eo pipefail # Exit the script if any statement returns error.
|
||||
|
||||
# This script is used for building Docker image with proper labels.
|
||||
#
|
||||
# Example usage:
|
||||
# $ ./build_image.sh <repository_version> <upper_constains_branch>
|
||||
#
|
||||
# To build from master branch (default):
|
||||
# $ ./build_image.sh
|
||||
# To build specific version run this script in the following way:
|
||||
# $ ./build_image.sh stable/queens
|
||||
# Building from specific commit:
|
||||
# $ ./build_image.sh cb7f226
|
||||
# When building from a tag monasca-common will be used in version available
|
||||
# in upper constraint file:
|
||||
# $ ./build_image.sh 2.5.0
|
||||
# To build image from Gerrit patch sets that is targeting branch stable/queens:
|
||||
# $ ./build_image.sh refs/changes/51/558751/1 stable/queens
|
||||
|
||||
[ -z "$DOCKER_IMAGE" ] && \
|
||||
DOCKER_IMAGE=$(\grep DOCKER_IMAGE Dockerfile | cut -f2 -d"=")
|
||||
|
||||
: "${REPO_VERSION:=$1}"
|
||||
[ -z "$REPO_VERSION" ] && \
|
||||
REPO_VERSION=$(\grep REPO_VERSION Dockerfile | cut -f2 -d"=")
|
||||
# Let's stick to more readable version and disable SC2001 here.
|
||||
# shellcheck disable=SC2001
|
||||
REPO_VERSION_CLEAN=$(echo "$REPO_VERSION" | sed 's|/|-|g')
|
||||
|
||||
[ -z "$APP_REPO" ] && APP_REPO=$(\grep APP_REPO Dockerfile | cut -f2 -d"=")
|
||||
GITHUB_REPO=$(echo "$APP_REPO" | sed 's/git.openstack.org/github.com/' | \
|
||||
sed 's/ssh:/https:/')
|
||||
|
||||
: "${CONSTRAINTS_BRANCH:=$2}"
|
||||
[ -z "$CONSTRAINTS_BRANCH" ] && \
|
||||
CONSTRAINTS_BRANCH=$(\grep CONSTRAINTS_BRANCH Dockerfile | cut -f2 -d"=")
|
||||
# When using stable version of repository use same stable constraints file.
|
||||
case "$REPO_VERSION" in
|
||||
*stable*)
|
||||
CONSTRAINTS_BRANCH_CLEAN="$REPO_VERSION"
|
||||
;;
|
||||
*)
|
||||
CONSTRAINTS_BRANCH_CLEAN="$CONSTRAINTS_BRANCH"
|
||||
;;
|
||||
esac
|
||||
|
||||
# Clone project to temporary directory for getting proper commit number from
|
||||
# branches and tags. We need this for setting proper image labels.
|
||||
# Docker does not allow to get any data from inside of system when building
|
||||
# image.
|
||||
TMP_DIR=$(mktemp -d)
|
||||
(
|
||||
cd "$TMP_DIR"
|
||||
# This many steps are needed to support gerrit patch sets.
|
||||
git init
|
||||
git remote add origin "$APP_REPO"
|
||||
git fetch origin "$REPO_VERSION"
|
||||
git reset --hard FETCH_HEAD
|
||||
)
|
||||
GIT_COMMIT=$(git -C "$TMP_DIR" rev-parse FETCH_HEAD)
|
||||
[ -z "${GIT_COMMIT}" ] && echo "No git commit hash found" && exit 1
|
||||
rm -rf "$TMP_DIR"
|
||||
|
||||
# TODO(Dobroslaw): find a way to set label monasca-common with version
|
||||
# we will be using with app.
|
||||
|
||||
CREATION_TIME=$(date -u +"%Y-%m-%dT%H:%M:%SZ")
|
||||
# Docker tags don't like colons so use shorter version of ISO 8601 for them.
|
||||
CREATION_TIME_SHORT=$(date -d "$CREATION_TIME" -u +"%Y%m%dT%H%M%SZ")
|
||||
|
||||
docker build --no-cache \
|
||||
--build-arg CREATION_TIME="$CREATION_TIME" \
|
||||
--build-arg GITHUB_REPO="$GITHUB_REPO" \
|
||||
--build-arg APP_REPO="$APP_REPO" \
|
||||
--build-arg REPO_VERSION="$REPO_VERSION" \
|
||||
--build-arg GIT_COMMIT="$GIT_COMMIT" \
|
||||
--build-arg CONSTRAINTS_BRANCH="$CONSTRAINTS_BRANCH_CLEAN" \
|
||||
--tag "$DOCKER_IMAGE":"$REPO_VERSION_CLEAN" \
|
||||
--tag "$DOCKER_IMAGE":"$REPO_VERSION_CLEAN"-"$CREATION_TIME_SHORT" .
|
1
docker/example/config_1.yml.j2
Normal file
1
docker/example/config_1.yml.j2
Normal file
@ -0,0 +1 @@
|
||||
kafka_uri: {{ KAFKA_URI }}
|
2
docker/example/config_2.yml.j2
Normal file
2
docker/example/config_2.yml.j2
Normal file
@ -0,0 +1,2 @@
|
||||
connection_string:
|
||||
"mysql+pymysql://{{ MYSQL_USER }}:{{ MYSQL_PASSWORD }}@{{ MYSQL_HOST }}/{{ MYSQL_DB }}"
|
20
docker/example/health_check.py
Normal file
20
docker/example/health_check.py
Normal file
@ -0,0 +1,20 @@
|
||||
#!/usr/bin/env python
|
||||
# coding=utf-8
|
||||
|
||||
# (C) Copyright 2018 FUJITSU LIMITED
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License. You may obtain
|
||||
# a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
"""Health check will returns 0 when service is working properly."""
|
||||
|
||||
# TODO(Dobroslaw): Fill me with health check magic.
|
28
docker/example/start.sh
Normal file
28
docker/example/start.sh
Normal file
@ -0,0 +1,28 @@
|
||||
#!/bin/sh
|
||||
# Starting script.
|
||||
# All checks you need to do before service could be safely started should
|
||||
# be added in this file.
|
||||
|
||||
set -e # Exit the script if any statement returns a non-true return value.
|
||||
|
||||
# Test services we need before starting our service.
|
||||
echo "Start script: waiting for needed services"
|
||||
python3 /kafka_wait_for_topics.py
|
||||
python3 /mysql_check.py
|
||||
|
||||
# Template all config files before start, it will use env variables.
|
||||
# Read usage examples: https://pypi.org/project/Templer/
|
||||
echo "Start script: creating config files from templates"
|
||||
templer /*.j2 /
|
||||
|
||||
# Start our service.
|
||||
# gunicorn --args
|
||||
echo "Start script: starting container"
|
||||
|
||||
# Allow server to stay alive in case of failure for 2 hours for debugging.
|
||||
RESULT=$?
|
||||
if [ $RESULT != 0 ] && [ "$STAY_ALIVE_ON_FAILURE" = "true" ]; then
|
||||
echo "Service died, waiting 120 min before exiting"
|
||||
sleep 7200
|
||||
fi
|
||||
exit $RESULT
|
145
docker/kafka_wait_for_topics.py
Normal file
145
docker/kafka_wait_for_topics.py
Normal file
@ -0,0 +1,145 @@
|
||||
#!/usr/bin/env python
|
||||
# coding=utf-8
|
||||
|
||||
# (C) Copyright 2017 Hewlett Packard Enterprise Development LP
|
||||
# (C) Copyright 2018 FUJITSU LIMITED
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License. You may obtain
|
||||
# a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
"""Wait for specific Kafka topics.
|
||||
|
||||
For using this script you need to set two environment variables:
|
||||
* `KAFKA_URI` for connection string to Kafka together with port.
|
||||
Example: `kafka:9092`, `192.168.10.6:9092`.
|
||||
* `KAFKA_WAIT_FOR_TOPICS` that contain topics that should exist in Kafka
|
||||
to consider it's working. Many topics should be separated with comma.
|
||||
Example: `retry-notifications,alarm-state-transitions`.
|
||||
|
||||
After making sure that this environment variables are set you can simply
|
||||
execute this script in the following way:
|
||||
`python3 kafka_wait_for_topics.py && ./start_service.sh`
|
||||
`python3 kafka_wait_for_topics.py || exit 1`
|
||||
|
||||
Additional environment variables available are:
|
||||
* `LOG_LEVEL` - default to `INFO`
|
||||
* `KAFKA_WAIT_RETRIES` - number of retries, default to `24`
|
||||
* `KAFKA_WAIT_INTERVAL` - in seconds, default to `5`
|
||||
"""
|
||||
|
||||
import logging
|
||||
import os
|
||||
import sys
|
||||
import time
|
||||
|
||||
from pykafka import KafkaClient
|
||||
from pykafka.exceptions import NoBrokersAvailableError
|
||||
|
||||
# Run this script only with Python 3
|
||||
if sys.version_info.major != 3:
|
||||
sys.stdout.write("Sorry, requires Python 3.x\n")
|
||||
sys.exit(1)
|
||||
|
||||
LOG_LEVEL = logging.getLevelName(os.environ.get('LOG_LEVEL', 'INFO'))
|
||||
logging.basicConfig(level=LOG_LEVEL)
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
KAFKA_HOSTS = os.environ.get('KAFKA_URI', 'kafka:9092')
|
||||
|
||||
REQUIRED_TOPICS = os.environ.get('KAFKA_WAIT_FOR_TOPICS', '') \
|
||||
.encode('utf-8').split(b',')
|
||||
|
||||
KAFKA_WAIT_RETRIES = int(os.environ.get('KAFKA_WAIT_RETRIES', '24'))
|
||||
KAFKA_WAIT_INTERVAL = int(os.environ.get('KAFKA_WAIT_INTERVAL', '5'))
|
||||
|
||||
|
||||
class TopicNoPartition(Exception):
|
||||
"""Raise when topic has no partitions."""
|
||||
|
||||
|
||||
class TopicNotFound(Exception):
|
||||
"""Raise when topic was not found."""
|
||||
|
||||
|
||||
def retry(retries=KAFKA_WAIT_RETRIES, delay=KAFKA_WAIT_INTERVAL,
|
||||
check_exceptions=()):
|
||||
"""Retry decorator."""
|
||||
def decorator(func):
|
||||
"""Decorator."""
|
||||
def f_retry(*args, **kwargs):
|
||||
"""Retry running function on exception after delay."""
|
||||
for i in range(1, retries + 1):
|
||||
try:
|
||||
return func(*args, **kwargs)
|
||||
# pylint: disable=W0703
|
||||
# We want to catch all exceptions here to retry.
|
||||
except check_exceptions + (Exception,) as exc:
|
||||
if i < retries:
|
||||
logger.info('Connection attempt %d of %d failed',
|
||||
i, retries)
|
||||
if isinstance(exc, check_exceptions):
|
||||
logger.debug('Caught known exception, retrying...',
|
||||
exc_info=True)
|
||||
else:
|
||||
logger.warn(
|
||||
'Caught unknown exception, retrying...',
|
||||
exc_info=True)
|
||||
else:
|
||||
logger.exception('Failed after %d attempts', retries)
|
||||
|
||||
raise
|
||||
|
||||
# No exception so wait before retrying
|
||||
time.sleep(delay)
|
||||
|
||||
return f_retry
|
||||
return decorator
|
||||
|
||||
|
||||
@retry(check_exceptions=(TopicNoPartition, TopicNotFound))
|
||||
def check_topics(client, req_topics):
|
||||
"""Check for existence of provided topics in Kafka."""
|
||||
client.update_cluster()
|
||||
logger.debug('Found topics: %r', client.topics.keys())
|
||||
|
||||
for req_topic in req_topics:
|
||||
if req_topic not in client.topics.keys():
|
||||
err_topic_not_found = 'Topic not found: {}'.format(req_topic)
|
||||
logger.warning(err_topic_not_found)
|
||||
raise TopicNotFound(err_topic_not_found)
|
||||
|
||||
topic = client.topics[req_topic]
|
||||
if not topic.partitions:
|
||||
err_topic_no_part = 'Topic has no partitions: {}'.format(req_topic)
|
||||
logger.warning(err_topic_no_part)
|
||||
raise TopicNoPartition(err_topic_no_part)
|
||||
|
||||
logger.info('Topic is ready: %s', req_topic)
|
||||
|
||||
|
||||
@retry(check_exceptions=(NoBrokersAvailableError,))
|
||||
def connect_kafka(hosts):
|
||||
"""Connect to Kafka with retries."""
|
||||
return KafkaClient(hosts=hosts)
|
||||
|
||||
|
||||
def main():
|
||||
"""Start main part of the wait script."""
|
||||
logger.info('Checking for available topics: %r', repr(REQUIRED_TOPICS))
|
||||
|
||||
client = connect_kafka(hosts=KAFKA_HOSTS)
|
||||
check_topics(client, REQUIRED_TOPICS)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
58
docker/mysql_check.py
Normal file
58
docker/mysql_check.py
Normal file
@ -0,0 +1,58 @@
|
||||
#!/usr/bin/env python
|
||||
# coding=utf-8
|
||||
|
||||
# (C) Copyright 2018 FUJITSU LIMITED
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License. You may obtain
|
||||
# a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
"""Health check for MySQL returns 0 when all checks works properly.
|
||||
|
||||
It's checking if requested database already exists.
|
||||
|
||||
After making sure that this environment variables are set you can simply
|
||||
execute this script in the following way:
|
||||
`python3 mysql_check.py && ./start_service.sh`
|
||||
`python3 mysql_check.py || exit 1`
|
||||
"""
|
||||
|
||||
import logging
|
||||
import os
|
||||
import sys
|
||||
|
||||
import pymysql
|
||||
|
||||
# Run this script only with Python 3
|
||||
if sys.version_info.major != 3:
|
||||
sys.stdout.write("Sorry, requires Python 3.x\n")
|
||||
sys.exit(1)
|
||||
|
||||
LOG_LEVEL = logging.getLevelName(os.environ.get('LOG_LEVEL', 'INFO'))
|
||||
logging.basicConfig(level=LOG_LEVEL)
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
MYSQL_HOST = os.environ.get('MYSQL_HOST', 'mysql')
|
||||
MYSQL_PORT = os.environ.get('MYSQL_HOST', 3306)
|
||||
MYSQL_USER = os.environ.get('MYSQL_USER', 'monapi')
|
||||
MYSQL_PASSWORD = os.environ.get('MYSQL_PASSWORD', 'password')
|
||||
MYSQL_DB = os.environ.get('MYSQL_DB', 'mon')
|
||||
|
||||
MYSQL_WAIT_RETRIES = int(os.environ.get('MYSQL_WAIT_RETRIES', '24'))
|
||||
MYSQL_WAIT_INTERVAL = int(os.environ.get('MYSQL_WAIT_INTERVAL', '5'))
|
||||
|
||||
# TODO(Dobroslaw): All checks and retry.
|
||||
db = pymysql.connect(
|
||||
host=MYSQL_HOST, port=MYSQL_PORT,
|
||||
user=MYSQL_USER, passwd=MYSQL_PASSWORD,
|
||||
db=MYSQL_DB
|
||||
)
|
36
docker/wait_for.sh
Normal file
36
docker/wait_for.sh
Normal file
@ -0,0 +1,36 @@
|
||||
#!/bin/sh
|
||||
|
||||
# This script will return 0 when on specific address (like 192.168.10.6:5000)
|
||||
# scanning will reveal that port is responding.
|
||||
#
|
||||
# Example usage:
|
||||
# ./wait_for.sh 192.168.10.6:5000 && ./start_service.sh
|
||||
# ./wait_for.sh 192.168.10.6:5000 || exit 1
|
||||
#
|
||||
# By default this script will check up to 24 times every 5 seconds.
|
||||
# You can overwrite this values with environment variables:
|
||||
# `WAIT_RETRIES`
|
||||
# `WAIT_INTERVAL`
|
||||
|
||||
: "${WAIT_RETRIES:=24}"
|
||||
: "${WAIT_INTERVAL:=5}"
|
||||
|
||||
wait_for() {
|
||||
echo "Waiting for $1 to listen on $2..."
|
||||
|
||||
for i in $(seq $WAIT_RETRIES)
|
||||
do
|
||||
nc -z "$1" "$2" && return
|
||||
echo "$1 not yet ready (attempt $i of $WAIT_RETRIES)"
|
||||
sleep "$WAIT_INTERVAL"
|
||||
done
|
||||
echo "$1 failed to become ready, exiting..."
|
||||
exit 1
|
||||
}
|
||||
|
||||
for var in "$@"
|
||||
do
|
||||
host=${var%:*}
|
||||
port=${var#*:}
|
||||
wait_for "$host" "$port"
|
||||
done
|
Loading…
x
Reference in New Issue
Block a user