Add docker image publishing

1) Refactor shipyard image build 2) Add shipyard docker image publishing Change-Id: I1a007ae21a88d6468bf42724e5def5cd04ebe48e
2018-05-28 14:08:52 +08:00 · 2018-05-28 14:08:52 +08:00 · 8a393dfc80
commit 8a393dfc80
parent d558bd6218
5 changed files with 260 additions and 57 deletions
--- a/.zuul.yaml
+++ b/.zuul.yaml
@ -33,6 +33,9 @@
        - airship-shipyard-pep8
        - airship-shipyard-security-bandit
        - airship-shipyard-image
+    post:
+      jobs:
+        - airship-shipyard-docker-build-post

 - nodeset:
    name: airship-shipyard-single-node
@ -130,6 +133,7 @@
      - tools/gate/playbooks/osh-infra-upgrade-host.yaml
      - tools/gate/playbooks/osh-infra-deploy-docker.yaml
    run: tools/gate/playbooks/run-image.yaml
+    nodeset: airship-shipyard-single-node
    required-projects:
      - openstack/openstack-helm-infra
    irrelevant-files:
@ -142,4 +146,48 @@
    description: |
      Run shipyard-image build.
    parent: airship-shipyard-image-base
-    nodeset: airship-shipyard-single-node
+    vars:
+      publish: false
+      tags:
+        dynamic:
+          patch_set: true
+
+- job:
+    name: airship-shipyard-docker-build-post
+    parent: airship-shipyard-image-base
+    secrets:
+      - airship_shipyard_quay_creds
+    vars:
+      publish: true
+      tags:
+        dynamic:
+          branch: true
+          commit: true
+        static:
+          - latest
+
+- secret:
+    name: airship_shipyard_quay_creds
+    data:
+      username: !encrypted/pkcs1-oaep
+        - k9WFg+z1sgabdmlWRa/6Cc37N2TxmzCGC23nNr/fC5eDf5z73D0XycpbMhvYZ4ZRqXuRD
+          Yl9kTaQ6yRiaUaQfHgqIR+8R5m/IrC+F57tageAiFljs1Yf895vA1TlpE0TCI8jUJs6M7
+          z8f87FkrTZhgVfOI4KNUb+g9EioEs8/sx5WEXt053lm18xhnh7VyDE9DpVtoZ2swqpXTG
+          DPqlWLi2SHBU9PVTmwtz+mRA9BCNPFS4vUL1EQoMir9gqwYrEV3aezNPUeiWqMq7HqaQd
+          ZSGdOmxUTsjPZnrvGP6cAe9fEDAcqB9hoO1EKkLTirGN/9sEpK96vmMyIiTU91fMUCy/L
+          tp94a22CvOPEgL5E59M2HI0IEYLkhN9pjyRm608MoO9kcm7vt6rFstJafDp8Sfh4Klk3v
+          WGkZi4CnILpvpOXEdvG0cXb7HYJroBe1dblaFtS+AZWNYVo/PmA/gZ2DIifEGHsfHJhvp
+          3cpIwBXY2BeP0lAtmHXf0afCP1aGNVlumKBz2zEhxT1hSxbmbjDuv7at/3kyI0KzuJQuU
+          fEDkHzF4U7XPW5FJudxqVew9wWxYap9DwWk5q6LdxI1/JKRyztNiUisk3o+I1LiKGfA0c
+          n+J3riTLmXl03zQXJRFNVdcexuolrtHNV5iTpN5P+/NT2SQXUj9Wmj44W7vz4g=
+      password: !encrypted/pkcs1-oaep
+        - ju6Q/Gr6Zp76tHJdag8Veejn2ovtTKtqXOa6UgCMN8kYdspOE0cUH9e6A80O2Sn3qwqiw
+          Y3cNWJQnK272Gmnm2r0tN8Y39l3G7n3dhbyimO4E/9NfMVeY/+GQEDSrlj+f7oxui99YS
+          gVfdA9L/fx4JCKRkqGOVT/IKMPTSRepcrS+R9o3kMFVtpz38G1CFfdnk2rTdLFVvgus5e
+          UY1Hougf37V/fJXDpf4NrCl//iKrJ0X8eafWkrVJ9H4x+Pni22zN54OjBMcle75fYQqxZ
+          VUFfKCedan9RvL6402NmoEY21WozoMmAPnoFe0I1XIiQecAaPs25snMrDJxX+prd4Lp6m
+          suyFlJcGHfvOaWIUmbBl2+OZwyZfl2LzCwYNR5OPbVZ8CjdJWWQ2e4Q8iRGbKqbA6cqUE
+          TXEnqmzobqfIR1Z9vS96kdAXEUo3MNcxyE600Vfwg0t4MyblYEXZjm+c7H8QCijM8Z8lX
+          580awP/H504kfbqMInMuq2Ac+08b6NPObpHf2Q5hunrkpn6/7GS2fzhmkMPfRriNI5lOr
+          VCl4jWDJDYhtKh29q/ILrtmajHwyCxQjSjNqZKy31iWfinnQDYNFmgzdRxwvoYN3qwU1w
+          eYlzieYFM/znu+LAXaDE17O+25FYe9lk1s96Bm3jhGVARWize+sQSCo/xBor2A=
--- a/25
+++ b/25
@ -14,11 +14,14 @@

 BUILD_CTX                  ?= src/bin
 DOCKER_REGISTRY            ?= quay.io
-IMAGE_PREFIX               ?= attcomdev
-IMAGE_TAG                  ?= latest
+IMAGE_PREFIX               ?= airshipit
+IMAGE_TAG                  ?= untagged
 HELM                       ?= helm
 LABEL                      ?= commit-id
 IMAGE_NAME                 := airflow shipyard
+PROXY                      ?= http://proxy.foo.com:8080
+USE_PROXY                  ?= false
+PUSH_IMAGE                 ?= false

 IMAGE:=${DOCKER_REGISTRY}/${IMAGE_PREFIX}/$(IMAGE_NAME):${IMAGE_TAG}
 IMAGE_DIR:=images/$(IMAGE_NAME)
@ -75,11 +78,25 @@ run:

 .PHONY: build_airflow
 build_airflow:
-	docker build -t $(IMAGE) --label $(LABEL) -f $(IMAGE_DIR)/Dockerfile $(IMAGE_DIR)
+ifeq ($(USE_PROXY), true)
+	docker build --network host -t $(IMAGE) --label $(LABEL) -f $(IMAGE_DIR)/Dockerfile $(IMAGE_DIR) --build-arg http_proxy=$(PROXY) --build-arg https_proxy=$(PROXY)
+else
+	docker build --network host -t $(IMAGE) --label $(LABEL) -f $(IMAGE_DIR)/Dockerfile $(IMAGE_DIR)
+endif
+ifeq ($(PUSH_IMAGE), true)
+	docker push $(IMAGE)
+endif

 .PHONY: build_shipyard
 build_shipyard:
-	docker build -t $(IMAGE) --label $(LABEL) -f $(IMAGE_DIR)/Dockerfile --build-arg ctx_base=$(BUILD_CTX) .
+ifeq ($(USE_PROXY), true)
+	docker build --network host -t $(IMAGE) --label $(LABEL) -f $(IMAGE_DIR)/Dockerfile --build-arg ctx_base=$(BUILD_CTX) . --build-arg http_proxy=$(PROXY) --build-arg https_proxy=$(PROXY)
+else
+	docker build --network host -t $(IMAGE) --label $(LABEL) -f $(IMAGE_DIR)/Dockerfile --build-arg ctx_base=$(BUILD_CTX) .
+endif
+ifeq ($(PUSH_IMAGE), true)
+	docker push $(IMAGE)
+endif

 .PHONY: clean
 clean:
--- a/tools/gate/playbooks/osh-infra-build.yaml
+++ b/tools/gate/playbooks/osh-infra-build.yaml
@ -22,15 +22,3 @@
    - build-helm-packages
  tags:
    - build-helm-packages
-
- hosts: all
-  vars_files:
-    - vars.yaml
-  vars:
-    work_dir: "{{ zuul.project.src_dir }}/{{ zuul_osh_infra_relative_path | default('') }}"
-  gather_facts: False
-  become: yes
-  roles:
-    - build-images
-  tags:
-    - build-images
--- a/tools/gate/roles/build-images/tasks/airship-shipyard.yaml
+++ b/tools/gate/roles/build-images/tasks/airship-shipyard.yaml
@ -12,46 +12,70 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.

- name: set zuul_site_mirror_fqdn from env var if not defined
-  when: zuul_site_mirror_fqdn is not defined
-  set_fact:
-    zuul_site_mirror_fqdn: "{{ lookup('env','zuul_site_mirror_fqdn') }}"
-
-# NOTE(portdirect): Until https://github.com/ansible/ansible/issues/21433 is
-# resolved, we build with a shell script to make use of the host network.
- name: Build Airship-Shipyard Image
+- name: Debug tag generation inputs
  block:
-    # NOTE(portdirect): We do this to ensure we are feeding the docker build
-    # a clean path to work with.
-    - name: Airship-Shipyard image build path
-      shell: cd "{{ work_dir }}"; pwd
-      register: airship_shipyard_path
-      vars:
-        zuul_airship_shipyard_relative_path: ../airship-shipyard
-        work_dir: "{{ zuul.project.src_dir }}/{{ zuul_airship_shipyard_relative_path | default('') }}"
+    - debug:
+        var: publish
+    - debug:
+        var: tags
+    - debug:
+        var: zuul
+    - debug:
+        msg: "{{ tags | to_json }}"

-    - name: Airship-Shipyard image build path
-      shell: |-
-              set -ex;
-              docker build \
-                --network host \
-                --force-rm \
-                --label zuul \
-                -f images/shipyard/Dockerfile \
-                --build-arg ctx_base=src/bin \
-                {% if zuul_site_mirror_fqdn is defined and zuul_site_mirror_fqdn %}
-                --build-arg UBUNTU_URL="http://{{ zuul_site_mirror_fqdn }}/ubuntu/" \
-                --build-arg ALLOW_UNAUTHENTICATED="true" \
-                --build-arg PIP_INDEX_URL="http://{{ zuul_site_mirror_fqdn }}/pypi/simple" \
-                --build-arg PIP_TRUSTED_HOST="{{ zuul_site_mirror_fqdn }}" \
-                {% endif %}
-                .
-      args:
-        chdir: "{{ airship_shipyard_path.stdout }}"
-        executable: /bin/bash
+- name: Determine tags
+  shell: echo '{{ tags | to_json }}' | python {{ zuul.project.src_dir }}/tools/image_tags.py
+  environment:
+    BRANCH: "{{ zuul.branch }}"
+    CHANGE: "{{ zuul.change }}"
+    COMMIT: "{{ zuul.newrev }}"
+    PATCHSET: "{{ zuul.patchset }}"
+  register: image_tags

-    - name: Register Shipyard Image ID
-      shell: |-
-              set -e;
-              echo $( docker images --filter label=zuul | awk '{print $3}' | head -2 | tail -1 )
-      register: shipyard_image_id
+- name: Debug computed tags
+  debug:
+    var: image_tags
+
+- name: Make images
+  when: not publish
+  block:
+    - make:
+        chdir: "{{ zuul.project.src_dir }}"
+        target: images
+        params:
+          IMAGE_TAG: "{{ item }}"
+      with_items: "{{ image_tags.stdout_lines }}"
+
+    - shell: "docker images"
+      register: docker_images
+
+    - debug:
+        var: docker_images
+
+  become: True
+
+- name: Publish images
+  block:
+    - docker_login:
+        username: "{{ airship_shipyard_quay_creds.username }}"
+        password: "{{ airship_shipyard_quay_creds.password }}"
+        registry_url: "https://quay.io/api/v1/"
+
+    - make:
+        chdir: "{{ zuul.project.src_dir }}"
+        target: images
+        params:
+          DOCKER_REGISTRY: "quay.io"
+          IMAGE_PREFIX: "airshipit"
+          IMAGE_TAG: "{{ item }}"
+          PUSH_IMAGE: "true"
+      with_items: "{{ image_tags.stdout_lines }}"
+
+    - shell: "docker images"
+      register: docker_images
+
+    - debug:
+        var: docker_images
+
+  when: publish
+  become: True
--- a/tools/image_tags.py
+++ b/tools/image_tags.py
@ -0,0 +1,126 @@
+#!/bin/python
+# Copyright 2018 AT&T Intellectual Property.  All other rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import json
+import logging
+import os
+import sys
+
+LOG = logging.getLogger(__name__)
+
+LOG_FORMAT = '%(asctime)s %(levelname)-8s %(name)s:%(funcName)s [%(lineno)3d] %(message)s'  # noqa
+
+
+class TagGenExeception(Exception):
+    pass
+
+
+def read_config(stream, env):
+    config = {}
+    try:
+        config['tags'] = json.load(stream)
+    except ValueError:
+        LOG.exception('Failed to decode JSON from input stream')
+        config['tags'] = {}
+
+    LOG.debug('Configuration after reading stream: %s', config)
+
+    config['context'] = {
+        'branch': env.get('BRANCH'),
+        'change': env.get('CHANGE'),
+        'commit': env.get('COMMIT'),
+        'ps': env.get('PATCHSET'),
+    }
+
+    LOG.info('Final configuration: %s', config)
+
+    return config
+
+
+def build_tags(config):
+    tags = config.get('tags', {}).get('static', [])
+    LOG.debug('Dynamic tags: %s', tags)
+    tags.extend(build_dynamic_tags(config))
+    LOG.info('All tags: %s', tags)
+    return tags
+
+
+def build_dynamic_tags(config):
+    dynamic_tags = []
+
+    dynamic_tags.extend(_build_branch_tag(config))
+    dynamic_tags.extend(_build_commit_tag(config))
+    dynamic_tags.extend(_build_ps_tag(config))
+
+    return dynamic_tags
+
+
+def _build_branch_tag(config):
+    if _valid_dg(config, 'branch'):
+        return [config['context']['branch']]
+    else:
+        return []
+
+
+def _build_commit_tag(config):
+    if _valid_dg(config, 'commit'):
+        return [config['context']['commit']]
+    else:
+        return []
+
+
+def _build_ps_tag(config):
+    if _valid_dg(config, 'patch_set', 'change') and _valid_dg(
+            config, 'patch_set', 'ps'):
+        return [
+            '%s-%s' % (config['context']['change'], config['context']['ps'])
+        ]
+    else:
+        return []
+
+
+def _valid_dg(config, dynamic_tag, context_name=None):
+    if context_name is None:
+        context_name = dynamic_tag
+
+    if config.get('tags', {}).get('dynamic', {}).get(dynamic_tag):
+        if config.get('context', {}).get(context_name):
+            return True
+        else:
+            raise TagGenExeception('Dynamic tag "%s" requested, but "%s"'
+                                   ' not found in context' % (dynamic_tag,
+                                                              context_name))
+    else:
+        return False
+
+
+def main():
+    config = read_config(sys.stdin, os.environ)
+    tags = build_tags(config)
+
+    for tag in tags:
+        print(tag)
+
+
+if __name__ == '__main__':
+    logging.basicConfig(format=LOG_FORMAT, level=logging.WARNING)
+    try:
+        main()
+    except TagGenExeception:
+        LOG.exception('Failed to generate tags')
+        sys.exit(1)
+    except:
+        LOG.exception('Unexpected exception')
+        sys.exit(2)