refactor: change images and zuul

We no longer use underscores in image names, we instead use dashes
because that is a sane pattern.

We also create a tool which updates zuul jobs automatically based on the
images and builds each image in it's own job (only if necessary when we
have Dockerfile changes).

Change-Id: I23437d65f8761dbea1f4ffed31ed87ff73ec9142

.zuul.yaml

@@ -1,74 +0,0 @@
-- job:
-    name: openstack-operator:images:build
-    parent: vexxhost-build-docker-image
-    provides: openstack-operator:image:operator
-    vars: &openstack_operator_images
-      docker_images:
-        - context: images/mcrouter
-          repository: vexxhost/mcrouter
-        - context: images/mcrouter_exporter
-          repository: vexxhost/mcrouter_exporter
-        - context: images/memcached
-          repository: vexxhost/memcached
-        - context: images/memcached_exporter
-          repository: vexxhost/memcached_exporter
-        - context: images/rabbitmq
-          repository: vexxhost/rabbitmq
-        - context: .
-          repository: vexxhost/openstack-operator
-
-- job:
-    name: openstack-operator:images:upload
-    parent: vexxhost-upload-docker-image
-    provides: openstack-operator:image:operator
-    vars: *openstack_operator_images
-
-- job:
-    name: openstack-operator:images:promote
-    parent: vexxhost-promote-docker-image
-    vars: *openstack_operator_images
-
-- job:
-    name: openstack-operator:linters:chart
-    parent: chart-testing-lint
-    vars:
-      zuul_work_dir: "{{ zuul.project.src_dir }}/chart"
-
-- job:
-    name: openstack-operator:functional
-    parent: devstack-tempest
-    requires:
-      - openstack-operator:images
-    pre-run: playbooks/functional/pre.yaml
-    run: playbooks/functional/run.yaml
-    post-run: playbooks/functional/post.yaml
-    vars:
-      devstack_services:
-        etcd3: false
-      docker_use_buildset_registry: true
-      minikube_dns_resolvers: ['1.1.1.1', '8.8.8.8']
-
-- project:
-    templates:
-      - publish-opendev-tox-docs
-    check:
-      jobs:
-        - golangci-lint
-        - golang-go-test
-        - openstack-operator:linters:chart
-        - openstack-operator:images:build
-        - openstack-operator:functional:
-            dependencies:
-              - openstack-operator:images:build
-    gate:
-      jobs:
-        - golangci-lint
-        - golang-go-test
-        - openstack-operator:linters:chart
-        - openstack-operator:images:upload
-        - openstack-operator:functional:
-            dependencies:
-              - openstack-operator:images:upload
-    promote:
-      jobs:
-        - openstack-operator:images:promote

Makefile

@@ -61,7 +61,7 @@ generate: controller-gen
 
 # Build the docker image
 docker-build: test
-	docker build . -t ${IMG}
+	docker build . -f images/openstack-operator/Dockerfile -t ${IMG}
 
 # Push the docker image
 docker-push:
@@ -87,8 +87,8 @@ endif
 # run-time images
 images:
 	docker build images/mcrouter -t vexxhost/mcrouter:latest
-	docker build images/mcrouter_exporter -t vexxhost/mcrouter_exporter:latest
+	docker build images/mcrouter-exporter -t vexxhost/mcrouter-exporter:latest
 	docker build images/memcached -t vexxhost/memcached:latest
-	docker build images/memcached_exporter -t vexxhost/memcached_exporter:latest
+	docker build images/memcached-exporter -t vexxhost/memcached-exporter:latest
 	docker build images/rabbitmq -t vexxhost/rabbitmq:latest
 

controllers/mcrouter_controller.go

@@ -107,7 +107,7 @@ func (r *McrouterReconciler) Reconcile(req ctrl.Request) (ctrl.Result, error) {
 											RunAsUser(999).
 											RunAsGroup(999),
 									),
-								builders.Container("exporter", "vexxhost/mcrouter_exporter:latest").
+								builders.Container("exporter", "vexxhost/mcrouter-exporter:latest").
 									Args("-mcrouter.address", "localhost:11211").
 									Port("metrics", 9442).HTTPProbe("metrics", "/metrics", 10, 30).
 									Resources(500, 128, 500, 2).

controllers/memcached_controller.go

@@ -111,7 +111,7 @@ func (r *MemcachedReconciler) Reconcile(req ctrl.Request) (ctrl.Result, error) {
 										builders.SecurityContext().
 											RunAsUser(1001),
 									),
-								builders.Container("exporter", "vexxhost/memcached_exporter:latest").
+								builders.Container("exporter", "vexxhost/memcached-exporter:latest").
 									Port("metrics", 9150).HTTPProbe("metrics", "/metrics", 10, 30).
 									Resources(500, 128, 500, 2).
 									SecurityContext(

hack/update-zuul-jobs.py

@@ -0,0 +1,108 @@
+#!/usr/bin/env python
+
+import copy
+import os
+import sys
+
+from ruamel import yaml
+
+
+yaml = yaml.YAML()
+images = os.listdir("images")
+
+build_jobs = []
+upload_jobs = []
+
+for image in images:
+    files = []
+    if image != 'openstack-operator':
+        files = ['^images/%s/.*' % image]
+    deps = ['opendev-buildset-registry']
+
+    job_vars = {
+        'docker_images': [
+            {
+                'context': 'images/%s' % image,
+                'repository': 'vexxhost/%s' % image,
+            }
+        ]
+    }
+    if image == 'openstack-operator':
+        job_vars['docker_images'][0]['context'] = '.'
+        job_vars['docker_images'][0]['dockerfile'] = 'images/openstack-operator/Dockerfile'
+
+    build_job = {
+        'job': {
+            'name': 'openstack-operator:images:build:%s' % image,
+            'parent': 'vexxhost-build-docker-image',
+            'provides': 'openstack-operator:image:%s' % image,
+            'dependencies': deps,
+            'vars': job_vars,
+        }
+    }
+
+    upload_job = {
+        'job': {
+            'name': 'openstack-operator:images:upload:%s' % image,
+            'parent': 'vexxhost-upload-docker-image',
+            'provides': 'openstack-operator:image:%s' % image,
+            'dependencies': deps,
+            'vars': job_vars,
+        }
+    }
+
+    promote_job = {
+        'job': {
+            'name': 'openstack-operator:images:promote:%s' % image,
+            'parent': 'vexxhost-promote-docker-image',
+            'vars': job_vars,
+        }
+    }
+
+    if files:
+        build_job['job']['files'] = files
+        upload_job['job']['files'] = files
+        promote_job['job']['files'] = files
+
+    project_config = {
+        'project': {
+            'check': {'jobs': [build_job['job']['name']]},
+            'gate': {'jobs': [upload_job['job']['name']]},
+            'promote': {'jobs': [promote_job['job']['name']]},
+        }
+    }
+
+    config = [
+        build_job,
+        upload_job,
+        promote_job,
+        project_config
+    ]
+
+    if image == 'openstack-operator':
+        build_jobs.append(build_job['job']['name'])
+        upload_jobs.append(upload_job['job']['name'])
+    else:
+        build_jobs.append({'name': build_job['job']['name'], 'soft': True})
+        upload_jobs.append({'name': upload_job['job']['name'], 'soft': True})
+
+    with open("zuul.d/%s-jobs.yaml" % image, "w+") as fd:
+        yaml.dump(config, fd)
+
+
+with open("zuul.d/functional-jobs.yaml") as fd:
+    data = yaml.load(fd)
+
+for obj in data:
+    if 'project' in obj:
+        for job in obj['project']['check']['jobs']:
+            if 'openstack-operator:functional' in job:
+                job['openstack-operator:functional']['dependencies'] = \
+                    build_jobs
+        for job in obj['project']['gate']['jobs']:
+            if 'openstack-operator:functional' in job:
+                job['openstack-operator:functional']['dependencies'] = \
+                    upload_jobs
+
+with open("zuul.d/functional-jobs.yaml", "w+") as fd:
+    yaml.dump(data, fd)

tox.ini

@@ -2,6 +2,12 @@
 minversion = 2.0
 skipsdist = True
 
+[testenv:update-zuul-jobs]
+deps =
+  ruamel.yaml
+commands =
+  {toxinidir}/hack/update-zuul-jobs.py
+
 [testenv:docs]
 deps =
   -r{toxinidir}/doc/requirements.txt

zuul.d/functional-jobs.yaml

@@ -0,0 +1,47 @@
+- job:
+    name: openstack-operator:functional
+    parent: devstack-tempest
+    requires:
+    - openstack-operator:images
+    pre-run: playbooks/functional/pre.yaml
+    run: playbooks/functional/run.yaml
+    post-run: playbooks/functional/post.yaml
+    vars:
+      devstack_services:
+        etcd3: false
+      docker_use_buildset_registry: true
+      minikube_dns_resolvers: [1.1.1.1, 8.8.8.8]
+
+- project:
+    check:
+      jobs:
+      - opendev-buildset-registry
+      - openstack-operator:functional:
+          dependencies:
+          - name: openstack-operator:images:build:rabbitmq
+            soft: true
+          - name: openstack-operator:images:build:memcached
+            soft: true
+          - name: openstack-operator:images:build:mcrouter
+            soft: true
+          - openstack-operator:images:build:openstack-operator
+          - name: openstack-operator:images:build:mcrouter-exporter
+            soft: true
+          - name: openstack-operator:images:build:memcached-exporter
+            soft: true
+    gate:
+      jobs:
+      - opendev-buildset-registry
+      - openstack-operator:functional:
+          dependencies:
+          - name: openstack-operator:images:upload:rabbitmq
+            soft: true
+          - name: openstack-operator:images:upload:memcached
+            soft: true
+          - name: openstack-operator:images:upload:mcrouter
+            soft: true
+          - openstack-operator:images:upload:openstack-operator
+          - name: openstack-operator:images:upload:mcrouter-exporter
+            soft: true
+          - name: openstack-operator:images:upload:memcached-exporter
+            soft: true

zuul.d/mcrouter-exporter-jobs.yaml

@@ -0,0 +1,34 @@
+- job:
+    name: openstack-operator:images:build:mcrouter-exporter
+    parent: vexxhost-build-docker-image
+    provides: openstack-operator:image:mcrouter-exporter
+    dependencies: &id001
+    - opendev-buildset-registry
+    vars: &id002
+      docker_images:
+      - context: images/mcrouter-exporter
+        repository: vexxhost/mcrouter-exporter
+    files: &id003
+    - ^images/mcrouter-exporter/.*
+- job:
+    name: openstack-operator:images:upload:mcrouter-exporter
+    parent: vexxhost-upload-docker-image
+    provides: openstack-operator:image:mcrouter-exporter
+    dependencies: *id001
+    vars: *id002
+    files: *id003
+- job:
+    name: openstack-operator:images:promote:mcrouter-exporter
+    parent: vexxhost-promote-docker-image
+    vars: *id002
+    files: *id003
+- project:
+    check:
+      jobs:
+      - openstack-operator:images:build:mcrouter-exporter
+    gate:
+      jobs:
+      - openstack-operator:images:upload:mcrouter-exporter
+    promote:
+      jobs:
+      - openstack-operator:images:promote:mcrouter-exporter

zuul.d/mcrouter-jobs.yaml

@@ -0,0 +1,34 @@
+- job:
+    name: openstack-operator:images:build:mcrouter
+    parent: vexxhost-build-docker-image
+    provides: openstack-operator:image:mcrouter
+    dependencies: &id001
+    - opendev-buildset-registry
+    vars: &id002
+      docker_images:
+      - context: images/mcrouter
+        repository: vexxhost/mcrouter
+    files: &id003
+    - ^images/mcrouter/.*
+- job:
+    name: openstack-operator:images:upload:mcrouter
+    parent: vexxhost-upload-docker-image
+    provides: openstack-operator:image:mcrouter
+    dependencies: *id001
+    vars: *id002
+    files: *id003
+- job:
+    name: openstack-operator:images:promote:mcrouter
+    parent: vexxhost-promote-docker-image
+    vars: *id002
+    files: *id003
+- project:
+    check:
+      jobs:
+      - openstack-operator:images:build:mcrouter
+    gate:
+      jobs:
+      - openstack-operator:images:upload:mcrouter
+    promote:
+      jobs:
+      - openstack-operator:images:promote:mcrouter

zuul.d/memcached-exporter-jobs.yaml

@@ -0,0 +1,34 @@
+- job:
+    name: openstack-operator:images:build:memcached-exporter
+    parent: vexxhost-build-docker-image
+    provides: openstack-operator:image:memcached-exporter
+    dependencies: &id001
+    - opendev-buildset-registry
+    vars: &id002
+      docker_images:
+      - context: images/memcached-exporter
+        repository: vexxhost/memcached-exporter
+    files: &id003
+    - ^images/memcached-exporter/.*
+- job:
+    name: openstack-operator:images:upload:memcached-exporter
+    parent: vexxhost-upload-docker-image
+    provides: openstack-operator:image:memcached-exporter
+    dependencies: *id001
+    vars: *id002
+    files: *id003
+- job:
+    name: openstack-operator:images:promote:memcached-exporter
+    parent: vexxhost-promote-docker-image
+    vars: *id002
+    files: *id003
+- project:
+    check:
+      jobs:
+      - openstack-operator:images:build:memcached-exporter
+    gate:
+      jobs:
+      - openstack-operator:images:upload:memcached-exporter
+    promote:
+      jobs:
+      - openstack-operator:images:promote:memcached-exporter

zuul.d/memcached-jobs.yaml

@@ -0,0 +1,34 @@
+- job:
+    name: openstack-operator:images:build:memcached
+    parent: vexxhost-build-docker-image
+    provides: openstack-operator:image:memcached
+    dependencies: &id001
+    - opendev-buildset-registry
+    vars: &id002
+      docker_images:
+      - context: images/memcached
+        repository: vexxhost/memcached
+    files: &id003
+    - ^images/memcached/.*
+- job:
+    name: openstack-operator:images:upload:memcached
+    parent: vexxhost-upload-docker-image
+    provides: openstack-operator:image:memcached
+    dependencies: *id001
+    vars: *id002
+    files: *id003
+- job:
+    name: openstack-operator:images:promote:memcached
+    parent: vexxhost-promote-docker-image
+    vars: *id002
+    files: *id003
+- project:
+    check:
+      jobs:
+      - openstack-operator:images:build:memcached
+    gate:
+      jobs:
+      - openstack-operator:images:upload:memcached
+    promote:
+      jobs:
+      - openstack-operator:images:promote:memcached

zuul.d/misc-jobs.yaml

@@ -0,0 +1,19 @@
+- job:
+    name: openstack-operator:linters:chart
+    parent: chart-testing-lint
+    vars:
+      zuul_work_dir: "{{ zuul.project.src_dir }}/chart"
+
+- project:
+    templates:
+      - publish-opendev-tox-docs
+    check:
+      jobs:
+        - golangci-lint
+        - golang-go-test
+        - openstack-operator:linters:chart
+    gate:
+      jobs:
+        - golangci-lint
+        - golang-go-test
+        - openstack-operator:linters:chart

zuul.d/openstack-operator-jobs.yaml

@@ -0,0 +1,31 @@
+- job:
+    name: openstack-operator:images:build:openstack-operator
+    parent: vexxhost-build-docker-image
+    provides: openstack-operator:image:openstack-operator
+    dependencies: &id001
+    - opendev-buildset-registry
+    vars: &id002
+      docker_images:
+      - context: .
+        repository: vexxhost/openstack-operator
+        dockerfile: images/openstack-operator/Dockerfile
+- job:
+    name: openstack-operator:images:upload:openstack-operator
+    parent: vexxhost-upload-docker-image
+    provides: openstack-operator:image:openstack-operator
+    dependencies: *id001
+    vars: *id002
+- job:
+    name: openstack-operator:images:promote:openstack-operator
+    parent: vexxhost-promote-docker-image
+    vars: *id002
+- project:
+    check:
+      jobs:
+      - openstack-operator:images:build:openstack-operator
+    gate:
+      jobs:
+      - openstack-operator:images:upload:openstack-operator
+    promote:
+      jobs:
+      - openstack-operator:images:promote:openstack-operator

zuul.d/rabbitmq-jobs.yaml

@@ -0,0 +1,34 @@
+- job:
+    name: openstack-operator:images:build:rabbitmq
+    parent: vexxhost-build-docker-image
+    provides: openstack-operator:image:rabbitmq
+    dependencies: &id001
+    - opendev-buildset-registry
+    vars: &id002
+      docker_images:
+      - context: images/rabbitmq
+        repository: vexxhost/rabbitmq
+    files: &id003
+    - ^images/rabbitmq/.*
+- job:
+    name: openstack-operator:images:upload:rabbitmq
+    parent: vexxhost-upload-docker-image
+    provides: openstack-operator:image:rabbitmq
+    dependencies: *id001
+    vars: *id002
+    files: *id003
+- job:
+    name: openstack-operator:images:promote:rabbitmq
+    parent: vexxhost-promote-docker-image
+    vars: *id002
+    files: *id003
+- project:
+    check:
+      jobs:
+      - openstack-operator:images:build:rabbitmq
+    gate:
+      jobs:
+      - openstack-operator:images:upload:rabbitmq
+    promote:
+      jobs:
+      - openstack-operator:images:promote:rabbitmq