New source for hyperkube binary definition

Now it's possible to use hyperkube Docker image to extract hyperkube binary.
Use case for this feature is kubelet/kubectl delivery in one binary(hyperkube)
which is built into Docker image. Promenade will extract hyperkube from Docker image,
create symlinks for kubelet/kubectl pointed to hyperkube. To do so promenade container
need to be configured to use Docker on the host where this container will be created.
This is happening only for script generation for genesis node. Later when promenade
will be started as a service pod inside ucp cluster it will generate scripts for joining nodes
by using cached hyperkube from /tmp.

Old way to delivery kubelet from tarball is still supported.

Configuration for the new method.

Need to export environment variables to properly configure Docker in Docker.
Docker socket should be provided as a mounted file inside promenade.
Also need to set temporary permissions for this socket during the build scripts stage.

Example:
DOCKER_SOCK="/var/run/docker.sock"
sudo chmod o+rw $DOCKER_SOCK
export DOCKER_HOST="unix:/${DOCKER_SOCK}"
export PROMENADE_TMP="abs_path_tmp_dir_on_host"
export PROMENADE_TMP_LOCAL="tmp_dir_inside_container"

After genesis scripts generation Docker socket permission should be turned back:
sudo chmod o-rw $DOCKER_SOCK

Change-Id: Ida22ea934fc551fec34df162d8147c8b9e630330

charts/promenade/templates/deployment-api.yaml

@@ -17,6 +17,8 @@ limitations under the License.
 {{- if .Values.manifests.deployment_api }}
 {{- $envAll := . }}
 {{- $labels := tuple $envAll "promenade" "api" | include "helm-toolkit.snippets.kubernetes_metadata_labels" -}}
+{{- $mounts_init_container := .Values.pod.mounts.promenade_api.init_container }}
+{{- $mounts_promenade_api := .Values.pod.mounts.promenade_api.promenade_api }}
 ---
 apiVersion: apps/v1
 kind: Deployment
@@ -47,6 +49,29 @@ spec:
         {{ .Values.labels.node_selector_key }}: {{ .Values.labels.node_selector_value }}
       serviceAccountName: promenade
       terminationGracePeriodSeconds: {{ .Values.pod.lifecycle.termination_grace_period.api.timeout | default "30" }}
+      initContainers:
+      - name: promenade-util
+        command:
+{{- if and $mounts_init_container.volumeMounts $mounts_promenade_api.volumeMounts }}
+          - "cp"
+  {{- range $mounts_init_container.volumeMounts }}
+    {{- if eq "hyperkube" (index . "name") }}
+          - {{ index . "mountPath" | quote }}
+    {{- end }}
+  {{- end }}
+  {{- range $mounts_promenade_api.volumeMounts }}
+    {{- if eq "cache" (index . "name") }}
+          - {{ index . "mountPath" | quote }}
+    {{- end }}
+  {{- end }}
+{{- else }}
+          - "true"
+{{- end }}
+        image: {{ .Values.images.tags.monitoring_image }}
+        imagePullPolicy: IfNotPresent
+        volumeMounts:
+{{ if $mounts_init_container.volumeMounts }}{{ toYaml $mounts_init_container.volumeMounts | indent 8 }}{{ end }}
+{{ if $mounts_promenade_api.volumeMounts }}{{ toYaml $mounts_promenade_api.volumeMounts | indent 8 }}{{ end }}
       containers:
         - name: promenade-api
           image: {{ .Values.images.tags.promenade }}
@@ -92,16 +117,15 @@ spec:
               mountPath: /etc/promenade/promenade.conf
               subPath: promenade.conf
               readOnly: true
-            - name: cache
-              mountPath: /tmp/cache
             - name: promenade-etc
               mountPath: /etc/promenade/policy.yaml
               subPath: policy.yaml
+{{ if $mounts_promenade_api.volumeMounts }}{{ toYaml $mounts_promenade_api.volumeMounts | indent 12 }}{{ end }}
       volumes:
         - name: promenade-etc
           configMap:
             name: promenade-etc
             defaultMode: 0444
-        - name: cache
-          emptyDir: {}
+{{ if $mounts_init_container.volumes }}{{ toYaml $mounts_init_container.volumes | indent 8 }}{{ end }}
+{{ if $mounts_promenade_api.volumes }}{{ toYaml $mounts_promenade_api.volumes | indent 8 }}{{ end }}
 {{- end }}

charts/promenade/templates/tests/test-promenade-api.yaml

@@ -17,7 +17,7 @@
 Test that the API is up and the health endpoint returns a 2XX code */}}
 {{- if .Values.manifests.test_promenade_api }}
 {{- $envAll := . }}
-{{- $mounts_promenade_api_init := .Values.pod.mounts.promenade_api.init_container }}
+{{- $mounts_promenade_api_test := .Values.pod.mounts.promenade_api.test_container }}
 ---
 apiVersion: v1
 kind: Pod
@@ -31,7 +31,7 @@ metadata:
 spec:
   restartPolicy: Never
   initContainers:
-{{ tuple $envAll "test" $mounts_promenade_api_init | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 8 }}
+{{ tuple $envAll "test" $mounts_promenade_api_test | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 8 }}
   containers:
     - name: "{{ .Release.Name }}-api-test"
       env:

charts/promenade/values.yaml

@@ -43,6 +43,7 @@ conf:
 
 images:
   tags:
+    monitoring_image: busybox:1.28.3
     promenade: quay.io/airshipit/promenade:latest
     ks_user: docker.io/openstackhelm/heat:newton
     ks_service: docker.io/openstackhelm/heat:newton
@@ -173,8 +174,23 @@ pod:
         default: kubernetes.io/hostname
   mounts:
     promenade_api:
-      init_container: null
+      test_container: null
+      init_container:
+        volumeMounts:
+          - name: hyperkube
+            mountPath: /hyperkube
+        volumes:
+          - name: hyperkube
+            hostPath:
+              path: /opt/kubernetes/bin/hyperkube
+              type: File
       promenade_api:
+        volumeMounts:
+          - name: cache
+            mountPath: /tmp/cache
+        volumes:
+          - name: cache
+            emptyDir: {}
   env:
     promenade_api:
 #      - name: http_proxy

examples/basic/HostSystem.yaml

@@ -12,9 +12,15 @@ data:
     kube-cgroup:
       enable: true
   files:
+    - path: /opt/kubernetes/bin/hyperkube
+      docker_image: gcr.io/google_containers/hyperkube-amd64:v1.11.6
+      file_path: /hyperkube
+      mode: 0555
     - path: /opt/kubernetes/bin/kubelet
-      tar_url: https://dl.k8s.io/v1.11.6/kubernetes-node-linux-amd64.tar.gz
-      tar_path: kubernetes/node/bin/kubelet
+      symlink: /opt/kubernetes/bin/hyperkube
+      mode: 0555
+    - path: /usr/local/bin/kubectl
+      symlink: /opt/kubernetes/bin/hyperkube
       mode: 0555
     - path: /etc/systemd/system/kube-cgroup.service
       content: |
@@ -64,12 +70,15 @@ data:
             su root root
             rotate 1
         }
+    - path: /etc/profile.d/kubeconfig.sh
+      mode: 0744
+      content: |-
+        export KUBECONFIG=/etc/kubernetes/admin/kubeconfig.yaml
   images:
+    monitoring_image: &busybox busybox:1.28.3
     haproxy: haproxy:1.8.3
     helm:
       helm: lachlanevenson/k8s-helm:v2.14.0
-    kubernetes:
-      kubectl: gcr.io/google_containers/hyperkube-amd64:v1.11.6
   packages:
     repositories:
       - deb http://apt.dockerproject.org/repo ubuntu-xenial main
@@ -112,5 +121,5 @@ data:
       socat: socat
   validation:
     pod_logs:
-      image: busybox:1.28.3
+      image: *busybox
 ...

examples/complete/HostSystem.yaml

@@ -9,9 +9,15 @@ metadata:
   storagePolicy: cleartext
 data:
   files:
+    - path: /opt/kubernetes/bin/hyperkube
+      docker_image: gcr.io/google_containers/hyperkube-amd64:v1.11.6
+      file_path: /hyperkube
+      mode: 0555
     - path: /opt/kubernetes/bin/kubelet
-      tar_url: https://dl.k8s.io/v1.11.6/kubernetes-node-linux-amd64.tar.gz
-      tar_path: kubernetes/node/bin/kubelet
+      symlink: /opt/kubernetes/bin/hyperkube
+      mode: 0555
+    - path: /usr/local/bin/kubectl
+      symlink: /opt/kubernetes/bin/hyperkube
       mode: 0555
     - path: /etc/logrotate.d/json-logrotate
       mode: 0444
@@ -30,12 +36,15 @@ data:
             su root root
             rotate 1
         }
+    - path: /etc/profile.d/kubeconfig.sh
+      mode: 0744
+      content: |-
+        export KUBECONFIG=/etc/kubernetes/admin/kubeconfig.yaml
   images:
+    monitoring_image: busybox:1.28.3
     haproxy: haproxy:1.8.3
     helm:
       helm: lachlanevenson/k8s-helm:v2.14.0
-    kubernetes:
-      kubectl: gcr.io/google_containers/hyperkube-amd64:v1.11.6
   packages:
     repositories:
       - deb http://apt.dockerproject.org/repo ubuntu-xenial main

promenade/builder.py

@@ -18,10 +18,11 @@ LOG = logging.getLogger(__name__)
 #   B108:hardcoded_tmp_directory
 # This cache needs to be shared by all forks within the same container, and so
 # must be at a well-known location.
+TMP_CACHE = '/tmp/cache'  # nosec
 CACHE_OPTS = {
     'cache.type': 'file',
-    'cache.data_dir': '/tmp/cache/data',  # nosec
-    'cache.lock_dir': '/tmp/cache/lock',  # nosec
+    'cache.data_dir': TMP_CACHE + '/data',  # nosec
+    'cache.lock_dir': TMP_CACHE + '/lock',  # nosec
 }
 
 CACHE = CacheManager(**parse_cache_config_options(CACHE_OPTS))
@@ -43,8 +44,16 @@ class Builder:
         self._file_cache = {}
         for file_spec in self._file_specs:
             path = file_spec['path']
+            islink = False
             if 'content' in file_spec:
                 data = file_spec['content']
+            elif 'docker_image' in file_spec:
+                data = _fetch_image_content(self.config.container_info,
+                                            file_spec['docker_image'],
+                                            file_spec['file_path'])
+            elif 'symlink' in file_spec:
+                data = file_spec['symlink']
+                islink = True
             elif 'tar_url' in file_spec:
                 data = _fetch_tar_content(file_spec['tar_url'],
                                           file_spec['tar_path'])
@@ -52,6 +61,7 @@ class Builder:
                 'path': path,
                 'data': data,
                 'mode': file_spec['mode'],
+                'islink': islink,
             }
 
     @property
@@ -61,6 +71,7 @@ class Builder:
             self.config.get_path('Genesis:files', []))
 
     def build_all(self, *, output_dir):
+        self.config.get_container_info()
         self.build_genesis(output_dir=output_dir)
         for node_document in self.config.iterate(
                 schema='promenade/KubernetesNode/v1'):
@@ -163,6 +174,33 @@ def _encrypt(cfg_dict, data):
             decrypt_teardown_command)
 
 
+# The following environment variables should be used
+# export DOCKER_HOST="unix://var/run/docker.sock"
+# export PROMENADE_TMP="tmp_dir_on_host"
+# export PROMENADE_TMP_LOCAL="tmp_dir_inside_container"
+# PROMENADE_TMP is the full path to temp dir from host
+# inside promenade container it should be bind to PROMENADE_TMP_LOCAL
+@CACHE.cache('fetch_image', expire=72 * 3600)
+def _fetch_image_content(config, image_url, file_path):
+    file_name = os.path.basename(file_path)
+    if config is None:
+        result_path = os.path.join(TMP_CACHE, file_name)
+        if not os.path.isfile(result_path):
+            raise Exception(
+                'ERROR: there is no container info and no file in cache')
+    else:
+        result_path = os.path.join(config['dir_local'], file_name)
+        client = config['client']
+        vol = {config['dir']: {'bind': config['dir_local'], 'mode': 'rw'}}
+        cmd = 'cp -v {} {}'.format(file_path, config['dir_local'])
+        image = client.images.pull(image_url)
+        output = client.containers.run(
+            image, command=cmd, auto_remove=True, volumes=vol)
+        LOG.debug(output)
+    f = open(result_path, 'rb')
+    return f.read()
+
+
 @CACHE.cache('fetch_tarball_content', expire=72 * 3600)
 def _fetch_tar_content(url, path):
     content = _fetch_tar_url(url)

promenade/config.py

@@ -1,7 +1,9 @@
 from . import exceptions, logging, validation
 from . import design_ref as dr
+import docker
 import jinja2
 import jsonpath_ng
+import os
 import yaml
 
 from deckhand.engine import layering
@@ -37,6 +39,7 @@ class Configuration:
                 raise exceptions.DeckhandException(str(e))
 
             LOG.info("Deckhand engine returned %d documents." % len(documents))
+        self.container_info = None
         self.debug = debug
         self.documents = documents
         self.leave_kubectl = leave_kubectl
@@ -113,6 +116,28 @@ class Configuration:
         for doc in self.iterate(*args, **kwargs):
             return doc
 
+    def get_container_info(self):
+        LOG.debug(
+            'Getting access to Docker via socket and getting mount points')
+        client = docker.from_env()
+        try:
+            client.ping()
+        except:
+            return
+        tmp_dir = os.getenv('PROMENADE_TMP')
+        if tmp_dir is None:
+            raise Exception('ERROR: undefined PROMENADE_TMP')
+        tmp_dir_local = os.getenv('PROMENADE_TMP_LOCAL')
+        if tmp_dir_local is None:
+            raise Exception('ERROR: undefined PROMENADE_TMP_LOCAL')
+        if not os.path.exists(tmp_dir_local):
+            raise Exception('ERROR: {} not found'.format(tmp_dir_local))
+        self.container_info = {
+            'client': client,
+            'dir': tmp_dir,
+            'dir_local': tmp_dir_local,
+        }
+
     def extract_genesis_config(self):
         LOG.debug('Extracting genesis config.')
         documents = []

promenade/schemas/HostSystem.yaml

@@ -39,6 +39,12 @@ data:
           $ref: '#/definitions/url'
         tar_path:
           $ref: '#/definitions/rel_path'
+        docker_image:
+          $ref: '#/definitions/url'
+        file_path:
+          $ref: '#/definitions/abs_path'
+        symlink:
+          $ref: '#/definitions/abs_path'
       required:
         - mode
         - path
@@ -46,12 +52,21 @@ data:
         - type: object
           required:
             - content
+        - type: object
+          required:
+            - symlink
         - type: object
           allOf:
             - type: object
               required:
                 - tar_url
                 - tar_path
+        - type: object
+          allOf:
+            - type: object
+              required:
+                - docker_image
+                - file_path
       additionalProperties: false
 
     image:
@@ -99,18 +114,12 @@ data:
           required:
             - helm
           additionalProperties: false
-        kubernetes:
-          type: object
-          properties:
-            kubectl:
-              $ref: '#/definitions/image'
-          required:
-            - kubectl
-          additionalProperties: false
+        monitoring_image:
+          $ref: '#/definitions/image'
       required:
         - haproxy
         - helm
-        - kubernetes
+        - monitoring_image
       additionalProperties: false
 
     packages:

promenade/tar_bundler.py

@@ -15,7 +15,7 @@ class TarBundler:
         self._tar_blob = io.BytesIO()
         self._tf = tarfile.open(fileobj=self._tar_blob, mode='w|gz')
 
-    def add(self, *, path, data, mode):
+    def add(self, *, path, data, mode, islink=False):
         if path.startswith('/'):
             path = path[1:]
 
@@ -37,7 +37,12 @@ class TarBundler:
         else:
             LOG.warning('Zero length file added to path=%s', path)
 
-        self._tf.addfile(tar_info, io.BytesIO(data_bytes))
+        if islink:
+            tar_info.type = tarfile.SYMTYPE
+            tar_info.linkname = data
+            self._tf.addfile(tar_info)
+        else:
+            self._tf.addfile(tar_info, io.BytesIO(data_bytes))
 
     def as_blob(self):
         self._tf.close()

promenade/templates/roles/common/usr/local/bin/kubectl

@@ -1,38 +0,0 @@
-#!/usr/bin/env bash
-
-set -e
-set -o pipefail
-
-ADDITIONAL_DOCKER_ARGS=
-TRANSLATE_CR=1
-
-for item in "$@"; do
-    if [[ $item =~ ^-[^-].*t.* ]]; then
-        ADDITIONAL_DOCKER_ARGS="$ADDITIONAL_DOCKER_ARGS -t"
-        TRANSLATE_CR=0
-        break
-    elif [[ $item == "--" ]]; then
-        break
-    fi
-done
-
-if [[ $TRANSLATE_CR == 1 ]]; then
-    docker run --rm -i \
-        $ADDITIONAL_DOCKER_ARGS \
-        --net host \
-        -v /etc/kubernetes/admin:/etc/kubernetes/admin \
-        -e KUBECONFIG=/etc/kubernetes/admin/kubeconfig.yaml \
-        {{ config['HostSystem:images.kubernetes.kubectl'] }} \
-            /kubectl \
-                "$@" \
-        | sed "s/\r//"
-else
-    exec docker run --rm -i \
-        $ADDITIONAL_DOCKER_ARGS \
-        --net host \
-        -v /etc/kubernetes/admin:/etc/kubernetes/admin \
-        -e KUBECONFIG=/etc/kubernetes/admin/kubeconfig.yaml \
-        {{ config['HostSystem:images.kubernetes.kubectl'] }} \
-            /kubectl \
-                "$@"
-fi

promenade/templates/roles/genesis/etc/kubernetes/manifests/bootstrap-armada.yaml

@@ -98,7 +98,7 @@ spec:
         - name: log
           mountPath: /tmp/log
     - name: monitor
-      image: {{ config['HostSystem:images.kubernetes.kubectl'] }}
+      image: {{ config['HostSystem:images.monitoring_image'] }}
       command:
         - /bin/sh
         - -c

requirements-direct.txt

@@ -1,5 +1,6 @@
 beaker==1.9.1
 click==6.7
+docker==3.7.2
 falcon==1.2.0
 jinja2==2.9.6
 jsonpath-ng==1.4.3

requirements-frozen.txt

@@ -1,32 +1,34 @@
 alembic==1.0.1
-amqp==2.3.2
+amqp==2.4.2
 asn1crypto==0.24.0
-attrs==18.2.0
+attrs==19.1.0
 Babel==2.6.0
 Beaker==1.9.1
-cachetools==3.0.0
-certifi==2018.10.15
-cffi==1.11.5
+cachetools==3.1.0
+certifi==2019.3.9
+cffi==1.12.3
 chardet==3.0.4
 click==6.7
-cliff==2.14.0
-cmd2==0.9.6
-colorama==0.4.0
+cliff==2.14.1
+cmd2==0.9.12
+colorama==0.4.1
 cryptography==2.3.1
-debtcollector==1.20.0
+debtcollector==1.21.0
 git+https://git.openstack.org/openstack/airship-deckhand@a76ffb66ae809c19281a6cf5c9414ed197d249b7
-decorator==4.3.0
+decorator==4.4.0
 deepdiff==3.3.0
-dnspython==1.15.0
-dogpile.cache==0.6.7
+dnspython==1.16.0
+docker==3.7.2
+docker-pycreds==0.4.0
+dogpile.cache==0.7.1
 eventlet==0.24.1
 extras==1.0.0
 falcon==1.2.0
-fasteners==0.14.1
+fasteners==0.15
 fixtures==3.0.0
 flake8==2.6.2
-futurist==1.8.0
-google-auth==1.5.1
+futurist==1.8.1
+google-auth==1.6.3
 greenlet==0.4.15
 hacking==1.1.0
 idna==2.6
@@ -34,34 +36,34 @@ ipaddress==1.0.22
 iso8601==0.1.12
 Jinja2==2.9.6
 jsonpath-ng==1.4.3
-jsonpickle==1.0
+jsonpickle==1.1
 jsonschema==2.6.0
 keystoneauth1==3.2.0
 keystonemiddleware==4.17.0
-kombu==4.2.1
+kombu==4.5.0
 kubernetes==3.0.0
 linecache2==1.0.0
-Mako==1.0.7
-MarkupSafe==1.1.0
+Mako==1.0.10
+MarkupSafe==1.1.1
 mccabe==0.5.3
 monotonic==1.5
-msgpack==0.5.6
+msgpack==0.6.1
 netaddr==0.7.19
-netifaces==0.10.7
+netifaces==0.10.9
 networkx==2.2
 oslo.cache==1.31.1
 oslo.concurrency==3.28.1
 oslo.config==6.6.2
 oslo.context==2.19.2
 oslo.db==4.41.1
-oslo.i18n==3.22.1
+oslo.i18n==3.23.1
 oslo.log==3.40.1
 oslo.messaging==9.1.1
 oslo.middleware==3.36.0
 oslo.policy==1.22.1
 oslo.serialization==2.28.1
-oslo.service==1.32.1
-oslo.utils==3.37.1
+oslo.service==1.39.0
+oslo.utils==3.40.2
 Paste==3.0.1
 PasteDeploy==1.5.2
 pbr==3.0.1
@@ -69,32 +71,32 @@ ply==3.11
 positional==1.2.1
 prettytable==0.7.2
 psycopg2==2.7.5
-pyasn1==0.4.4
-pyasn1-modules==0.2.2
-pycadf==2.8.0
+pyasn1==0.4.5
+pyasn1-modules==0.2.5
+pycadf==2.9.0
 pycodestyle==2.0.0
 pycparser==2.19
 pyflakes==1.2.3
 pyinotify==0.9.6
-pyparsing==2.3.0
+pyparsing==2.4.0
 pyperclip==1.7.0
 python-barbicanclient==4.7.0
-python-dateutil==2.7.5
-python-editor==1.0.3
+python-dateutil==2.8.0
+python-editor==1.0.4
 python-keystoneclient==3.18.0
 python-memcached==1.59
 python-mimeparse==1.6.0
-pytz==2018.7
+pytz==2019.1
 PyYAML==3.12
 repoze.lru==0.7
 requests==2.18.4
-rfc3986==1.1.0
+rfc3986==1.3.2
 Routes==2.4.1
 rsa==4.0
 six==1.11.0
-SQLAlchemy==1.2.13
-sqlalchemy-migrate==0.11.0
-sqlparse==0.2.4
+SQLAlchemy==1.3.3
+sqlalchemy-migrate==0.12.0
+sqlparse==0.3.0
 statsd==3.3.0
 stevedore==1.30.0
 Tempita==0.5.2
@@ -105,9 +107,10 @@ traceback2==1.4.0
 unittest2==1.1.0
 urllib3==1.24
 uWSGI==2.0.15
-vine==1.1.4
+vine==1.3.0
 wcwidth==0.1.7
-WebOb==1.8.3
-websocket-client==0.40.0
+WebOb==1.8.5
+websocket-client==0.56.0
 Werkzeug==0.14.1
-wrapt==1.10.11
+wrapt==1.11.1
+yappi==1.0

tests/unit/builder_data/simple/HostSystem.yaml

@@ -35,11 +35,10 @@ data:
             rotate 1
         }
   images:
+    monitoring_image: &busybox busybox:1.28.3
     haproxy: haproxy:1.8.3
     helm:
       helm: lachlanevenson/k8s-helm:v2.14.0
-    kubernetes:
-      kubectl: gcr.io/google_containers/hyperkube-amd64:v1.11.6
   packages:
     repositories:
       - deb http://apt.dockerproject.org/repo ubuntu-xenial main
@@ -82,5 +81,5 @@ data:
       socat: socat=1.7.3.1-1
   validation:
     pod_logs:
-      image: busybox:1.28.3
+      image: *busybox
 ...

tools/g2/lib/config.sh

@@ -6,6 +6,7 @@ export NGINX_DIR="${TEMP_DIR}/nginx"
 export NGINX_URL="http://192.168.77.1:7777"
 export PROMENADE_BASE_URL="http://promenade-api.ucp.svc.cluster.local"
 export PROMENADE_DEBUG=${PROMENADE_DEBUG:-0}
+export PROMENADE_TMP_LOCAL=${PROMENADE_TMP_LOCAL:-tmp_bin}
 export PROMENADE_ENCRYPTION_KEY=${PROMENADE_ENCRYPTION_KEY:-testkey}
 export REGISTRY_DATA_DIR=${REGISTRY_DATA_DIR:-/mnt/registry}
 export VIRSH_POOL=${VIRSH_POOL:-promenade}

tools/g2/lib/kube.sh

@@ -9,7 +9,7 @@ kubectl_cmd() {
 
     shift
 
-    ssh_cmd "${VIA}" kubectl "${@}"
+    ssh_cmd "${VIA}" kubectl --kubeconfig /etc/kubernetes/admin/kubeconfig.yaml "${@}"
 }
 
 kubectl_wait_for_pod() {

tools/g2/lib/registry.sh

@@ -60,6 +60,12 @@ registry_up() {
         docker rm -fv "${REGISTRY_ID}" &>> "${LOG_FILE}"
     fi
 
+    cur=$(grep registry /etc/hosts | cut -f1 -d:)
+    if [ -n "$cur" ]; then
+        sudo sed -i "s/.*registry/127.0.0.1 registry/g" /etc/hosts
+    else
+        echo "127.0.0.1 registry" | sudo tee -a /etc/hosts
+    fi
     if [[ -z ${RUNNING_REGISTRY_ID} ]]; then
         log Starting docker registry
         docker run -d \

tools/g2/stages/build-scripts.sh

@@ -8,10 +8,23 @@ cd "${TEMP_DIR}"
 mkdir scripts
 chmod 777 scripts
 
+PROMENADE_TMP_LOCAL="$(basename "$PROMENADE_TMP_LOCAL")"
+PROMENADE_TMP="${TEMP_DIR}/${PROMENADE_TMP_LOCAL}"
+mkdir -p "$PROMENADE_TMP"
+chmod 777 "$PROMENADE_TMP"
+
+DOCKER_SOCK="/var/run/docker.sock"
+sudo chmod o+rw $DOCKER_SOCK
+
 log Building scripts
 docker run --rm -t \
     -w /target \
     -v "${TEMP_DIR}:/target" \
+    -v "${PROMENADE_TMP}:/${PROMENADE_TMP_LOCAL}" \
+    -v "${DOCKER_SOCK}:${DOCKER_SOCK}" \
+    -e "DOCKER_HOST=unix:/${DOCKER_SOCK}" \
+    -e "PROMENADE_TMP=${PROMENADE_TMP}" \
+    -e "PROMENADE_TMP_LOCAL=/${PROMENADE_TMP_LOCAL}" \
     -e "PROMENADE_DEBUG=${PROMENADE_DEBUG}" \
     -e "PROMENADE_ENCRYPTION_KEY=${PROMENADE_ENCRYPTION_KEY}" \
     "${IMAGE_PROMENADE}" \
@@ -20,3 +33,5 @@ docker run --rm -t \
                 --validators \
                 -o scripts \
                 config/*.yaml
+
+sudo chmod o-rw $DOCKER_SOCK

tools/setup_gate.sh

@@ -47,6 +47,17 @@ sudo apt-get remove -q -y docker docker-engine docker.io
 sudo apt-get install -q -y --no-install-recommends \
     docker-ce
 
+# Set up proxy when using docker_image in yamls
+sudo mkdir -p /etc/systemd/system/docker.service.d/
+cat << EOF | sudo tee /etc/systemd/system/docker.service.d/proxy.conf
+[Service]
+Environment="HTTP_PROXY=${HTTP_PROXY}"
+Environment="HTTPS_PROXY=${HTTPS_PROXY}"
+Environment="NO_PROXY=${NO_PROXY}"
+EOF
+sudo systemctl daemon-reload
+sudo systemctl restart docker
+
 log_stage_header "Joining User Groups"
 for grp in docker libvirtd libvirt; do
     if ! groups | grep $grp > /dev/null; then