From 4b025bbf2f8fac81165691ed29b2a05bee4dc57c Mon Sep 17 00:00:00 2001
From: "James E. Blair" <jim@acmegating.com>
Date: Mon, 17 Mar 2025 14:29:09 -0700
Subject: [PATCH] Add upload-image-swift role
This is a near-copy of the swift-upload-image role in
opendev/zuul-providers. This is to share the opendev-developed role
with the wider zuul community as we start to test zuul-launcher.
Change-Id: I95ef921f08fabae9fce2e57884905b11b7d0c1cb
---
doc/source/dib-roles.rst | 1 +
roles/upload-image-swift/README.rst | 58 ++++++
roles/upload-image-swift/defaults/main.yaml | 5 +
roles/upload-image-swift/library/__init__.py | 0
.../library/image_upload_swift.py | 168 ++++++++++++++++++
roles/upload-image-swift/tasks/main.yaml | 56 ++++++
6 files changed, 288 insertions(+)
create mode 100644 roles/upload-image-swift/README.rst
create mode 100644 roles/upload-image-swift/defaults/main.yaml
create mode 100644 roles/upload-image-swift/library/__init__.py
create mode 100644 roles/upload-image-swift/library/image_upload_swift.py
create mode 100644 roles/upload-image-swift/tasks/main.yaml
diff --git a/doc/source/dib-roles.rst b/doc/source/dib-roles.rst
index 691b8f4b5..d83e4c4c8 100644
--- a/doc/source/dib-roles.rst
+++ b/doc/source/dib-roles.rst
@@ -4,3 +4,4 @@ Diskimage-Builder Roles
.. zuul:autorole:: ensure-dib
.. zuul:autorole:: build-diskimage
.. zuul:autorole:: convert-diskimage
+.. zuul:autorole:: upload-image-swift
diff --git a/roles/upload-image-swift/README.rst b/roles/upload-image-swift/README.rst
new file mode 100644
index 000000000..f31a9fbc8
--- /dev/null
+++ b/roles/upload-image-swift/README.rst
@@ -0,0 +1,58 @@
+Upload a filesystem image to a swift container
+
+This uploads a filesystem image (for example, one built by diskimage
+builder) to an OpenStack Object Store (Swift) container. The role
+returns an artifact to Zuul suitable for use by the zuul-launcher.
+
+**Role Variables**
+
+.. zuul:rolevar:: upload_image_swift_cloud_config
+
+ Complex argument which contains the cloud configuration in
+ os-cloud-config (clouds.yaml) format. It is expected that this
+ argument comes from a `Secret`.
+
+.. zuul:rolevar:: upload_image_swift_container
+
+ This role will create containers which do not already exist.
+
+ Note that you will want to set this to a value that uniquely
+ identifies your Zuul installation if using shared object stores that
+ require globally unique container names. For example if using a
+ public cloud whose Swift API is provided by Ceph.
+
+ The container should be dedicated to image uploads so that the
+ "delete_after" option may be safely used.
+
+.. zuul:rolevar:: upload_image_swift_delete_after
+ :default: 0
+
+ Number of seconds to delete objects after upload. Default is 0
+ (disabled). This will tell swift to delete the file automatically,
+ but if that fails, the next run of the role will attempt to delete
+ any objects in the bucket older than this time.
+
+.. zuul:rolevar:: upload_image_swift_image_name
+ :default: `{{ build_diskimage_image_name }}`
+
+ The Zuul image name for use by zuul-launcher (e.g., `debian-bookworm`).
+
+.. zuul:rolevar:: upload_image_swift_format
+
+ The image format (e.g., `qcow2`).
+
+.. zuul:rolevar:: upload_image_swift_extension
+ :default: `{{ upload_image_swift_format }}`
+
+ The extension to use when uploading (only used in the default
+ values for the following variables.
+
+.. zuul:rolevar:: upload_image_swift_filename
+ :default: `{{ build_diskimage_image_root }}/{{ build_diskimage_image_name }}.{{ upload_image_swift_extension }}`
+
+ The path of the local file to upload.
+
+.. zuul:rolevar:: upload_image_swift_name
+ :default: `{{ zuul.build }}-{{ build_diskimage_image_name }}.{{ upload_image_swift_extension }}`
+
+ The object name to use when uploading.
diff --git a/roles/upload-image-swift/defaults/main.yaml b/roles/upload-image-swift/defaults/main.yaml
new file mode 100644
index 000000000..3e42c766b
--- /dev/null
+++ b/roles/upload-image-swift/defaults/main.yaml
@@ -0,0 +1,5 @@
+upload_image_swift_image_name: '{{ build_diskimage_image_name }}'
+upload_image_swift_delete_after: 0
+upload_image_swift_filename: '{{ build_diskimage_image_root }}/{{ build_diskimage_image_name }}.{{ upload_image_swift_extension }}'
+upload_image_swift_name: '{{ zuul.build }}-{{ build_diskimage_image_name }}.{{ upload_image_swift_extension }}'
+upload_image_swift_extension: '{{ upload_image_swift_format }}'
diff --git a/roles/upload-image-swift/library/__init__.py b/roles/upload-image-swift/library/__init__.py
new file mode 100644
index 000000000..e69de29bb
diff --git a/roles/upload-image-swift/library/image_upload_swift.py b/roles/upload-image-swift/library/image_upload_swift.py
new file mode 100644
index 000000000..44081b0a5
--- /dev/null
+++ b/roles/upload-image-swift/library/image_upload_swift.py
@@ -0,0 +1,168 @@
+# Copyright 2014 Rackspace Australia
+# Copyright 2018 Red Hat, Inc
+# Copyright 2024 Acme Gating, LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+import argparse
+import concurrent.futures
+import datetime
+import logging
+import os
+import sys
+import traceback
+
+import openstack
+import requests.exceptions
+import keystoneauth1.exceptions
+
+from ansible.module_utils.basic import AnsibleModule
+
+SEGMENT_SIZE = 500000000 # 500MB
+
+
+def get_cloud(cloud):
+ if isinstance(cloud, dict):
+ config = openstack.config.loader.OpenStackConfig().get_one(**cloud)
+ return openstack.connection.Connection(
+ config=config,
+ pool_executor=concurrent.futures.ThreadPoolExecutor(
+ max_workers=10
+ ))
+ else:
+ return openstack.connect(cloud=cloud)
+
+
+def _add_etag_to_manifest(self, *args, **kw):
+ return
+
+
+def prune(cloud, container, delete_after):
+ # In case the automatic expiration doesn't work, manually prune old uploads
+ if not delete_after:
+ return
+ target = (datetime.datetime.now(datetime.UTC) -
+ datetime.timedelta(seconds=delete_after))
+ endpoint = cloud.object_store.get_endpoint()
+ url = os.path.join(endpoint, container)
+ for obj in cloud.object_store.objects(container):
+ ts = datetime.datetime.fromisoformat(obj['last_modified'])
+ ts = ts.replace(tzinfo=datetime.UTC)
+ if ts < target:
+ path = os.path.join(url, obj.name)
+ try:
+ cloud.session.delete(path)
+ except keystoneauth1.exceptions.http.NotFound:
+ pass
+
+
+def run(cloud, container, filename, name, delete_after=None):
+ # Monkey-patch sdk so that the SLO upload does not add the etag;
+ # this works around an issue with rackspace-flex.
+ cloud.object_store._add_etag_to_manifest = _add_etag_to_manifest
+ prune(cloud, container, delete_after)
+ headers = {}
+ if delete_after:
+ headers['X-Delete-After'] = str(delete_after)
+ endpoint = cloud.object_store.get_endpoint()
+ cloud.object_store.create_object(
+ container,
+ name=name,
+ filename=filename,
+ segment_size=SEGMENT_SIZE,
+ **headers)
+ url = os.path.join(endpoint, container, name)
+ return url
+
+
+def ansible_main():
+ module = AnsibleModule(
+ argument_spec=dict(
+ cloud=dict(required=True, type='raw'),
+ container=dict(required=True, type='str'),
+ filename=dict(required=True, type='path'),
+ name=dict(required=True, type='str'),
+ delete_after=dict(type='int'),
+ )
+ )
+
+ p = module.params
+ cloud = get_cloud(p.get('cloud'))
+ try:
+ url = run(
+ cloud,
+ p.get('container'),
+ p.get('filename'),
+ p.get('name'),
+ delete_after=p.get('delete_after'),
+ )
+ except (keystoneauth1.exceptions.http.HttpError,
+ requests.exceptions.RequestException):
+ s = "Error uploading to %s.%s" % (cloud.name, cloud.config.region_name)
+ s += "\n" + traceback.format_exc()
+ module.fail_json(
+ changed=False,
+ msg=s,
+ cloud=cloud.name,
+ region_name=cloud.config.region_name)
+ module.exit_json(
+ changed=True,
+ url=url,
+ )
+
+
+def cli_main():
+ parser = argparse.ArgumentParser(
+ description="Upload image to swift"
+ )
+ parser.add_argument('--verbose', action='store_true',
+ help='show debug information')
+ parser.add_argument('cloud',
+ help='Name of the cloud to use when uploading')
+ parser.add_argument('container',
+ help='Name of the container to use when uploading')
+ parser.add_argument('filename',
+ help='the file to upload')
+ parser.add_argument('name',
+ help='the object name')
+ parser.add_argument('--delete-after',
+ help='Number of seconds to delete object after '
+ 'upload. Default is 3 days (259200 seconds) '
+ 'and if set to 0 X-Delete-After will not be set',
+ type=int)
+
+ args = parser.parse_args()
+
+ if args.verbose:
+ logging.basicConfig(level=logging.DEBUG)
+ # Set requests log level accordingly
+ logging.getLogger("requests").setLevel(logging.DEBUG)
+ logging.getLogger("keystoneauth").setLevel(logging.INFO)
+ logging.getLogger("stevedore").setLevel(logging.INFO)
+ logging.captureWarnings(True)
+
+ url = run(
+ get_cloud(args.cloud),
+ args.container,
+ args.filename,
+ args.name,
+ delete_after=args.delete_after,
+ )
+ print(url)
+
+
+if __name__ == '__main__':
+ if not sys.stdin.isatty():
+ ansible_main()
+ else:
+ cli_main()
diff --git a/roles/upload-image-swift/tasks/main.yaml b/roles/upload-image-swift/tasks/main.yaml
new file mode 100644
index 000000000..5fe6ca768
--- /dev/null
+++ b/roles/upload-image-swift/tasks/main.yaml
@@ -0,0 +1,56 @@
+# Run the checksums in the background while we're uploading
+- name: Get sha256 hash
+ stat:
+ path: '{{ upload_image_swift_filename }}'
+ checksum_algorithm: sha256
+ async: 600
+ poll: 0
+ register: sha256_task
+
+- name: Get md5 hash
+ stat:
+ path: '{{ upload_image_swift_filename }}'
+ checksum_algorithm: md5
+ async: 600
+ poll: 0
+ register: md5_task
+
+- name: Upload image to swift
+ no_log: true
+ upload_image_swift:
+ cloud: '{{ upload_image_swift_cloud_config }}'
+ container: '{{ upload_image_swift_container }}'
+ filename: '{{ upload_image_swift_filename }}'
+ name: '{{ upload_image_swift_name }}'
+ delete_after: '{{ upload_image_swift_delete_after }}'
+ register: upload_results
+
+- name: Wait for sha256
+ async_status:
+ jid: "{{ sha256_task.ansible_job_id }}"
+ register: sha256
+ until: sha256.finished
+ retries: 1
+ delay: 10
+
+- name: Wait for md5
+ async_status:
+ jid: "{{ md5_task.ansible_job_id }}"
+ register: md5
+ until: md5.finished
+ retries: 1
+ delay: 10
+
+- name: Return artifact to Zuul
+ zuul_return:
+ data:
+ zuul:
+ artifacts:
+ - name: '{{ upload_image_swift_format }} image'
+ url: '{{ upload_results.url }}'
+ metadata:
+ type: 'zuul_image'
+ image_name: '{{ upload_image_swift_image_name }}'
+ format: '{{ upload_image_swift_format }}'
+ sha256: '{{ sha256.stat.checksum }}'
+ md5sum: '{{ md5.stat.checksum }}'