Merge "Add upload-image-swift role"

doc/source/dib-roles.rst

@@ -4,3 +4,4 @@ Diskimage-Builder Roles
 .. zuul:autorole:: ensure-dib
 .. zuul:autorole:: build-diskimage
 .. zuul:autorole:: convert-diskimage
+.. zuul:autorole:: upload-image-swift

roles/upload-image-swift/README.rst

@@ -0,0 +1,58 @@
+Upload a filesystem image to a swift container
+
+This uploads a filesystem image (for example, one built by diskimage
+builder) to an OpenStack Object Store (Swift) container.  The role
+returns an artifact to Zuul suitable for use by the zuul-launcher.
+
+**Role Variables**
+
+.. zuul:rolevar:: upload_image_swift_cloud_config
+
+   Complex argument which contains the cloud configuration in
+   os-cloud-config (clouds.yaml) format.  It is expected that this
+   argument comes from a `Secret`.
+
+.. zuul:rolevar:: upload_image_swift_container
+
+   This role will create containers which do not already exist.
+
+   Note that you will want to set this to a value that uniquely
+   identifies your Zuul installation if using shared object stores that
+   require globally unique container names. For example if using a
+   public cloud whose Swift API is provided by Ceph.
+
+   The container should be dedicated to image uploads so that the
+   "delete_after" option may be safely used.
+
+.. zuul:rolevar:: upload_image_swift_delete_after
+   :default: 0
+
+   Number of seconds to delete objects after upload. Default is 0
+   (disabled).  This will tell swift to delete the file automatically,
+   but if that fails, the next run of the role will attempt to delete
+   any objects in the bucket older than this time.
+
+.. zuul:rolevar:: upload_image_swift_image_name
+   :default: `{{ build_diskimage_image_name }}`
+
+   The Zuul image name for use by zuul-launcher (e.g., `debian-bookworm`).
+
+.. zuul:rolevar:: upload_image_swift_format
+
+   The image format (e.g., `qcow2`).
+
+.. zuul:rolevar:: upload_image_swift_extension
+   :default: `{{ upload_image_swift_format }}`
+
+   The extension to use when uploading (only used in the default
+   values for the following variables.
+
+.. zuul:rolevar:: upload_image_swift_filename
+   :default: `{{ build_diskimage_image_root }}/{{ build_diskimage_image_name }}.{{ upload_image_swift_extension }}`
+
+   The path of the local file to upload.
+
+.. zuul:rolevar:: upload_image_swift_name
+   :default: `{{ zuul.build }}-{{ build_diskimage_image_name }}.{{ upload_image_swift_extension }}`
+
+   The object name to use when uploading.

roles/upload-image-swift/defaults/main.yaml

@@ -0,0 +1,5 @@
+upload_image_swift_image_name: '{{ build_diskimage_image_name }}'
+upload_image_swift_delete_after: 0
+upload_image_swift_filename: '{{ build_diskimage_image_root }}/{{ build_diskimage_image_name }}.{{ upload_image_swift_extension }}'
+upload_image_swift_name: '{{ zuul.build }}-{{ build_diskimage_image_name }}.{{ upload_image_swift_extension }}'
+upload_image_swift_extension: '{{ upload_image_swift_format }}'

roles/upload-image-swift/library/image_upload_swift.py

@@ -0,0 +1,168 @@
+# Copyright 2014 Rackspace Australia
+# Copyright 2018 Red Hat, Inc
+# Copyright 2024 Acme Gating, LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+import argparse
+import concurrent.futures
+import datetime
+import logging
+import os
+import sys
+import traceback
+
+import openstack
+import requests.exceptions
+import keystoneauth1.exceptions
+
+from ansible.module_utils.basic import AnsibleModule
+
+SEGMENT_SIZE = 500000000  # 500MB
+
+
+def get_cloud(cloud):
+    if isinstance(cloud, dict):
+        config = openstack.config.loader.OpenStackConfig().get_one(**cloud)
+        return openstack.connection.Connection(
+            config=config,
+            pool_executor=concurrent.futures.ThreadPoolExecutor(
+                max_workers=10
+            ))
+    else:
+        return openstack.connect(cloud=cloud)
+
+
+def _add_etag_to_manifest(self, *args, **kw):
+    return
+
+
+def prune(cloud, container, delete_after):
+    # In case the automatic expiration doesn't work, manually prune old uploads
+    if not delete_after:
+        return
+    target = (datetime.datetime.now(datetime.UTC) -
+              datetime.timedelta(seconds=delete_after))
+    endpoint = cloud.object_store.get_endpoint()
+    url = os.path.join(endpoint, container)
+    for obj in cloud.object_store.objects(container):
+        ts = datetime.datetime.fromisoformat(obj['last_modified'])
+        ts = ts.replace(tzinfo=datetime.UTC)
+        if ts < target:
+            path = os.path.join(url, obj.name)
+            try:
+                cloud.session.delete(path)
+            except keystoneauth1.exceptions.http.NotFound:
+                pass
+
+
+def run(cloud, container, filename, name, delete_after=None):
+    # Monkey-patch sdk so that the SLO upload does not add the etag;
+    # this works around an issue with rackspace-flex.
+    cloud.object_store._add_etag_to_manifest = _add_etag_to_manifest
+    prune(cloud, container, delete_after)
+    headers = {}
+    if delete_after:
+        headers['X-Delete-After'] = str(delete_after)
+    endpoint = cloud.object_store.get_endpoint()
+    cloud.object_store.create_object(
+        container,
+        name=name,
+        filename=filename,
+        segment_size=SEGMENT_SIZE,
+        **headers)
+    url = os.path.join(endpoint, container, name)
+    return url
+
+
+def ansible_main():
+    module = AnsibleModule(
+        argument_spec=dict(
+            cloud=dict(required=True, type='raw'),
+            container=dict(required=True, type='str'),
+            filename=dict(required=True, type='path'),
+            name=dict(required=True, type='str'),
+            delete_after=dict(type='int'),
+        )
+    )
+
+    p = module.params
+    cloud = get_cloud(p.get('cloud'))
+    try:
+        url = run(
+            cloud,
+            p.get('container'),
+            p.get('filename'),
+            p.get('name'),
+            delete_after=p.get('delete_after'),
+        )
+    except (keystoneauth1.exceptions.http.HttpError,
+            requests.exceptions.RequestException):
+        s = "Error uploading to %s.%s" % (cloud.name, cloud.config.region_name)
+        s += "\n" + traceback.format_exc()
+        module.fail_json(
+            changed=False,
+            msg=s,
+            cloud=cloud.name,
+            region_name=cloud.config.region_name)
+    module.exit_json(
+        changed=True,
+        url=url,
+    )
+
+
+def cli_main():
+    parser = argparse.ArgumentParser(
+        description="Upload image to swift"
+    )
+    parser.add_argument('--verbose', action='store_true',
+                        help='show debug information')
+    parser.add_argument('cloud',
+                        help='Name of the cloud to use when uploading')
+    parser.add_argument('container',
+                        help='Name of the container to use when uploading')
+    parser.add_argument('filename',
+                        help='the file to upload')
+    parser.add_argument('name',
+                        help='the object name')
+    parser.add_argument('--delete-after',
+                        help='Number of seconds to delete object after '
+                             'upload. Default is 3 days (259200 seconds) '
+                             'and if set to 0 X-Delete-After will not be set',
+                        type=int)
+
+    args = parser.parse_args()
+
+    if args.verbose:
+        logging.basicConfig(level=logging.DEBUG)
+        # Set requests log level accordingly
+        logging.getLogger("requests").setLevel(logging.DEBUG)
+        logging.getLogger("keystoneauth").setLevel(logging.INFO)
+        logging.getLogger("stevedore").setLevel(logging.INFO)
+        logging.captureWarnings(True)
+
+    url = run(
+        get_cloud(args.cloud),
+        args.container,
+        args.filename,
+        args.name,
+        delete_after=args.delete_after,
+    )
+    print(url)
+
+
+if __name__ == '__main__':
+    if not sys.stdin.isatty():
+        ansible_main()
+    else:
+        cli_main()

roles/upload-image-swift/tasks/main.yaml

@@ -0,0 +1,56 @@
+# Run the checksums in the background while we're uploading
+- name: Get sha256 hash
+  stat:
+    path: '{{ upload_image_swift_filename }}'
+    checksum_algorithm: sha256
+  async: 600
+  poll: 0
+  register: sha256_task
+
+- name: Get md5 hash
+  stat:
+    path: '{{ upload_image_swift_filename }}'
+    checksum_algorithm: md5
+  async: 600
+  poll: 0
+  register: md5_task
+
+- name: Upload image to swift
+  no_log: true
+  upload_image_swift:
+    cloud: '{{ upload_image_swift_cloud_config }}'
+    container: '{{ upload_image_swift_container }}'
+    filename: '{{ upload_image_swift_filename }}'
+    name: '{{ upload_image_swift_name }}'
+    delete_after: '{{ upload_image_swift_delete_after }}'
+  register: upload_results
+
+- name: Wait for sha256
+  async_status:
+    jid: "{{ sha256_task.ansible_job_id }}"
+  register: sha256
+  until: sha256.finished
+  retries: 1
+  delay: 10
+
+- name: Wait for md5
+  async_status:
+    jid: "{{ md5_task.ansible_job_id }}"
+  register: md5
+  until: md5.finished
+  retries: 1
+  delay: 10
+
+- name: Return artifact to Zuul
+  zuul_return:
+    data:
+      zuul:
+        artifacts:
+          - name: '{{ upload_image_swift_format }} image'
+            url: '{{ upload_results.url }}'
+            metadata:
+              type: 'zuul_image'
+              image_name: '{{ upload_image_swift_image_name }}'
+              format: '{{ upload_image_swift_format }}'
+              sha256: '{{ sha256.stat.checksum }}'
+              md5sum: '{{ md5.stat.checksum }}'