diff --git a/doc/source/dib-roles.rst b/doc/source/dib-roles.rst index 691b8f4b5..d83e4c4c8 100644 --- a/doc/source/dib-roles.rst +++ b/doc/source/dib-roles.rst @@ -4,3 +4,4 @@ Diskimage-Builder Roles .. zuul:autorole:: ensure-dib .. zuul:autorole:: build-diskimage .. zuul:autorole:: convert-diskimage +.. zuul:autorole:: upload-image-swift diff --git a/roles/upload-image-swift/README.rst b/roles/upload-image-swift/README.rst new file mode 100644 index 000000000..f31a9fbc8 --- /dev/null +++ b/roles/upload-image-swift/README.rst @@ -0,0 +1,58 @@ +Upload a filesystem image to a swift container + +This uploads a filesystem image (for example, one built by diskimage +builder) to an OpenStack Object Store (Swift) container. The role +returns an artifact to Zuul suitable for use by the zuul-launcher. + +**Role Variables** + +.. zuul:rolevar:: upload_image_swift_cloud_config + + Complex argument which contains the cloud configuration in + os-cloud-config (clouds.yaml) format. It is expected that this + argument comes from a `Secret`. + +.. zuul:rolevar:: upload_image_swift_container + + This role will create containers which do not already exist. + + Note that you will want to set this to a value that uniquely + identifies your Zuul installation if using shared object stores that + require globally unique container names. For example if using a + public cloud whose Swift API is provided by Ceph. + + The container should be dedicated to image uploads so that the + "delete_after" option may be safely used. + +.. zuul:rolevar:: upload_image_swift_delete_after + :default: 0 + + Number of seconds to delete objects after upload. Default is 0 + (disabled). This will tell swift to delete the file automatically, + but if that fails, the next run of the role will attempt to delete + any objects in the bucket older than this time. + +.. zuul:rolevar:: upload_image_swift_image_name + :default: `{{ build_diskimage_image_name }}` + + The Zuul image name for use by zuul-launcher (e.g., `debian-bookworm`). + +.. zuul:rolevar:: upload_image_swift_format + + The image format (e.g., `qcow2`). + +.. zuul:rolevar:: upload_image_swift_extension + :default: `{{ upload_image_swift_format }}` + + The extension to use when uploading (only used in the default + values for the following variables. + +.. zuul:rolevar:: upload_image_swift_filename + :default: `{{ build_diskimage_image_root }}/{{ build_diskimage_image_name }}.{{ upload_image_swift_extension }}` + + The path of the local file to upload. + +.. zuul:rolevar:: upload_image_swift_name + :default: `{{ zuul.build }}-{{ build_diskimage_image_name }}.{{ upload_image_swift_extension }}` + + The object name to use when uploading. diff --git a/roles/upload-image-swift/defaults/main.yaml b/roles/upload-image-swift/defaults/main.yaml new file mode 100644 index 000000000..3e42c766b --- /dev/null +++ b/roles/upload-image-swift/defaults/main.yaml @@ -0,0 +1,5 @@ +upload_image_swift_image_name: '{{ build_diskimage_image_name }}' +upload_image_swift_delete_after: 0 +upload_image_swift_filename: '{{ build_diskimage_image_root }}/{{ build_diskimage_image_name }}.{{ upload_image_swift_extension }}' +upload_image_swift_name: '{{ zuul.build }}-{{ build_diskimage_image_name }}.{{ upload_image_swift_extension }}' +upload_image_swift_extension: '{{ upload_image_swift_format }}' diff --git a/roles/upload-image-swift/library/__init__.py b/roles/upload-image-swift/library/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/roles/upload-image-swift/library/image_upload_swift.py b/roles/upload-image-swift/library/image_upload_swift.py new file mode 100644 index 000000000..44081b0a5 --- /dev/null +++ b/roles/upload-image-swift/library/image_upload_swift.py @@ -0,0 +1,168 @@ +# Copyright 2014 Rackspace Australia +# Copyright 2018 Red Hat, Inc +# Copyright 2024 Acme Gating, LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import argparse +import concurrent.futures +import datetime +import logging +import os +import sys +import traceback + +import openstack +import requests.exceptions +import keystoneauth1.exceptions + +from ansible.module_utils.basic import AnsibleModule + +SEGMENT_SIZE = 500000000 # 500MB + + +def get_cloud(cloud): + if isinstance(cloud, dict): + config = openstack.config.loader.OpenStackConfig().get_one(**cloud) + return openstack.connection.Connection( + config=config, + pool_executor=concurrent.futures.ThreadPoolExecutor( + max_workers=10 + )) + else: + return openstack.connect(cloud=cloud) + + +def _add_etag_to_manifest(self, *args, **kw): + return + + +def prune(cloud, container, delete_after): + # In case the automatic expiration doesn't work, manually prune old uploads + if not delete_after: + return + target = (datetime.datetime.now(datetime.UTC) - + datetime.timedelta(seconds=delete_after)) + endpoint = cloud.object_store.get_endpoint() + url = os.path.join(endpoint, container) + for obj in cloud.object_store.objects(container): + ts = datetime.datetime.fromisoformat(obj['last_modified']) + ts = ts.replace(tzinfo=datetime.UTC) + if ts < target: + path = os.path.join(url, obj.name) + try: + cloud.session.delete(path) + except keystoneauth1.exceptions.http.NotFound: + pass + + +def run(cloud, container, filename, name, delete_after=None): + # Monkey-patch sdk so that the SLO upload does not add the etag; + # this works around an issue with rackspace-flex. + cloud.object_store._add_etag_to_manifest = _add_etag_to_manifest + prune(cloud, container, delete_after) + headers = {} + if delete_after: + headers['X-Delete-After'] = str(delete_after) + endpoint = cloud.object_store.get_endpoint() + cloud.object_store.create_object( + container, + name=name, + filename=filename, + segment_size=SEGMENT_SIZE, + **headers) + url = os.path.join(endpoint, container, name) + return url + + +def ansible_main(): + module = AnsibleModule( + argument_spec=dict( + cloud=dict(required=True, type='raw'), + container=dict(required=True, type='str'), + filename=dict(required=True, type='path'), + name=dict(required=True, type='str'), + delete_after=dict(type='int'), + ) + ) + + p = module.params + cloud = get_cloud(p.get('cloud')) + try: + url = run( + cloud, + p.get('container'), + p.get('filename'), + p.get('name'), + delete_after=p.get('delete_after'), + ) + except (keystoneauth1.exceptions.http.HttpError, + requests.exceptions.RequestException): + s = "Error uploading to %s.%s" % (cloud.name, cloud.config.region_name) + s += "\n" + traceback.format_exc() + module.fail_json( + changed=False, + msg=s, + cloud=cloud.name, + region_name=cloud.config.region_name) + module.exit_json( + changed=True, + url=url, + ) + + +def cli_main(): + parser = argparse.ArgumentParser( + description="Upload image to swift" + ) + parser.add_argument('--verbose', action='store_true', + help='show debug information') + parser.add_argument('cloud', + help='Name of the cloud to use when uploading') + parser.add_argument('container', + help='Name of the container to use when uploading') + parser.add_argument('filename', + help='the file to upload') + parser.add_argument('name', + help='the object name') + parser.add_argument('--delete-after', + help='Number of seconds to delete object after ' + 'upload. Default is 3 days (259200 seconds) ' + 'and if set to 0 X-Delete-After will not be set', + type=int) + + args = parser.parse_args() + + if args.verbose: + logging.basicConfig(level=logging.DEBUG) + # Set requests log level accordingly + logging.getLogger("requests").setLevel(logging.DEBUG) + logging.getLogger("keystoneauth").setLevel(logging.INFO) + logging.getLogger("stevedore").setLevel(logging.INFO) + logging.captureWarnings(True) + + url = run( + get_cloud(args.cloud), + args.container, + args.filename, + args.name, + delete_after=args.delete_after, + ) + print(url) + + +if __name__ == '__main__': + if not sys.stdin.isatty(): + ansible_main() + else: + cli_main() diff --git a/roles/upload-image-swift/tasks/main.yaml b/roles/upload-image-swift/tasks/main.yaml new file mode 100644 index 000000000..5fe6ca768 --- /dev/null +++ b/roles/upload-image-swift/tasks/main.yaml @@ -0,0 +1,56 @@ +# Run the checksums in the background while we're uploading +- name: Get sha256 hash + stat: + path: '{{ upload_image_swift_filename }}' + checksum_algorithm: sha256 + async: 600 + poll: 0 + register: sha256_task + +- name: Get md5 hash + stat: + path: '{{ upload_image_swift_filename }}' + checksum_algorithm: md5 + async: 600 + poll: 0 + register: md5_task + +- name: Upload image to swift + no_log: true + upload_image_swift: + cloud: '{{ upload_image_swift_cloud_config }}' + container: '{{ upload_image_swift_container }}' + filename: '{{ upload_image_swift_filename }}' + name: '{{ upload_image_swift_name }}' + delete_after: '{{ upload_image_swift_delete_after }}' + register: upload_results + +- name: Wait for sha256 + async_status: + jid: "{{ sha256_task.ansible_job_id }}" + register: sha256 + until: sha256.finished + retries: 1 + delay: 10 + +- name: Wait for md5 + async_status: + jid: "{{ md5_task.ansible_job_id }}" + register: md5 + until: md5.finished + retries: 1 + delay: 10 + +- name: Return artifact to Zuul + zuul_return: + data: + zuul: + artifacts: + - name: '{{ upload_image_swift_format }} image' + url: '{{ upload_results.url }}' + metadata: + type: 'zuul_image' + image_name: '{{ upload_image_swift_image_name }}' + format: '{{ upload_image_swift_format }}' + sha256: '{{ sha256.stat.checksum }}' + md5sum: '{{ md5.stat.checksum }}'