diff --git a/bindep.txt b/bindep.txt index a4176d7..9e9a0c1 100644 --- a/bindep.txt +++ b/bindep.txt @@ -1,7 +1,11 @@ musl-dev [compile test platform:apk] make [compile test platform:apk] linux-headers [compile test platform:apk] -gcc [compile test] +# Git only needed for temporary sha256 git repo +git +# gcc normally only needed for compile time, but temporarily runtime +# for sha256 +gcc g++ [compile test platform:apk platform:dpkg] gcc-c++ [compile test platform:rpm] libssl-dev [compile test platform:dpkg] @@ -9,5 +13,7 @@ openssl-devel [compile test platform:rpm] libressl-dev [compile test platform:apk] libffi-dev [compile test platform:dpkg platform:apk] libffi-devel [compile test platform:rpm] -python3-dev [compile test platform:dpkg platform:apk] -python3-devel [compile test platform:rpm] +# python3-dev normally only needed for compile time, but temporarily runtime +# for sha256 +python3-dev [platform:dpkg platform:apk] +python3-devel [platform:rpm] diff --git a/requirements.txt b/requirements.txt index 37e8500..302ccea 100644 --- a/requirements.txt +++ b/requirements.txt @@ -6,5 +6,9 @@ routes requests openstacksdk python-dateutil -rehash +# Temporary until upstream releases support for py 3.11: +git+https://github.com/jeblair/sha256 +# cython and wheel temporary since for the temp sha256 repo install phase +cython +wheel pyjwt>=2.0.0,<3.0.0 diff --git a/zuul_registry/hasher.py b/zuul_registry/hasher.py new file mode 100644 index 0000000..f9acddd --- /dev/null +++ b/zuul_registry/hasher.py @@ -0,0 +1,65 @@ +# Copyright 2024 Acme Gating, LLC +# +# This module is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This software is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this software. If not, see . + +import base64 + +import sha256 + + +class ResumableSha256: + def __init__(self): + self.hasher = sha256.sha256() + self.buffer = b'' + + def update(self, data): + if self.buffer is None: + raise Exception("Unable to update: hash is complete") + if self.buffer: + data = self.buffer + data + self.buffer = b'' + extra_len = len(data) % 64 + if extra_len: + self.buffer = data[-extra_len:] + data = data[:-extra_len] + self.hasher.update(data) + + def get_state(self): + hstate = self.hasher.state + return { + 'hash': base64.encodebytes(hstate[0]).decode('ascii'), + 'counter': hstate[1], + 'buffer': base64.encodebytes(self.buffer).decode('ascii'), + } + + def set_state(self, state): + hstate = ( + base64.decodebytes(state['hash'].encode('ascii')), + state['counter'], + ) + self.hasher.state = hstate + self.buffer = base64.decodebytes(state['buffer'].encode('ascii')) + + def finish(self): + if self.buffer: + self.hasher.update(self.buffer) + self.buffer = None + + def digest(self): + self.finish() + return self.hasher.digest() + + def hexdigest(self): + self.finish() + return self.hasher.hexdigest() diff --git a/zuul_registry/storage.py b/zuul_registry/storage.py index f944b35..1fef65b 100644 --- a/zuul_registry/storage.py +++ b/zuul_registry/storage.py @@ -13,17 +13,17 @@ # You should have received a copy of the GNU General Public License # along with this software. If not, see . -import base64 import json import logging import os import queue -import rehash import hashlib import threading import time from uuid import uuid4 +from . import hasher + class UploadRecord: """Information about an upload. @@ -49,7 +49,7 @@ class UploadRecord: def __init__(self): self.chunks = [] - self.hasher = rehash.sha256() + self.hasher = hasher.ResumableSha256() @property def count(self): @@ -66,15 +66,10 @@ class UploadRecord: def load(self, data): data = json.loads(data.decode('utf8')) self.chunks = data['chunks'] - hash_state = data['hash_state'] - hash_state['md_data'] = base64.decodebytes( - hash_state['md_data'].encode('ascii')) - self.hasher.__setstate__(hash_state) + self.hasher.set_state(data['hash_state']) def dump(self): - hash_state = self.hasher.__getstate__() - hash_state['md_data'] = base64.encodebytes( - hash_state['md_data']).decode('ascii') + hash_state = self.hasher.get_state() data = dict(chunks=self.chunks, hash_state=hash_state) return json.dumps(data).encode('utf8')