From 334dd378c968e674da3fad6a73cb324f09c4ad5c Mon Sep 17 00:00:00 2001 From: Martin Kopec Date: Wed, 20 Apr 2022 10:52:00 +0200 Subject: [PATCH] Fix compatibility issues A lot of things have changed at once (or have been ignored in refstack for some time) and in order to keep refstack functioning this commit addressed the following: * Update nodejs jobs so that we keep testing refstack with the newest nodejs * Update jobs for Zed development cycle * Add support for py3.10 * Fix 2009327 story - refstack is now compatible with jsonschema>=3.2.0 (schema definition got fixed) * Drop py3.6 and 3.7 (upstream tooling as e.g. devstack isn't supporting those anymore) * Fix mysql setup script used to run functional unit tests * Add WebTest dependency needed by unit tests * Update alembic and jsonschema version to at least currently recomended versions (by constraints) Task: 43761 Story: 2009327 Change-Id: I37bb7dc520119bdab280d62e23922641ef7658ce --- .zuul.yaml | 12 +++++------- refstack/api/validators.py | 5 ++--- refstack/db/migrations/alembic/utils.py | 8 +++++++- requirements.txt | 6 ++---- setup-mysql-tests.sh | 14 ++++++++++---- setup.cfg | 4 +--- test-requirements.txt | 1 + tox.ini | 2 +- 8 files changed, 29 insertions(+), 23 deletions(-) diff --git a/.zuul.yaml b/.zuul.yaml index 381369fd..de124890 100644 --- a/.zuul.yaml +++ b/.zuul.yaml @@ -1,17 +1,15 @@ - project: templates: - - nodejs6-jobs + - nodejs16-jobs - openstack-cover-jobs - - openstack-python3-yoga-jobs + - openstack-python3-zed-jobs check: jobs: - - refstack-tox-functional: - nodeset: openstack-single-node-bionic + - refstack-tox-functional - opendev-tox-docs gate: jobs: - - refstack-tox-functional: - nodeset: openstack-single-node-bionic + - refstack-tox-functional - opendev-tox-docs promote: jobs: @@ -19,7 +17,7 @@ - job: name: refstack-tox-functional - parent: openstack-tox + parent: openstack-tox-with-sudo description: | Run functional tests for an OpenStack Python project under cPython 3. Uses tox with the ``functional`` environment. diff --git a/refstack/api/validators.py b/refstack/api/validators.py index 016085c0..c96b587f 100644 --- a/refstack/api/validators.py +++ b/refstack/api/validators.py @@ -98,7 +98,7 @@ class TestResultValidator(BaseValidator): 'duration_seconds': {'type': 'integer'}, 'results': { 'type': 'array', - 'items': [{ + 'items': { 'type': 'object', 'properties': { 'name': {'type': 'string'}, @@ -107,8 +107,7 @@ class TestResultValidator(BaseValidator): 'format': 'uuid_hex' } } - }] - + } } }, 'required': ['cpid', 'duration_seconds', 'results'], diff --git a/refstack/db/migrations/alembic/utils.py b/refstack/db/migrations/alembic/utils.py index a1c6d5c6..c8a4ed1c 100644 --- a/refstack/db/migrations/alembic/utils.py +++ b/refstack/db/migrations/alembic/utils.py @@ -18,7 +18,13 @@ import os from alembic import config as alembic_conf from alembic.operations import Operations import alembic.migration as alembic_migration -from collections import Iterable +try: + # Python 3.10 and above + from collections.abc import Iterable +except ImportError: + # Python <= 3.9, this is deprecated since Python 3.3 and it's + # removed in Python 3.10 + from collections import Iterable from oslo_config import cfg from sqlalchemy import text diff --git a/requirements.txt b/requirements.txt index 915723c5..d49dc26b 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,5 @@ SQLAlchemy>=0.8.3 -alembic +alembic>=1.8.0 beaker beautifulsoup4 cryptography>=3.0 # BSD/Apache-2.0 @@ -11,9 +11,7 @@ oslo.utils>=3.16.0 # Apache-2.0 pecan>=0.8.2 requests>=2.2.0,!=2.4.0 requests-cache>=0.4.9,<0.6.0 -# jsonschema is capped due to the following bug: -# https://storyboard.openstack.org/#!/story/2009327 -jsonschema==3.2.0 +jsonschema>=4.7.0 PyJWT>=2.0.0 # MIT WebOb>=1.7.1 # MIT PyMySQL>=0.6.2,!=0.6.4 diff --git a/setup-mysql-tests.sh b/setup-mysql-tests.sh index 190ed90e..136285a2 100755 --- a/setup-mysql-tests.sh +++ b/setup-mysql-tests.sh @@ -28,15 +28,21 @@ fi # Else setup mysql base for tests. # Start MySQL process for tests MYSQL_DATA=`mktemp -d /tmp/refstack-mysql-XXXXX` +ls -lshd ${MYSQL_DATA} mkfifo ${MYSQL_DATA}/out # On systems like Fedora here's where mysqld can be found PATH=$PATH:/usr/libexec -mysqld --no-defaults --datadir=${MYSQL_DATA} --pid-file=${MYSQL_DATA}/mysql.pid --socket=${MYSQL_DATA}/mysql.socket --skip-networking --skip-grant-tables &> ${MYSQL_DATA}/out & +MYSQL_SOCKET="/var/run/mysqld/mysqld.sock" +sudo chown -R mysql:mysql ${MYSQL_DATA} +mysqld --initialize-insecure --basedir=${MYSQL_DATA} --datadir=${MYSQL_DATA}/data --pid-file=${MYSQL_DATA}/mysql.pid --socket=${MYSQL_SOCKET}/ --skip-networking --skip-grant-tables &> ${MYSQL_DATA}/out & # Wait for MySQL to start listening to connections wait_for_line "mysqld: ready for connections." ${MYSQL_DATA}/out -export REFSTACK_TEST_MYSQL_URL="mysql+pymysql://root@localhost/test?unix_socket=${MYSQL_DATA}/mysql.socket&charset=utf8" -mysql --no-defaults -S ${MYSQL_DATA}/mysql.socket -e 'set @@global.show_compatibility_56=ON;' > /dev/null 2>&1 -mysql --no-defaults -S ${MYSQL_DATA}/mysql.socket -e 'CREATE DATABASE test;' +sudo mysql -S ${MYSQL_SOCKET} -e 'set @@global.show_compatibility_56=ON;' > /dev/null 2>&1 +sudo mysql -S ${MYSQL_SOCKET} -e 'CREATE DATABASE test;' +sudo mysql -S ${MYSQL_SOCKET} -e "CREATE USER 'refstack'@'localhost' IDENTIFIED BY 'ref_pass';" +sudo mysql -S ${MYSQL_SOCKET} -e "GRANT ALL PRIVILEGES ON test . * TO 'refstack'@'localhost';" +sudo mysql -S ${MYSQL_SOCKET} -e "FLUSH PRIVILEGES;" +export REFSTACK_TEST_MYSQL_URL="mysql+pymysql://refstack:ref_pass@localhost/test?unix_socket=${MYSQL_SOCKET}&charset=utf8" # Yield execution to venv command $* diff --git a/setup.cfg b/setup.cfg index a94a1500..f248bc49 100644 --- a/setup.cfg +++ b/setup.cfg @@ -6,7 +6,7 @@ description-file = author = OpenStack author-email = openstack-discuss@lists.openstack.org home-page = https://refstack.openstack.org -python_requires = >=3.6 +python_requires = >=3.8 classifier = Environment :: OpenStack Intended Audience :: Developers @@ -15,8 +15,6 @@ classifier = Operating System :: POSIX :: Linux Programming Language :: Python Programming Language :: Python :: 3 - Programming Language :: Python :: 3.6 - Programming Language :: Python :: 3.7 Programming Language :: Python :: 3.8 Programming Language :: Python :: 3 :: Only Programming Language :: Python :: Implementation :: CPython diff --git a/test-requirements.txt b/test-requirements.txt index e5f32af4..e10e9be8 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -10,3 +10,4 @@ stestr>=1.1.0 # Apache-2.0 testtools>=0.9.34 pep257>=0.5.0 PyMySQL>=0.6.2,!=0.6.4 +WebTest>=3.0.0 diff --git a/tox.ini b/tox.ini index d623dc21..344a2fca 100644 --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,5 @@ [tox] -envlist = py36,py38,pep8,pip-check-reqs +envlist = py3,pep8,pip-check-reqs minversion = 3.18 skipsdist = True