From 34c30b9fa539cb18cf01335c435f0841dc588530 Mon Sep 17 00:00:00 2001 From: Aurelio Jargas Date: Sat, 8 Feb 2025 23:45:01 +0100 Subject: [PATCH] Add role: `ensure-python-command`, refactor similar roles This role ensures that a specific pip-installable command is available. Example usage: - role: ensure-python-command vars: ensure_python_command_name: poetry ensure_python_command_version: ==1.8.5 # omit to install latest In this case, if the `poetry` command is not already available, pip will install it in a new venv. Either way, after running this role, the `ensure_python_command_executable` variable will hold the full path to the command. We already have similar roles for specific commands: - ensure-nox - ensure-poetry - ensure-pyproject-build - ensure-tox - ensure-twine - ensure-uv These roles are essentially copies of each other with different command names. This new role consolidates that code. The existing roles now act as wrappers that just set variables and call the new role. > Note: The `ensure-tox` role has not been refactored due to exclusive > legacy code related to Python 2, which must be removed first. The new role introduces three variables to replace the overloaded `ensure__executable` variable from the other roles: - `ensure_python_command_name` (input, command name) - `ensure_python_command_existing` (input, existing path for the command) - `ensure_python_command_executable` (output, detected/installed path) This separation avoids using the same variable as both input and output, which can cause issues due to Ansible's variable precedence rules: https://docs.ansible.com/ansible/latest/playbook_guide/playbooks_variables.html Understanding variable precedence ... 19. set_facts / registered vars 20. role (and include_role) params ... Since we use `set_fact` inside the role, it is ineffective when the same variable is also passed as a role parameter :/ I'm not adding tests for the new role because its functionality is already covered by the existing tests for all the refactored roles: - test-playbooks/ensure-nox.yaml - test-playbooks/ensure-poetry.yaml - test-playbooks/ensure-pyproject-build.yaml - test-playbooks/ensure-twine.yaml - test-playbooks/ensure-uv.yaml Change-Id: Idd970cb31bd928576bca3602ce96fbc491ecdb60 --- doc/source/python-roles.rst | 1 + roles/ensure-nox/defaults/main.yaml | 1 - roles/ensure-nox/tasks/main.yaml | 40 +++-------- roles/ensure-poetry/defaults/main.yaml | 1 - roles/ensure-poetry/tasks/main.yaml | 50 +++---------- .../ensure-pyproject-build/defaults/main.yaml | 1 - roles/ensure-pyproject-build/tasks/main.yaml | 51 +++---------- roles/ensure-python-command/README.rst | 71 +++++++++++++++++++ .../ensure-python-command/defaults/main.yaml | 5 ++ roles/ensure-python-command/tasks/main.yaml | 64 +++++++++++++++++ roles/ensure-python-command/vars/main.yaml | 2 + roles/ensure-twine/defaults/main.yaml | 1 - roles/ensure-twine/tasks/main.yaml | 50 +++---------- roles/ensure-uv/defaults/main.yaml | 1 - roles/ensure-uv/tasks/main.yaml | 50 +++---------- zuul-tests.d/python-jobs.yaml | 5 ++ 16 files changed, 199 insertions(+), 195 deletions(-) create mode 100644 roles/ensure-python-command/README.rst create mode 100644 roles/ensure-python-command/defaults/main.yaml create mode 100644 roles/ensure-python-command/tasks/main.yaml create mode 100644 roles/ensure-python-command/vars/main.yaml diff --git a/doc/source/python-roles.rst b/doc/source/python-roles.rst index 66a4c8e89..eeb8c1bc0 100644 --- a/doc/source/python-roles.rst +++ b/doc/source/python-roles.rst @@ -7,6 +7,7 @@ Python Roles .. zuul:autorole:: ensure-if-python .. zuul:autorole:: ensure-nox .. zuul:autorole:: ensure-pip +.. zuul:autorole:: ensure-python-command .. zuul:autorole:: ensure-poetry .. zuul:autorole:: ensure-pyproject-build .. zuul:autorole:: ensure-python diff --git a/roles/ensure-nox/defaults/main.yaml b/roles/ensure-nox/defaults/main.yaml index e54288fa3..4ae6845b8 100644 --- a/roles/ensure-nox/defaults/main.yaml +++ b/roles/ensure-nox/defaults/main.yaml @@ -1,3 +1,2 @@ -nox_executable: nox ensure_nox_version: '' nox_venv_path: '{{ ansible_user_dir }}/.local/nox' diff --git a/roles/ensure-nox/tasks/main.yaml b/roles/ensure-nox/tasks/main.yaml index de8aa9b97..2849f8fd8 100644 --- a/roles/ensure-nox/tasks/main.yaml +++ b/roles/ensure-nox/tasks/main.yaml @@ -1,34 +1,14 @@ -- name: Install pip +- name: Check and install nox if necessary include_role: - name: ensure-pip + name: ensure-python-command + vars: + ensure_python_command_name: nox + ensure_python_command_version: "{{ ensure_nox_version }}" + ensure_python_command_existing: "{{ nox_executable | default('') }}" + ensure_python_command_venv_path: "{{ nox_venv_path }}" + ensure_python_command_global_symlink: false # not supported -- name: Check if nox is installed - shell: | - command -v {{ nox_executable }} {{ nox_venv_path }}/bin/nox || exit 1 - args: - executable: /bin/bash - register: nox_preinstalled - failed_when: false - -- name: Export preinstalled nox_exectuable +- name: Export nox_executable path set_fact: - nox_executable: '{{ nox_preinstalled.stdout_lines[0] }}' + nox_executable: "{{ ensure_python_command_executable }}" cacheable: true - when: nox_preinstalled.rc == 0 - -- name: Install nox to local env - when: nox_preinstalled.rc != 0 - block: - - name: Create local venv - command: '{{ ensure_pip_virtualenv_command }} {{ nox_venv_path }}' - - - name: Install nox to local venv - command: '{{ nox_venv_path }}/bin/pip install nox{{ ensure_nox_version }}' - - - name: Export installed nox_executable path - set_fact: - nox_executable: '{{ nox_venv_path }}/bin/nox' - cacheable: true - -- name: Output nox version - command: "{{ nox_executable }} --version" diff --git a/roles/ensure-poetry/defaults/main.yaml b/roles/ensure-poetry/defaults/main.yaml index 9ce9c23f4..ba3d6739b 100644 --- a/roles/ensure-poetry/defaults/main.yaml +++ b/roles/ensure-poetry/defaults/main.yaml @@ -1,4 +1,3 @@ ensure_poetry_global_symlink: false ensure_poetry_version: "" -ensure_poetry_executable: poetry ensure_poetry_venv_path: "{{ ansible_user_dir }}/.local/poetry" diff --git a/roles/ensure-poetry/tasks/main.yaml b/roles/ensure-poetry/tasks/main.yaml index 011104910..504fd9fd2 100644 --- a/roles/ensure-poetry/tasks/main.yaml +++ b/roles/ensure-poetry/tasks/main.yaml @@ -1,44 +1,14 @@ -- name: Install pip +- name: Check and install poetry if necessary include_role: - name: ensure-pip + name: ensure-python-command + vars: + ensure_python_command_name: poetry + ensure_python_command_version: "{{ ensure_poetry_version }}" + ensure_python_command_existing: "{{ ensure_poetry_executable | default('') }}" + ensure_python_command_venv_path: "{{ ensure_poetry_venv_path }}" + ensure_python_command_global_symlink: "{{ ensure_poetry_global_symlink }}" -- name: Check if poetry is installed - shell: | - command -v {{ ensure_poetry_executable }} {{ ensure_poetry_venv_path }}/bin/poetry || exit 1 - args: - executable: /bin/bash - register: poetry_preinstalled - failed_when: false - -- name: Export preinstalled ensure_poetry_executable +- name: Export ensure_poetry_executable path set_fact: - ensure_poetry_executable: "{{ poetry_preinstalled.stdout_lines[0] }}" + ensure_poetry_executable: "{{ ensure_python_command_executable }}" cacheable: true - when: poetry_preinstalled.rc == 0 - -- name: Install poetry to local env - when: poetry_preinstalled.rc != 0 - block: - - name: Create local venv - command: "{{ ensure_pip_virtualenv_command }} {{ ensure_poetry_venv_path }}" - - - name: Install poetry to local venv - command: "{{ ensure_poetry_venv_path }}/bin/pip install poetry{{ ensure_poetry_version }}" - - - name: Export installed ensure_poetry_executable path - set_fact: - ensure_poetry_executable: "{{ ensure_poetry_venv_path }}/bin/poetry" - cacheable: true - -- name: Output poetry version - command: "{{ ensure_poetry_executable }} --version" - -- name: Make global symlink - when: - - ensure_poetry_global_symlink - - ensure_poetry_executable != '/usr/local/bin/poetry' - file: - state: link - src: "{{ ensure_poetry_executable }}" - dest: /usr/local/bin/poetry - become: yes diff --git a/roles/ensure-pyproject-build/defaults/main.yaml b/roles/ensure-pyproject-build/defaults/main.yaml index d944d4035..1375912a3 100644 --- a/roles/ensure-pyproject-build/defaults/main.yaml +++ b/roles/ensure-pyproject-build/defaults/main.yaml @@ -1,4 +1,3 @@ ensure_pyproject_build_global_symlink: false ensure_pyproject_build_version: "" -ensure_pyproject_build_executable: pyproject-build ensure_pyproject_build_venv_path: "{{ ansible_user_dir }}/.local/pyproject-build" diff --git a/roles/ensure-pyproject-build/tasks/main.yaml b/roles/ensure-pyproject-build/tasks/main.yaml index f46d4c799..8131e4647 100644 --- a/roles/ensure-pyproject-build/tasks/main.yaml +++ b/roles/ensure-pyproject-build/tasks/main.yaml @@ -1,44 +1,15 @@ -- name: Install pip +- name: Check and install pyproject-build if necessary include_role: - name: ensure-pip + name: ensure-python-command + vars: + ensure_python_command_name: pyproject-build + ensure_python_command_package: build + ensure_python_command_version: "{{ ensure_pyproject_build_version }}" + ensure_python_command_existing: "{{ ensure_pyproject_build_executable | default('') }}" + ensure_python_command_venv_path: "{{ ensure_pyproject_build_venv_path }}" + ensure_python_command_global_symlink: "{{ ensure_pyproject_build_global_symlink }}" -- name: Check if pyproject-build is installed - shell: | - command -v {{ ensure_pyproject_build_executable }} {{ ensure_pyproject_build_venv_path }}/bin/pyproject-build || exit 1 - args: - executable: /bin/bash - register: pyproject_build_preinstalled - failed_when: false - -- name: Export preinstalled ensure_pyproject_build_executable +- name: Export ensure_pyproject_build_executable path set_fact: - ensure_pyproject_build_executable: "{{ pyproject_build_preinstalled.stdout_lines[0] }}" + ensure_pyproject_build_executable: "{{ ensure_python_command_executable }}" cacheable: true - when: pyproject_build_preinstalled.rc == 0 - -- name: Install pyproject-build to local env - when: pyproject_build_preinstalled.rc != 0 - block: - - name: Create local venv - command: "{{ ensure_pip_virtualenv_command }} {{ ensure_pyproject_build_venv_path }}" - - - name: Install pyproject-build to local venv - command: "{{ ensure_pyproject_build_venv_path }}/bin/pip install build{{ ensure_pyproject_build_version }}" - - - name: Export installed ensure_pyproject_build_executable path - set_fact: - ensure_pyproject_build_executable: "{{ ensure_pyproject_build_venv_path }}/bin/pyproject-build" - cacheable: true - -- name: Output pyproject-build version - command: "{{ ensure_pyproject_build_executable }} --version" - -- name: Make global symlink - when: - - ensure_pyproject_build_global_symlink - - ensure_pyproject_build_executable != '/usr/local/bin/pyproject-build' - file: - state: link - src: "{{ ensure_pyproject_build_executable }}" - dest: /usr/local/bin/pyproject-build - become: yes diff --git a/roles/ensure-python-command/README.rst b/roles/ensure-python-command/README.rst new file mode 100644 index 000000000..a73d697a2 --- /dev/null +++ b/roles/ensure-python-command/README.rst @@ -0,0 +1,71 @@ +Ensure a pip-installed command is available + +This role checks for the specified command, and if not found, installs +it via ``pip`` into a virtual environment for the current user. + +The minimal required input is the command name. Additionally, you can +specify a version or a path to a local venv, among other things. + +Example: + +.. code-block:: yaml + + - role: ensure-python-command + vars: + ensure_python_command_name: poetry + ensure_python_command_version: ==1.8.5 # omit to install latest + +In this case, if the ``poetry`` command is not already available, pip +will install it in a new venv. Either way, after running this role, the +``ensure_python_command_executable`` variable will hold the full path to +the command. + +**Role Variables** + +.. zuul:rolevar:: ensure_python_command_name + + Required. The name of the command to ensure is available. + +.. zuul:rolevar:: ensure_python_command_package + :default: {{ ensure_python_command_name }} + + The name of the Python package that provides the desired command. + Defaults to the command name, since this is usually the case. + Set this variable when they differ. + +.. zuul:rolevar:: ensure_python_command_version + :default: '' + + The version specifier to select the version of the package to install. + If omitted, the latest version will be installed. + +.. zuul:rolevar:: ensure_python_command_existing + :default: '' + + Look for an existing command at this specific path. For example, if your base + image pre-installs the command in an out-of-path environment, set this so the + role does not attempt to install the command again. + +.. zuul:rolevar:: ensure_python_command_venv_path + :default: {{ ansible_user_dir }}/.local/{{ ensure_python_command_package }} + + Directory for the Python venv where the package should be installed. + +.. zuul:rolevar:: ensure_python_command_global_symlink + :default: False + + Install a symlink to the command executable into ``/usr/local/bin/``. + + This can be useful when scripts need to be run that expect to find the + command in a more standard location and plumbing through the value + of ``ensure_python_command_executable`` would be onerous. + + Setting this requires root access, so should only be done in + circumstances where root access is available. + +**Output Variables** + +.. zuul:rolevar:: ensure_python_command_executable + + The full path to the command executable, whether it was detected or + installed by the role. diff --git a/roles/ensure-python-command/defaults/main.yaml b/roles/ensure-python-command/defaults/main.yaml new file mode 100644 index 000000000..52b75fffc --- /dev/null +++ b/roles/ensure-python-command/defaults/main.yaml @@ -0,0 +1,5 @@ +ensure_python_command_global_symlink: false +ensure_python_command_version: "" +ensure_python_command_existing: "" +ensure_python_command_package: "{{ ensure_python_command_name }}" +ensure_python_command_venv_path: "{{ ansible_user_dir }}/.local/{{ ensure_python_command_package }}" diff --git a/roles/ensure-python-command/tasks/main.yaml b/roles/ensure-python-command/tasks/main.yaml new file mode 100644 index 000000000..86446a3c1 --- /dev/null +++ b/roles/ensure-python-command/tasks/main.yaml @@ -0,0 +1,64 @@ +- name: Install pip + include_role: + name: ensure-pip + +- name: Check if the command name is set and non-empty + fail: + msg: Required variable ensure_python_command_name is not set or empty + when: ensure_python_command_name is not defined or not ensure_python_command_name.strip() + +# Part 1 +# Check if the command is already available. If not, pip-install it to a local venv. +# Either way, save the command's full path to the output variable. + +- name: Check if {{ ensure_python_command_name }} is available + # 1. Full path optionally informed by the role caller + # 2. Ansible user $PATH (bare command name) + # 3. Local venv (installed from a previous run for this role) + shell: | + command -v \ + {{ ensure_python_command_existing }} \ + {{ ensure_python_command_name | quote }} \ + {{ ensure_python_command_venv_executable | quote }} \ + || exit 1 + args: + executable: /bin/bash + register: ensure_python_command_preinstalled + failed_when: false + +- name: Export preinstalled ensure_python_command_executable + set_fact: + ensure_python_command_executable: "{{ ensure_python_command_preinstalled.stdout_lines[0] }}" + cacheable: true + when: ensure_python_command_preinstalled.rc == 0 + +- name: Install {{ ensure_python_command_package }} to local env + when: ensure_python_command_preinstalled.rc != 0 + block: + - name: Create local venv + command: "{{ ensure_pip_virtualenv_command }} {{ ensure_python_command_venv_path }}" + + - name: Install {{ ensure_python_command_package }} to local venv + command: "{{ ensure_python_command_venv_path }}/bin/pip install {{ ensure_python_command_package }}{{ ensure_python_command_version }}" + + - name: Export installed ensure_python_command_executable path + set_fact: + ensure_python_command_executable: "{{ ensure_python_command_venv_executable }}" + cacheable: true + +# Part 2 +# Try to show the command's version and maybe also create a symlink to it in /usr/local/bin + +- name: Output {{ ensure_python_command_name }} version + command: "{{ ensure_python_command_executable }} --version" + failed_when: false + +- name: Make global symlink + when: + - ensure_python_command_global_symlink + - ensure_python_command_executable != ensure_python_command_global_symlink_path + file: + state: link + src: "{{ ensure_python_command_executable }}" + dest: "{{ ensure_python_command_global_symlink_path }}" + become: yes diff --git a/roles/ensure-python-command/vars/main.yaml b/roles/ensure-python-command/vars/main.yaml new file mode 100644 index 000000000..d3fd7f665 --- /dev/null +++ b/roles/ensure-python-command/vars/main.yaml @@ -0,0 +1,2 @@ +ensure_python_command_global_symlink_path: /usr/local/bin/{{ ensure_python_command_name }} +ensure_python_command_venv_executable: "{{ ensure_python_command_venv_path }}/bin/{{ ensure_python_command_name }}" diff --git a/roles/ensure-twine/defaults/main.yaml b/roles/ensure-twine/defaults/main.yaml index 160d497ac..1554904a9 100644 --- a/roles/ensure-twine/defaults/main.yaml +++ b/roles/ensure-twine/defaults/main.yaml @@ -1,5 +1,4 @@ ensure_twine_global_symlink: false # version 6.1.0 is breaking test-release-openstack CI job ensure_twine_version: ">1.12.0,!=6.1.0" -pypi_twine_executable: twine ensure_twine_venv_path: "{{ ansible_user_dir }}/.local/twine" diff --git a/roles/ensure-twine/tasks/main.yaml b/roles/ensure-twine/tasks/main.yaml index 3727fc4ab..636ac4922 100644 --- a/roles/ensure-twine/tasks/main.yaml +++ b/roles/ensure-twine/tasks/main.yaml @@ -1,44 +1,14 @@ -- name: Install pip +- name: Check and install twine if necessary include_role: - name: ensure-pip + name: ensure-python-command + vars: + ensure_python_command_name: twine + ensure_python_command_version: "{{ ensure_twine_version }}" + ensure_python_command_existing: "{{ pypi_twine_executable | default('') }}" + ensure_python_command_venv_path: "{{ ensure_twine_venv_path }}" + ensure_python_command_global_symlink: "{{ ensure_twine_global_symlink }}" -- name: Check if twine is installed - shell: | - command -v {{ pypi_twine_executable }} {{ ensure_twine_venv_path }}/bin/twine || exit 1 - args: - executable: /bin/bash - register: twine_preinstalled - failed_when: false - -- name: Export preinstalled pypi_twine_executable +- name: Export pypi_twine_executable path set_fact: - pypi_twine_executable: "{{ twine_preinstalled.stdout_lines[0] }}" + pypi_twine_executable: "{{ ensure_python_command_executable }}" cacheable: true - when: twine_preinstalled.rc == 0 - -- name: Install twine to local env - when: twine_preinstalled.rc != 0 - block: - - name: Create local venv - command: "{{ ensure_pip_virtualenv_command }} {{ ensure_twine_venv_path }}" - - - name: Install twine to local venv - command: "{{ ensure_twine_venv_path }}/bin/pip install twine{{ ensure_twine_version }}" - - - name: Export installed pypi_twine_executable path - set_fact: - pypi_twine_executable: "{{ ensure_twine_venv_path }}/bin/twine" - cacheable: true - -- name: Output twine version - command: "{{ pypi_twine_executable }} --version" - -- name: Make global symlink - when: - - ensure_twine_global_symlink - - pypi_twine_executable != '/usr/local/bin/twine' - file: - state: link - src: "{{ pypi_twine_executable }}" - dest: /usr/local/bin/twine - become: yes diff --git a/roles/ensure-uv/defaults/main.yaml b/roles/ensure-uv/defaults/main.yaml index a6060f028..9863f17ff 100644 --- a/roles/ensure-uv/defaults/main.yaml +++ b/roles/ensure-uv/defaults/main.yaml @@ -1,4 +1,3 @@ ensure_uv_global_symlink: false ensure_uv_version: "" -ensure_uv_executable: uv ensure_uv_venv_path: "{{ ansible_user_dir }}/.local/uv" diff --git a/roles/ensure-uv/tasks/main.yaml b/roles/ensure-uv/tasks/main.yaml index 87dd44e8a..b56f1a86d 100644 --- a/roles/ensure-uv/tasks/main.yaml +++ b/roles/ensure-uv/tasks/main.yaml @@ -1,44 +1,14 @@ -- name: Install pip +- name: Check and install uv if necessary include_role: - name: ensure-pip + name: ensure-python-command + vars: + ensure_python_command_name: uv + ensure_python_command_version: "{{ ensure_uv_version }}" + ensure_python_command_existing: "{{ ensure_uv_executable | default('') }}" + ensure_python_command_venv_path: "{{ ensure_uv_venv_path }}" + ensure_python_command_global_symlink: "{{ ensure_uv_global_symlink }}" -- name: Check if uv is installed - shell: | - command -v {{ ensure_uv_executable }} {{ ensure_uv_venv_path }}/bin/uv || exit 1 - args: - executable: /bin/bash - register: uv_preinstalled - failed_when: false - -- name: Export preinstalled ensure_uv_executable +- name: Export ensure_uv_executable path set_fact: - ensure_uv_executable: "{{ uv_preinstalled.stdout_lines[0] }}" + ensure_uv_executable: "{{ ensure_python_command_executable }}" cacheable: true - when: uv_preinstalled.rc == 0 - -- name: Install uv to local env - when: uv_preinstalled.rc != 0 - block: - - name: Create local venv - command: "{{ ensure_pip_virtualenv_command }} {{ ensure_uv_venv_path }}" - - - name: Install uv to local venv - command: "{{ ensure_uv_venv_path }}/bin/pip install uv{{ ensure_uv_version }}" - - - name: Export installed ensure_uv_executable path - set_fact: - ensure_uv_executable: "{{ ensure_uv_venv_path }}/bin/uv" - cacheable: true - -- name: Output uv version - command: "{{ ensure_uv_executable }} --version" - -- name: Make global symlink - when: - - ensure_uv_global_symlink - - ensure_uv_executable != '/usr/local/bin/uv' - file: - state: link - src: "{{ ensure_uv_executable }}" - dest: /usr/local/bin/uv - become: yes diff --git a/zuul-tests.d/python-jobs.yaml b/zuul-tests.d/python-jobs.yaml index a4ee1d420..8a96f9aa3 100644 --- a/zuul-tests.d/python-jobs.yaml +++ b/zuul-tests.d/python-jobs.yaml @@ -3,6 +3,7 @@ description: Test the ensure-nox role files: - roles/ensure-nox/.* + - roles/ensure-python-command/.* - test-playbooks/ensure-nox.yaml run: test-playbooks/ensure-nox.yaml tags: all-platforms @@ -149,6 +150,7 @@ description: Test the ensure-poetry role files: - roles/ensure-poetry/.* + - roles/ensure-python-command/.* - test-playbooks/ensure-poetry.yaml run: test-playbooks/ensure-poetry.yaml tags: all-platforms @@ -218,6 +220,7 @@ description: Test the ensure-pyproject-build role files: - roles/ensure-pyproject-build/.* + - roles/ensure-python-command/.* - test-playbooks/ensure-pyproject-build.yaml run: test-playbooks/ensure-pyproject-build.yaml tags: all-platforms @@ -287,6 +290,7 @@ description: Test the ensure-twine role files: - roles/ensure-twine/.* + - roles/ensure-python-command/.* - test-playbooks/ensure-twine.yaml run: test-playbooks/ensure-twine.yaml tags: all-platforms @@ -434,6 +438,7 @@ description: Test the ensure-uv role files: - roles/ensure-uv/.* + - roles/ensure-python-command/.* - test-playbooks/ensure-uv.yaml run: test-playbooks/ensure-uv.yaml tags: all-platforms