Add role: ensure-python-command, refactor similar roles

This role ensures that a specific pip-installable command is
available.

Example usage:

    - role: ensure-python-command
      vars:
        ensure_python_command_name: poetry
        ensure_python_command_version: ==1.8.5  # omit to install latest

In this case, if the `poetry` command is not already available, pip will
install it in a new venv. Either way, after running this role, the
`ensure_python_command_executable` variable will hold the full path to
the command.

We already have similar roles for specific commands:

- ensure-nox
- ensure-poetry
- ensure-pyproject-build
- ensure-tox
- ensure-twine
- ensure-uv

These roles are essentially copies of each other with different command
names. This new role consolidates that code. The existing roles now act
as wrappers that just set variables and call the new role.

> Note: The `ensure-tox` role has not been refactored due to exclusive
> legacy code related to Python 2, which must be removed first.

The new role introduces three variables to replace the overloaded
`ensure_<command>_executable` variable from the other roles:

- `ensure_python_command_name` (input, command name)
- `ensure_python_command_existing` (input, existing path for the command)
- `ensure_python_command_executable` (output, detected/installed path)

This separation avoids using the same variable as both input and output,
which can cause issues due to Ansible's variable precedence rules:

    https://docs.ansible.com/ansible/latest/playbook_guide/playbooks_variables.html

    Understanding variable precedence

    ...
    19. set_facts / registered vars
    20. role (and include_role) params
    ...

Since we use `set_fact` inside the role, it is ineffective when the same
variable is also passed as a role parameter :/

I'm not adding tests for the new role because its functionality is
already covered by the existing tests for all the refactored roles:

- test-playbooks/ensure-nox.yaml
- test-playbooks/ensure-poetry.yaml
- test-playbooks/ensure-pyproject-build.yaml
- test-playbooks/ensure-twine.yaml
- test-playbooks/ensure-uv.yaml

Change-Id: Idd970cb31bd928576bca3602ce96fbc491ecdb60

doc/source/python-roles.rst

@@ -7,6 +7,7 @@ Python Roles
 .. zuul:autorole:: ensure-if-python
 .. zuul:autorole:: ensure-nox
 .. zuul:autorole:: ensure-pip
+.. zuul:autorole:: ensure-python-command
 .. zuul:autorole:: ensure-poetry
 .. zuul:autorole:: ensure-pyproject-build
 .. zuul:autorole:: ensure-python

roles/ensure-nox/defaults/main.yaml

@@ -1,3 +1,2 @@
-nox_executable: nox
 ensure_nox_version: ''
 nox_venv_path: '{{ ansible_user_dir }}/.local/nox'

roles/ensure-nox/tasks/main.yaml

@@ -1,34 +1,14 @@
-- name: Install pip
+- name: Check and install nox if necessary
   include_role:
-    name: ensure-pip
+    name: ensure-python-command
+  vars:
+    ensure_python_command_name: nox
+    ensure_python_command_version: "{{ ensure_nox_version }}"
+    ensure_python_command_existing: "{{ nox_executable | default('') }}"
+    ensure_python_command_venv_path: "{{ nox_venv_path }}"
+    ensure_python_command_global_symlink: false  # not supported
 
-- name: Check if nox is installed
-  shell: |
-    command -v {{ nox_executable }} {{ nox_venv_path }}/bin/nox || exit 1
-  args:
-    executable: /bin/bash
-  register: nox_preinstalled
-  failed_when: false
-
-- name: Export preinstalled nox_exectuable
+- name: Export nox_executable path
   set_fact:
-    nox_executable: '{{ nox_preinstalled.stdout_lines[0] }}'
+    nox_executable: "{{ ensure_python_command_executable }}"
     cacheable: true
-  when: nox_preinstalled.rc == 0
-
-- name: Install nox to local env
-  when: nox_preinstalled.rc != 0
-  block:
-    - name: Create local venv
-      command: '{{ ensure_pip_virtualenv_command }} {{ nox_venv_path }}'
-
-    - name: Install nox to local venv
-      command: '{{ nox_venv_path }}/bin/pip install nox{{ ensure_nox_version }}'
-
-    - name: Export installed nox_executable path
-      set_fact:
-        nox_executable: '{{ nox_venv_path }}/bin/nox'
-        cacheable: true
-
-- name: Output nox version
-  command: "{{ nox_executable }} --version"

roles/ensure-poetry/defaults/main.yaml

@@ -1,4 +1,3 @@
 ensure_poetry_global_symlink: false
 ensure_poetry_version: ""
-ensure_poetry_executable: poetry
 ensure_poetry_venv_path: "{{ ansible_user_dir }}/.local/poetry"

roles/ensure-poetry/tasks/main.yaml

@@ -1,44 +1,14 @@
-- name: Install pip
+- name: Check and install poetry if necessary
   include_role:
-    name: ensure-pip
+    name: ensure-python-command
+  vars:
+    ensure_python_command_name: poetry
+    ensure_python_command_version: "{{ ensure_poetry_version }}"
+    ensure_python_command_existing: "{{ ensure_poetry_executable | default('') }}"
+    ensure_python_command_venv_path: "{{ ensure_poetry_venv_path }}"
+    ensure_python_command_global_symlink: "{{ ensure_poetry_global_symlink }}"
 
-- name: Check if poetry is installed
-  shell: |
-    command -v {{ ensure_poetry_executable }} {{ ensure_poetry_venv_path }}/bin/poetry || exit 1
-  args:
-    executable: /bin/bash
-  register: poetry_preinstalled
-  failed_when: false
-
-- name: Export preinstalled ensure_poetry_executable
+- name: Export ensure_poetry_executable path
   set_fact:
-    ensure_poetry_executable: "{{ poetry_preinstalled.stdout_lines[0] }}"
+    ensure_poetry_executable: "{{ ensure_python_command_executable }}"
     cacheable: true
-  when: poetry_preinstalled.rc == 0
-
-- name: Install poetry to local env
-  when: poetry_preinstalled.rc != 0
-  block:
-    - name: Create local venv
-      command: "{{ ensure_pip_virtualenv_command }} {{ ensure_poetry_venv_path }}"
-
-    - name: Install poetry to local venv
-      command: "{{ ensure_poetry_venv_path }}/bin/pip install poetry{{ ensure_poetry_version }}"
-
-    - name: Export installed ensure_poetry_executable path
-      set_fact:
-        ensure_poetry_executable: "{{ ensure_poetry_venv_path }}/bin/poetry"
-        cacheable: true
-
-- name: Output poetry version
-  command: "{{ ensure_poetry_executable }} --version"
-
-- name: Make global symlink
-  when:
-    - ensure_poetry_global_symlink
-    - ensure_poetry_executable != '/usr/local/bin/poetry'
-  file:
-    state: link
-    src: "{{ ensure_poetry_executable }}"
-    dest: /usr/local/bin/poetry
-  become: yes

roles/ensure-pyproject-build/defaults/main.yaml

@@ -1,4 +1,3 @@
 ensure_pyproject_build_global_symlink: false
 ensure_pyproject_build_version: ""
-ensure_pyproject_build_executable: pyproject-build
 ensure_pyproject_build_venv_path: "{{ ansible_user_dir }}/.local/pyproject-build"

roles/ensure-pyproject-build/tasks/main.yaml

@@ -1,44 +1,15 @@
-- name: Install pip
+- name: Check and install pyproject-build if necessary
   include_role:
-    name: ensure-pip
+    name: ensure-python-command
+  vars:
+    ensure_python_command_name: pyproject-build
+    ensure_python_command_package: build
+    ensure_python_command_version: "{{ ensure_pyproject_build_version }}"
+    ensure_python_command_existing: "{{ ensure_pyproject_build_executable | default('') }}"
+    ensure_python_command_venv_path: "{{ ensure_pyproject_build_venv_path }}"
+    ensure_python_command_global_symlink: "{{ ensure_pyproject_build_global_symlink }}"
 
-- name: Check if pyproject-build is installed
-  shell: |
-    command -v {{ ensure_pyproject_build_executable }} {{ ensure_pyproject_build_venv_path }}/bin/pyproject-build || exit 1
-  args:
-    executable: /bin/bash
-  register: pyproject_build_preinstalled
-  failed_when: false
-
-- name: Export preinstalled ensure_pyproject_build_executable
+- name: Export ensure_pyproject_build_executable path
   set_fact:
-    ensure_pyproject_build_executable: "{{ pyproject_build_preinstalled.stdout_lines[0] }}"
+    ensure_pyproject_build_executable: "{{ ensure_python_command_executable }}"
     cacheable: true
-  when: pyproject_build_preinstalled.rc == 0
-
-- name: Install pyproject-build to local env
-  when: pyproject_build_preinstalled.rc != 0
-  block:
-    - name: Create local venv
-      command: "{{ ensure_pip_virtualenv_command }} {{ ensure_pyproject_build_venv_path }}"
-
-    - name: Install pyproject-build to local venv
-      command: "{{ ensure_pyproject_build_venv_path }}/bin/pip install build{{ ensure_pyproject_build_version }}"
-
-    - name: Export installed ensure_pyproject_build_executable path
-      set_fact:
-        ensure_pyproject_build_executable: "{{ ensure_pyproject_build_venv_path }}/bin/pyproject-build"
-        cacheable: true
-
-- name: Output pyproject-build version
-  command: "{{ ensure_pyproject_build_executable }} --version"
-
-- name: Make global symlink
-  when:
-    - ensure_pyproject_build_global_symlink
-    - ensure_pyproject_build_executable != '/usr/local/bin/pyproject-build'
-  file:
-    state: link
-    src: "{{ ensure_pyproject_build_executable }}"
-    dest: /usr/local/bin/pyproject-build
-  become: yes

roles/ensure-python-command/README.rst

@@ -0,0 +1,71 @@
+Ensure a pip-installed command is available
+
+This role checks for the specified command, and if not found, installs
+it via ``pip`` into a virtual environment for the current user.
+
+The minimal required input is the command name. Additionally, you can
+specify a version or a path to a local venv, among other things.
+
+Example:
+
+.. code-block:: yaml
+
+   - role: ensure-python-command
+     vars:
+       ensure_python_command_name: poetry
+       ensure_python_command_version: ==1.8.5  # omit to install latest
+
+In this case, if the ``poetry`` command is not already available, pip
+will install it in a new venv. Either way, after running this role, the
+``ensure_python_command_executable`` variable will hold the full path to
+the command.
+
+**Role Variables**
+
+.. zuul:rolevar:: ensure_python_command_name
+
+   Required. The name of the command to ensure is available.
+
+.. zuul:rolevar:: ensure_python_command_package
+   :default: {{ ensure_python_command_name }}
+
+   The name of the Python package that provides the desired command.
+   Defaults to the command name, since this is usually the case.
+   Set this variable when they differ.
+
+.. zuul:rolevar:: ensure_python_command_version
+   :default: ''
+
+   The version specifier to select the version of the package to install.
+   If omitted, the latest version will be installed.
+
+.. zuul:rolevar:: ensure_python_command_existing
+   :default: ''
+
+   Look for an existing command at this specific path. For example, if your base
+   image pre-installs the command in an out-of-path environment, set this so the
+   role does not attempt to install the command again.
+
+.. zuul:rolevar:: ensure_python_command_venv_path
+   :default: {{ ansible_user_dir }}/.local/{{ ensure_python_command_package }}
+
+   Directory for the Python venv where the package should be installed.
+
+.. zuul:rolevar:: ensure_python_command_global_symlink
+   :default: False
+
+   Install a symlink to the command executable into ``/usr/local/bin/``.
+
+   This can be useful when scripts need to be run that expect to find the
+   command in a more standard location and plumbing through the value
+   of ``ensure_python_command_executable`` would be onerous.
+
+   Setting this requires root access, so should only be done in
+   circumstances where root access is available.
+
+**Output Variables**
+
+.. zuul:rolevar:: ensure_python_command_executable
+
+   The full path to the command executable, whether it was detected or
+   installed by the role.

roles/ensure-python-command/defaults/main.yaml

@@ -0,0 +1,5 @@
+ensure_python_command_global_symlink: false
+ensure_python_command_version: ""
+ensure_python_command_existing: ""
+ensure_python_command_package: "{{ ensure_python_command_name }}"
+ensure_python_command_venv_path: "{{ ansible_user_dir }}/.local/{{ ensure_python_command_package }}"

roles/ensure-python-command/tasks/main.yaml

@@ -0,0 +1,64 @@
+- name: Install pip
+  include_role:
+    name: ensure-pip
+
+- name: Check if the command name is set and non-empty
+  fail:
+    msg: Required variable ensure_python_command_name is not set or empty
+  when: ensure_python_command_name is not defined or not ensure_python_command_name.strip()
+
+# Part 1
+# Check if the command is already available. If not, pip-install it to a local venv.
+# Either way, save the command's full path to the output variable.
+
+- name: Check if {{ ensure_python_command_name }} is available
+  # 1. Full path optionally informed by the role caller
+  # 2. Ansible user $PATH (bare command name)
+  # 3. Local venv (installed from a previous run for this role)
+  shell: |
+    command -v \
+    {{ ensure_python_command_existing }} \
+    {{ ensure_python_command_name | quote }} \
+    {{ ensure_python_command_venv_executable | quote }} \
+    || exit 1
+  args:
+    executable: /bin/bash
+  register: ensure_python_command_preinstalled
+  failed_when: false
+
+- name: Export preinstalled ensure_python_command_executable
+  set_fact:
+    ensure_python_command_executable: "{{ ensure_python_command_preinstalled.stdout_lines[0] }}"
+    cacheable: true
+  when: ensure_python_command_preinstalled.rc == 0
+
+- name: Install {{ ensure_python_command_package }} to local env
+  when: ensure_python_command_preinstalled.rc != 0
+  block:
+    - name: Create local venv
+      command: "{{ ensure_pip_virtualenv_command }} {{ ensure_python_command_venv_path }}"
+
+    - name: Install {{ ensure_python_command_package }} to local venv
+      command: "{{ ensure_python_command_venv_path }}/bin/pip install {{ ensure_python_command_package }}{{ ensure_python_command_version }}"
+
+    - name: Export installed ensure_python_command_executable path
+      set_fact:
+        ensure_python_command_executable: "{{ ensure_python_command_venv_executable }}"
+        cacheable: true
+
+# Part 2
+# Try to show the command's version and maybe also create a symlink to it in /usr/local/bin
+
+- name: Output {{ ensure_python_command_name }} version
+  command: "{{ ensure_python_command_executable }} --version"
+  failed_when: false
+
+- name: Make global symlink
+  when:
+    - ensure_python_command_global_symlink
+    - ensure_python_command_executable != ensure_python_command_global_symlink_path
+  file:
+    state: link
+    src: "{{ ensure_python_command_executable }}"
+    dest: "{{ ensure_python_command_global_symlink_path }}"
+  become: yes

roles/ensure-python-command/vars/main.yaml

@@ -0,0 +1,2 @@
+ensure_python_command_global_symlink_path: /usr/local/bin/{{ ensure_python_command_name }}
+ensure_python_command_venv_executable: "{{ ensure_python_command_venv_path }}/bin/{{ ensure_python_command_name }}"

roles/ensure-twine/defaults/main.yaml

@@ -1,5 +1,4 @@
 ensure_twine_global_symlink: false
 # version 6.1.0 is breaking test-release-openstack CI job
 ensure_twine_version: ">1.12.0,!=6.1.0"
-pypi_twine_executable: twine
 ensure_twine_venv_path: "{{ ansible_user_dir }}/.local/twine"

roles/ensure-twine/tasks/main.yaml

@@ -1,44 +1,14 @@
-- name: Install pip
+- name: Check and install twine if necessary
   include_role:
-    name: ensure-pip
+    name: ensure-python-command
+  vars:
+    ensure_python_command_name: twine
+    ensure_python_command_version: "{{ ensure_twine_version }}"
+    ensure_python_command_existing: "{{ pypi_twine_executable | default('') }}"
+    ensure_python_command_venv_path: "{{ ensure_twine_venv_path }}"
+    ensure_python_command_global_symlink: "{{ ensure_twine_global_symlink }}"
 
-- name: Check if twine is installed
-  shell: |
-    command -v {{ pypi_twine_executable }} {{ ensure_twine_venv_path }}/bin/twine || exit 1
-  args:
-    executable: /bin/bash
-  register: twine_preinstalled
-  failed_when: false
-
-- name: Export preinstalled pypi_twine_executable
+- name: Export pypi_twine_executable path
   set_fact:
-    pypi_twine_executable: "{{ twine_preinstalled.stdout_lines[0] }}"
+    pypi_twine_executable: "{{ ensure_python_command_executable }}"
     cacheable: true
-  when: twine_preinstalled.rc == 0
-
-- name: Install twine to local env
-  when: twine_preinstalled.rc != 0
-  block:
-    - name: Create local venv
-      command: "{{ ensure_pip_virtualenv_command }} {{ ensure_twine_venv_path }}"
-
-    - name: Install twine to local venv
-      command: "{{ ensure_twine_venv_path }}/bin/pip install twine{{ ensure_twine_version }}"
-
-    - name: Export installed pypi_twine_executable path
-      set_fact:
-        pypi_twine_executable: "{{ ensure_twine_venv_path }}/bin/twine"
-        cacheable: true
-
-- name: Output twine version
-  command: "{{ pypi_twine_executable }} --version"
-
-- name: Make global symlink
-  when:
-    - ensure_twine_global_symlink
-    - pypi_twine_executable != '/usr/local/bin/twine'
-  file:
-    state: link
-    src: "{{ pypi_twine_executable }}"
-    dest: /usr/local/bin/twine
-  become: yes

roles/ensure-uv/defaults/main.yaml

@@ -1,4 +1,3 @@
 ensure_uv_global_symlink: false
 ensure_uv_version: ""
-ensure_uv_executable: uv
 ensure_uv_venv_path: "{{ ansible_user_dir }}/.local/uv"

roles/ensure-uv/tasks/main.yaml

@@ -1,44 +1,14 @@
-- name: Install pip
+- name: Check and install uv if necessary
   include_role:
-    name: ensure-pip
+    name: ensure-python-command
+  vars:
+    ensure_python_command_name: uv
+    ensure_python_command_version: "{{ ensure_uv_version }}"
+    ensure_python_command_existing: "{{ ensure_uv_executable | default('') }}"
+    ensure_python_command_venv_path: "{{ ensure_uv_venv_path }}"
+    ensure_python_command_global_symlink: "{{ ensure_uv_global_symlink }}"
 
-- name: Check if uv is installed
-  shell: |
-    command -v {{ ensure_uv_executable }} {{ ensure_uv_venv_path }}/bin/uv || exit 1
-  args:
-    executable: /bin/bash
-  register: uv_preinstalled
-  failed_when: false
-
-- name: Export preinstalled ensure_uv_executable
+- name: Export ensure_uv_executable path
   set_fact:
-    ensure_uv_executable: "{{ uv_preinstalled.stdout_lines[0] }}"
+    ensure_uv_executable: "{{ ensure_python_command_executable }}"
     cacheable: true
-  when: uv_preinstalled.rc == 0
-
-- name: Install uv to local env
-  when: uv_preinstalled.rc != 0
-  block:
-    - name: Create local venv
-      command: "{{ ensure_pip_virtualenv_command }} {{ ensure_uv_venv_path }}"
-
-    - name: Install uv to local venv
-      command: "{{ ensure_uv_venv_path }}/bin/pip install uv{{ ensure_uv_version }}"
-
-    - name: Export installed ensure_uv_executable path
-      set_fact:
-        ensure_uv_executable: "{{ ensure_uv_venv_path }}/bin/uv"
-        cacheable: true
-
-- name: Output uv version
-  command: "{{ ensure_uv_executable }} --version"
-
-- name: Make global symlink
-  when:
-    - ensure_uv_global_symlink
-    - ensure_uv_executable != '/usr/local/bin/uv'
-  file:
-    state: link
-    src: "{{ ensure_uv_executable }}"
-    dest: /usr/local/bin/uv
-  become: yes

zuul-tests.d/python-jobs.yaml

@@ -3,6 +3,7 @@
     description: Test the ensure-nox role
     files:
       - roles/ensure-nox/.*
+      - roles/ensure-python-command/.*
       - test-playbooks/ensure-nox.yaml
     run: test-playbooks/ensure-nox.yaml
     tags: all-platforms
@@ -149,6 +150,7 @@
     description: Test the ensure-poetry role
     files:
       - roles/ensure-poetry/.*
+      - roles/ensure-python-command/.*
       - test-playbooks/ensure-poetry.yaml
     run: test-playbooks/ensure-poetry.yaml
     tags: all-platforms
@@ -218,6 +220,7 @@
     description: Test the ensure-pyproject-build role
     files:
       - roles/ensure-pyproject-build/.*
+      - roles/ensure-python-command/.*
       - test-playbooks/ensure-pyproject-build.yaml
     run: test-playbooks/ensure-pyproject-build.yaml
     tags: all-platforms
@@ -287,6 +290,7 @@
     description: Test the ensure-twine role
     files:
       - roles/ensure-twine/.*
+      - roles/ensure-python-command/.*
       - test-playbooks/ensure-twine.yaml
     run: test-playbooks/ensure-twine.yaml
     tags: all-platforms
@@ -434,6 +438,7 @@
     description: Test the ensure-uv role
     files:
       - roles/ensure-uv/.*
+      - roles/ensure-python-command/.*
       - test-playbooks/ensure-uv.yaml
     run: test-playbooks/ensure-uv.yaml
     tags: all-platforms