From 8aa8086ab21de38b255238012a3b536ef05954fd Mon Sep 17 00:00:00 2001 From: Dmitry Tantsur Date: Wed, 3 Jun 2020 11:12:26 +0200 Subject: [PATCH] Switch to using virtual envs by default They're already used in the CI, and chances are high that non-venv installation is broken on at least some supported systems. Change-Id: Ie3c93a0a1537658eaccf69da3f64d0fc43f292d1 --- doc/source/install/index.rst | 51 ++++++++++++++----- doc/source/install/virtualenv.rst | 15 +----- playbooks/ci/run.yaml | 3 +- .../defaults/main.yml | 2 +- .../bifrost-create-vm-nodes/defaults/main.yml | 2 +- .../bifrost-ironic-install/defaults/main.yml | 2 +- .../defaults/main.yml | 2 +- .../notes/venv-default-895f7b633803297f.yaml | 9 ++++ scripts/install-deps.sh | 5 ++ scripts/test-bifrost.sh | 10 ++-- zuul.d/bifrost-jobs.yaml | 8 --- 11 files changed, 60 insertions(+), 49 deletions(-) create mode 100644 releasenotes/notes/venv-default-895f7b633803297f.yaml diff --git a/doc/source/install/index.rst b/doc/source/install/index.rst index e07f80861..cbc3ee841 100644 --- a/doc/source/install/index.rst +++ b/doc/source/install/index.rst @@ -106,14 +106,12 @@ Installation of Ansible Installation of Ansible can take place using the provided environment setup script located at ``scripts/env-setup.sh`` which is present in the bifrost repository. This may also be used if you already have ansible, as it will -install ansible and various dependencies to ``~/.local`` in order to avoid -overwriting or conflicting with a system-wide Ansible installation. +install ansible and various dependencies to a virtual environment in order +to avoid overwriting or conflicting with a system-wide Ansible installation. -If you use ``env-setup.sh``, ansible will be installed along -with its missing Python dependencies into user's ``~/.local`` directory. - -Alternatively, if you have a working Ansible installation, -under normal circumstances the installation playbook can be executed. +Alternatively, if you have a working Ansible installation, under normal +circumstances the installation playbook can be executed, but you will need +to configure the `Virtual environment`_. .. note:: All testing takes place utilizing the ``scripts/env-setup.sh`` script. Please feel free to submit @@ -121,6 +119,31 @@ under normal circumstances the installation playbook can be executed. to OpenStack Gerrit for any issues encountered if you choose to directly invoke the playbooks without using ``env-setup.sh``. +Virtual environment +------------------- + +To avoid conflicts between Python packages installed from source and system +packages, Bifrost defaults to installing everything to a virtual environment. +``scripts/env-setup.sh`` will automatically create a virtual environment in +``/opt/stack/bifrost`` if it does not exist. + +If you want to enable system-wide installing, set ``ENABLE_VENV`` to ``false`` +before calling ``env-setup.sh``:: + + export ENABLE_VENV=false + +.. warning:: + This is not recommended and not tested. Future versions of Bifrost may + remove support of running outside of a virtual environment. + +If you want to relocate the virtual environment, export the ``VENV`` variable +before calling ``env-setup.sh``:: + + export VENV=/path/to/my/venv + +If you're using the ansible playbooks directly (without the helper scripts), +set the ``enable_venv`` and ``bifrost_venv_dir`` variables accordingly. + Pre-installation settings ------------------------- @@ -175,15 +198,11 @@ Dependencies In order to really get started, you must install dependencies. -If you used the ``env-setup.sh`` environment setup script:: +The ``env-setup.sh`` script automatically invokes ``install-deps.sh`` and +creates a virtual environment for you:: bash ./scripts/env-setup.sh - export PATH=${HOME}/.local/bin:${PATH} - cd playbooks - -Otherwise:: - - pip install -r requirements.txt + source /opt/stack/bifrost/bin/activate cd playbooks Once the dependencies are in-place, you can execute the ansible playbook to @@ -262,4 +281,8 @@ Advanced Topics keystone offline-install + +.. toctree:: + :hidden: + virtualenv diff --git a/doc/source/install/virtualenv.rst b/doc/source/install/virtualenv.rst index 8b20e6c70..3c733eee7 100644 --- a/doc/source/install/virtualenv.rst +++ b/doc/source/install/virtualenv.rst @@ -1,17 +1,4 @@ Virtualenv Installation Support =============================== -Bifrost can be used with a Python virtual environment. At present, -this feature is experimental, so it's disabled by default. If you -would like to use a virtual environment, you'll need to modify the -install steps slightly. To set up the virtual environment and install -ansible into it, run ``env-setup.sh`` as follows:: - - export VENV=/opt/stack/bifrost - ./scripts/env-setup.sh - -Then run the install playbook with the following arguments:: - - ansible-playbook -vvvv -i inventory/target install.yaml - -This will install ironic and its dependencies into the virtual environment. +Virtual environments are now used by default, see :doc:`/install/index`. diff --git a/playbooks/ci/run.yaml b/playbooks/ci/run.yaml index 53439ae34..0ae150373 100644 --- a/playbooks/ci/run.yaml +++ b/playbooks/ci/run.yaml @@ -10,6 +10,5 @@ UPPER_CONSTRAINTS_FILE: "{{ ansible_user_dir }}/{{ zuul.projects['opendev.org/openstack/requirements'].src_dir }}/upper-constraints.txt" WORKSPACE: "{{ ansible_user_dir }}/src/opendev.org" USE_DHCP: "{{ use_dhcp | default(false) | bool | lower }}" - USE_VENV: "{{ use_venv | default(true) | bool | lower }}" + ENABLE_VENV: "{{ use_venv | default(true) | bool | lower }}" ZUUL_BRANCH: "{{ zuul.branch }}" - VENV: "/opt/stack/bifrost" diff --git a/playbooks/roles/bifrost-create-dib-image/defaults/main.yml b/playbooks/roles/bifrost-create-dib-image/defaults/main.yml index 8ff8e4724..64b5bf3b8 100644 --- a/playbooks/roles/bifrost-create-dib-image/defaults/main.yml +++ b/playbooks/roles/bifrost-create-dib-image/defaults/main.yml @@ -20,7 +20,7 @@ ipa_git_folder: /opt/stack/ironic-python-agent reqs_git_folder: /opt/stack/requirements ipa_builder_git_folder: /opt/stack/ironic-python-agent-builder # Settings related to installing bifrost in a virtual environment -enable_venv: false +enable_venv: true bifrost_venv_dir: "{{ lookup('env', 'VENV') | default('/opt/stack/bifrost') }}" bifrost_venv_env: VIRTUAL_ENV: "{{ bifrost_venv_dir }}" diff --git a/playbooks/roles/bifrost-create-vm-nodes/defaults/main.yml b/playbooks/roles/bifrost-create-vm-nodes/defaults/main.yml index 6bdd941b1..9ddf773cf 100644 --- a/playbooks/roles/bifrost-create-vm-nodes/defaults/main.yml +++ b/playbooks/roles/bifrost-create-vm-nodes/defaults/main.yml @@ -34,7 +34,7 @@ test_vm_machine: "pc-1.0" # NOTE(pas-ha) not really tested with non-local qemu connections test_vm_libvirt_uri: "{{ lookup('env', 'LIBVIRT_CONNECT_URI') | default('qemu:///system', true) }}" # Settings related to installing bifrost in a virtual environment -enable_venv: false +enable_venv: true bifrost_venv_dir: "{{ lookup('env', 'VENV') | default('/opt/stack/bifrost') }}" bifrost_venv_env: VIRTUAL_ENV: "{{ bifrost_venv_dir }}" diff --git a/playbooks/roles/bifrost-ironic-install/defaults/main.yml b/playbooks/roles/bifrost-ironic-install/defaults/main.yml index 1935bd2a7..d0d4ec3bb 100644 --- a/playbooks/roles/bifrost-ironic-install/defaults/main.yml +++ b/playbooks/roles/bifrost-ironic-install/defaults/main.yml @@ -223,7 +223,7 @@ power_off_after_inspection: false download_ipxe: false # Settings related to installing bifrost in a virtual environment -enable_venv: false +enable_venv: true bifrost_venv_dir: "{{ lookup('env', 'VENV') | default('/opt/stack/bifrost') }}" bifrost_venv_env: VIRTUAL_ENV: "{{ bifrost_venv_dir }}" diff --git a/playbooks/roles/bifrost-keystone-install/defaults/main.yml b/playbooks/roles/bifrost-keystone-install/defaults/main.yml index ff67e518f..08a1217aa 100644 --- a/playbooks/roles/bifrost-keystone-install/defaults/main.yml +++ b/playbooks/roles/bifrost-keystone-install/defaults/main.yml @@ -31,7 +31,7 @@ file_url_port: 8080 http_boot_folder: /httpboot # Settings related to installing bifrost in a virtual environment -enable_venv: false +enable_venv: true bifrost_venv_dir: "{{ lookup('env', 'VENV') | default('/opt/stack/bifrost') }}" bifrost_venv_env: VIRTUAL_ENV: "{{ bifrost_venv_dir }}" diff --git a/releasenotes/notes/venv-default-895f7b633803297f.yaml b/releasenotes/notes/venv-default-895f7b633803297f.yaml new file mode 100644 index 000000000..8382c05e8 --- /dev/null +++ b/releasenotes/notes/venv-default-895f7b633803297f.yaml @@ -0,0 +1,9 @@ +--- +upgrade: + - | + All packages are now installed in a virtual environment in + ``/opt/stack/bifrost`` by default instead of system-wide. +deprecations: + - | + Support for system-wide installation of packages is deprecated, untested + and may be removed in a future release. diff --git a/scripts/install-deps.sh b/scripts/install-deps.sh index 17ab64c44..6f4528b87 100644 --- a/scripts/install-deps.sh +++ b/scripts/install-deps.sh @@ -6,6 +6,11 @@ declare -A PKG_MAP # workaround: for latest bindep to work, it needs to use en_US local export LANG=c +ENABLE_VENV=${ENABLE_VENV:-true} +if [[ "$ENABLE_VENV" != false ]]; then + export VENV=${VENV:-/opt/stack/bifrost} +fi + CHECK_CMD_PKGS=( python3-devel python3 diff --git a/scripts/test-bifrost.sh b/scripts/test-bifrost.sh index 55ee5096b..14a1ee51e 100755 --- a/scripts/test-bifrost.sh +++ b/scripts/test-bifrost.sh @@ -7,11 +7,11 @@ SCRIPT_HOME="$(cd "$(dirname "$0")" && pwd)" BIFROST_HOME=$SCRIPT_HOME/.. ANSIBLE_INSTALL_ROOT=${ANSIBLE_INSTALL_ROOT:-/opt/stack} USE_DHCP="${USE_DHCP:-false}" -USE_VENV="${USE_VENV:-true}" BUILD_IMAGE="${BUILD_IMAGE:-false}" BAREMETAL_DATA_FILE=${BAREMETAL_DATA_FILE:-'/tmp/baremetal.json'} ENABLE_KEYSTONE="${ENABLE_KEYSTONE:-false}" ZUUL_BRANCH=${ZUUL_BRANCH:-} +ENABLE_VENV=${ENABLE_VENV:-true} # Set defaults for ansible command-line options to drive the different # tests. @@ -42,7 +42,6 @@ PROVISION_WAIT_TIMEOUT=${PROVISION_WAIT_TIMEOUT:-900} NOAUTH_MODE=true CLOUD_CONFIG="" WAIT_FOR_DEPLOY=true -ENABLE_VENV=true # Get OS information source /etc/os-release || source /usr/lib/os-release @@ -54,19 +53,16 @@ if [ "$ZUUL_BRANCH" != "" ]; then VM_SETUP_EXTRA="-e test_vm_storage_pool_path=/opt/libvirt/images" fi -if [ ${USE_VENV} = "true" ]; then - export VENV=/opt/stack/bifrost - $SCRIPT_HOME/env-setup.sh +source $SCRIPT_HOME/env-setup.sh +if [ ${ENABLE_VENV} = "true" ]; then # Note(cinerama): activate is not compatible with "set -u"; # disable it just for this line. set +u source ${VENV}/bin/activate set -u ANSIBLE=${VENV}/bin/ansible-playbook - ENABLE_VENV="true" ANSIBLE_PYTHON_INTERP=${VENV}/bin/python3 else - $SCRIPT_HOME/env-setup.sh ANSIBLE=${HOME}/.local/bin/ansible-playbook ANSIBLE_PYTHON_INTERP=$(which python3) fi diff --git a/zuul.d/bifrost-jobs.yaml b/zuul.d/bifrost-jobs.yaml index 3e43c2ec2..d3479b0df 100644 --- a/zuul.d/bifrost-jobs.yaml +++ b/zuul.d/bifrost-jobs.yaml @@ -55,8 +55,6 @@ name: bifrost-integration-dhcp-opensuse-15 parent: bifrost-integration-dhcp nodeset: opensuse-15 - vars: - use_venv: true - job: name: bifrost-integration-dhcp-debian-buster @@ -84,8 +82,6 @@ name: bifrost-integration-dibipa-debian-opensuse-15 parent: bifrost-integration-dibipa-debian nodeset: opensuse-15 - vars: - use_venv: true - job: name: bifrost-integration-tinyipa @@ -130,12 +126,8 @@ name: bifrost-integration-tinyipa-opensuse-15 parent: bifrost-integration-tinyipa nodeset: opensuse-15 - vars: - use_venv: true - job: name: bifrost-integration-tinyipa-fedora-latest parent: bifrost-integration-tinyipa nodeset: fedora-latest - vars: - use_venv: true