opendev/system-config
Code Issues Proposed changes

Switch python-builder/python-base to pip wheel

Browse Source 
A recent change in pip wheel cache behavior had upstream pip indicating
that we really should be using pip wheel instead. The reason we weren't
using pip wheel appears to be that we wanted to infer what top level
wheel to install via contents of a dir separate from our wheel output
dir/wheel cache. Using pip wheel implies everything gets flattened into
one location. We deal with this by having the build tool write all of
the top level wheels we care about into a separate location. Later we
can install all of the top level wheels while pointing find links at the
larger set of deps in the dir created by pip wheel.

Change-Id: Id9c674c1ec6fe5e72534549082e3adda9e286fd5
This commit is contained in:
Clark Boylan 2022-10-20 07:50:57 -07:00
parent 4884d3b858
commit 521df0eda0
2 changed files with 26 additions and 22 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

37
docker/python-builder/scripts/assemble
View File

@ -48,29 +48,30 @@ function install_wheels {
# that sdist which is exactly what we want. This triggers code
# generation steps such as are found in zuul, since the sequencing
# otherwise happens in a way that makes wheel content copying unhappy.
# `pip wheel` isn't used here because it puts all of the output
# in the output dir and not the wheel cache, so it's not
# possible to tell what is the wheel for the project and
# what is the wheel cache.
/tmp/venv/bin/python3 -m build -o /output/wheels ./
#
# This is a separate step before we use pip wheel so that we can write
# the top level wheels to install into a different location than all of
# our dependencies.
/tmp/venv/bin/python3 -m build -o /output/toplevel_wheels ./
# Install everything so that the wheel cache is populated with
# transitive depends. If a requirements.txt file exists, install
# it directly so that people can use git url syntax to do things
# like pick up patched but unreleased versions of dependencies.
# Pip wheel everything to build a collection of all wheels for all
# transitive depends. This ensures that the final install on the prod
# images can avoid installing any build depends. If a requirements.txt
# file exists, start there so that people can use git url syntax to do
# things like pick up patched but unreleased versions of dependencies.
# Only do this for the main package (i.e. only write requirements
# once).
if [ -f /tmp/src/requirements.txt ] && [ ! -f /output/requirements.txt ] ; then
/tmp/venv/bin/pip install $CONSTRAINTS --cache-dir=/output/wheels -r /tmp/src/requirements.txt
/tmp/venv/bin/pip wheel $CONSTRAINTS --wheel-dir=/output/wheels -r /tmp/src/requirements.txt
cp /tmp/src/requirements.txt /output/requirements.txt
fi
/tmp/venv/bin/pip install $CONSTRAINTS --cache-dir=/output/wheels /output/wheels/*whl
/tmp/venv/bin/pip wheel $CONSTRAINTS --wheel-dir=/output/wheels /output/toplevel_wheels/*whl
# Install each of the extras so that we collect all possibly
# needed wheels in the wheel cache. get-extras-packages also
# Pip wheel each of the extras so that we collect all possibly
# needed wheels in the wheel output dir. get-extras-packages also
# writes out the req files into /output/$extra/requirements.txt.
for req in $(get-extras-packages) ; do
/tmp/venv/bin/pip install $CONSTRAINTS --cache-dir=/output/wheels "$req"
/tmp/venv/bin/pip wheel $CONSTRAINTS --wheel-dir=/output/wheels "$req"
done
}
@ -98,10 +99,10 @@ if [ -f /tmp/src/upper-constraints.txt ] ; then
CONSTRAINTS="-c /tmp/src/upper-constraints.txt"
fi
# If we got a list of packages, install them, otherwise install the
# main package.
# If we got a list of packages, get wheels for them, otherwise do
# this for the main package.
if [[ $PACKAGES ]] ; then
/tmp/venv/bin/pip install $CONSTRAINTS --cache-dir=/output/wheels $PACKAGES
/tmp/venv/bin/pip wheel $CONSTRAINTS --wheel-dir=/output/wheels $PACKAGES
for package in $PACKAGES ; do
echo "$package" >> /output/packages.txt
done
@ -113,7 +114,7 @@ else
install_wheels
fi
# go through ZUUL_SIBLINGS, if any, and build those wheels too
# Go through ZUUL_SIBLINGS, if any, and build those wheels too
for sibling in ${ZUUL_SIBLINGS:-}; do
pushd .zuul-siblings/${sibling}
install_wheels

11
docker/python-builder/scripts/install-from-bindep
View File

@ -29,7 +29,10 @@ fi
# to do things like pick up patched but unreleased versions
# of dependencies.
if [ -f /output/requirements.txt ] ; then
pip install $CONSTRAINTS --cache-dir=/output/wheels -r /output/requirements.txt
# --find-links will point pip at all of the wheels the assemble script
# downloaded or built. --no-index ensures that pip will only refer to
# the local set of wheels when looking for packages to install.
pip install $CONSTRAINTS --no-index --find-links=/output/wheels -r /output/requirements.txt
fi
# Add any requested extras to the list of things to install
@ -41,10 +44,10 @@ done
if [ -f /output/packages.txt ] ; then
# If a package list was passed to assemble, install that in the final
# image.
pip install $CONSTRAINTS --cache-dir=/output/wheels -r /output/packages.txt $EXTRAS
pip install $CONSTRAINTS --no-index --find-links=/output/wheels -r /output/packages.txt $EXTRAS
else
# Install the wheels.
pip install $CONSTRAINTS --cache-dir=/output/wheels /output/wheels/*.whl $EXTRAS
# Install the top level wheels.
pip install $CONSTRAINTS --no-index --find-links=/output/wheels /output/toplevel_wheels/*.whl $EXTRAS
fi
# clean up after ourselves