opendev/pypi-mirror
Code Issues Proposed changes

Start building wheels in the mirror

Browse Source 
The mirror should build wheels, because they're awesome. However, because
the wheel format does not do fine-grained enough selection on
things like the high likelihood that Ubuntu precise and Ubuntu raring
might have different underlying C libraries for python 2.7, we need to
put our built wheels into a structure that allows them to be used
safely.

lsb_release -r -i is used because Centos lists lsb_release -c as "Final"
which is not particularly helpful. This will wind up with identifying
strings like Ubuntu-12.04 and Centos-6.4 - which should be good enough
for what we need. A call to a shell is used so that the same logic can
be easily duplicated in select-mirror, which is shell, to produce the
appropriate pip.conf file.

Change-Id: Icf360c921dbc357dbeb43386c6e8f51bd5f993ff
This commit is contained in:
Monty Taylor 2013-08-09 19:37:57 -03:00
parent 92a1f3a57c
commit 384c02d41f
2 changed files with 104 additions and 35 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

138
jeepyb/cmd/run_mirror.py
View File

@ -158,11 +158,25 @@ class Mirror(object):
def build_mirror(self, mirror): def build_mirror(self, mirror):
print("Building mirror: %s" % mirror['name']) print("Building mirror: %s" % mirror['name'])
pip_format = ("%s install -U %s --exists-action=w " pip_format = (
"--download-cache=%s --build %s -r %s") "%(pip)s install -U %(extra_args)s --exists-action=w --use-wheel"
venv_format = ("virtualenv --clear --extra-search-dir=%s %s") " --download-cache=%(download_cache)s"
upgrade_format = ("%s install -U --exists-action=w " " --build %(build_dir)s -f %(find_links)s"
"--download-cache=%s --build %s %s") " -r %(requirements_file)s")
venv_format = (
"virtualenv --clear --extra-search-dir=%(extra_search_dir)s"
" %(venv_dir)s")
upgrade_format = (
"%(pip)s install -U --use-wheel --exists-action=w"
" --download-cache=%(download_cache)s --build %(build_dir)s"
" -f %(find_links)s %(requirement)s")
wheel_file_format = (
"%(pip)s wheel --download-cache=%(download_cache)s"
" --use-wheel --wheel-dir %(wheel_dir)s -f %(find_links)s"
" -r %(requirements_file)s")
wheel_format = (
"%(pip)s wheel --download-cache=%(download_cache)s"
" -f %(find_links)s --wheel-dir %(wheel_dir)s %(requirement)s")
workdir = tempfile.mkdtemp() workdir = tempfile.mkdtemp()
reqs = os.path.join(workdir, "reqs") reqs = os.path.join(workdir, "reqs")
@ -174,11 +188,11 @@ class Mirror(object):
'projects') 'projects')
pip_cache_dir = os.path.join(self.config['cache-root'], pip_cache_dir = os.path.join(self.config['cache-root'],
'pip', mirror['name']) 'pip', mirror['name'])
wheelhouse = os.path.join(self.config['cache-root'], "wheelhouse")
if not self.args.noop: if not self.args.noop:
if not os.path.exists(project_cache_dir): for new_dir in (project_cache_dir, pip_cache_dir, wheelhouse):
os.makedirs(project_cache_dir) if not os.path.exists(new_dir):
if not os.path.exists(pip_cache_dir): os.makedirs(new_dir)
os.makedirs(pip_cache_dir)
for project in mirror['projects']: for project in mirror['projects']:
print("Updating repository: %s" % project) print("Updating repository: %s" % project)
@ -218,17 +232,31 @@ class Mirror(object):
if os.path.exists(requires_file): if os.path.exists(requires_file):
reqlist.append(requires_file) reqlist.append(requires_file)
if reqlist: if reqlist:
out = self.run_command(venv_format % out = self.run_command(
(pip_cache_dir, venv)) venv_format % dict(
out = self.run_command(upgrade_format % extra_search_dir=pip_cache_dir, venv_dir=venv))
(pip, pip_cache_dir, # Need to do these separately. If you attempt to upgrade
build, "setuptools")) # setuptools with something else, you can get into a
out = self.run_command(upgrade_format % # situation where distribute has been upgraded, but pip
(pip, pip_cache_dir, # attemps to install something else before installing
build, "pip")) # the setuptools replacement. The safest thing is to
out = self.run_command(upgrade_format % # simply upgrade setuptools by itself.
(pip, pip_cache_dir, # There is a current theory that pip 1.4 may solve
build, "virtualenv")) # the setuptools upgrade issues, so upgrading that first
# is a good idea
for requirement in [
"pip", "setuptools", "wheel", "virtualenv"]:
self.run_command(
upgrade_format % dict(
pip=pip, download_cache=pip_cache_dir,
build_dir=build, find_links=wheelhouse,
requirement=requirement))
for requirement in ["pip", "setuptools", "virtualenv"]:
self.run_command(
wheel_format % dict(
pip=pip, download_cache=pip_cache_dir,
find_links=wheelhouse, wheel_dir=wheelhouse,
requirement=requirement))
if os.path.exists(build): if os.path.exists(build):
shutil.rmtree(build) shutil.rmtree(build)
new_reqs = self.process_http_requirements(reqlist, new_reqs = self.process_http_requirements(reqlist,
@ -237,9 +265,16 @@ class Mirror(object):
(reqfp, reqfn) = tempfile.mkstemp() (reqfp, reqfn) = tempfile.mkstemp()
os.write(reqfp, '\n'.join(new_reqs)) os.write(reqfp, '\n'.join(new_reqs))
os.close(reqfp) os.close(reqfp)
out = self.run_command(pip_format % out = self.run_command(
(pip, "", pip_cache_dir, wheel_file_format % dict(
build, reqfn)) pip=pip, download_cache=pip_cache_dir,
find_links=wheelhouse, wheel_dir=wheelhouse,
requirements_file=reqfn))
out = self.run_command(
pip_format % dict(
pip=pip, extra_args="",
download_cache=pip_cache_dir, build_dir=build,
find_links=wheelhouse, requirements_file=reqfn))
if "\nSuccessfully installed " not in out: if "\nSuccessfully installed " not in out:
sys.stderr.write("Installing pip requires for %s:%s " sys.stderr.write("Installing pip requires for %s:%s "
"failed.\n%s\n" % "failed.\n%s\n" %
@ -256,13 +291,20 @@ class Mirror(object):
for r in requires: for r in requires:
reqfd.write(r + "\n") reqfd.write(r + "\n")
reqfd.close() reqfd.close()
out = self.run_command(venv_format % out = self.run_command(venv_format % dict(
(pip_cache_dir, venv)) extra_search_dir=pip_cache_dir, venv_dir=venv))
if os.path.exists(build): if os.path.exists(build):
shutil.rmtree(build) shutil.rmtree(build)
out = self.run_command(pip_format % out = self.run_command(
(pip, "--no-install", wheel_file_format % dict(
pip_cache_dir, build, reqs)) pip=pip, download_cache=pip_cache_dir,
find_links=wheelhouse, wheel_dir=wheelhouse,
requirements_file=reqs))
out = self.run_command(
pip_format % dict(
pip=pip, extra_args="--no-install",
download_cache=pip_cache_dir, build_dir=build,
find_links=wheelhouse, requirements_file=reqs))
if "\nSuccessfully downloaded " not in out: if "\nSuccessfully downloaded " not in out:
sys.stderr.write("Downloading pip requires for " sys.stderr.write("Downloading pip requires for "
"%s:%s failed.\n%s\n" % "%s:%s failed.\n%s\n" %
@ -273,23 +315,27 @@ class Mirror(object):
print("no requirements") print("no requirements")
shutil.rmtree(workdir) shutil.rmtree(workdir)
def _get_distro(self):
out = self.run_command('lsb_release -i -r -s')
return out.strip().replace('\n', '-')
def process_cache(self, mirror): def process_cache(self, mirror):
if self.args.noop: if self.args.noop:
return return
self._write_tarball_mirror(mirror)
self._write_wheel_mirror(mirror)
def _write_tarball_mirror(self, mirror):
pip_cache_dir = os.path.join(self.config['cache-root'], pip_cache_dir = os.path.join(self.config['cache-root'],
'pip', mirror['name']) 'pip', mirror['name'])
destination_mirror = mirror['output'] destination_mirror = mirror['output']
PACKAGE_VERSION_RE = re.compile(r'(.*)-[0-9]') PACKAGE_VERSION_RE = re.compile(r'(.*)-[0-9]')
full_html_line = "<a href='{dir}/{name}'>{name}</a><br />\n"
packages = {} packages = {}
package_count = 0 package_count = 0
if not os.path.exists(destination_mirror):
os.makedirs(destination_mirror)
for filename in os.listdir(pip_cache_dir): for filename in os.listdir(pip_cache_dir):
if filename.endswith('content-type'): if filename.endswith('content-type'):
continue continue
@ -304,9 +350,32 @@ class Mirror(object):
package_name = name_match.group(1) package_name = name_match.group(1)
version_list = packages.get(package_name, {}) version_list = packages.get(package_name, {})
version_list[tarball] = filename version_list[tarball] = os.path.join(pip_cache_dir, filename)
packages[package_name] = version_list packages[package_name] = version_list
package_count = package_count + 1 package_count = package_count + 1
self._write_mirror(destination_mirror, packages, package_count)
def _write_wheel_mirror(self, mirror):
distro = self._get_distro()
wheelhouse = os.path.join(self.config['cache-root'], "wheelhouse")
wheel_destination_mirror = os.path.join(mirror['output'], distro)
packages = {}
package_count = 0
for filename in os.listdir(wheelhouse):
package_name = filename.split('-')[0].replace('_', '-')
version_list = packages.get(package_name, {})
version_list[filename] = os.path.join(wheelhouse, filename)
packages[package_name] = version_list
package_count = package_count + 1
self._write_mirror(wheel_destination_mirror, packages, package_count)
def _write_mirror(self, destination_mirror, packages, package_count):
full_html_line = "<a href='{dir}/{name}'>{name}</a><br />\n"
if not os.path.exists(destination_mirror):
os.makedirs(destination_mirror)
full_html = open(os.path.join(destination_mirror, ".full.html"), 'w') full_html = open(os.path.join(destination_mirror, ".full.html"), 'w')
simple_html = open(os.path.join(destination_mirror, ".index.html"), simple_html = open(os.path.join(destination_mirror, ".index.html"),
@ -330,8 +399,7 @@ class Mirror(object):
index.write("""<html><head> index.write("""<html><head>
<title>%s &ndash; PyPI Mirror</title> <title>%s &ndash; PyPI Mirror</title>
</head><body>\n""" % package_name) </head><body>\n""" % package_name)
for tarball, filename in versions.items(): for tarball, source_path in versions.items():
source_path = os.path.join(pip_cache_dir, filename)
destination_path = os.path.join(destination_dir, destination_path = os.path.join(destination_dir,
tarball) tarball)
dot_destination_path = os.path.join(destination_dir, dot_destination_path = os.path.join(destination_dir,

1
requirements.txt
View File

@ -6,3 +6,4 @@ pyyaml
pkginfo pkginfo
PyRSS2Gen PyRSS2Gen
python-swiftclient python-swiftclient
pip>=1.4