opendev/pypi-mirror
Code Issues Proposed changes

Be more agressive trying to install requirements.

Browse Source 
When -r requirements.txt is used, either then entire file works
or the entire file doesn't. This means that if there is a hiccup
with any of the files, we lose the entire run instead of just
the one file.

By iterating over the list and installing as many as we can, then
processing the freeze, we get as many of them as we can.

This is especially helpful when combined with the upcoming pbr/jeepyb
gate, where the lack of a requirement in the gate will show up when
the thing that needs it can't install it. If we get everything else,
the error message in that case will be much more clear as to why.

Change-Id: I27606b92a8be2605fa30362aaab51e65a21770bc
This commit is contained in:
Monty Taylor 2013-06-23 14:43:02 -04:00
parent e8d897d329
commit f4ed0c3120
1 changed files with 31 additions and 33 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

64
jeepyb/cmd/run_mirror.py
View File

@ -158,7 +158,7 @@ class Mirror(object):
def build_mirror(self, mirror):
print("Building mirror: %s" % mirror['name'])
pip_format = ("%s install -M -U %s --exists-action=w "
"--download-cache=%s --build %s -r %s")
"--download-cache=%s --build %s %s")
venv_format = ("virtualenv --clear --extra-search-dir=%s %s")
workdir = tempfile.mkdtemp()
@ -219,41 +219,39 @@ class Mirror(object):
new_reqs = self.process_http_requirements(reqlist,
pip_cache_dir,
pip)
(reqfp, reqfn) = tempfile.mkstemp()
os.write(reqfp, '\n'.join(new_reqs))
os.close(reqfp)
for req in new_reqs:
out = self.run_command(pip_format %
(pip, "", pip_cache_dir,
build, req))
if "\nSuccessfully installed " not in out:
sys.stderr.write(
"Installing pip requires for %s:%s "
"failed.\n%s\n" % (project, branch, out))
print("pip install did not indicate success")
freeze = self.run_command("%s freeze -l" % pip)
requires = self.find_pkg_info(build)
reqfd = open(reqs, "w")
for line in freeze.split("\n"):
if line.startswith("-e ") or (
"==" in line and " " not in line):
requires.add(line)
for r in requires:
reqfd.write(r + "\n")
reqfd.close()
out = self.run_command(venv_format %
(pip_cache_dir, venv))
if os.path.exists(build):
shutil.rmtree(build)
out = self.run_command(pip_format %
(pip, "", pip_cache_dir,
build, reqfn))
if "\nSuccessfully installed " not in out:
sys.stderr.write("Installing pip requires for %s:%s "
"failed.\n%s\n" %
(pip, "--no-install",
pip_cache_dir, build, reqs))
if "\nSuccessfully downloaded " not in out:
sys.stderr.write("Downloading pip requires for "
"%s:%s failed.\n%s\n" %
(project, branch, out))
print("pip install did not indicate success")
else:
freeze = self.run_command("%s freeze -l" % pip)
requires = self.find_pkg_info(build)
reqfd = open(reqs, "w")
for line in freeze.split("\n"):
if line.startswith("-e ") or (
"==" in line and " " not in line):
requires.add(line)
for r in requires:
reqfd.write(r + "\n")
reqfd.close()
out = self.run_command(venv_format %
(pip_cache_dir, venv))
if os.path.exists(build):
shutil.rmtree(build)
out = self.run_command(pip_format %
(pip, "--no-install",
pip_cache_dir, build, reqs))
if "\nSuccessfully downloaded " not in out:
sys.stderr.write("Downloading pip requires for "
"%s:%s failed.\n%s\n" %
(project, branch, out))
print("pip install did not indicate success")
print("cached:\n%s" % freeze)
print("cached:\n%s" % freeze)
else:
print("no requirements")
shutil.rmtree(workdir)