From f4ed0c3120852c2a59af8ebc4fdf657672787d67 Mon Sep 17 00:00:00 2001 From: Monty Taylor Date: Sun, 23 Jun 2013 14:43:02 -0400 Subject: [PATCH] Be more agressive trying to install requirements. When -r requirements.txt is used, either then entire file works or the entire file doesn't. This means that if there is a hiccup with any of the files, we lose the entire run instead of just the one file. By iterating over the list and installing as many as we can, then processing the freeze, we get as many of them as we can. This is especially helpful when combined with the upcoming pbr/jeepyb gate, where the lack of a requirement in the gate will show up when the thing that needs it can't install it. If we get everything else, the error message in that case will be much more clear as to why. Change-Id: I27606b92a8be2605fa30362aaab51e65a21770bc --- jeepyb/cmd/run_mirror.py | 64 +++++++++++++++++++--------------------- 1 file changed, 31 insertions(+), 33 deletions(-) diff --git a/jeepyb/cmd/run_mirror.py b/jeepyb/cmd/run_mirror.py index 8e84479..13bdd83 100644 --- a/jeepyb/cmd/run_mirror.py +++ b/jeepyb/cmd/run_mirror.py @@ -158,7 +158,7 @@ class Mirror(object): def build_mirror(self, mirror): print("Building mirror: %s" % mirror['name']) pip_format = ("%s install -M -U %s --exists-action=w " - "--download-cache=%s --build %s -r %s") + "--download-cache=%s --build %s %s") venv_format = ("virtualenv --clear --extra-search-dir=%s %s") workdir = tempfile.mkdtemp() @@ -219,41 +219,39 @@ class Mirror(object): new_reqs = self.process_http_requirements(reqlist, pip_cache_dir, pip) - (reqfp, reqfn) = tempfile.mkstemp() - os.write(reqfp, '\n'.join(new_reqs)) - os.close(reqfp) + + for req in new_reqs: + out = self.run_command(pip_format % + (pip, "", pip_cache_dir, + build, req)) + if "\nSuccessfully installed " not in out: + sys.stderr.write( + "Installing pip requires for %s:%s " + "failed.\n%s\n" % (project, branch, out)) + print("pip install did not indicate success") + freeze = self.run_command("%s freeze -l" % pip) + requires = self.find_pkg_info(build) + reqfd = open(reqs, "w") + for line in freeze.split("\n"): + if line.startswith("-e ") or ( + "==" in line and " " not in line): + requires.add(line) + for r in requires: + reqfd.write(r + "\n") + reqfd.close() + out = self.run_command(venv_format % + (pip_cache_dir, venv)) + if os.path.exists(build): + shutil.rmtree(build) out = self.run_command(pip_format % - (pip, "", pip_cache_dir, - build, reqfn)) - if "\nSuccessfully installed " not in out: - sys.stderr.write("Installing pip requires for %s:%s " - "failed.\n%s\n" % + (pip, "--no-install", + pip_cache_dir, build, reqs)) + if "\nSuccessfully downloaded " not in out: + sys.stderr.write("Downloading pip requires for " + "%s:%s failed.\n%s\n" % (project, branch, out)) print("pip install did not indicate success") - else: - freeze = self.run_command("%s freeze -l" % pip) - requires = self.find_pkg_info(build) - reqfd = open(reqs, "w") - for line in freeze.split("\n"): - if line.startswith("-e ") or ( - "==" in line and " " not in line): - requires.add(line) - for r in requires: - reqfd.write(r + "\n") - reqfd.close() - out = self.run_command(venv_format % - (pip_cache_dir, venv)) - if os.path.exists(build): - shutil.rmtree(build) - out = self.run_command(pip_format % - (pip, "--no-install", - pip_cache_dir, build, reqs)) - if "\nSuccessfully downloaded " not in out: - sys.stderr.write("Downloading pip requires for " - "%s:%s failed.\n%s\n" % - (project, branch, out)) - print("pip install did not indicate success") - print("cached:\n%s" % freeze) + print("cached:\n%s" % freeze) else: print("no requirements") shutil.rmtree(workdir)