6d976477df
When the user gives a folder to the lookup, it's expected to work, whatever the content of the folder is. In the Zuul v3 case, we are symlinking the role that are checked out as part of the Zuul dependency resolution, for example when Depends-On is used. If we don't follow symlinks, those checkout roles aren't part of the repo build process, and won't be build. Change-Id: Ic09f6a9ee28e3f7919226f22b803ee1a1b8ad53d
764 lines
26 KiB
Python
764 lines
26 KiB
Python
# Copyright 2014, Rackspace US, Inc.
|
|
#
|
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
# you may not use this file except in compliance with the License.
|
|
# You may obtain a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
# See the License for the specific language governing permissions and
|
|
# limitations under the License.
|
|
#
|
|
# (c) 2014, Kevin Carter <kevin.carter@rackspace.com>
|
|
|
|
import os
|
|
import re
|
|
import traceback
|
|
import yaml
|
|
|
|
from ansible.errors import AnsibleError
|
|
from ansible.plugins.lookup import LookupBase
|
|
|
|
|
|
try:
|
|
basestring
|
|
except NameError:
|
|
basestring = str
|
|
|
|
|
|
# Used to keep track of git package parts as various files are processed
|
|
GIT_PACKAGE_DEFAULT_PARTS = dict()
|
|
|
|
|
|
# Role based package indexes
|
|
ROLE_DISTRO_BREAKOUT_PACKAGES = dict()
|
|
ROLE_BREAKOUT_REQUIREMENTS = dict()
|
|
ROLE_PACKAGES = dict()
|
|
ROLE_REQUIREMENTS = dict()
|
|
|
|
|
|
REQUIREMENTS_FILE_TYPES = [
|
|
'test-requirements.txt',
|
|
'dev-requirements.txt',
|
|
'requirements.txt',
|
|
'global-requirements.txt',
|
|
'global-requirement-pins.txt'
|
|
]
|
|
|
|
|
|
# List of variable names that could be used within the yaml files that
|
|
# represent lists of python packages.
|
|
BUILT_IN_PIP_PACKAGE_VARS = [
|
|
'service_pip_dependencies',
|
|
'pip_common_packages',
|
|
'pip_container_packages',
|
|
'pip_packages'
|
|
]
|
|
|
|
BUILT_IN_DISTRO_PACKAGE_VARS = [
|
|
'distro_packages',
|
|
'apt_packages',
|
|
'yum_packages'
|
|
]
|
|
|
|
|
|
PACKAGE_MAPPING = {
|
|
'packages': set(),
|
|
'remote_packages': set(),
|
|
'remote_package_parts': list(),
|
|
'role_packages': dict(),
|
|
'role_project_groups': dict(),
|
|
'distro_packages': set()
|
|
}
|
|
|
|
|
|
def map_base_and_remote_packages(package, package_map):
|
|
"""Determine whether a package is a base package or a remote package
|
|
and add to the appropriate set.
|
|
|
|
:type package: ``str``
|
|
:type package_map: ``dict``
|
|
"""
|
|
def check_for_ignore(p):
|
|
p_parts = GIT_PACKAGE_DEFAULT_PARTS.get(p)
|
|
if p_parts:
|
|
fragments = p_parts.get('fragments', '') or ''
|
|
if 'ignorerequirements=True' not in fragments:
|
|
package_map['packages'].add(p)
|
|
else:
|
|
package_map['packages'].add(p)
|
|
|
|
if package.startswith(('http:', 'https:', 'git+')):
|
|
if '@' not in package:
|
|
check_for_ignore(p=package)
|
|
else:
|
|
git_parts = git_pip_link_parse(package)
|
|
package_name = git_parts[-2]
|
|
if not package_name:
|
|
package_name = git_pip_link_parse(package)[0]
|
|
|
|
for rpkg in list(package_map['remote_packages']):
|
|
rpkg_name = git_pip_link_parse(rpkg)[-2]
|
|
if not rpkg_name:
|
|
rpkg_name = git_pip_link_parse(package)[0]
|
|
|
|
if rpkg_name == package_name:
|
|
package_map['remote_packages'].remove(rpkg)
|
|
package_map['remote_packages'].add(package)
|
|
break
|
|
else:
|
|
package_map['remote_packages'].add(package)
|
|
else:
|
|
check_for_ignore(p=package)
|
|
|
|
|
|
def parse_remote_package_parts(package_map):
|
|
"""Parse parts of each remote package and add them to
|
|
the remote_package_parts list.
|
|
|
|
:type package_map: ``dict``
|
|
"""
|
|
keys = [
|
|
'name',
|
|
'version',
|
|
'fragment',
|
|
'url',
|
|
'original',
|
|
'egg_name',
|
|
'project_group'
|
|
]
|
|
remote_pkg_parts = [
|
|
dict(
|
|
zip(
|
|
keys, git_pip_link_parse(i)
|
|
)
|
|
) for i in package_map['remote_packages']
|
|
]
|
|
package_map['remote_package_parts'].extend(remote_pkg_parts)
|
|
package_map['remote_package_parts'] = list(
|
|
dict(
|
|
(i['name'], i)
|
|
for i in package_map['remote_package_parts']
|
|
).values()
|
|
)
|
|
|
|
|
|
def map_role_packages(package_map):
|
|
"""Add and sort packages belonging to a role to the role_packages dict.
|
|
|
|
:type package_map: ``dict``
|
|
"""
|
|
for k, v in ROLE_PACKAGES.items():
|
|
role_pkgs = package_map['role_packages'][k] = list()
|
|
package_map['role_project_groups'][k] = v.pop('project_group', 'all')
|
|
for pkg_list in v.values():
|
|
role_pkgs.extend(pkg_list)
|
|
else:
|
|
package_map['role_packages'][k] = sorted(set(role_pkgs))
|
|
|
|
|
|
def map_base_package_details(package_map):
|
|
"""Parse package version and marker requirements and add to the
|
|
base packages set.
|
|
|
|
:type package_map: ``dict``
|
|
"""
|
|
check_pkgs = dict()
|
|
base_packages = sorted(list(package_map['packages']))
|
|
for pkg in base_packages:
|
|
name, versions, markers = _pip_requirement_split(pkg)
|
|
if versions and markers:
|
|
versions = '%s;%s' % (versions, markers)
|
|
elif not versions and markers:
|
|
versions = ';%s' % markers
|
|
|
|
if name in check_pkgs:
|
|
if versions and not check_pkgs[name]:
|
|
check_pkgs[name] = versions
|
|
else:
|
|
check_pkgs[name] = versions
|
|
else:
|
|
return_pkgs = list()
|
|
for k, v in check_pkgs.items():
|
|
if v:
|
|
return_pkgs.append('%s%s' % (k, v))
|
|
else:
|
|
return_pkgs.append(k)
|
|
package_map['packages'] = set(return_pkgs)
|
|
|
|
|
|
def git_pip_link_parse(repo):
|
|
"""Return a tuple containing the parts of a git repository.
|
|
|
|
Example parsing a standard git repo:
|
|
>>> git_pip_link_parse('git+https://github.com/username/repo-name@tag')
|
|
('repo-name',
|
|
'tag',
|
|
None,
|
|
'https://github.com/username/repo',
|
|
'git+https://github.com/username/repo@tag',
|
|
'repo_name')
|
|
|
|
Example parsing a git repo that uses an installable from a subdirectory:
|
|
>>> git_pip_link_parse(
|
|
... 'git+https://github.com/username/repo@tag#egg=plugin.name'
|
|
... '&subdirectory=remote_path/plugin.name'
|
|
... )
|
|
('plugin.name',
|
|
'tag',
|
|
'remote_path/plugin.name',
|
|
'https://github.com/username/repo',
|
|
'git+https://github.com/username/repo@tag#egg=plugin.name&'
|
|
'subdirectory=remote_path/plugin.name',
|
|
'plugin.name')
|
|
|
|
:param repo: git repo string to parse.
|
|
:type repo: ``str``
|
|
:returns: ``tuple``
|
|
"""'meta'
|
|
|
|
def _meta_return(meta_data, item):
|
|
"""Return the value of an item in meta data."""
|
|
|
|
return meta_data.lstrip('#').split('%s=' % item)[-1].split('&')[0]
|
|
|
|
_git_url = repo.split('+')
|
|
if len(_git_url) >= 2:
|
|
_git_url = _git_url[1]
|
|
else:
|
|
_git_url = _git_url[0]
|
|
|
|
git_branch_sha = _git_url.split('@')
|
|
if len(git_branch_sha) > 2:
|
|
branch = git_branch_sha.pop()
|
|
url = '@'.join(git_branch_sha)
|
|
elif len(git_branch_sha) > 1:
|
|
url, branch = git_branch_sha
|
|
else:
|
|
url = git_branch_sha[0]
|
|
branch = 'master'
|
|
|
|
egg_name = name = os.path.basename(url.rstrip('/'))
|
|
egg_name = egg_name.replace('-', '_')
|
|
|
|
_branch = branch.split('#')
|
|
branch = _branch[0]
|
|
|
|
plugin_path = None
|
|
# Determine if the package is a plugin type
|
|
if len(_branch) > 1:
|
|
if 'subdirectory=' in _branch[-1]:
|
|
plugin_path = _meta_return(_branch[-1], 'subdirectory')
|
|
name = os.path.basename(plugin_path)
|
|
|
|
if 'egg=' in _branch[-1]:
|
|
egg_name = _meta_return(_branch[-1], 'egg')
|
|
egg_name = egg_name.replace('-', '_')
|
|
|
|
if 'gitname=' in _branch[-1]:
|
|
name = _meta_return(_branch[-1], 'gitname')
|
|
|
|
project_group = 'all'
|
|
if 'projectgroup=' in _branch[-1]:
|
|
project_group = _meta_return(_branch[-1], 'projectgroup')
|
|
|
|
return name.lower(), branch, plugin_path, url, repo, egg_name, project_group
|
|
|
|
|
|
def _pip_requirement_split(requirement):
|
|
"""Split pip versions from a given requirement.
|
|
|
|
The method will return the package name, versions, and any markers.
|
|
|
|
:type requirement: ``str``
|
|
:returns: ``tuple``
|
|
"""
|
|
version_descriptors = "(>=|<=|>|<|==|~=|!=)"
|
|
requirement = requirement.split(';')
|
|
requirement_info = re.split(r'%s\s*' % version_descriptors, requirement[0])
|
|
name = requirement_info[0]
|
|
marker = None
|
|
if len(requirement) > 1:
|
|
marker = requirement[-1]
|
|
versions = None
|
|
if len(requirement_info) > 1:
|
|
versions = ''.join(requirement_info[1:])
|
|
|
|
return name, versions, marker
|
|
|
|
|
|
class DependencyFileProcessor(object):
|
|
def __init__(self, local_path):
|
|
"""Find required files.
|
|
|
|
:type local_path: ``str``
|
|
:return:
|
|
"""
|
|
self.pip = dict()
|
|
self.pip['git_package'] = list()
|
|
self.pip['py_package'] = list()
|
|
self.pip['git_data'] = list()
|
|
self.git_pip_install = 'git+%s@%s'
|
|
self.file_names = self._get_files(path=local_path)
|
|
|
|
# Process everything simply by calling the method
|
|
self._process_files()
|
|
|
|
def _py_pkg_extend(self, packages, py_package=None):
|
|
if py_package is None:
|
|
py_package = self.pip['py_package']
|
|
|
|
for pkg in packages:
|
|
pkg_name = _pip_requirement_split(pkg)[0]
|
|
for py_pkg in py_package:
|
|
py_pkg_name = _pip_requirement_split(py_pkg)[0]
|
|
if pkg_name == py_pkg_name:
|
|
py_package.remove(py_pkg)
|
|
else:
|
|
norm_pkgs = [i.lower() for i in packages if not i.startswith('{{')]
|
|
py_package.extend(norm_pkgs)
|
|
return py_package
|
|
|
|
@staticmethod
|
|
def _filter_files(file_names, ext):
|
|
"""Filter the files and return a sorted list.
|
|
:type file_names:
|
|
:type ext: ``str`` or ``tuple``
|
|
:returns: ``list``
|
|
"""
|
|
_file_names = list()
|
|
file_name_words = ['/defaults/', '/vars/', '/user_']
|
|
file_name_words.extend(REQUIREMENTS_FILE_TYPES)
|
|
for file_name in file_names:
|
|
if file_name.endswith(ext):
|
|
if any(i in file_name for i in file_name_words):
|
|
_file_names.append(file_name)
|
|
else:
|
|
return _file_names
|
|
|
|
@staticmethod
|
|
def _get_files(path):
|
|
"""Return a list of all files in the defaults/repo_packages directory.
|
|
|
|
:type path: ``str``
|
|
:returns: ``list``
|
|
"""
|
|
paths = os.walk(os.path.abspath(path), followlinks=True)
|
|
files = list()
|
|
for fpath, _, afiles in paths:
|
|
for afile in afiles:
|
|
files.append(os.path.join(fpath, afile))
|
|
else:
|
|
return files
|
|
|
|
def _check_plugins(self, git_repo_plugins, git_data):
|
|
"""Check if the git url is a plugin type.
|
|
|
|
:type git_repo_plugins: ``dict``
|
|
:type git_data: ``dict``
|
|
"""
|
|
for repo_plugin in git_repo_plugins:
|
|
strip_plugin_path = repo_plugin['package'].lstrip('/')
|
|
plugin = '%s/%s' % (
|
|
repo_plugin['path'].strip('/'),
|
|
strip_plugin_path
|
|
)
|
|
|
|
name = git_data['name'] = os.path.basename(strip_plugin_path)
|
|
git_data['egg_name'] = name.replace('-', '_')
|
|
package = self.git_pip_install % (
|
|
git_data['repo'], git_data['branch']
|
|
)
|
|
package += '#egg=%s' % git_data['egg_name']
|
|
package += '&subdirectory=%s' % plugin
|
|
package += '&gitname=%s' % name
|
|
if git_data['fragments']:
|
|
package += '&%s' % git_data['fragments']
|
|
|
|
self.pip['git_data'].append(git_data)
|
|
self.pip['git_package'].append(package)
|
|
|
|
if name not in GIT_PACKAGE_DEFAULT_PARTS:
|
|
GIT_PACKAGE_DEFAULT_PARTS[name] = git_data.copy()
|
|
else:
|
|
GIT_PACKAGE_DEFAULT_PARTS[name].update(git_data.copy())
|
|
|
|
@staticmethod
|
|
def _check_defaults(git_data, name, item):
|
|
"""Check if a default exists and use it if an item is undefined.
|
|
|
|
:type git_data: ``dict``
|
|
:type name: ``str``
|
|
:type item: ``str``
|
|
"""
|
|
if not git_data[item] and name in GIT_PACKAGE_DEFAULT_PARTS:
|
|
check_item = GIT_PACKAGE_DEFAULT_PARTS[name].get(item)
|
|
if check_item:
|
|
git_data[item] = check_item
|
|
|
|
def _process_git(self, loaded_yaml, git_item, yaml_file_name):
|
|
"""Process git repos.
|
|
|
|
:type loaded_yaml: ``dict``
|
|
:type git_item: ``str``
|
|
"""
|
|
git_data = dict()
|
|
if git_item.split('_')[0] == 'git':
|
|
prefix = ''
|
|
else:
|
|
prefix = '%s_' % git_item.split('_git_repo')[0].replace('.', '_')
|
|
|
|
# Set the various variable definitions
|
|
repo_var = prefix + 'git_repo'
|
|
name_var = prefix + 'git_package_name'
|
|
branch_var = prefix + 'git_install_branch'
|
|
fragment_var = prefix + 'git_install_fragments'
|
|
plugins_var = prefix + 'repo_plugins'
|
|
group_var = prefix + 'git_project_group'
|
|
|
|
# get the repo definition
|
|
git_data['repo'] = loaded_yaml.get(repo_var)
|
|
group = git_data['project_group'] = loaded_yaml.get(group_var, 'all')
|
|
|
|
# get the repo name definition
|
|
name = git_data['name'] = loaded_yaml.get(name_var)
|
|
if not name:
|
|
# NOTE: strip off trailing /, .git, or .git/
|
|
name = git_data['name'] = os.path.basename(
|
|
re.sub(r'(\/$|\.git(\/)?$)', '', git_data['repo'])
|
|
)
|
|
git_data['egg_name'] = name.replace('-', '_')
|
|
|
|
# This conditional is set to ensure we're only processing git
|
|
# repos from the defaults file when those same repos are not
|
|
# being set in the repo_packages files.
|
|
if '/defaults/main' in yaml_file_name:
|
|
if name in GIT_PACKAGE_DEFAULT_PARTS:
|
|
return
|
|
|
|
# get the repo branch definition
|
|
git_data['branch'] = loaded_yaml.get(branch_var)
|
|
self._check_defaults(git_data, name, 'branch')
|
|
if not git_data['branch']:
|
|
git_data['branch'] = 'master'
|
|
|
|
package = self.git_pip_install % (git_data['repo'], git_data['branch'])
|
|
|
|
# get the repo fragment definitions, if any
|
|
git_data['fragments'] = loaded_yaml.get(fragment_var)
|
|
self._check_defaults(git_data, name, 'fragments')
|
|
|
|
package += '#egg=%s' % git_data['egg_name']
|
|
package += '&gitname=%s' % name
|
|
package += '&projectgroup=%s' % group
|
|
if git_data['fragments']:
|
|
package += '&%s' % git_data['fragments']
|
|
|
|
self.pip['git_package'].append(package)
|
|
self.pip['git_data'].append(git_data.copy())
|
|
|
|
# Set the default package parts to track data during the run
|
|
if name not in GIT_PACKAGE_DEFAULT_PARTS:
|
|
GIT_PACKAGE_DEFAULT_PARTS[name] = git_data.copy()
|
|
else:
|
|
GIT_PACKAGE_DEFAULT_PARTS[name].update(git_data)
|
|
|
|
# get the repo plugin definitions, if any
|
|
git_data['plugins'] = loaded_yaml.get(plugins_var)
|
|
self._check_defaults(git_data, name, 'plugins')
|
|
if git_data['plugins']:
|
|
self._check_plugins(
|
|
git_repo_plugins=git_data['plugins'],
|
|
git_data=git_data
|
|
)
|
|
|
|
def _package_build_index(self, packages, role_name, var_name, pkg_index,
|
|
project_group='all', var_file_name=None,
|
|
pip_packages=True):
|
|
if pip_packages:
|
|
self._py_pkg_extend(packages)
|
|
|
|
if role_name:
|
|
if role_name in pkg_index:
|
|
role_pkgs = pkg_index[role_name]
|
|
else:
|
|
role_pkgs = pkg_index[role_name] = dict()
|
|
|
|
role_pkgs['project_group'] = project_group
|
|
|
|
if var_file_name:
|
|
_name = os.path.splitext(os.path.basename(var_file_name))[0]
|
|
if _name in pkg_index[role_name]:
|
|
file_name_index = pkg_index[role_name][_name]
|
|
else:
|
|
file_name_index = pkg_index[role_name][_name] = dict()
|
|
pkgs = file_name_index.get(var_name, list())
|
|
pkgs = self._py_pkg_extend(packages, pkgs)
|
|
file_name_index[var_name] = sorted(set(pkgs))
|
|
else:
|
|
pkgs = role_pkgs.get(var_name, list())
|
|
pkgs.extend(packages)
|
|
if 'pip' in var_name:
|
|
pkgs = [i.lower() for i in pkgs if not i.startswith('{{')]
|
|
else:
|
|
pkgs = [i for i in pkgs if not i.startswith('{{')]
|
|
if pkgs:
|
|
pkg_index[role_name][var_name] = sorted(set(pkgs))
|
|
else:
|
|
for k, v in pkg_index.items():
|
|
for item_name in v.keys():
|
|
if var_name == item_name:
|
|
pkg_index[k][item_name] = self._py_pkg_extend(
|
|
packages,
|
|
pkg_index[k][item_name]
|
|
)
|
|
|
|
def _process_files(self):
|
|
"""Process all of the requirement files."""
|
|
self._process_files_defaults()
|
|
self._process_files_requirements()
|
|
|
|
def _process_files_defaults(self):
|
|
"""Process files."""
|
|
for file_name in self._filter_files(self.file_names, ('yaml', 'yml')):
|
|
with open(file_name, 'r') as f:
|
|
# If there is an exception loading the file continue
|
|
# and if the loaded_config is None continue. This makes
|
|
# no bad config gets passed to the rest of the process.
|
|
try:
|
|
loaded_config = yaml.safe_load(f.read())
|
|
except Exception: # Broad exception so everything is caught
|
|
continue
|
|
else:
|
|
if not loaded_config or not isinstance(loaded_config, dict):
|
|
continue
|
|
|
|
if 'roles' in file_name:
|
|
_role_name = file_name.split('roles%s' % os.sep)[-1]
|
|
role_name = _role_name.split(os.sep)[0]
|
|
else:
|
|
role_name = None
|
|
|
|
for key, value in loaded_config.items():
|
|
if key.endswith('role_project_group'):
|
|
project_group = value
|
|
break
|
|
else:
|
|
project_group = 'all'
|
|
|
|
if role_name is not None:
|
|
PACKAGE_MAPPING['role_project_groups'][role_name] = project_group
|
|
for key, values in loaded_config.items():
|
|
key = key.lower()
|
|
if key.endswith('git_repo'):
|
|
self._process_git(
|
|
loaded_yaml=loaded_config,
|
|
git_item=key,
|
|
yaml_file_name=file_name
|
|
)
|
|
# Process pip packages
|
|
self._process_packages(
|
|
pkg_constant=BUILT_IN_PIP_PACKAGE_VARS,
|
|
pkg_breakout_index=ROLE_BREAKOUT_REQUIREMENTS,
|
|
pkg_role_index=ROLE_PACKAGES,
|
|
pkg_var_name=key,
|
|
packages=values,
|
|
role_name=role_name,
|
|
project_group=project_group
|
|
)
|
|
# Process distro packages
|
|
self._process_packages(
|
|
pkg_constant=BUILT_IN_DISTRO_PACKAGE_VARS,
|
|
pkg_breakout_index=ROLE_DISTRO_BREAKOUT_PACKAGES,
|
|
pkg_role_index=dict(), # this is not used here
|
|
pkg_var_name=key,
|
|
packages=values,
|
|
role_name=role_name,
|
|
project_group=project_group,
|
|
role_index=False,
|
|
var_file_name=file_name,
|
|
pip_packages=False
|
|
)
|
|
|
|
def _process_packages(self, pkg_constant, pkg_breakout_index,
|
|
pkg_role_index, pkg_var_name, packages, role_name,
|
|
project_group, role_index=True, var_file_name=None,
|
|
pip_packages=True):
|
|
"""Process variables to build the package data structures.
|
|
|
|
:param pkg_constant: CONSTANT used to validate package names
|
|
:type pkg_constant: ``list``
|
|
:param pkg_breakout_index: CONSTANT used to store indexed packages
|
|
:type pkg_breakout_index: ``dict``
|
|
:param pkg_role_index: CONSTANT used to store role indexed packages
|
|
:type pkg_role_index: ``dict``
|
|
:param pkg_var_name: package variable name
|
|
:type pkg_var_name: ``str``
|
|
:param packages: list of packages to index
|
|
:type packages: ``list``
|
|
:param role_name: Name of the role where the packages came from
|
|
:type role_name: ``str``
|
|
:param project_group: Name of the group being indexed
|
|
:type project_group: ``str``
|
|
:param role_index: Enable or disable the use of the role index
|
|
:type role_index: ``bool``
|
|
:param var_file_name: Variable file name used to index packages
|
|
:type var_file_name: ``str``
|
|
:param pip_packages: Enable or disable pip index types
|
|
:type pip_packages: ``bool``
|
|
"""
|
|
if [i for i in pkg_constant if i in pkg_var_name]:
|
|
if 'proprietary' in pkg_var_name:
|
|
return
|
|
|
|
self._package_build_index(
|
|
packages=packages,
|
|
role_name=role_name,
|
|
var_name=pkg_var_name,
|
|
pkg_index=pkg_breakout_index,
|
|
project_group=project_group,
|
|
var_file_name=var_file_name,
|
|
pip_packages=pip_packages
|
|
)
|
|
|
|
if not role_index:
|
|
return
|
|
elif 'optional' in pkg_var_name:
|
|
return
|
|
else:
|
|
self._package_build_index(
|
|
packages=packages,
|
|
role_name=role_name,
|
|
var_name=pkg_var_name,
|
|
pkg_index=pkg_role_index,
|
|
project_group=project_group,
|
|
var_file_name=var_file_name,
|
|
pip_packages=pip_packages
|
|
)
|
|
|
|
def _process_files_requirements(self):
|
|
"""Process requirements files."""
|
|
return_list = self._filter_files(self.file_names, 'txt')
|
|
for file_name in return_list:
|
|
base_name = os.path.basename(file_name)
|
|
if base_name in REQUIREMENTS_FILE_TYPES:
|
|
index = REQUIREMENTS_FILE_TYPES.index(base_name)
|
|
return_list.remove(file_name)
|
|
return_list.insert(index, file_name)
|
|
else:
|
|
for file_name in return_list:
|
|
if file_name.endswith('other-requirements.txt'):
|
|
continue
|
|
elif file_name.endswith('bindep.txt'):
|
|
continue
|
|
elif 'roles' in file_name:
|
|
_role_name = file_name.split('roles%s' % os.sep)[-1]
|
|
role_name = _role_name.split(os.sep)[0]
|
|
else:
|
|
role_name = 'default'
|
|
with open(file_name, 'r') as f:
|
|
packages = [
|
|
i.split()[0].lower() for i in f.read().splitlines()
|
|
if i
|
|
if not i.startswith('#')
|
|
]
|
|
base_file_name = os.path.basename(file_name)
|
|
if base_file_name.endswith('test-requirements.txt'):
|
|
continue
|
|
if base_file_name.endswith('global-requirement-pins.txt'):
|
|
self._package_build_index(
|
|
packages=packages,
|
|
role_name='global_pins',
|
|
var_name='pinned_packages',
|
|
pkg_index=ROLE_REQUIREMENTS,
|
|
project_group='all'
|
|
)
|
|
self._package_build_index(
|
|
packages=packages,
|
|
role_name=role_name,
|
|
var_name='txt_file_packages',
|
|
pkg_index=ROLE_REQUIREMENTS,
|
|
project_group='all'
|
|
)
|
|
|
|
|
|
def _abs_path(path):
|
|
return os.path.abspath(
|
|
os.path.expanduser(
|
|
path
|
|
)
|
|
)
|
|
|
|
|
|
class LookupModule(LookupBase):
|
|
def __init__(self, basedir=None, **kwargs):
|
|
"""Run the lookup module.
|
|
|
|
:type basedir:
|
|
:type kwargs:
|
|
"""
|
|
|
|
def run(self, terms, variables=None, **kwargs):
|
|
"""Run the main application.
|
|
|
|
:type terms: ``str``
|
|
:type variables: ``str``
|
|
:type kwargs: ``dict``
|
|
:returns: ``list``
|
|
"""
|
|
if isinstance(terms, str):
|
|
terms = [terms]
|
|
|
|
return_data = PACKAGE_MAPPING
|
|
|
|
for term in terms:
|
|
return_list = list()
|
|
try:
|
|
dfp = DependencyFileProcessor(
|
|
local_path=_abs_path(str(term))
|
|
)
|
|
return_list.extend(dfp.pip['py_package'])
|
|
return_list.extend(dfp.pip['git_package'])
|
|
except Exception as exp:
|
|
raise AnsibleError(
|
|
'lookup_plugin.py_pkgs(%s) returned "%s" error "%s"' % (
|
|
term,
|
|
str(exp),
|
|
traceback.format_exc()
|
|
)
|
|
)
|
|
|
|
for item in return_list:
|
|
map_base_and_remote_packages(item, return_data)
|
|
else:
|
|
parse_remote_package_parts(return_data)
|
|
else:
|
|
map_role_packages(return_data)
|
|
map_base_package_details(return_data)
|
|
# Sort everything within the returned data
|
|
for key, value in return_data.items():
|
|
if isinstance(value, (list, set)):
|
|
try:
|
|
if all(isinstance(item, dict) for item in value):
|
|
return_data[key] = sorted(value, key = lambda k: k['name'])
|
|
else:
|
|
return_data[key] = sorted(value)
|
|
except TypeError:
|
|
return_data[key] = value
|
|
return_data['role_requirement_files'] = ROLE_REQUIREMENTS
|
|
return_data['role_requirements'] = ROLE_BREAKOUT_REQUIREMENTS
|
|
_dp = return_data['role_distro_packages'] = ROLE_DISTRO_BREAKOUT_PACKAGES
|
|
for k, v in PACKAGE_MAPPING['role_project_groups'].items():
|
|
if k in _dp:
|
|
_dp[k]['project_group'] = v
|
|
return [return_data]
|
|
|
|
# Used for testing and debuging usage: `python plugins/lookups/py_pkgs.py ../`
|
|
if __name__ == '__main__':
|
|
import sys
|
|
import json
|
|
print(json.dumps(LookupModule().run(terms=sys.argv[1:]), indent=4, sort_keys=True))
|