Clean oslo dependencies files
Modules flakes, version, rootwrap and setup are no longer available or needed. The modules install_venv_common is now synced as a script. Change-Id: I0724638cd7bb51b0deadde2d66a35de9a3f7b605
This commit is contained in:
Ghe Rivero
parent
40c447b680
commit
97e01aa062
@ -7,10 +7,8 @@ module=db.sqlalchemy
|
||||
module=eventlet_backdoor
|
||||
module=excutils
|
||||
module=fileutils
|
||||
module=flakes
|
||||
module=gettextutils
|
||||
module=importutils
|
||||
module=install_venv_common
|
||||
module=jsonutils
|
||||
module=local
|
||||
module=lockutils
|
||||
@ -22,11 +20,11 @@ module=notifier
|
||||
module=periodic_task
|
||||
module=policy
|
||||
module=processutils
|
||||
module=rootwrap
|
||||
module=rpc
|
||||
module=setup
|
||||
module=timeutils
|
||||
module=version
|
||||
|
||||
# Tools
|
||||
script=tools/install_venv_common
|
||||
|
||||
# The base module to hold the copy of openstack.common
|
||||
base=tuskar
|
||||
|
||||
@ -1,16 +0,0 @@
|
||||
# vim: tabstop=4 shiftwidth=4 softtabstop=4
|
||||
|
||||
# Copyright (c) 2011 OpenStack Foundation.
|
||||
# All Rights Reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License. You may obtain
|
||||
# a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
@ -1,130 +0,0 @@
|
||||
#!/usr/bin/env python
|
||||
# vim: tabstop=4 shiftwidth=4 softtabstop=4
|
||||
|
||||
# Copyright (c) 2011 OpenStack Foundation.
|
||||
# All Rights Reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License. You may obtain
|
||||
# a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
"""Root wrapper for OpenStack services
|
||||
|
||||
Filters which commands a service is allowed to run as another user.
|
||||
|
||||
To use this with tuskar, you should set the following in
|
||||
tuskar.conf:
|
||||
rootwrap_config=/etc/tuskar/rootwrap.conf
|
||||
|
||||
You also need to let the tuskar user run tuskar-rootwrap
|
||||
as root in sudoers:
|
||||
tuskar ALL = (root) NOPASSWD: /usr/bin/tuskar-rootwrap
|
||||
/etc/tuskar/rootwrap.conf *
|
||||
|
||||
Service packaging should deploy .filters files only on nodes where
|
||||
they are needed, to avoid allowing more than is necessary.
|
||||
"""
|
||||
|
||||
from __future__ import print_function
|
||||
|
||||
import ConfigParser
|
||||
import logging
|
||||
import os
|
||||
import pwd
|
||||
import signal
|
||||
import subprocess
|
||||
import sys
|
||||
|
||||
|
||||
RC_UNAUTHORIZED = 99
|
||||
RC_NOCOMMAND = 98
|
||||
RC_BADCONFIG = 97
|
||||
RC_NOEXECFOUND = 96
|
||||
|
||||
|
||||
def _subprocess_setup():
|
||||
# Python installs a SIGPIPE handler by default. This is usually not what
|
||||
# non-Python subprocesses expect.
|
||||
signal.signal(signal.SIGPIPE, signal.SIG_DFL)
|
||||
|
||||
|
||||
def _exit_error(execname, message, errorcode, log=True):
|
||||
print("%s: %s" % (execname, message))
|
||||
if log:
|
||||
logging.error(message)
|
||||
sys.exit(errorcode)
|
||||
|
||||
|
||||
def main():
|
||||
# Split arguments, require at least a command
|
||||
execname = sys.argv.pop(0)
|
||||
if len(sys.argv) < 2:
|
||||
_exit_error(execname, "No command specified", RC_NOCOMMAND, log=False)
|
||||
|
||||
configfile = sys.argv.pop(0)
|
||||
userargs = sys.argv[:]
|
||||
|
||||
# Add ../ to sys.path to allow running from branch
|
||||
possible_topdir = os.path.normpath(os.path.join(os.path.abspath(execname),
|
||||
os.pardir, os.pardir))
|
||||
if os.path.exists(os.path.join(possible_topdir, "tuskar", "__init__.py")):
|
||||
sys.path.insert(0, possible_topdir)
|
||||
|
||||
from tuskar.openstack.common.rootwrap import wrapper
|
||||
|
||||
# Load configuration
|
||||
try:
|
||||
rawconfig = ConfigParser.RawConfigParser()
|
||||
rawconfig.read(configfile)
|
||||
config = wrapper.RootwrapConfig(rawconfig)
|
||||
except ValueError as exc:
|
||||
msg = "Incorrect value in %s: %s" % (configfile, exc.message)
|
||||
_exit_error(execname, msg, RC_BADCONFIG, log=False)
|
||||
except ConfigParser.Error:
|
||||
_exit_error(execname, "Incorrect configuration file: %s" % configfile,
|
||||
RC_BADCONFIG, log=False)
|
||||
|
||||
if config.use_syslog:
|
||||
wrapper.setup_syslog(execname,
|
||||
config.syslog_log_facility,
|
||||
config.syslog_log_level)
|
||||
|
||||
# Execute command if it matches any of the loaded filters
|
||||
filters = wrapper.load_filters(config.filters_path)
|
||||
try:
|
||||
filtermatch = wrapper.match_filter(filters, userargs,
|
||||
exec_dirs=config.exec_dirs)
|
||||
if filtermatch:
|
||||
command = filtermatch.get_command(userargs,
|
||||
exec_dirs=config.exec_dirs)
|
||||
if config.use_syslog:
|
||||
logging.info("(%s > %s) Executing %s (filter match = %s)" % (
|
||||
os.getlogin(), pwd.getpwuid(os.getuid())[0],
|
||||
command, filtermatch.name))
|
||||
|
||||
obj = subprocess.Popen(command,
|
||||
stdin=sys.stdin,
|
||||
stdout=sys.stdout,
|
||||
stderr=sys.stderr,
|
||||
preexec_fn=_subprocess_setup,
|
||||
env=filtermatch.get_environment(userargs))
|
||||
obj.wait()
|
||||
sys.exit(obj.returncode)
|
||||
|
||||
except wrapper.FilterMatchNotExecutable as exc:
|
||||
msg = ("Executable not found: %s (filter match = %s)"
|
||||
% (exc.match.exec_path, exc.match.name))
|
||||
_exit_error(execname, msg, RC_NOEXECFOUND, log=config.use_syslog)
|
||||
|
||||
except wrapper.NoFilterMatched:
|
||||
msg = ("Unauthorized command: %s (no filter matched)"
|
||||
% ' '.join(userargs))
|
||||
_exit_error(execname, msg, RC_UNAUTHORIZED, log=config.use_syslog)
|
||||
@ -1,228 +0,0 @@
|
||||
# vim: tabstop=4 shiftwidth=4 softtabstop=4
|
||||
|
||||
# Copyright (c) 2011 OpenStack Foundation.
|
||||
# All Rights Reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License. You may obtain
|
||||
# a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
import os
|
||||
import re
|
||||
|
||||
|
||||
class CommandFilter(object):
|
||||
"""Command filter only checking that the 1st argument matches exec_path."""
|
||||
|
||||
def __init__(self, exec_path, run_as, *args):
|
||||
self.name = ''
|
||||
self.exec_path = exec_path
|
||||
self.run_as = run_as
|
||||
self.args = args
|
||||
self.real_exec = None
|
||||
|
||||
def get_exec(self, exec_dirs=[]):
|
||||
"""Returns existing executable, or empty string if none found."""
|
||||
if self.real_exec is not None:
|
||||
return self.real_exec
|
||||
self.real_exec = ""
|
||||
if self.exec_path.startswith('/'):
|
||||
if os.access(self.exec_path, os.X_OK):
|
||||
self.real_exec = self.exec_path
|
||||
else:
|
||||
for binary_path in exec_dirs:
|
||||
expanded_path = os.path.join(binary_path, self.exec_path)
|
||||
if os.access(expanded_path, os.X_OK):
|
||||
self.real_exec = expanded_path
|
||||
break
|
||||
return self.real_exec
|
||||
|
||||
def match(self, userargs):
|
||||
"""Only check that the first argument (command) matches exec_path."""
|
||||
return os.path.basename(self.exec_path) == userargs[0]
|
||||
|
||||
def get_command(self, userargs, exec_dirs=[]):
|
||||
"""Returns command to execute (with sudo -u if run_as != root)."""
|
||||
to_exec = self.get_exec(exec_dirs=exec_dirs) or self.exec_path
|
||||
if (self.run_as != 'root'):
|
||||
# Used to run commands at lesser privileges
|
||||
return ['sudo', '-u', self.run_as, to_exec] + userargs[1:]
|
||||
return [to_exec] + userargs[1:]
|
||||
|
||||
def get_environment(self, userargs):
|
||||
"""Returns specific environment to set, None if none."""
|
||||
return None
|
||||
|
||||
|
||||
class RegExpFilter(CommandFilter):
|
||||
"""Command filter doing regexp matching for every argument."""
|
||||
|
||||
def match(self, userargs):
|
||||
# Early skip if command or number of args don't match
|
||||
if (len(self.args) != len(userargs)):
|
||||
# DENY: argument numbers don't match
|
||||
return False
|
||||
# Compare each arg (anchoring pattern explicitly at end of string)
|
||||
for (pattern, arg) in zip(self.args, userargs):
|
||||
try:
|
||||
if not re.match(pattern + '$', arg):
|
||||
break
|
||||
except re.error:
|
||||
# DENY: Badly-formed filter
|
||||
return False
|
||||
else:
|
||||
# ALLOW: All arguments matched
|
||||
return True
|
||||
|
||||
# DENY: Some arguments did not match
|
||||
return False
|
||||
|
||||
|
||||
class PathFilter(CommandFilter):
|
||||
"""Command filter checking that path arguments are within given dirs
|
||||
|
||||
One can specify the following constraints for command arguments:
|
||||
1) pass - pass an argument as is to the resulting command
|
||||
2) some_str - check if an argument is equal to the given string
|
||||
3) abs path - check if a path argument is within the given base dir
|
||||
|
||||
A typical rootwrapper filter entry looks like this:
|
||||
# cmdname: filter name, raw command, user, arg_i_constraint [, ...]
|
||||
chown: PathFilter, /bin/chown, root, nova, /var/lib/images
|
||||
|
||||
"""
|
||||
|
||||
def match(self, userargs):
|
||||
command, arguments = userargs[0], userargs[1:]
|
||||
|
||||
equal_args_num = len(self.args) == len(arguments)
|
||||
exec_is_valid = super(PathFilter, self).match(userargs)
|
||||
args_equal_or_pass = all(
|
||||
arg == 'pass' or arg == value
|
||||
for arg, value in zip(self.args, arguments)
|
||||
if not os.path.isabs(arg) # arguments not specifying abs paths
|
||||
)
|
||||
paths_are_within_base_dirs = all(
|
||||
os.path.commonprefix([arg, os.path.realpath(value)]) == arg
|
||||
for arg, value in zip(self.args, arguments)
|
||||
if os.path.isabs(arg) # arguments specifying abs paths
|
||||
)
|
||||
|
||||
return (equal_args_num and
|
||||
exec_is_valid and
|
||||
args_equal_or_pass and
|
||||
paths_are_within_base_dirs)
|
||||
|
||||
def get_command(self, userargs, exec_dirs=[]):
|
||||
command, arguments = userargs[0], userargs[1:]
|
||||
|
||||
# convert path values to canonical ones; copy other args as is
|
||||
args = [os.path.realpath(value) if os.path.isabs(arg) else value
|
||||
for arg, value in zip(self.args, arguments)]
|
||||
|
||||
return super(PathFilter, self).get_command([command] + args,
|
||||
exec_dirs)
|
||||
|
||||
|
||||
class DnsmasqFilter(CommandFilter):
|
||||
"""Specific filter for the dnsmasq call (which includes env)."""
|
||||
|
||||
CONFIG_FILE_ARG = 'CONFIG_FILE'
|
||||
|
||||
def match(self, userargs):
|
||||
if (userargs[0] == 'env' and
|
||||
userargs[1].startswith(self.CONFIG_FILE_ARG) and
|
||||
userargs[2].startswith('NETWORK_ID=') and
|
||||
userargs[3] == 'dnsmasq'):
|
||||
return True
|
||||
return False
|
||||
|
||||
def get_command(self, userargs, exec_dirs=[]):
|
||||
to_exec = self.get_exec(exec_dirs=exec_dirs) or self.exec_path
|
||||
dnsmasq_pos = userargs.index('dnsmasq')
|
||||
return [to_exec] + userargs[dnsmasq_pos + 1:]
|
||||
|
||||
def get_environment(self, userargs):
|
||||
env = os.environ.copy()
|
||||
env[self.CONFIG_FILE_ARG] = userargs[1].split('=')[-1]
|
||||
env['NETWORK_ID'] = userargs[2].split('=')[-1]
|
||||
return env
|
||||
|
||||
|
||||
class DeprecatedDnsmasqFilter(DnsmasqFilter):
|
||||
"""Variant of dnsmasq filter to support old-style FLAGFILE."""
|
||||
CONFIG_FILE_ARG = 'FLAGFILE'
|
||||
|
||||
|
||||
class KillFilter(CommandFilter):
|
||||
"""Specific filter for the kill calls.
|
||||
1st argument is the user to run /bin/kill under
|
||||
2nd argument is the location of the affected executable
|
||||
Subsequent arguments list the accepted signals (if any)
|
||||
|
||||
This filter relies on /proc to accurately determine affected
|
||||
executable, so it will only work on procfs-capable systems (not OSX).
|
||||
"""
|
||||
|
||||
def __init__(self, *args):
|
||||
super(KillFilter, self).__init__("/bin/kill", *args)
|
||||
|
||||
def match(self, userargs):
|
||||
if userargs[0] != "kill":
|
||||
return False
|
||||
args = list(userargs)
|
||||
if len(args) == 3:
|
||||
# A specific signal is requested
|
||||
signal = args.pop(1)
|
||||
if signal not in self.args[1:]:
|
||||
# Requested signal not in accepted list
|
||||
return False
|
||||
else:
|
||||
if len(args) != 2:
|
||||
# Incorrect number of arguments
|
||||
return False
|
||||
if len(self.args) > 1:
|
||||
# No signal requested, but filter requires specific signal
|
||||
return False
|
||||
try:
|
||||
command = os.readlink("/proc/%d/exe" % int(args[1]))
|
||||
# NOTE(yufang521247): /proc/PID/exe may have '\0' on the
|
||||
# end, because python doen't stop at '\0' when read the
|
||||
# target path.
|
||||
command = command.split('\0')[0]
|
||||
# NOTE(dprince): /proc/PID/exe may have ' (deleted)' on
|
||||
# the end if an executable is updated or deleted
|
||||
if command.endswith(" (deleted)"):
|
||||
command = command[:command.rindex(" ")]
|
||||
if command != self.args[0]:
|
||||
# Affected executable does not match
|
||||
return False
|
||||
except (ValueError, OSError):
|
||||
# Incorrect PID
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
class ReadFileFilter(CommandFilter):
|
||||
"""Specific filter for the utils.read_file_as_root call."""
|
||||
|
||||
def __init__(self, file_path, *args):
|
||||
self.file_path = file_path
|
||||
super(ReadFileFilter, self).__init__("/bin/cat", "root", *args)
|
||||
|
||||
def match(self, userargs):
|
||||
if userargs[0] != 'cat':
|
||||
return False
|
||||
if userargs[1] != self.file_path:
|
||||
return False
|
||||
if len(userargs) != 2:
|
||||
return False
|
||||
return True
|
||||
@ -1,149 +0,0 @@
|
||||
# vim: tabstop=4 shiftwidth=4 softtabstop=4
|
||||
|
||||
# Copyright (c) 2011 OpenStack Foundation.
|
||||
# All Rights Reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License. You may obtain
|
||||
# a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
|
||||
import ConfigParser
|
||||
import logging
|
||||
import logging.handlers
|
||||
import os
|
||||
import string
|
||||
|
||||
from tuskar.openstack.common.rootwrap import filters
|
||||
|
||||
|
||||
class NoFilterMatched(Exception):
|
||||
"""This exception is raised when no filter matched."""
|
||||
pass
|
||||
|
||||
|
||||
class FilterMatchNotExecutable(Exception):
|
||||
"""
|
||||
This exception is raised when a filter matched but no executable was
|
||||
found.
|
||||
"""
|
||||
def __init__(self, match=None, **kwargs):
|
||||
self.match = match
|
||||
|
||||
|
||||
class RootwrapConfig(object):
|
||||
|
||||
def __init__(self, config):
|
||||
# filters_path
|
||||
self.filters_path = config.get("DEFAULT", "filters_path").split(",")
|
||||
|
||||
# exec_dirs
|
||||
if config.has_option("DEFAULT", "exec_dirs"):
|
||||
self.exec_dirs = config.get("DEFAULT", "exec_dirs").split(",")
|
||||
else:
|
||||
# Use system PATH if exec_dirs is not specified
|
||||
self.exec_dirs = os.environ["PATH"].split(':')
|
||||
|
||||
# syslog_log_facility
|
||||
if config.has_option("DEFAULT", "syslog_log_facility"):
|
||||
v = config.get("DEFAULT", "syslog_log_facility")
|
||||
facility_names = logging.handlers.SysLogHandler.facility_names
|
||||
self.syslog_log_facility = getattr(logging.handlers.SysLogHandler,
|
||||
v, None)
|
||||
if self.syslog_log_facility is None and v in facility_names:
|
||||
self.syslog_log_facility = facility_names.get(v)
|
||||
if self.syslog_log_facility is None:
|
||||
raise ValueError('Unexpected syslog_log_facility: %s' % v)
|
||||
else:
|
||||
default_facility = logging.handlers.SysLogHandler.LOG_SYSLOG
|
||||
self.syslog_log_facility = default_facility
|
||||
|
||||
# syslog_log_level
|
||||
if config.has_option("DEFAULT", "syslog_log_level"):
|
||||
v = config.get("DEFAULT", "syslog_log_level")
|
||||
self.syslog_log_level = logging.getLevelName(v.upper())
|
||||
if (self.syslog_log_level == "Level %s" % v.upper()):
|
||||
raise ValueError('Unexepected syslog_log_level: %s' % v)
|
||||
else:
|
||||
self.syslog_log_level = logging.ERROR
|
||||
|
||||
# use_syslog
|
||||
if config.has_option("DEFAULT", "use_syslog"):
|
||||
self.use_syslog = config.getboolean("DEFAULT", "use_syslog")
|
||||
else:
|
||||
self.use_syslog = False
|
||||
|
||||
|
||||
def setup_syslog(execname, facility, level):
|
||||
rootwrap_logger = logging.getLogger()
|
||||
rootwrap_logger.setLevel(level)
|
||||
handler = logging.handlers.SysLogHandler(address='/dev/log',
|
||||
facility=facility)
|
||||
handler.setFormatter(logging.Formatter(
|
||||
os.path.basename(execname) + ': %(message)s'))
|
||||
rootwrap_logger.addHandler(handler)
|
||||
|
||||
|
||||
def build_filter(class_name, *args):
|
||||
"""Returns a filter object of class class_name."""
|
||||
if not hasattr(filters, class_name):
|
||||
logging.warning("Skipping unknown filter class (%s) specified "
|
||||
"in filter definitions" % class_name)
|
||||
return None
|
||||
filterclass = getattr(filters, class_name)
|
||||
return filterclass(*args)
|
||||
|
||||
|
||||
def load_filters(filters_path):
|
||||
"""Load filters from a list of directories."""
|
||||
filterlist = []
|
||||
for filterdir in filters_path:
|
||||
if not os.path.isdir(filterdir):
|
||||
continue
|
||||
for filterfile in os.listdir(filterdir):
|
||||
filterconfig = ConfigParser.RawConfigParser()
|
||||
filterconfig.read(os.path.join(filterdir, filterfile))
|
||||
for (name, value) in filterconfig.items("Filters"):
|
||||
filterdefinition = [string.strip(s) for s in value.split(',')]
|
||||
newfilter = build_filter(*filterdefinition)
|
||||
if newfilter is None:
|
||||
continue
|
||||
newfilter.name = name
|
||||
filterlist.append(newfilter)
|
||||
return filterlist
|
||||
|
||||
|
||||
def match_filter(filter_list, userargs, exec_dirs=[]):
|
||||
"""
|
||||
Checks user command and arguments through command filters and
|
||||
returns the first matching filter.
|
||||
Raises NoFilterMatched if no filter matched.
|
||||
Raises FilterMatchNotExecutable if no executable was found for the
|
||||
best filter match.
|
||||
"""
|
||||
first_not_executable_filter = None
|
||||
|
||||
for f in filter_list:
|
||||
if f.match(userargs):
|
||||
# Try other filters if executable is absent
|
||||
if not f.get_exec(exec_dirs=exec_dirs):
|
||||
if not first_not_executable_filter:
|
||||
first_not_executable_filter = f
|
||||
continue
|
||||
# Otherwise return matching filter for execution
|
||||
return f
|
||||
|
||||
if first_not_executable_filter:
|
||||
# A filter matched, but no executable was found for it
|
||||
raise FilterMatchNotExecutable(match=first_not_executable_filter)
|
||||
|
||||
# No filter matched
|
||||
raise NoFilterMatched()
|
||||
@ -1,367 +0,0 @@
|
||||
# vim: tabstop=4 shiftwidth=4 softtabstop=4
|
||||
|
||||
# Copyright 2011 OpenStack Foundation.
|
||||
# Copyright 2012-2013 Hewlett-Packard Development Company, L.P.
|
||||
# All Rights Reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License. You may obtain
|
||||
# a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
"""
|
||||
Utilities with minimum-depends for use in setup.py
|
||||
"""
|
||||
|
||||
from __future__ import print_function
|
||||
|
||||
import email
|
||||
import os
|
||||
import re
|
||||
import subprocess
|
||||
import sys
|
||||
|
||||
from setuptools.command import sdist
|
||||
|
||||
|
||||
def parse_mailmap(mailmap='.mailmap'):
|
||||
mapping = {}
|
||||
if os.path.exists(mailmap):
|
||||
with open(mailmap, 'r') as fp:
|
||||
for l in fp:
|
||||
try:
|
||||
canonical_email, alias = re.match(
|
||||
r'[^#]*?(<.+>).*(<.+>).*', l).groups()
|
||||
except AttributeError:
|
||||
continue
|
||||
mapping[alias] = canonical_email
|
||||
return mapping
|
||||
|
||||
|
||||
def _parse_git_mailmap(git_dir, mailmap='.mailmap'):
|
||||
mailmap = os.path.join(os.path.dirname(git_dir), mailmap)
|
||||
return parse_mailmap(mailmap)
|
||||
|
||||
|
||||
def canonicalize_emails(changelog, mapping):
|
||||
"""Takes in a string and an email alias mapping and replaces all
|
||||
instances of the aliases in the string with their real email.
|
||||
"""
|
||||
for alias, email_address in mapping.iteritems():
|
||||
changelog = changelog.replace(alias, email_address)
|
||||
return changelog
|
||||
|
||||
|
||||
# Get requirements from the first file that exists
|
||||
def get_reqs_from_files(requirements_files):
|
||||
for requirements_file in requirements_files:
|
||||
if os.path.exists(requirements_file):
|
||||
with open(requirements_file, 'r') as fil:
|
||||
return fil.read().split('\n')
|
||||
return []
|
||||
|
||||
|
||||
def parse_requirements(requirements_files=['requirements.txt',
|
||||
'tools/pip-requires']):
|
||||
requirements = []
|
||||
for line in get_reqs_from_files(requirements_files):
|
||||
# For the requirements list, we need to inject only the portion
|
||||
# after egg= so that distutils knows the package it's looking for
|
||||
# such as:
|
||||
# -e git://github.com/openstack/nova/master#egg=nova
|
||||
if re.match(r'\s*-e\s+', line):
|
||||
requirements.append(re.sub(r'\s*-e\s+.*#egg=(.*)$', r'\1',
|
||||
line))
|
||||
# such as:
|
||||
# http://github.com/openstack/nova/zipball/master#egg=nova
|
||||
elif re.match(r'\s*https?:', line):
|
||||
requirements.append(re.sub(r'\s*https?:.*#egg=(.*)$', r'\1',
|
||||
line))
|
||||
# -f lines are for index locations, and don't get used here
|
||||
elif re.match(r'\s*-f\s+', line):
|
||||
pass
|
||||
# argparse is part of the standard library starting with 2.7
|
||||
# adding it to the requirements list screws distro installs
|
||||
elif line == 'argparse' and sys.version_info >= (2, 7):
|
||||
pass
|
||||
else:
|
||||
requirements.append(line)
|
||||
|
||||
return requirements
|
||||
|
||||
|
||||
def parse_dependency_links(requirements_files=['requirements.txt',
|
||||
'tools/pip-requires']):
|
||||
dependency_links = []
|
||||
# dependency_links inject alternate locations to find packages listed
|
||||
# in requirements
|
||||
for line in get_reqs_from_files(requirements_files):
|
||||
# skip comments and blank lines
|
||||
if re.match(r'(\s*#)|(\s*$)', line):
|
||||
continue
|
||||
# lines with -e or -f need the whole line, minus the flag
|
||||
if re.match(r'\s*-[ef]\s+', line):
|
||||
dependency_links.append(re.sub(r'\s*-[ef]\s+', '', line))
|
||||
# lines that are only urls can go in unmolested
|
||||
elif re.match(r'\s*https?:', line):
|
||||
dependency_links.append(line)
|
||||
return dependency_links
|
||||
|
||||
|
||||
def _run_shell_command(cmd, throw_on_error=False):
|
||||
if os.name == 'nt':
|
||||
output = subprocess.Popen(["cmd.exe", "/C", cmd],
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.PIPE)
|
||||
else:
|
||||
output = subprocess.Popen(["/bin/sh", "-c", cmd],
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.PIPE)
|
||||
out = output.communicate()
|
||||
if output.returncode and throw_on_error:
|
||||
raise Exception("%s returned %d" % cmd, output.returncode)
|
||||
if not out:
|
||||
return None
|
||||
return out[0].strip() or None
|
||||
|
||||
|
||||
def _get_git_directory():
|
||||
parent_dir = os.path.dirname(__file__)
|
||||
while True:
|
||||
git_dir = os.path.join(parent_dir, '.git')
|
||||
if os.path.exists(git_dir):
|
||||
return git_dir
|
||||
parent_dir, child = os.path.split(parent_dir)
|
||||
if not child: # reached to root dir
|
||||
return None
|
||||
|
||||
|
||||
def write_git_changelog():
|
||||
"""Write a changelog based on the git changelog."""
|
||||
new_changelog = 'ChangeLog'
|
||||
git_dir = _get_git_directory()
|
||||
if not os.getenv('SKIP_WRITE_GIT_CHANGELOG'):
|
||||
if git_dir:
|
||||
git_log_cmd = 'git --git-dir=%s log' % git_dir
|
||||
changelog = _run_shell_command(git_log_cmd)
|
||||
mailmap = _parse_git_mailmap(git_dir)
|
||||
with open(new_changelog, "w") as changelog_file:
|
||||
changelog_file.write(canonicalize_emails(changelog, mailmap))
|
||||
else:
|
||||
open(new_changelog, 'w').close()
|
||||
|
||||
|
||||
def generate_authors():
|
||||
"""Create AUTHORS file using git commits."""
|
||||
jenkins_email = 'jenkins@review.(openstack|stackforge).org'
|
||||
old_authors = 'AUTHORS.in'
|
||||
new_authors = 'AUTHORS'
|
||||
git_dir = _get_git_directory()
|
||||
if not os.getenv('SKIP_GENERATE_AUTHORS'):
|
||||
if git_dir:
|
||||
# don't include jenkins email address in AUTHORS file
|
||||
git_log_cmd = ("git --git-dir=" + git_dir +
|
||||
" log --format='%aN <%aE>' | sort -u | "
|
||||
"egrep -v '" + jenkins_email + "'")
|
||||
changelog = _run_shell_command(git_log_cmd)
|
||||
signed_cmd = ("git --git-dir=" + git_dir +
|
||||
" log | grep -i Co-authored-by: | sort -u")
|
||||
signed_entries = _run_shell_command(signed_cmd)
|
||||
if signed_entries:
|
||||
new_entries = "\n".join(
|
||||
[signed.split(":", 1)[1].strip()
|
||||
for signed in signed_entries.split("\n") if signed])
|
||||
changelog = "\n".join((changelog, new_entries))
|
||||
mailmap = _parse_git_mailmap(git_dir)
|
||||
with open(new_authors, 'w') as new_authors_fh:
|
||||
new_authors_fh.write(canonicalize_emails(changelog, mailmap))
|
||||
if os.path.exists(old_authors):
|
||||
with open(old_authors, "r") as old_authors_fh:
|
||||
new_authors_fh.write('\n' + old_authors_fh.read())
|
||||
else:
|
||||
open(new_authors, 'w').close()
|
||||
|
||||
|
||||
_rst_template = """%(heading)s
|
||||
%(underline)s
|
||||
|
||||
.. automodule:: %(module)s
|
||||
:members:
|
||||
:undoc-members:
|
||||
:show-inheritance:
|
||||
"""
|
||||
|
||||
|
||||
def get_cmdclass():
|
||||
"""Return dict of commands to run from setup.py."""
|
||||
|
||||
cmdclass = dict()
|
||||
|
||||
def _find_modules(arg, dirname, files):
|
||||
for filename in files:
|
||||
if filename.endswith('.py') and filename != '__init__.py':
|
||||
arg["%s.%s" % (dirname.replace('/', '.'),
|
||||
filename[:-3])] = True
|
||||
|
||||
class LocalSDist(sdist.sdist):
|
||||
"""Builds the ChangeLog and Authors files from VC first."""
|
||||
|
||||
def run(self):
|
||||
write_git_changelog()
|
||||
generate_authors()
|
||||
# sdist.sdist is an old style class, can't use super()
|
||||
sdist.sdist.run(self)
|
||||
|
||||
cmdclass['sdist'] = LocalSDist
|
||||
|
||||
# If Sphinx is installed on the box running setup.py,
|
||||
# enable setup.py to build the documentation, otherwise,
|
||||
# just ignore it
|
||||
try:
|
||||
from sphinx.setup_command import BuildDoc
|
||||
|
||||
class LocalBuildDoc(BuildDoc):
|
||||
|
||||
builders = ['html', 'man']
|
||||
|
||||
def generate_autoindex(self):
|
||||
print("**Autodocumenting from %s" % os.path.abspath(os.curdir))
|
||||
modules = {}
|
||||
option_dict = self.distribution.get_option_dict('build_sphinx')
|
||||
source_dir = os.path.join(option_dict['source_dir'][1], 'api')
|
||||
if not os.path.exists(source_dir):
|
||||
os.makedirs(source_dir)
|
||||
for pkg in self.distribution.packages:
|
||||
if '.' not in pkg:
|
||||
os.path.walk(pkg, _find_modules, modules)
|
||||
module_list = modules.keys()
|
||||
module_list.sort()
|
||||
autoindex_filename = os.path.join(source_dir, 'autoindex.rst')
|
||||
with open(autoindex_filename, 'w') as autoindex:
|
||||
autoindex.write(""".. toctree::
|
||||
:maxdepth: 1
|
||||
|
||||
""")
|
||||
for module in module_list:
|
||||
output_filename = os.path.join(source_dir,
|
||||
"%s.rst" % module)
|
||||
heading = "The :mod:`%s` Module" % module
|
||||
underline = "=" * len(heading)
|
||||
values = dict(module=module, heading=heading,
|
||||
underline=underline)
|
||||
|
||||
print("Generating %s" % output_filename)
|
||||
with open(output_filename, 'w') as output_file:
|
||||
output_file.write(_rst_template % values)
|
||||
autoindex.write(" %s.rst\n" % module)
|
||||
|
||||
def run(self):
|
||||
if not os.getenv('SPHINX_DEBUG'):
|
||||
self.generate_autoindex()
|
||||
|
||||
for builder in self.builders:
|
||||
self.builder = builder
|
||||
self.finalize_options()
|
||||
self.project = self.distribution.get_name()
|
||||
self.version = self.distribution.get_version()
|
||||
self.release = self.distribution.get_version()
|
||||
BuildDoc.run(self)
|
||||
|
||||
class LocalBuildLatex(LocalBuildDoc):
|
||||
builders = ['latex']
|
||||
|
||||
cmdclass['build_sphinx'] = LocalBuildDoc
|
||||
cmdclass['build_sphinx_latex'] = LocalBuildLatex
|
||||
except ImportError:
|
||||
pass
|
||||
|
||||
return cmdclass
|
||||
|
||||
|
||||
def _get_revno(git_dir):
|
||||
"""Return the number of commits since the most recent tag.
|
||||
|
||||
We use git-describe to find this out, but if there are no
|
||||
tags then we fall back to counting commits since the beginning
|
||||
of time.
|
||||
"""
|
||||
describe = _run_shell_command(
|
||||
"git --git-dir=%s describe --always" % git_dir)
|
||||
if "-" in describe:
|
||||
return describe.rsplit("-", 2)[-2]
|
||||
|
||||
# no tags found
|
||||
revlist = _run_shell_command(
|
||||
"git --git-dir=%s rev-list --abbrev-commit HEAD" % git_dir)
|
||||
return len(revlist.splitlines())
|
||||
|
||||
|
||||
def _get_version_from_git(pre_version):
|
||||
"""Return a version which is equal to the tag that's on the current
|
||||
revision if there is one, or tag plus number of additional revisions
|
||||
if the current revision has no tag."""
|
||||
|
||||
git_dir = _get_git_directory()
|
||||
if git_dir:
|
||||
if pre_version:
|
||||
try:
|
||||
return _run_shell_command(
|
||||
"git --git-dir=" + git_dir + " describe --exact-match",
|
||||
throw_on_error=True).replace('-', '.')
|
||||
except Exception:
|
||||
sha = _run_shell_command(
|
||||
"git --git-dir=" + git_dir + " log -n1 --pretty=format:%h")
|
||||
return "%s.a%s.g%s" % (pre_version, _get_revno(git_dir), sha)
|
||||
else:
|
||||
return _run_shell_command(
|
||||
"git --git-dir=" + git_dir + " describe --always").replace(
|
||||
'-', '.')
|
||||
return None
|
||||
|
||||
|
||||
def _get_version_from_pkg_info(package_name):
|
||||
"""Get the version from PKG-INFO file if we can."""
|
||||
try:
|
||||
pkg_info_file = open('PKG-INFO', 'r')
|
||||
except (IOError, OSError):
|
||||
return None
|
||||
try:
|
||||
pkg_info = email.message_from_file(pkg_info_file)
|
||||
except email.MessageError:
|
||||
return None
|
||||
# Check to make sure we're in our own dir
|
||||
if pkg_info.get('Name', None) != package_name:
|
||||
return None
|
||||
return pkg_info.get('Version', None)
|
||||
|
||||
|
||||
def get_version(package_name, pre_version=None):
|
||||
"""Get the version of the project. First, try getting it from PKG-INFO, if
|
||||
it exists. If it does, that means we're in a distribution tarball or that
|
||||
install has happened. Otherwise, if there is no PKG-INFO file, pull the
|
||||
version from git.
|
||||
|
||||
We do not support setup.py version sanity in git archive tarballs, nor do
|
||||
we support packagers directly sucking our git repo into theirs. We expect
|
||||
that a source tarball be made from our git repo - or that if someone wants
|
||||
to make a source tarball from a fork of our repo with additional tags in it
|
||||
that they understand and desire the results of doing that.
|
||||
"""
|
||||
version = os.environ.get("OSLO_PACKAGE_VERSION", None)
|
||||
if version:
|
||||
return version
|
||||
version = _get_version_from_pkg_info(package_name)
|
||||
if version:
|
||||
return version
|
||||
version = _get_version_from_git(pre_version)
|
||||
if version:
|
||||
return version
|
||||
raise Exception("Versioning for this project requires either an sdist"
|
||||
" tarball, or access to an upstream git repository.")
|
||||
@ -1,94 +0,0 @@
|
||||
|
||||
# Copyright 2012 OpenStack Foundation
|
||||
# Copyright 2012-2013 Hewlett-Packard Development Company, L.P.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License. You may obtain
|
||||
# a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
"""
|
||||
Utilities for consuming the version from pkg_resources.
|
||||
"""
|
||||
|
||||
import pkg_resources
|
||||
|
||||
|
||||
class VersionInfo(object):
|
||||
|
||||
def __init__(self, package):
|
||||
"""Object that understands versioning for a package
|
||||
:param package: name of the python package, such as glance, or
|
||||
python-glanceclient
|
||||
"""
|
||||
self.package = package
|
||||
self.release = None
|
||||
self.version = None
|
||||
self._cached_version = None
|
||||
|
||||
def __str__(self):
|
||||
"""Make the VersionInfo object behave like a string."""
|
||||
return self.version_string()
|
||||
|
||||
def __repr__(self):
|
||||
"""Include the name."""
|
||||
return "VersionInfo(%s:%s)" % (self.package, self.version_string())
|
||||
|
||||
def _get_version_from_pkg_resources(self):
|
||||
"""Get the version of the package from the pkg_resources record
|
||||
associated with the package."""
|
||||
try:
|
||||
requirement = pkg_resources.Requirement.parse(self.package)
|
||||
provider = pkg_resources.get_provider(requirement)
|
||||
return provider.version
|
||||
except pkg_resources.DistributionNotFound:
|
||||
# The most likely cause for this is running tests in a tree
|
||||
# produced from a tarball where the package itself has not been
|
||||
# installed into anything. Revert to setup-time logic.
|
||||
from tuskar.openstack.common import setup
|
||||
return setup.get_version(self.package)
|
||||
|
||||
def release_string(self):
|
||||
"""Return the full version of the package including suffixes indicating
|
||||
VCS status.
|
||||
"""
|
||||
if self.release is None:
|
||||
self.release = self._get_version_from_pkg_resources()
|
||||
|
||||
return self.release
|
||||
|
||||
def version_string(self):
|
||||
"""Return the short version minus any alpha/beta tags."""
|
||||
if self.version is None:
|
||||
parts = []
|
||||
for part in self.release_string().split('.'):
|
||||
if part[0].isdigit():
|
||||
parts.append(part)
|
||||
else:
|
||||
break
|
||||
self.version = ".".join(parts)
|
||||
|
||||
return self.version
|
||||
|
||||
# Compatibility functions
|
||||
canonical_version_string = version_string
|
||||
version_string_with_vcs = release_string
|
||||
|
||||
def cached_version_string(self, prefix=""):
|
||||
"""Generate an object which will expand in a string context to
|
||||
the results of version_string(). We do this so that don't
|
||||
call into pkg_resources every time we start up a program when
|
||||
passing version information into the CONF constructor, but
|
||||
rather only do the calculation when and if a version is requested
|
||||
"""
|
||||
if not self._cached_version:
|
||||
self._cached_version = "%s%s" % (prefix,
|
||||
self.version_string())
|
||||
return self._cached_version
|
||||
Loading…
x
Reference in New Issue
Block a user