x/terracotta
Code Issues Proposed changes

Merge "Fix_config_invoke"

Browse Source
This commit is contained in:
Jenkins 2015-05-06 09:41:42 +00:00 committed by Gerrit Code Review
commit f009826949
6 changed files with 101 additions and 78 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

24
etc/terracotta.conf.sample
View File

@ -1,4 +1,4 @@
[default] [DEFAULT]
# The name of the host running the global manager # The name of the host running the global manager
global_manager_host = controller global_manager_host = controller
@ -9,6 +9,12 @@ global_manager_port = 60080
# cleaner in seconds # cleaner in seconds
db_cleaner_interval = 7200 db_cleaner_interval = 7200
# The admin user name for authentication with Nova using Keystone
os_admin_user = user
# The admin password for authentication with Nova using Keystone
os_admin_password = userpassword
# The directory used by the data collector to store the data on the # The directory used by the data collector to store the data on the
# resource usage by the VMs running on the host # resource usage by the VMs running on the host
local_data_directory = /var/lib/terracotta local_data_directory = /var/lib/terracotta
@ -25,6 +31,10 @@ data_collector_data_length = 100
# The network bandwidth in MB/s available for VM migration # The network bandwidth in MB/s available for VM migration
network_migration_bandwidth = 10 network_migration_bandwidth = 10
# The time interval between subsequent invocations of the data
# collector in seconds
data_collector_interval = 300
[api] [api]
# Terracotta API server host # Terracotta API server host
host = 0.0.0.0 host = 0.0.0.0
@ -51,13 +61,7 @@ auth_enable = True
vm_instance_directory = /var/lib/nova/instances vm_instance_directory = /var/lib/nova/instances
# The admin tenant name for authentication with Nova using Keystone # The admin tenant name for authentication with Nova using Keystone
os_admin_tenant_name = admin os_admin_tenant_name = tenantname
# The admin user name for authentication with Nova using Keystone
os_admin_user = admin
# The admin password for authentication with Nova using Keystone
os_admin_password = adminpassword
# The OpenStack authentication URL # The OpenStack authentication URL
os_auth_url = http://controller:5000/v2.0/ os_auth_url = http://controller:5000/v2.0/
@ -133,10 +137,6 @@ algorithm_vm_selection_factory = terracotta.locals.vm_selection.algorithms.minim
algorithm_vm_selection_parameters = {"last_n": 2} algorithm_vm_selection_parameters = {"last_n": 2}
[collector] [collector]
# The time interval between subsequent invocations of the data
# collector in seconds
data_collector_interval = 300
# The threshold on the overall (all cores) utilization of the physical # The threshold on the overall (all cores) utilization of the physical
# CPU of a host, above which the host is considered to be overloaded. # CPU of a host, above which the host is considered to be overloaded.
# This is used for logging host overloads into the database. # This is used for logging host overloads into the database.

110
terracotta/config.py
View File

@ -34,21 +34,35 @@ default_opts = [
cfg.StrOpt('global_manager_host', default='controller', cfg.StrOpt('global_manager_host', default='controller',
help='The name of the host running the global manager'), help='The name of the host running the global manager'),
cfg.IntOpt('global_manager_port', default=60080, cfg.IntOpt('global_manager_port', default=60080,
help='The port of the REST web service exposed by the global manager'), help='The port of the REST web service exposed by the global '
'manager'),
cfg.IntOpt('db_cleaner_interval', default=7200, cfg.IntOpt('db_cleaner_interval', default=7200,
help='The time interval between subsequent invocations of the database'), help='The time interval between subsequent invocations of the '
'database'),
cfg.StrOpt('os_admin_user', default='user',
help='The admin user name for authentication '
'with Nova using Keystone'),
cfg.StrOpt('os_admin_password', default='userpassword',
help='The admin password for authentication '
'with Nova using Keystone'),
cfg.StrOpt('local_data_directory', default='/var/lib/terracotta', cfg.StrOpt('local_data_directory', default='/var/lib/terracotta',
help='The directory used by the data collector to store the data on the' help='The directory used by the data collector to store the '
'resource usage by the VMs running on the host'), 'data on the resource usage by the VMs running on the '
'host'),
cfg.FloatOpt('host_cpu_usable_by_vms', default=1.0, cfg.FloatOpt('host_cpu_usable_by_vms', default=1.0,
help='The threshold on the overall (all cores) utilization of the physical ' help='The threshold on the overall (all cores) utilization '
'CPU of a host that can be allocated to VMs.'), 'of the physical CPU of a host that can be allocated to '
'VMs.'),
cfg.IntOpt('data_collector_data_length', default=100, cfg.IntOpt('data_collector_data_length', default=100,
help='The number of the latest data values stored locally ' help='The number of the latest data values stored locally '
'by the data collector and passed to the underload / ' 'by the data collector and passed to the underload / '
'overload detection and VM placement algorithms'), 'overload detection and VM placement algorithms'),
cfg.FloatOpt('network_migration_bandwidth', default=10, cfg.FloatOpt('network_migration_bandwidth', default=10,
help='The network bandwidth in MB/s available for VM migration') help='The network bandwidth in MB/s available for '
'VM migration'),
cfg.IntOpt('data_collector_interval', default=300,
help='The time interval between subsequent invocations '
'of the data collector in seconds')
] ]
api_opts = [ api_opts = [
@ -57,7 +71,8 @@ api_opts = [
] ]
pecan_opts = [ pecan_opts = [
cfg.StrOpt('root', default='terracotta.api.controllers.root.RootController', cfg.StrOpt('root', default='terracotta.api.'
'controllers.root.RootController',
help='Pecan root controller'), help='Pecan root controller'),
cfg.ListOpt('modules', default=["terracotta.api"], cfg.ListOpt('modules', default=["terracotta.api"],
help='A list of modules where pecan will search for ' help='A list of modules where pecan will search for '
@ -85,18 +100,13 @@ global_manager_opts = [
cfg.StrOpt('os_admin_tenant_name', default='tenantname', cfg.StrOpt('os_admin_tenant_name', default='tenantname',
help='The admin tenant name for authentication ' help='The admin tenant name for authentication '
'with Nova using Keystone'), 'with Nova using Keystone'),
cfg.StrOpt('os_admin_user', default='username',
help='The admin user name for authentication '
'with Nova using Keystone'),
cfg.StrOpt('os_admin_password', default='adminpassword',
help='The admin password for authentication '
'with Nova using Keystone'),
cfg.StrOpt('os_auth_url', default='http://controller:5000/v2.0/', cfg.StrOpt('os_auth_url', default='http://controller:5000/v2.0/',
help='The OpenStack authentication URL'), help='The OpenStack authentication URL'),
cfg.ListOpt('compute_hosts', default=['compute1', 'compute2', 'compute3'], cfg.ListOpt('compute_hosts', default=['compute1', 'compute2', 'compute3'],
help='A coma-separated list of compute host names'), help='A coma-separated list of compute host names'),
cfg.BoolOpt('block_migration', default=True, cfg.BoolOpt('block_migration', default=True,
help='Whether to use block migration (includes disk migration)'), help='Whether to use block migration'
'(includes disk migration)'),
cfg.StrOpt('compute_user', default='terracotta', cfg.StrOpt('compute_user', default='terracotta',
help='The user name for connecting to the compute hosts ' help='The user name for connecting to the compute hosts '
'to switch them into the sleep mode'), 'to switch them into the sleep mode'),
@ -104,19 +114,24 @@ global_manager_opts = [
help='The password of the user account used for connecting ' help='The password of the user account used for connecting '
'to the compute hosts to switch them into the sleep mode'), 'to the compute hosts to switch them into the sleep mode'),
cfg.StrOpt('sleep_command', default='pm-suspend', cfg.StrOpt('sleep_command', default='pm-suspend',
help='A shell command used to switch a host into the sleep mode, the' help='A shell command used to switch a host into the sleep '
'compute_user must have permissions to execute this command'), 'mode, the compute_user must have permissions to execute '
'this command'),
cfg.StrOpt('ether_wake_interface', default='eth0', cfg.StrOpt('ether_wake_interface', default='eth0',
help='The network interface to send a magic packet from ' help='The network interface to send a magic packet from '
'using ether-wake'), 'using ether-wake'),
cfg.StrOpt('algorithm_vm_placement_factory', cfg.StrOpt('algorithm_vm_placement_factory',
default='terracotta.globals.vm_placement.bin_packing.best_fit_decreasing_factory', default='terracotta.globals.vm_placement.bin_packing.'
help='The fully qualified name of a Python factory function that returns a' 'best_fit_decreasing_factory',
'function implementing a VM placement algorithm'), help='The fully qualified name of a Python factory function '
'that returns a function implementing a VM placement '
'algorithm'),
cfg.DictOpt('algorithm_vm_placement_parameters', cfg.DictOpt('algorithm_vm_placement_parameters',
default={'cpu_threshold': 0.8, 'ram_threshold': 0.95, 'last_n_vm_cpu': 2}, default={'cpu_threshold': 0.8,
help='A JSON encoded parameters, which will be parsed and passed to the' 'ram_threshold': 0.95,
'specified VM placement algorithm factory') 'last_n_vm_cpu': 2},
help='A JSON encoded parameters, which will be parsed and '
'passed to the specified VM placement algorithm factory')
] ]
local_manager_opts = [ local_manager_opts = [
@ -124,17 +139,22 @@ local_manager_opts = [
help='The time interval between subsequent invocations ' help='The time interval between subsequent invocations '
'of the local manager in seconds'), 'of the local manager in seconds'),
cfg.StrOpt('algorithm_underload_detection_factory', cfg.StrOpt('algorithm_underload_detection_factory',
default='terracotta.locals.underload.trivial.last_n_average_threshold_factory', default='terracotta.locals.underload.trivial.last_n_average '
help='The fully qualified name of a Python factory function that returns a' 'threshold_factory',
'function implementing an underload detection algorithm'), help='The fully qualified name of a Python factory function '
'that returns a function implementing an underload '
'detection algorithm'),
cfg.DictOpt('algorithm_underload_detection_parameters', cfg.DictOpt('algorithm_underload_detection_parameters',
default={'threshold': 0.5, 'n': 2}, default={'threshold': 0.5,
help='A JSON encoded parameters, which will be parsed and passed to the' 'n': 2},
'specified underload detection algorithm factory'), help='A JSON encoded parameters, which will be parsed and '
'passed to the specified underload detection algorithm '
'factory'),
cfg.StrOpt('algorithm_overload_detection_factory', cfg.StrOpt('algorithm_overload_detection_factory',
default='terracotta.locals.overload.mhod.core.mhod_factory', default='terracotta.locals.overload.mhod.core.mhod_factory',
help='The fully qualified name of a Python factory function that returns a' help='The fully qualified name of a Python factory function '
'function implementing an overload detection algorithm'), 'that returns a function implementing an overload '
'detection algorithm'),
cfg.DictOpt('algorithm_overload_detection_parameters', cfg.DictOpt('algorithm_overload_detection_parameters',
default={'state_config': [0.8], default={'state_config': [0.8],
'otf': 0.1, 'otf': 0.1,
@ -142,31 +162,33 @@ local_manager_opts = [
'window_sizes': [30, 40, 50, 60, 70, 80, 90, 100], 'window_sizes': [30, 40, 50, 60, 70, 80, 90, 100],
'bruteforce_step': 0.5, 'bruteforce_step': 0.5,
'learning_steps': 10}, 'learning_steps': 10},
help='A JSON encoded parameters, which will be parsed and passed to the' help='A JSON encoded parameters, which will be parsed and '
'specified overload detection algorithm factory'), 'passed to the specified overload detection algorithm '
'factory'),
cfg.StrOpt('algorithm_vm_selection_factory', cfg.StrOpt('algorithm_vm_selection_factory',
default='terracotta.locals.vm_selection.algorithms.minimum_migration_time_max_cpu_factory', default='terracotta.locals.vm_selection.algorithms.'
help='The fully qualified name of a Python factory function that returns a' 'minimum_migration_time_max_cpu_factory',
'function implementing a VM selection algorithm'), help='The fully qualified name of a Python factory function '
'that returns a function implementing a VM selection '
'algorithm'),
cfg.DictOpt('algorithm_vm_selection_parameters', cfg.DictOpt('algorithm_vm_selection_parameters',
default={'last_n': 2}, default={'last_n': 2},
help='A JSON encoded parameters, which will be parsed and passed to the' help='A JSON encoded parameters, which will be parsed and '
'specified VM selection algorithm factory') 'passed to the specified VM selection algorithm factory')
] ]
collector_opts = [ collector_opts = [
cfg.IntOpt('data_collector_interval', default=300,
help='The time interval between subsequent invocations '
'of the data collector in seconds'),
cfg.FloatOpt('host_cpu_overload_threshold', default=0.8, cfg.FloatOpt('host_cpu_overload_threshold', default=0.8,
help='The threshold on the overall (all cores) utilization ' help='The threshold on the overall (all cores) utilization '
'of the physical CPU of a host, above which the host ' 'of the physical CPU of a host, above which the host '
'is considered to be overloaded.' 'is considered to be overloaded.'
'This is used for logging host overloads into the database.') 'This is used for logging host overloads into the'
' database.')
] ]
database_opts = [ database_opts = [
cfg.StrOpt('sql_connection', default='mysql://terracotta:terracottapassword@controller/terracotta', cfg.StrOpt('sql_connection', default='mysql://terracotta:terracottapasswd@'
'controller/terracotta',
help='The host name and credentials for connecting ' help='The host name and credentials for connecting '
'to the MySQL database specified in the format ' 'to the MySQL database specified in the format '
'supported by SQLAlchemy') 'supported by SQLAlchemy')
@ -175,7 +197,7 @@ database_opts = [
CONF = cfg.CONF CONF = cfg.CONF
CONF.register_opts(pecan_opts, group='pecan') CONF.register_opts(pecan_opts, group='pecan')
CONF.register_opts(default_opts, group='default') CONF.register_opts(default_opts, group='DEFAULT')
CONF.register_opts(api_opts, group='api') CONF.register_opts(api_opts, group='api')
CONF.register_opts(global_manager_opts, group='global_manager') CONF.register_opts(global_manager_opts, group='global_manager')
CONF.register_opts(local_manager_opts, group='local_manager') CONF.register_opts(local_manager_opts, group='local_manager')

26
terracotta/globals/manager.py
View File

@ -284,13 +284,13 @@ class GlobalManager(object):
'nova': client.Client(2, 'nova': client.Client(2,
CONF.os_admin_user, CONF.os_admin_user,
CONF.os_admin_password, CONF.os_admin_password,
CONF.os_admin_tenant_name, CONF.global_manager.os_admin_tenant_name,
CONF.os_auth_url, CONF.global_manager.os_auth_url,
service_type="compute"), service_type="compute"),
'hashed_username': sha1(CONF.os_admin_user).hexdigest(), 'hashed_username': sha1(CONF.os_admin_user).hexdigest(),
'hashed_password': sha1(CONF.os_admin_password).hexdigest(), 'hashed_password': sha1(CONF.os_admin_password).hexdigest(),
'compute_hosts': common.parse_compute_hosts( 'compute_hosts': common.parse_compute_hosts(
CONF.compute_hosts), CONF.global_manager.compute_hosts),
'host_macs': {}} 'host_macs': {}}
def switch_hosts_on(self, hosts): def switch_hosts_on(self, hosts):
@ -302,7 +302,7 @@ class GlobalManager(object):
command = '{0} -i {1} {2}'.format( command = '{0} -i {1} {2}'.format(
etherwake, etherwake,
CONF.ether_wake_interface, CONF.global_manager.ether_wake_interface,
self.state['host_macs'][host]) self.state['host_macs'][host])
LOG.debug('Calling: %s', command) LOG.debug('Calling: %s', command)
@ -420,10 +420,10 @@ class GlobalManager(object):
if 'vm_placement' not in self.state: if 'vm_placement' not in self.state:
vm_placement_params = common.parse_parameters( vm_placement_params = common.parse_parameters(
CONF.algorithm_vm_placement_parameters) CONF.global_manager.algorithm_vm_placement_parameters)
vm_placement_state = None vm_placement_state = None
vm_placement = common.call_function_by_name( vm_placement = common.call_function_by_name(
CONF.algorithm_vm_placement_factory, CONF.global_manager.algorithm_vm_placement_factory,
[time_step, [time_step,
migration_time, migration_time,
vm_placement_params]) vm_placement_params])
@ -460,14 +460,14 @@ class GlobalManager(object):
LOG.info('Started underload VM migrations') LOG.info('Started underload VM migrations')
migrate_vms(self.state['db'], migrate_vms(self.state['db'],
self.state['nova'], self.state['nova'],
CONF.vm_instance_directory, CONF.global_manager.vm_instance_directory,
placement, placement,
CONF.block_migration) CONF.global_manager.block_migration)
LOG.info('Completed underload VM migrations') LOG.info('Completed underload VM migrations')
if hosts_to_deactivate: if hosts_to_deactivate:
switch_hosts_off(self.state['db'], switch_hosts_off(self.state['db'],
CONF.sleep_command, CONF.global_manager.sleep_command,
hosts_to_deactivate) hosts_to_deactivate)
LOG.info('Completed processing an underload request') LOG.info('Completed processing an underload request')
@ -572,10 +572,10 @@ class GlobalManager(object):
if 'vm_placement' not in state: if 'vm_placement' not in state:
vm_placement_params = common.parse_parameters( vm_placement_params = common.parse_parameters(
CONF.algorithm_vm_placement_parameters) CONF.global_manager.algorithm_vm_placement_parameters)
vm_placement_state = None vm_placement_state = None
vm_placement = common.call_function_by_name( vm_placement = common.call_function_by_name(
CONF.algorithm_vm_placement_factory, CONF.global_manager.algorithm_vm_placement_factory,
[time_step, [time_step,
migration_time, migration_time,
vm_placement_params]) vm_placement_params])
@ -608,9 +608,9 @@ class GlobalManager(object):
LOG.info('Started overload VM migrations') LOG.info('Started overload VM migrations')
migrate_vms(self.state['db'], migrate_vms(self.state['db'],
self.state['nova'], self.state['nova'],
CONF.vm_instance_directory, CONF.global_manager.vm_instance_directory,
placement, placement,
CONF.block_migration) CONF.global_manager.block_migration)
LOG.info('Completed overload VM migrations') LOG.info('Completed overload VM migrations')
LOG.info('Completed processing an overload request') LOG.info('Completed processing an overload request')
return state return state

2
terracotta/locals/collector.py
View File

@ -159,7 +159,7 @@ class Collector(periodic_task.PeriodicTasks):
'vir_connection': vir_connection, 'vir_connection': vir_connection,
'hostname': hostname, 'hostname': hostname,
'host_cpu_overload_threshold': 'host_cpu_overload_threshold':
CONF.host_cpu_overload_threshold * \ CONF.collector.host_cpu_overload_threshold * \
host_cpu_usable_by_vms, host_cpu_usable_by_vms,
'physical_cpus': physical_cpus, 'physical_cpus': physical_cpus,
'physical_cpu_mhz': host_cpu_mhz, 'physical_cpu_mhz': host_cpu_mhz,

12
terracotta/locals/manager.py
View File

@ -224,9 +224,9 @@ class LocalManager(periodic_task.PeriodicTasks):
if 'underload_detection' not in state: if 'underload_detection' not in state:
underload_detection_params = common.parse_parameters( underload_detection_params = common.parse_parameters(
CONF.algorithm_underload_detection_parameters) CONF.local_manager.algorithm_underload_detection_parameters)
underload_detection = common.call_function_by_name( underload_detection = common.call_function_by_name(
CONF.algorithm_underload_detection_factory, CONF.local_manager.algorithm_underload_detection_factory,
[time_step, [time_step,
migration_time, migration_time,
underload_detection_params]) underload_detection_params])
@ -234,9 +234,9 @@ class LocalManager(periodic_task.PeriodicTasks):
state['underload_detection_state'] = {} state['underload_detection_state'] = {}
overload_detection_params = common.parse_parameters( overload_detection_params = common.parse_parameters(
CONF.algorithm_overload_detection_parameters) CONF.local_manager.algorithm_overload_detection_parameters)
overload_detection = common.call_function_by_name( overload_detection = common.call_function_by_name(
CONF.algorithm_overload_detection_factory, CONF.local_manager.algorithm_overload_detection_factory,
[time_step, [time_step,
migration_time, migration_time,
overload_detection_params]) overload_detection_params])
@ -244,9 +244,9 @@ class LocalManager(periodic_task.PeriodicTasks):
state['overload_detection_state'] = {} state['overload_detection_state'] = {}
vm_selection_params = common.parse_parameters( vm_selection_params = common.parse_parameters(
CONF.algorithm_vm_selection_parameters) CONF.local_manager.algorithm_vm_selection_parameters)
vm_selection = common.call_function_by_name( vm_selection = common.call_function_by_name(
CONF.algorithm_vm_selection_factory, CONF.local_manager.algorithm_vm_selection_factory,
[time_step, [time_step,
migration_time, migration_time,
vm_selection_params]) vm_selection_params])

5
terracotta/utils/db_utils.py
View File

@ -23,6 +23,7 @@ from terracotta import db as database
LOG = logging.getLogger(__name__) LOG = logging.getLogger(__name__)
CONF = cfg.CONF
def init_db(): def init_db():
@ -31,7 +32,7 @@ def init_db():
:param sql_connection: A database connection URL. :param sql_connection: A database connection URL.
:return: The initialized database. :return: The initialized database.
""" """
engine = create_engine(CONF.sql_connection) engine = create_engine(CONF.database.sql_connection)
metadata = MetaData() metadata = MetaData()
metadata.bind = engine metadata.bind = engine
@ -91,5 +92,5 @@ def init_db():
vm_resource_usage, vm_migrations, host_states, vm_resource_usage, vm_migrations, host_states,
host_overload) host_overload)
LOG.debug('Initialized a DB connection to %s', CONF.sql_connection) LOG.debug('Initialized a DB connection to %s', CONF.database.sql_connection)
return db return db