x/terracotta
Code Issues Proposed changes

Merge "Fix_config_invoke"

Browse Source
This commit is contained in:
Jenkins 2015-05-06 09:41:42 +00:00 committed by Gerrit Code Review
commit f009826949
6 changed files with 101 additions and 78 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

24
etc/terracotta.conf.sample
View File

@ -1,4 +1,4 @@
[default]
[DEFAULT]
# The name of the host running the global manager
global_manager_host = controller
@ -9,6 +9,12 @@ global_manager_port = 60080
# cleaner in seconds
db_cleaner_interval = 7200
# The admin user name for authentication with Nova using Keystone
os_admin_user = user
# The admin password for authentication with Nova using Keystone
os_admin_password = userpassword
# The directory used by the data collector to store the data on the
# resource usage by the VMs running on the host
local_data_directory = /var/lib/terracotta
@ -25,6 +31,10 @@ data_collector_data_length = 100
# The network bandwidth in MB/s available for VM migration
network_migration_bandwidth = 10
# The time interval between subsequent invocations of the data
# collector in seconds
data_collector_interval = 300
[api]
# Terracotta API server host
host = 0.0.0.0
@ -51,13 +61,7 @@ auth_enable = True
vm_instance_directory = /var/lib/nova/instances
# The admin tenant name for authentication with Nova using Keystone
os_admin_tenant_name = admin
# The admin user name for authentication with Nova using Keystone
os_admin_user = admin
# The admin password for authentication with Nova using Keystone
os_admin_password = adminpassword
os_admin_tenant_name = tenantname
# The OpenStack authentication URL
os_auth_url = http://controller:5000/v2.0/
@ -133,10 +137,6 @@ algorithm_vm_selection_factory = terracotta.locals.vm_selection.algorithms.minim
algorithm_vm_selection_parameters = {"last_n": 2}
[collector]
# The time interval between subsequent invocations of the data
# collector in seconds
data_collector_interval = 300
# The threshold on the overall (all cores) utilization of the physical
# CPU of a host, above which the host is considered to be overloaded.
# This is used for logging host overloads into the database.

110
terracotta/config.py
View File

@ -34,21 +34,35 @@ default_opts = [
cfg.StrOpt('global_manager_host', default='controller',
help='The name of the host running the global manager'),
cfg.IntOpt('global_manager_port', default=60080,
help='The port of the REST web service exposed by the global manager'),
help='The port of the REST web service exposed by the global '
'manager'),
cfg.IntOpt('db_cleaner_interval', default=7200,
help='The time interval between subsequent invocations of the database'),
help='The time interval between subsequent invocations of the '
'database'),
cfg.StrOpt('os_admin_user', default='user',
help='The admin user name for authentication '
'with Nova using Keystone'),
cfg.StrOpt('os_admin_password', default='userpassword',
help='The admin password for authentication '
'with Nova using Keystone'),
cfg.StrOpt('local_data_directory', default='/var/lib/terracotta',
help='The directory used by the data collector to store the data on the'
'resource usage by the VMs running on the host'),
help='The directory used by the data collector to store the '
'data on the resource usage by the VMs running on the '
'host'),
cfg.FloatOpt('host_cpu_usable_by_vms', default=1.0,
help='The threshold on the overall (all cores) utilization of the physical '
'CPU of a host that can be allocated to VMs.'),
help='The threshold on the overall (all cores) utilization '
'of the physical CPU of a host that can be allocated to '
'VMs.'),
cfg.IntOpt('data_collector_data_length', default=100,
help='The number of the latest data values stored locally '
'by the data collector and passed to the underload / '
'overload detection and VM placement algorithms'),
cfg.FloatOpt('network_migration_bandwidth', default=10,
help='The network bandwidth in MB/s available for VM migration')
help='The network bandwidth in MB/s available for '
'VM migration'),
cfg.IntOpt('data_collector_interval', default=300,
help='The time interval between subsequent invocations '
'of the data collector in seconds')
]
api_opts = [
@ -57,7 +71,8 @@ api_opts = [
]
pecan_opts = [
cfg.StrOpt('root', default='terracotta.api.controllers.root.RootController',
cfg.StrOpt('root', default='terracotta.api.'
'controllers.root.RootController',
help='Pecan root controller'),
cfg.ListOpt('modules', default=["terracotta.api"],
help='A list of modules where pecan will search for '
@ -85,18 +100,13 @@ global_manager_opts = [
cfg.StrOpt('os_admin_tenant_name', default='tenantname',
help='The admin tenant name for authentication '
'with Nova using Keystone'),
cfg.StrOpt('os_admin_user', default='username',
help='The admin user name for authentication '
'with Nova using Keystone'),
cfg.StrOpt('os_admin_password', default='adminpassword',
help='The admin password for authentication '
'with Nova using Keystone'),
cfg.StrOpt('os_auth_url', default='http://controller:5000/v2.0/',
help='The OpenStack authentication URL'),
cfg.ListOpt('compute_hosts', default=['compute1', 'compute2', 'compute3'],
help='A coma-separated list of compute host names'),
cfg.BoolOpt('block_migration', default=True,
help='Whether to use block migration (includes disk migration)'),
help='Whether to use block migration'
'(includes disk migration)'),
cfg.StrOpt('compute_user', default='terracotta',
help='The user name for connecting to the compute hosts '
'to switch them into the sleep mode'),
@ -104,19 +114,24 @@ global_manager_opts = [
help='The password of the user account used for connecting '
'to the compute hosts to switch them into the sleep mode'),
cfg.StrOpt('sleep_command', default='pm-suspend',
help='A shell command used to switch a host into the sleep mode, the'
'compute_user must have permissions to execute this command'),
help='A shell command used to switch a host into the sleep '
'mode, the compute_user must have permissions to execute '
'this command'),
cfg.StrOpt('ether_wake_interface', default='eth0',
help='The network interface to send a magic packet from '
'using ether-wake'),
cfg.StrOpt('algorithm_vm_placement_factory',
default='terracotta.globals.vm_placement.bin_packing.best_fit_decreasing_factory',
help='The fully qualified name of a Python factory function that returns a'
'function implementing a VM placement algorithm'),
default='terracotta.globals.vm_placement.bin_packing.'
'best_fit_decreasing_factory',
help='The fully qualified name of a Python factory function '
'that returns a function implementing a VM placement '
'algorithm'),
cfg.DictOpt('algorithm_vm_placement_parameters',
default={'cpu_threshold': 0.8, 'ram_threshold': 0.95, 'last_n_vm_cpu': 2},
help='A JSON encoded parameters, which will be parsed and passed to the'
'specified VM placement algorithm factory')
default={'cpu_threshold': 0.8,
'ram_threshold': 0.95,
'last_n_vm_cpu': 2},
help='A JSON encoded parameters, which will be parsed and '
'passed to the specified VM placement algorithm factory')
]
local_manager_opts = [
@ -124,17 +139,22 @@ local_manager_opts = [
help='The time interval between subsequent invocations '
'of the local manager in seconds'),
cfg.StrOpt('algorithm_underload_detection_factory',
default='terracotta.locals.underload.trivial.last_n_average_threshold_factory',
help='The fully qualified name of a Python factory function that returns a'
'function implementing an underload detection algorithm'),
default='terracotta.locals.underload.trivial.last_n_average '
'threshold_factory',
help='The fully qualified name of a Python factory function '
'that returns a function implementing an underload '
'detection algorithm'),
cfg.DictOpt('algorithm_underload_detection_parameters',
default={'threshold': 0.5, 'n': 2},
help='A JSON encoded parameters, which will be parsed and passed to the'
'specified underload detection algorithm factory'),
default={'threshold': 0.5,
'n': 2},
help='A JSON encoded parameters, which will be parsed and '
'passed to the specified underload detection algorithm '
'factory'),
cfg.StrOpt('algorithm_overload_detection_factory',
default='terracotta.locals.overload.mhod.core.mhod_factory',
help='The fully qualified name of a Python factory function that returns a'
'function implementing an overload detection algorithm'),
help='The fully qualified name of a Python factory function '
'that returns a function implementing an overload '
'detection algorithm'),
cfg.DictOpt('algorithm_overload_detection_parameters',
default={'state_config': [0.8],
'otf': 0.1,
@ -142,31 +162,33 @@ local_manager_opts = [
'window_sizes': [30, 40, 50, 60, 70, 80, 90, 100],
'bruteforce_step': 0.5,
'learning_steps': 10},
help='A JSON encoded parameters, which will be parsed and passed to the'
'specified overload detection algorithm factory'),
help='A JSON encoded parameters, which will be parsed and '
'passed to the specified overload detection algorithm '
'factory'),
cfg.StrOpt('algorithm_vm_selection_factory',
default='terracotta.locals.vm_selection.algorithms.minimum_migration_time_max_cpu_factory',
help='The fully qualified name of a Python factory function that returns a'
'function implementing a VM selection algorithm'),
default='terracotta.locals.vm_selection.algorithms.'
'minimum_migration_time_max_cpu_factory',
help='The fully qualified name of a Python factory function '
'that returns a function implementing a VM selection '
'algorithm'),
cfg.DictOpt('algorithm_vm_selection_parameters',
default={'last_n': 2},
help='A JSON encoded parameters, which will be parsed and passed to the'
'specified VM selection algorithm factory')
help='A JSON encoded parameters, which will be parsed and '
'passed to the specified VM selection algorithm factory')
]
collector_opts = [
cfg.IntOpt('data_collector_interval', default=300,
help='The time interval between subsequent invocations '
'of the data collector in seconds'),
cfg.FloatOpt('host_cpu_overload_threshold', default=0.8,
help='The threshold on the overall (all cores) utilization '
'of the physical CPU of a host, above which the host '
'is considered to be overloaded.'
'This is used for logging host overloads into the database.')
'This is used for logging host overloads into the'
' database.')
]
database_opts = [
cfg.StrOpt('sql_connection', default='mysql://terracotta:terracottapassword@controller/terracotta',
cfg.StrOpt('sql_connection', default='mysql://terracotta:terracottapasswd@'
'controller/terracotta',
help='The host name and credentials for connecting '
'to the MySQL database specified in the format '
'supported by SQLAlchemy')
@ -175,7 +197,7 @@ database_opts = [
CONF = cfg.CONF
CONF.register_opts(pecan_opts, group='pecan')
CONF.register_opts(default_opts, group='default')
CONF.register_opts(default_opts, group='DEFAULT')
CONF.register_opts(api_opts, group='api')
CONF.register_opts(global_manager_opts, group='global_manager')
CONF.register_opts(local_manager_opts, group='local_manager')

26
terracotta/globals/manager.py
View File

@ -284,13 +284,13 @@ class GlobalManager(object):
'nova': client.Client(2,
CONF.os_admin_user,
CONF.os_admin_password,
CONF.os_admin_tenant_name,
CONF.os_auth_url,
CONF.global_manager.os_admin_tenant_name,
CONF.global_manager.os_auth_url,
service_type="compute"),
'hashed_username': sha1(CONF.os_admin_user).hexdigest(),
'hashed_password': sha1(CONF.os_admin_password).hexdigest(),
'compute_hosts': common.parse_compute_hosts(
CONF.compute_hosts),
CONF.global_manager.compute_hosts),
'host_macs': {}}
def switch_hosts_on(self, hosts):
@ -302,7 +302,7 @@ class GlobalManager(object):
command = '{0} -i {1} {2}'.format(
etherwake,
CONF.ether_wake_interface,
CONF.global_manager.ether_wake_interface,
self.state['host_macs'][host])
LOG.debug('Calling: %s', command)
@ -420,10 +420,10 @@ class GlobalManager(object):
if 'vm_placement' not in self.state:
vm_placement_params = common.parse_parameters(
CONF.algorithm_vm_placement_parameters)
CONF.global_manager.algorithm_vm_placement_parameters)
vm_placement_state = None
vm_placement = common.call_function_by_name(
CONF.algorithm_vm_placement_factory,
CONF.global_manager.algorithm_vm_placement_factory,
[time_step,
migration_time,
vm_placement_params])
@ -460,14 +460,14 @@ class GlobalManager(object):
LOG.info('Started underload VM migrations')
migrate_vms(self.state['db'],
self.state['nova'],
CONF.vm_instance_directory,
CONF.global_manager.vm_instance_directory,
placement,
CONF.block_migration)
CONF.global_manager.block_migration)
LOG.info('Completed underload VM migrations')
if hosts_to_deactivate:
switch_hosts_off(self.state['db'],
CONF.sleep_command,
CONF.global_manager.sleep_command,
hosts_to_deactivate)
LOG.info('Completed processing an underload request')
@ -572,10 +572,10 @@ class GlobalManager(object):
if 'vm_placement' not in state:
vm_placement_params = common.parse_parameters(
CONF.algorithm_vm_placement_parameters)
CONF.global_manager.algorithm_vm_placement_parameters)
vm_placement_state = None
vm_placement = common.call_function_by_name(
CONF.algorithm_vm_placement_factory,
CONF.global_manager.algorithm_vm_placement_factory,
[time_step,
migration_time,
vm_placement_params])
@ -608,9 +608,9 @@ class GlobalManager(object):
LOG.info('Started overload VM migrations')
migrate_vms(self.state['db'],
self.state['nova'],
CONF.vm_instance_directory,
CONF.global_manager.vm_instance_directory,
placement,
CONF.block_migration)
CONF.global_manager.block_migration)
LOG.info('Completed overload VM migrations')
LOG.info('Completed processing an overload request')
return state

2
terracotta/locals/collector.py
View File

@ -159,7 +159,7 @@ class Collector(periodic_task.PeriodicTasks):
'vir_connection': vir_connection,
'hostname': hostname,
'host_cpu_overload_threshold':
CONF.host_cpu_overload_threshold * \
CONF.collector.host_cpu_overload_threshold * \
host_cpu_usable_by_vms,
'physical_cpus': physical_cpus,
'physical_cpu_mhz': host_cpu_mhz,

12
terracotta/locals/manager.py
View File

@ -224,9 +224,9 @@ class LocalManager(periodic_task.PeriodicTasks):
if 'underload_detection' not in state:
underload_detection_params = common.parse_parameters(
CONF.algorithm_underload_detection_parameters)
CONF.local_manager.algorithm_underload_detection_parameters)
underload_detection = common.call_function_by_name(
CONF.algorithm_underload_detection_factory,
CONF.local_manager.algorithm_underload_detection_factory,
[time_step,
migration_time,
underload_detection_params])
@ -234,9 +234,9 @@ class LocalManager(periodic_task.PeriodicTasks):
state['underload_detection_state'] = {}
overload_detection_params = common.parse_parameters(
CONF.algorithm_overload_detection_parameters)
CONF.local_manager.algorithm_overload_detection_parameters)
overload_detection = common.call_function_by_name(
CONF.algorithm_overload_detection_factory,
CONF.local_manager.algorithm_overload_detection_factory,
[time_step,
migration_time,
overload_detection_params])
@ -244,9 +244,9 @@ class LocalManager(periodic_task.PeriodicTasks):
state['overload_detection_state'] = {}
vm_selection_params = common.parse_parameters(
CONF.algorithm_vm_selection_parameters)
CONF.local_manager.algorithm_vm_selection_parameters)
vm_selection = common.call_function_by_name(
CONF.algorithm_vm_selection_factory,
CONF.local_manager.algorithm_vm_selection_factory,
[time_step,
migration_time,
vm_selection_params])

5
terracotta/utils/db_utils.py
View File

@ -23,6 +23,7 @@ from terracotta import db as database
LOG = logging.getLogger(__name__)
CONF = cfg.CONF
def init_db():
@ -31,7 +32,7 @@ def init_db():
:param sql_connection: A database connection URL.
:return: The initialized database.
"""
engine = create_engine(CONF.sql_connection)
engine = create_engine(CONF.database.sql_connection)
metadata = MetaData()
metadata.bind = engine
@ -91,5 +92,5 @@ def init_db():
vm_resource_usage, vm_migrations, host_states,
host_overload)
LOG.debug('Initialized a DB connection to %s', CONF.sql_connection)
LOG.debug('Initialized a DB connection to %s', CONF.database.sql_connection)
return db