From 65c2c4418ca51581de25917d094c96fef4ffa5f0 Mon Sep 17 00:00:00 2001 From: Michael Krotscheck Date: Fri, 17 Oct 2014 16:22:58 -0700 Subject: [PATCH] Plugins may now register cron workers. This adds a crontab plugin hook to StoryBoard, allowing a plugin developer to run periodic events. Example use cases include: - Summary emails. - Periodic report generation. - Synchronization check points. Plugins are expected to provide their own execution interval and configuration indicator. The management of cron workers is implemented as its own cron plugin as a sample, and unit tests for all components are provided. Change-Id: I3aa466e183f1faede9493123510ee11feb55e7aa --- doc/source/extending/index.rst | 96 +++++ doc/source/extending/plugin_cron.rst | 88 +++++ doc/source/index.rst | 9 + etc/storyboard.conf.sample | 6 + requirements.txt | 4 +- setup.cfg | 4 +- storyboard/api/app.py | 4 + storyboard/plugin/cron/__init__.py | 71 ++++ storyboard/plugin/cron/base.py | 125 ++++++ storyboard/plugin/cron/manager.py | 133 +++++++ storyboard/tests/plugin/cron/__init__.py | 0 storyboard/tests/plugin/cron/mock_plugin.py | 56 +++ storyboard/tests/plugin/cron/test_base.py | 123 ++++++ storyboard/tests/plugin/cron/test_manager.py | 382 +++++++++++++++++++ 14 files changed, 1099 insertions(+), 2 deletions(-) create mode 100644 doc/source/extending/index.rst create mode 100644 doc/source/extending/plugin_cron.rst create mode 100644 storyboard/plugin/cron/__init__.py create mode 100644 storyboard/plugin/cron/base.py create mode 100644 storyboard/plugin/cron/manager.py create mode 100644 storyboard/tests/plugin/cron/__init__.py create mode 100644 storyboard/tests/plugin/cron/mock_plugin.py create mode 100644 storyboard/tests/plugin/cron/test_base.py create mode 100644 storyboard/tests/plugin/cron/test_manager.py diff --git a/doc/source/extending/index.rst b/doc/source/extending/index.rst new file mode 100644 index 00000000..171ce526 --- /dev/null +++ b/doc/source/extending/index.rst @@ -0,0 +1,96 @@ +============================== +Extending StoryBoard: Overview +============================== + +StoryBoard provides many extension points that allow you to customize its +functionality to your own needs. All of these are implemented using +`stevedore `_, so that installing them is +simple, straightforward, and independent of the storyboard core libraries. +StoryBoard itself makes use of these extension points, +providing several 'in-branch' plugins that you may use as a template for your +own work. + +Getting Started +--------------- + +Registering your extensions +``````````````````````````` + +Stevedore uses setup entry point hooks to determine which plugins are +available. To begin, you should register your implementation classes in your +setup.cfg file. For example:: + + [entry_points] + storyboard.plugin.user_preferences = + my-plugin-config = my.namespace.plugin:UserPreferences + storyboard.worker.task = + my-plugin-worker = my.namespace.plugin:EventWorker + storyboard.plugin.cron = + my-plugin-cron = my.namespace.plugin:CronWorker + +Configuring and enabling your extensions +```````````````````````````````````````` + +Every plugin type builds on `storyboard.plugin.base:PluginBase`, +which supports your plugin's configuration. Upon creation, +storyboard's global configuration is injected into the plugin as the `config` +property. With this object, it is left to the developer to implement the +`enabled()` method, which informs storyboard that it has all it needs to +operate. + +An example basic plugin:: + + class BasicPlugin(PluginBase): + + def enabled(self): + return self.config.my_config_property + +Each extension hook may also add its own requirements, which will be detailed +below. + +Available Extension Points +-------------------------- + +User Preferences +```````````````` + +The simplest, and perhaps most important, extension point, +allows you to inject default preferences into storyboard. These will be made +available via the API for consumption by the webclient, +however you will need to consume those preferences yourself:: + + [entry_points] + storyboard.plugin.user_preferences = + my-plugin-config = my.namespace.plugin:UserPreferences + +To learn how to write a user preference plugin, please contribute to this +documentation. + +Cron Workers +```````````` + +Frequently you will need to perform time-based, repeated actions within +storyboard, such as maintenance. By creating and installing a cron +plugin, StoryBoard will manage and maintain your crontab registration for you:: + + [entry_points] + storyboard.plugin.cron = + my-plugin-cron = my.namespace.plugin:CronWorker + +To learn how to write a cron plugin, `read more here <./plugin_cron.html>`_. + +Event Workers +````````````` + +If you would like your plugin to react to a specific API event in storyboard, +you can write a plugin to do so. This plugin will receive notification +whenever a POST, PUT, or DELETE action occurs on the API, +and your plugin can decide how to process each event in an asynchronous +thread which will not impact the stability of the API:: + + [entry_points] + storyboard.worker.task = + my-plugin-worker = my.namespace.plugin:EventWorker + +To learn how to write a user preference plugin, please contribute to this +documentation. diff --git a/doc/source/extending/plugin_cron.rst b/doc/source/extending/plugin_cron.rst new file mode 100644 index 00000000..9e16ade1 --- /dev/null +++ b/doc/source/extending/plugin_cron.rst @@ -0,0 +1,88 @@ +================================== +Extending StoryBoard: Cron Plugins +================================== + +Overview +-------- + +StoryBoard requires the occasional periodic task, to support things like +cleanup and maintenance. It does this by directly managing its own crontab +entries, and extension hooks are available for you to add your own +functionality. Crontab entries are checked every 5 minutes, +with new entries added and old/orphaned entries removed. Note that this +monitoring is only active while the storyboard api is running. As soon as the +API is shut down, all cron plugins are shut down as well. + +When your plugin is executed, it is done so via `storyboard-cron` which +bootstraps configuration and storyboard. It does not maintain state +between runs, and terminates as soon as your code finishes. + +We DO NOT recommend you use this extension mechanism to create long running +processes. Upon the execution of your plugin's `run()` method, +you will be provided with the time it was last executed, as well as the current +timestamp. Please limit your plugin's execution scope to events that occurred +within that time frame, and exit after. + +Cron Plugin Quickstart +---------------------- + +Step 1: Create a new python project using setuptools +#################################################### + +This is left as an exercise to the reader. Don't forget to include storyboard +as a requirement. + +Step 2: Implement your plugin +############################# + +Add a registered entry point in your plugin's `setup.cfg`. The name should be +reasonably unique:: + + [entry_points] + storyboard.plugin.cron = + my-plugin-cron = my.namespace.plugin:CronWorker + +Then, implement your plugin by extending `CronPluginBase`. You may register +your own configuration groups, please see +`oslo.config `_ +for more details.:: + + from storyboard.plugin.cron.base import CronPluginBase + + class MyCronPlugin(CronPluginBase): + + def enabled(self): + '''This method should return whether the plugin is enabled and + configured. It has access to self.config, which is a reference to + storyboard's global configuration object. + ''' + return True + + def interval(self): + '''This method should return the crontab interval for this + plugin's execution schedule. It is used verbatim. + ''' + return "? * * * *" + + def run(self, start_time, end_time): + '''Execute your plugin. The provided parameters are the start and + end times of the time window for which this particular execution + is responsible. + + This particular implementation simply deletes oauth tokens that + are older than one week. + ''' + + lastweek = datetime.utcnow() - timedelta(weeks=1) + + query = api_base.model_query(AccessToken) + query = query.filter(AccessToken.expires_at < lastweek) + query.delete() + + +Step 3: Install your plugin +########################### +Finally, install your plugin, which may require you switch into storyboard's +virtual environment. Pip should automatically register your plugin:: + + pip install my-storyboard-plugin diff --git a/doc/source/index.rst b/doc/source/index.rst index 07709fa5..d80a8d1f 100644 --- a/doc/source/index.rst +++ b/doc/source/index.rst @@ -48,6 +48,15 @@ Developer docs contributing webclient +Extending StoryBoard +-------------------- + +.. toctree:: + :maxdepth: 1 + + Overview + Plugins: Cron Workers + Client API Reference -------------------- diff --git a/etc/storyboard.conf.sample b/etc/storyboard.conf.sample index 98e14ec9..855caf52 100644 --- a/etc/storyboard.conf.sample +++ b/etc/storyboard.conf.sample @@ -54,6 +54,12 @@ lock_path = $state_path/lock # and subscriptions. # enable_notifications = True +[cron] +# Storyboard's cron management configuration + +# Enable or disable cron (Default disabled) +# enable = true + [cors] # W3C CORS configuration. For more information, see http://www.w3.org/TR/cors/ diff --git a/requirements.txt b/requirements.txt index b68c12b3..5a219e0f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -17,4 +17,6 @@ WSME>=0.6 sqlalchemy-migrate>=0.8.2,!=0.8.4 SQLAlchemy-FullText-Search eventlet>=0.13.0 -stevedore>=1.0.0 \ No newline at end of file +stevedore>=1.0.0 +python-crontab>=1.8.1 +tzlocal>=1.1.2 \ No newline at end of file diff --git a/setup.cfg b/setup.cfg index ca57b16b..d348230d 100644 --- a/setup.cfg +++ b/setup.cfg @@ -36,10 +36,12 @@ console_scripts = storyboard-worker-daemon = storyboard.worker.daemon:run storyboard-db-manage = storyboard.db.migration.cli:main storyboard-migrate = storyboard.migrate.cli:main + storyboard-cron = storyboard.plugin.cron:main storyboard.worker.task = subscription = storyboard.worker.task.subscription:Subscription storyboard.plugin.user_preferences = - +storyboard.plugin.cron = + cron-management = storyboard.plugin.cron.manager:CronManager [build_sphinx] source-dir = doc/source diff --git a/storyboard/api/app.py b/storyboard/api/app.py index 01f04bb0..8524aa4e 100644 --- a/storyboard/api/app.py +++ b/storyboard/api/app.py @@ -30,6 +30,7 @@ from storyboard.api.v1.search import search_engine from storyboard.notifications.notification_hook import NotificationHook from storyboard.openstack.common.gettextutils import _ # noqa from storyboard.openstack.common import log +from storyboard.plugin.cron import load_crontab from storyboard.plugin.user_preferences import initialize_user_preferences CONF = cfg.CONF @@ -95,6 +96,9 @@ def setup_app(pecan_config=None): # Load user preference plugins initialize_user_preferences() + # Initialize crontab + load_crontab() + # Setup notifier if CONF.enable_notifications: hooks.append(NotificationHook()) diff --git a/storyboard/plugin/cron/__init__.py b/storyboard/plugin/cron/__init__.py new file mode 100644 index 00000000..db29fd87 --- /dev/null +++ b/storyboard/plugin/cron/__init__.py @@ -0,0 +1,71 @@ +# Copyright (c) 2014 Hewlett-Packard Development Company, L.P. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. See the License for the specific language governing permissions and +# limitations under the License. + +import atexit + +from oslo.config import cfg +from storyboard.openstack.common import log +from storyboard.plugin.base import StoryboardPluginLoader +from storyboard.plugin.cron.manager import CronManager + + +LOG = log.getLogger(__name__) +CONF = cfg.CONF + +CRON_OPTS = [ + cfg.StrOpt("plugin", + default="storyboard.plugin.cron.manager:CronManager", + help="The name of the cron plugin to execute.") +] + + +def main(): + """Run a specific cron plugin from the commandline. Used by the system's + crontab to target different plugins on different execution intervals. + """ + CONF.register_cli_opts(CRON_OPTS) + CONF(project='storyboard') + log.setup('storyboard') + + loader = StoryboardPluginLoader(namespace="storyboard.plugin.cron") + + if loader.extensions: + loader.map(execute_plugin, CONF.plugin) + + +def execute_plugin(ext, name): + """Private handler method that checks individual loaded plugins. + """ + plugin_name = ext.obj.get_name() + if name == plugin_name: + LOG.info("Executing cron plugin: %s" % (plugin_name,)) + ext.obj.execute() + + +def load_crontab(): + """Initialize all registered crontab plugins.""" + + # We cheat here - crontab plugin management is implemented as a crontab + # plugin itself, so we create a single instance to kick things off, + # which will then add itself to recheck periodically. + manager_plugin = CronManager(CONF) + if manager_plugin.enabled(): + manager_plugin.execute() + atexit.register(unload_crontab, manager_plugin) + else: + unload_crontab(manager_plugin) + + +def unload_crontab(manager_plugin): + manager_plugin.remove() diff --git a/storyboard/plugin/cron/base.py b/storyboard/plugin/cron/base.py new file mode 100644 index 00000000..e8b66131 --- /dev/null +++ b/storyboard/plugin/cron/base.py @@ -0,0 +1,125 @@ +# Copyright (c) 2014 Hewlett-Packard Development Company, L.P. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. See the License for the specific language governing permissions and +# limitations under the License. + + +import abc +import calendar +import datetime +import os +import pytz +import six + +from storyboard.common.working_dir import get_working_directory +from storyboard.openstack.common import log +import storyboard.plugin.base as plugin_base + + +LOG = log.getLogger(__name__) + + +@six.add_metaclass(abc.ABCMeta) +class CronPluginBase(plugin_base.PluginBase): + """Base class for a plugin that executes business logic on a time + interval. In order to prevent processing overlap on long-running + processes that may exceed the tick interval, the plugin will be provided + with the time range for which it is responsible. + + It is likely that multiple instances of a plugin may be running + simultaneously, as a previous execution may not have finished processing + by the time the next one is started. Please ensure that your plugin + operates in a time bounded, thread safe manner. + """ + + @abc.abstractmethod + def run(self, start_time, end_time): + """Execute a periodic task. + + :param start_time: The last time the plugin was run. + :param end_time: The current timestamp. + :return: Nothing. + """ + + def _get_file_mtime(self, path, date=None): + """Retrieve the date of this plugin's last_run file. If a date is + provided, it will also update the file's date before returning that + date. + + :param path: The path of the file to retreive. + :param date: A datetime to use to set as the mtime of the file. + :return: The mtime of the file. + """ + + # Get our timezones. + utc_tz = pytz.utc + + # If the file doesn't exist, create it with a sane base time. + if not os.path.exists(path): + base_time = datetime.datetime \ + .utcfromtimestamp(0) \ + .replace(tzinfo=utc_tz) + with open(path, 'a'): + base_timestamp = calendar.timegm(base_time.timetuple()) + os.utime(path, (base_timestamp, base_timestamp)) + + # If a date was passed, use it to update the file. + if date: + # If the date does not have a timezone, throw an exception. + # That's bad practice and makes our date/time conversions + # impossible. + if not date.tzinfo: + raise TypeError("Please include a timezone when passing" + " datetime instances") + + with open(path, 'a'): + mtimestamp = calendar.timegm(date + .astimezone(utc_tz) + .timetuple()) + os.utime(path, (mtimestamp, mtimestamp)) + + # Retrieve the file's last mtime. + pid_info = os.stat(path) + return datetime.datetime \ + .fromtimestamp(pid_info.st_mtime, utc_tz) + + def execute(self): + """Execute this cron plugin, first by determining its own working + directory, then calculating the appropriate runtime interval, + and finally executing the run() method. + """ + + plugin_name = self.get_name() + working_directory = get_working_directory() + cron_directory = os.path.join(working_directory, 'cron') + if not os.path.exists(cron_directory): + os.makedirs(cron_directory) + + lr_file = os.path.join(cron_directory, plugin_name) + + now = pytz.utc.localize(datetime.datetime.utcnow()) + + start_time = self._get_file_mtime(path=lr_file) + end_time = self._get_file_mtime(path=lr_file, + date=now) + self.run(start_time, end_time) + + @abc.abstractmethod + def interval(self): + """The plugin's cron interval, as a string. + + :return: The cron interval. Example: "* * * * *" + """ + + def get_name(self): + """A simple name for this plugin.""" + return self.__module__ + ":" + self.__class__.__name__ diff --git a/storyboard/plugin/cron/manager.py b/storyboard/plugin/cron/manager.py new file mode 100644 index 00000000..3a425eb3 --- /dev/null +++ b/storyboard/plugin/cron/manager.py @@ -0,0 +1,133 @@ +# Copyright (c) 2014 Hewlett-Packard Development Company, L.P. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. See the License for the specific language governing permissions and +# limitations under the License. + +from crontab import CronTab +from oslo.config import cfg + +from storyboard.openstack.common import log +from storyboard.plugin.base import StoryboardPluginLoader +from storyboard.plugin.cron.base import CronPluginBase + + +LOG = log.getLogger(__name__) + +CRON_MANAGEMENT_OPTS = [ + cfg.BoolOpt('enable', + default=False, + help='Enable StoryBoard\'s Crontab management.') +] +CONF = cfg.CONF +CONF.register_opts(CRON_MANAGEMENT_OPTS, 'cron') + + +class CronManager(CronPluginBase): + """A Cron Plugin serves both as the manager for other storyboard + cron tasks, and as an example plugin for storyboard. It checks every + 5 minutes or so to see what storyboard cron plugins are registered vs. + running, and enables/disables them accordingly. + """ + + def __init__(self, config, tabfile=None): + super(CronManager, self).__init__(config=config) + + self.tabfile = tabfile + + def enabled(self): + """Indicate whether this plugin is enabled. This indicates whether + this plugin alone is runnable, as opposed to the entire cron system. + """ + return self.config.cron.enable + + def interval(self): + """This plugin executes every 5 minutes. + + :return: "*/5 * * * *" + """ + return "*/5 * * * *" + + def run(self, start_time, end_time): + """Execute a periodic task. + + :param start_time: The last time the plugin was run. + :param end_time: The current timestamp. + """ + + # First, go through the stevedore registration and update the plugins + # we know about. + loader = StoryboardPluginLoader(namespace="storyboard.plugin.cron") + handled_plugins = dict() + if loader.extensions: + loader.map(self._manage_plugins, handled_plugins) + + # Now manually go through the cron list and remove anything that + # isn't registered. + cron = CronTab(tabfile=self.tabfile) + not_handled = lambda x: x.comment not in handled_plugins + jobs = filter(not_handled, cron.find_command('storyboard-cron')) + cron.remove(*jobs) + cron.write() + + def _manage_plugins(self, ext, handled_plugins=dict()): + """Adds a plugin instance to crontab.""" + plugin = ext.obj + + cron = CronTab(tabfile=self.tabfile) + plugin_name = plugin.get_name() + plugin_interval = plugin.interval() + command = "storyboard-cron --plugin %s" % (plugin_name,) + + # Pull any existing jobs. + job = None + for item in cron.find_comment(plugin_name): + LOG.info("Found existing cron job: %s" % (plugin_name,)) + job = item + job.set_command(command) + job.set_comment(plugin_name) + job.setall(plugin_interval) + break + + if not job: + LOG.info("Adding cron job: %s" % (plugin_name,)) + job = cron.new(command=command, comment=plugin_name) + job.setall(plugin_interval) + + # Update everything. + job.set_command(command) + job.set_comment(plugin_name) + job.setall(plugin_interval) + + # This code us usually not triggered, because the stevedore plugin + # loader harness already filters based on the results of the + # enabled() method, however we're keeping it in here since plugin + # loading and individual plugin functionality are independent, and may + # change independently. + if plugin.enabled(): + job.enable() + else: + LOG.info("Disabled cron plugin: %s", (plugin_name,)) + job.enable(False) + + # Remember the state of this plugin + handled_plugins[plugin_name] = True + + # Save it. + cron.write() + + def remove(self): + """Remove all storyboard cron extensions. + """ + # Flush all orphans + cron = CronTab(tabfile=self.tabfile) + cron.remove_all(command='storyboard-cron') + cron.write() diff --git a/storyboard/tests/plugin/cron/__init__.py b/storyboard/tests/plugin/cron/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/storyboard/tests/plugin/cron/mock_plugin.py b/storyboard/tests/plugin/cron/mock_plugin.py new file mode 100644 index 00000000..c615c1c8 --- /dev/null +++ b/storyboard/tests/plugin/cron/mock_plugin.py @@ -0,0 +1,56 @@ +# Copyright (c) 2014 Hewlett-Packard Development Company, L.P. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. See the License for the specific language governing permissions and +# limitations under the License. + +from oslo.config import cfg + +import storyboard.plugin.cron.base as plugin_base + + +CONF = cfg.CONF + + +class MockPlugin(plugin_base.CronPluginBase): + """A mock cron plugin for testing.""" + + def __init__(self, config, is_enabled=True, + plugin_interval="0 0 1 1 0"): + """Create a new instance of the base plugin, with some sane defaults. + The default cron interval is '0:00 on January 1st if a Sunday', which + should ensure that the manipulation of the cron environment on the test + machine does not actually execute anything. + """ + super(MockPlugin, self).__init__(config) + self.is_enabled = is_enabled + self.plugin_interval = plugin_interval + + def enabled(self): + """Return our enabled value.""" + return self.is_enabled + + def run(self, start_time, end_time): + """Stores the data to a global variable so we can test it. + + :param working_dir: Path to a working directory your plugin can use. + :param start_time: The last time the plugin was run. + :param end_time: The current timestamp. + :return: Nothing. + """ + self.last_invocation_parameters = (start_time, end_time) + + def interval(self): + """The plugin's cron interval, as a string. + + :return: The cron interval. Example: "* * * * *" + """ + return self.plugin_interval diff --git a/storyboard/tests/plugin/cron/test_base.py b/storyboard/tests/plugin/cron/test_base.py new file mode 100644 index 00000000..d336001a --- /dev/null +++ b/storyboard/tests/plugin/cron/test_base.py @@ -0,0 +1,123 @@ +# Copyright (c) 2014 Hewlett-Packard Development Company, L.P. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. See the License for the specific language governing permissions and +# limitations under the License. + +import calendar +import datetime +import os +import pytz +import shutil +import tzlocal + +from storyboard.common.working_dir import get_working_directory +import storyboard.tests.base as base +from storyboard.tests.plugin.cron.mock_plugin import MockPlugin + + +class TestCronPluginBase(base.TestCase): + """Test the abstract plugin core.""" + + def setUp(self): + super(TestCronPluginBase, self).setUp() + + # Create the stamp directory + working_directory = get_working_directory() + cron_directory = os.path.join(working_directory, 'cron') + if not os.path.exists(cron_directory): + os.makedirs(cron_directory) + + def tearDown(self): + super(TestCronPluginBase, self).tearDown() + + # remove the stamp directory + working_directory = get_working_directory() + cron_directory = os.path.join(working_directory, 'cron') + shutil.rmtree(cron_directory) + + def test_get_name(self): + """Test that the plugin can name itself.""" + plugin = MockPlugin(dict()) + self.assertEqual("storyboard.tests.plugin.cron.mock_plugin:MockPlugin", + plugin.get_name()) + + def test_mtime(self): + """Assert that the mtime utility function always returns UTC dates, + yet correctly translates dates to systime. + """ + sys_tz = tzlocal.get_localzone() + + # Generate the plugin and build our file path + plugin = MockPlugin(dict()) + plugin_name = plugin.get_name() + working_dir = get_working_directory() + last_run_path = os.path.join(working_dir, 'cron', plugin_name) + + # Call the mtime method, ensuring that it is created. + self.assertFalse(os.path.exists(last_run_path)) + creation_mtime = plugin._get_file_mtime(last_run_path) + self.assertTrue(os.path.exists(last_run_path)) + + # Assert that the returned timezone is UTC. + self.assertEquals(pytz.utc, creation_mtime.tzinfo) + + # Assert that the creation time equals UTC 0. + creation_time = calendar.timegm(creation_mtime.timetuple()) + self.assertEqual(0, creation_time.real) + + # Assert that we can update the time. + updated_mtime = datetime.datetime(year=2000, month=1, day=1, hour=1, + minute=1, second=1, tzinfo=pytz.utc) + updated_result = plugin._get_file_mtime(last_run_path, updated_mtime) + self.assertEqual(updated_mtime, updated_result) + updated_stat = os.stat(last_run_path) + updated_time_from_file = datetime.datetime \ + .fromtimestamp(updated_stat.st_mtime, tz=sys_tz) + self.assertEqual(updated_mtime, updated_time_from_file) + + # Assert that passing a system timezone datetime is still applicable + # and comparable. + updated_sysmtime = datetime.datetime(year=2000, month=1, day=1, hour=1, + minute=1, second=1, + tzinfo=sys_tz) + updated_sysresult = plugin._get_file_mtime(last_run_path, + updated_sysmtime) + self.assertEqual(updated_sysmtime, updated_sysresult) + self.assertEqual(pytz.utc, updated_sysresult.tzinfo) + + def test_execute(self): + """Assert that the public execution method correctly builds the + plugin API's input parameters. + """ + + # Generate the plugin and simulate a previous execution + plugin = MockPlugin(dict()) + plugin_name = plugin.get_name() + working_directory = get_working_directory() + + last_run_path = os.path.join(working_directory, 'cron', plugin_name) + last_run_date = datetime.datetime(year=2000, month=1, day=1, + hour=12, minute=0, second=0, + microsecond=0, tzinfo=pytz.utc) + plugin._get_file_mtime(last_run_path, last_run_date) + + # Execute the plugin + plugin.execute() + + # Current timestamp, remove microseconds so that we don't run into + # execution time delay problems. + now = pytz.utc.localize(datetime.datetime.utcnow()) \ + .replace(microsecond=0) + + # Check the plugin's params. + self.assertEqual(last_run_date, plugin.last_invocation_parameters[0]) + self.assertEqual(now, plugin.last_invocation_parameters[1]) diff --git a/storyboard/tests/plugin/cron/test_manager.py b/storyboard/tests/plugin/cron/test_manager.py new file mode 100644 index 00000000..91e775de --- /dev/null +++ b/storyboard/tests/plugin/cron/test_manager.py @@ -0,0 +1,382 @@ +# Copyright (c) 2014 Hewlett-Packard Development Company, L.P. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. See the License for the specific language governing permissions and +# limitations under the License. + + +import os +import tempfile + +import crontab +from oslo.config import cfg +from stevedore.extension import Extension + +import storyboard.plugin.base as plugin_base +import storyboard.plugin.cron.manager as cronmanager +import storyboard.tests.base as base +from storyboard.tests.plugin.cron.mock_plugin import MockPlugin + + +CONF = cfg.CONF + + +class TestCronManager(base.TestCase): + def setUp(self): + super(TestCronManager, self).setUp() + + (user, self.tabfile) = tempfile.mkstemp(prefix='cron_') + + # Flush the crontab before test. + cron = crontab.CronTab(tabfile=self.tabfile) + cron.remove_all(command='storyboard-cron') + cron.write() + + CONF.register_opts(cronmanager.CRON_MANAGEMENT_OPTS, 'cron') + CONF.set_override('enable', True, group='cron') + + def tearDown(self): + super(TestCronManager, self).tearDown() + CONF.clear_override('enable', group='cron') + + # Flush the crontab after test. + cron = crontab.CronTab(tabfile=self.tabfile) + cron.remove_all(command='storyboard-cron') + cron.write() + + os.remove(self.tabfile) + + def test_enabled(self): + """This plugin must be enabled if the configuration tells it to be + enabled. + """ + enabled_plugin = cronmanager.CronManager(CONF, tabfile=self.tabfile) + self.assertTrue(enabled_plugin.enabled()) + + CONF.set_override('enable', False, group='cron') + enabled_plugin = cronmanager.CronManager(CONF) + self.assertFalse(enabled_plugin.enabled()) + CONF.clear_override('enable', group='cron') + + def test_interval(self): + """Assert that the cron manager runs every 5 minutes.""" + + plugin = cronmanager.CronManager(CONF, tabfile=self.tabfile) + self.assertEqual("*/5 * * * *", plugin.interval()) + + def test_manage_plugins(self): + """Assert that the cron manager adds plugins to crontab.""" + + mock_plugin = MockPlugin(dict()) + mock_plugin_name = mock_plugin.get_name() + mock_extensions = [Extension('test_one', None, None, mock_plugin)] + + loader = plugin_base.StoryboardPluginLoader.make_test_instance( + mock_extensions, namespace='storyboard.plugin.testing' + ) + + # Run the add_plugin routine. + manager = cronmanager.CronManager(CONF, tabfile=self.tabfile) + loader.map(manager._manage_plugins) + + # Manually test the crontab. + self.assertCronLength(1, command='storyboard-cron') + self.assertCronContains( + command='storyboard-cron --plugin %s' % (mock_plugin_name,), + comment=mock_plugin.get_name(), + interval=mock_plugin.interval(), + enabled=mock_plugin.enabled() + ) + + def test_manage_disabled_plugin(self): + """Assert that a disabled plugin is added to the system crontab, + but disabled. While we don't anticipate this feature to ever be + triggered (since the plugin loader won't present disabled plugins), + it's still a good safety net. + """ + mock_plugin = MockPlugin(dict(), is_enabled=False) + mock_plugin_name = mock_plugin.get_name() + mock_extensions = [Extension('test_one', None, None, mock_plugin)] + + loader = plugin_base.StoryboardPluginLoader.make_test_instance( + mock_extensions, namespace='storyboard.plugin.testing' + ) + + # Run the add_plugin routine. + manager = cronmanager.CronManager(CONF, tabfile=self.tabfile) + loader.map(manager._manage_plugins) + + # Manually test the crontab. + self.assertCronLength(1, command='storyboard-cron') + self.assertCronContains( + command='storyboard-cron --plugin %s' % (mock_plugin_name,), + comment=mock_plugin.get_name(), + interval=mock_plugin.interval(), + enabled=mock_plugin.enabled() + ) + + def test_manage_existing_update(self): + """Assert that a plugin whose signature changes is appropriately + updated in the system crontab. + """ + mock_plugin = MockPlugin(dict(), + plugin_interval="*/10 * * * *", + is_enabled=False) + mock_plugin_name = mock_plugin.get_name() + mock_extensions = [Extension('test_one', None, None, mock_plugin)] + + loader = plugin_base.StoryboardPluginLoader.make_test_instance( + mock_extensions, namespace='storyboard.plugin.testing' + ) + + # Run the add_plugin routine. + manager = cronmanager.CronManager(CONF, tabfile=self.tabfile) + loader.map(manager._manage_plugins) + + # Manually test the crontab. + self.assertCronLength(1, command='storyboard-cron') + self.assertCronContains( + command='storyboard-cron --plugin %s' % (mock_plugin_name,), + comment=mock_plugin.get_name(), + interval=mock_plugin.interval(), + enabled=mock_plugin.enabled() + ) + + # Update the plugin and re-run the loader + mock_plugin.plugin_interval = "*/5 * * * *" + loader.map(manager._manage_plugins) + + # re-test the crontab. + self.assertCronLength(1, command='storyboard-cron') + self.assertCronContains( + command='storyboard-cron --plugin %s' % (mock_plugin_name,), + comment=mock_plugin.get_name(), + interval=mock_plugin.interval(), + enabled=mock_plugin.enabled() + ) + + def test_remove_plugin(self): + """Assert that the remove() method on the manager removes plugins from + the crontab. + """ + mock_plugin = MockPlugin(dict(), is_enabled=False) + mock_plugin_name = mock_plugin.get_name() + mock_extensions = [Extension('test_one', None, None, mock_plugin)] + + loader = plugin_base.StoryboardPluginLoader.make_test_instance( + mock_extensions, namespace='storyboard.plugin.testing' + ) + + # Run the add_plugin routine. + manager = cronmanager.CronManager(CONF, tabfile=self.tabfile) + loader.map(manager._manage_plugins) + + # Manually test the crontab. + self.assertCronLength(1, command='storyboard-cron') + self.assertCronContains( + command='storyboard-cron --plugin %s' % (mock_plugin_name,), + comment=mock_plugin.get_name(), + interval=mock_plugin.interval(), + enabled=mock_plugin.enabled() + ) + + # Now run the manager's remove method. + manager.remove() + + # Make sure we don't leave anything behind. + self.assertCronLength(0, command='storyboard-cron') + + def test_remove_only_storyboard(self): + """Assert that the remove() method manager only removes storyboard + plugins, and not others. + """ + # Create a test job. + cron = crontab.CronTab(tabfile=self.tabfile) + job = cron.new(command='echo 1', comment='echo_test') + job.setall("0 0 */10 * *") + cron.write() + + # Create a plugin and have the manager add it to cron. + mock_plugin = MockPlugin(dict(), is_enabled=False) + mock_extensions = [Extension('test_one', None, None, mock_plugin)] + + loader = plugin_base.StoryboardPluginLoader.make_test_instance( + mock_extensions, + namespace='storyboard.plugin.testing' + ) + manager = cronmanager.CronManager(CONF, tabfile=self.tabfile) + loader.map(manager._manage_plugins) + + # Assert that there's two jobs in our cron. + self.assertCronLength(1, command='storyboard-cron') + self.assertCronLength(1, comment='echo_test') + + # Call manager remove. + manager.remove() + + # Check crontab. + self.assertCronLength(0, command='storyboard-cron') + self.assertCronLength(1, comment='echo_test') + + # Clean up after ourselves. + cron = crontab.CronTab(tabfile=self.tabfile) + cron.remove_all(comment='echo_test') + cron.write() + + def test_remove_not_there(self): + """Assert that the remove() method is idempotent and can happen if + we're already unregistered. + """ + manager = cronmanager.CronManager(CONF, tabfile=self.tabfile) + manager.remove() + + def test_execute(self): + """Test that execute() method adds plugins.""" + + # Actually run the real cronmanager. + manager = cronmanager.CronManager(CONF, tabfile=self.tabfile) + manager.execute() + + # We're expecting 1 in-branch plugins. + self.assertCronLength(1, command='storyboard-cron') + + def test_execute_update(self): + """Test that execute() method updates plugins.""" + + # Manually create an instance of a known plugin with a time interval + # that doesn't match what the plugin wants. + manager = cronmanager.CronManager(CONF, tabfile=self.tabfile) + manager_name = manager.get_name() + manager_command = "storyboard-cron --plugin %s" % (manager_name,) + manager_comment = manager_name + manager_old_interval = "0 0 */2 * *" + + cron = crontab.CronTab(tabfile=self.tabfile) + job = cron.new( + command=manager_command, + comment=manager_comment + ) + job.enable(False) + job.setall(manager_old_interval) + cron.write() + + # Run the manager + manager.execute() + + # Check a new crontab to see what we find. + self.assertCronLength(1, command=manager_command) + + cron = crontab.CronTab(tabfile=self.tabfile) + for job in cron.find_command(manager_command): + self.assertNotEqual(manager_old_interval, job.slices) + self.assertEqual(manager.interval(), job.slices) + self.assertTrue(job.enabled) + + # Cleanup after ourselves. + manager.remove() + + # Assert that things are gone. + self.assertCronLength(0, command='storyboard-cron') + + def test_execute_remove_orphans(self): + """Test that execute() method removes orphaned/deregistered plugins.""" + + # Manually create an instance of a plugin that's not in our default + # stevedore registration + plugin = MockPlugin(dict()) + plugin_name = plugin.get_name() + plugin_command = "storyboard-cron --plugin %s" % (plugin_name,) + plugin_comment = plugin_name + plugin_interval = plugin.interval() + + cron = crontab.CronTab(tabfile=self.tabfile) + job = cron.new( + command=plugin_command, + comment=plugin_comment + ) + job.enable(False) + job.setall(plugin_interval) + cron.write() + + # Run the manager + manager = cronmanager.CronManager(CONF, tabfile=self.tabfile) + manager.execute() + + # Check a new crontab to see what we find. + cron = crontab.CronTab(tabfile=self.tabfile) + self.assertCronLength(0, command=plugin_command) + self.assertCronLength(1, command='storyboard-cron') + + # Cleanup after ourselves. + manager.remove() + + # Assert that things are gone. + self.assertCronLength(0, command='storyboard-cron') + + def test_execute_add_new(self): + """Test that execute() method adds newly registered plugins.""" + + # Manuall add the cron manager + manager = cronmanager.CronManager(CONF, tabfile=self.tabfile) + manager_name = manager.get_name() + manager_command = "storyboard-cron --plugin %s" % (manager_name,) + manager_comment = manager_name + + cron = crontab.CronTab(tabfile=self.tabfile) + job = cron.new( + command=manager_command, + comment=manager_comment + ) + job.enable(manager.enabled()) + job.setall(manager.interval()) + cron.write() + + # Run the manager + manager = cronmanager.CronManager(CONF, tabfile=self.tabfile) + manager.execute() + + # Check a new crontab to see what we find. + self.assertCronLength(1, command='storyboard-cron') + + # Cleanup after ourselves. + manager.remove() + + # Assert that things are gone. + self.assertCronLength(0, command='storyboard-cron') + + def assertCronLength(self, length=0, command=None, comment=None): + cron = crontab.CronTab(tabfile=self.tabfile) + if command: + self.assertEqual(length, + len(list(cron.find_command(command)))) + elif comment: + self.assertEqual(length, + len(list(cron.find_comment(comment)))) + else: + self.assertEqual(0, length) + + def assertCronContains(self, command, comment, interval, enabled=True): + cron = crontab.CronTab(tabfile=self.tabfile) + found = False + + for job in cron.find_comment(comment): + if job.command != command: + continue + elif job.comment != comment: + continue + elif job.enabled != enabled: + continue + elif str(job.slices) != interval: + continue + else: + found = True + break + self.assertTrue(found)