From 4a61c383fbeb6c03457c83bc60985258b800d032 Mon Sep 17 00:00:00 2001 From: Rodion Promyshlennikov Date: Fri, 6 May 2016 11:46:16 +0300 Subject: [PATCH] Add destructive tests Add tests to verify plugin work after network disaster in the whole cluster and only on plugin node. Change-Id: I2c1d11ecdff362f77d1c2da7d9ade1113c10174b --- stacklight_tests/helpers/helpers.py | 35 +++++++ .../influxdb_grafana/test_destructive.py | 94 +++++++++++++++++++ 2 files changed, 129 insertions(+) create mode 100644 stacklight_tests/influxdb_grafana/test_destructive.py diff --git a/stacklight_tests/helpers/helpers.py b/stacklight_tests/helpers/helpers.py index 88a999e..ca52456 100644 --- a/stacklight_tests/helpers/helpers.py +++ b/stacklight_tests/helpers/helpers.py @@ -13,6 +13,7 @@ # under the License. import os +import time import urllib2 from devops.helpers import helpers @@ -139,3 +140,37 @@ class PluginHelper(object): logger.info('Wait a %s node offline status', devops_node.name) helpers.wait(lambda: not self.fuel_web.get_nailgun_node_by_devops_node( devops_node)['online'], timeout=60 * 5, timeout_msg=msg) + + @staticmethod + def block_network_by_interface(interface): + if interface.network.is_blocked: + raise Exception('Network {0} is blocked'.format(interface)) + else: + interface.network.block() + + @staticmethod + def unblock_network_by_interface(interface): + if interface.network.is_blocked: + interface.network.unblock() + else: + raise Exception( + 'Network {0} was not blocked'.format(interface)) + + def emulate_whole_network_disaster(self, delay_before_recover=5 * 60, + wait_become_online=True): + + nodes = [node for node in self.env.d_env.get_nodes() + if node.driver.node_active(node)] + + networks_interfaces = nodes[1].interfaces + + for interface in networks_interfaces: + self.block_network_by_interface(interface) + + time.sleep(delay_before_recover) + + for interface in networks_interfaces: + self.unblock_network_by_interface(interface) + + if wait_become_online: + self.fuel_web.wait_nodes_get_online_state(nodes[1:]) diff --git a/stacklight_tests/influxdb_grafana/test_destructive.py b/stacklight_tests/influxdb_grafana/test_destructive.py new file mode 100644 index 0000000..46ac207 --- /dev/null +++ b/stacklight_tests/influxdb_grafana/test_destructive.py @@ -0,0 +1,94 @@ +# Copyright 2016 Mirantis, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from fuelweb_test.helpers.decorators import log_snapshot_after_test +from proboscis import test + +from stacklight_tests.influxdb_grafana import api +from stacklight_tests.influxdb_grafana import test_smoke_bvt + + +@test(groups=["plugins"]) +class TestDestructiveInfluxdbPlugin(api.InfluxdbPluginApi): + """Class for testing plugin failover after network disaster.""" + + @test(depends_on=[ + test_smoke_bvt.TestInfluxdbPlugin.deploy_ha_influxdb_grafana_plugin], + groups=["check_disaster_influxdb_grafana", "influxdb_grafana", + "destructive", "check_failover_network_all_influxdb_grafana"]) + @log_snapshot_after_test + def emulate_network_disaster_whole_cluster_influxdb_grafana_plugin(self): + """Verify that the backends and dashboards recover + after a network interruption in the whole cluster. + + Scenario: + 1. Revert snapshot with 9 deployed nodes in HA configuration + 2. Simulate network interruption in the whole cluster + 3. Wait for at least 7 minutes before recover network availability + 4. Recover network availability + 5. Wait while all services are started + 6. Run OSTF + 7. Check that plugin is working + 8. Check that data continues to be pushed by the various nodes + once the network interruption has ended + + Duration 40m + """ + + self.env.revert_snapshot("deploy_ha_influxdb_grafana_plugin") + + self.helpers.emulate_whole_network_disaster( + delay_before_recover=7 * 60) + + self.wait_plugin_online() + + self.check_plugin_online() + + self.helpers.run_ostf(should_fail=1) + + self.env.make_snapshot( + "emulate_network_disaster_whole_cluster_influxdb_grafana_plugin") + + @test(depends_on=[ + test_smoke_bvt.TestInfluxdbPlugin.deploy_influxdb_grafana_plugin], + groups=["check_disaster_influxdb_grafana", "influxdb_grafana", + "destructive", "check_failover_network_node_influxdb_grafana"]) + @log_snapshot_after_test + def emulate_network_disaster_on_influxdb_grafana_plugin_node(self): + """Verify that the backends and dashboards recover after + a network failure on plugin node. + + Scenario: + 1. Revert snapshot with 3 deployed nodes + 2. Simulate network interruption on plugin node + 3. Wait for at least 30 seconds before recover network availability + 4. Recover network availability + 5. Run OSTF + 6. Check that plugin is working + Duration 20m + """ + self.env.revert_snapshot("deploy_influxdb_grafana_plugin") + + with self.fuel_web.get_ssh_for_nailgun_node( + self.get_influxdb_master_node()) as remote: + self.remote_ops.simulate_network_interrupt_on_node(remote) + + self.wait_plugin_online() + + self.check_plugin_online() + + self.helpers.run_ostf() + + self.env.make_snapshot( + "emulate_network_disaster_on_influxdb_grafana_plugin_node")