Fix reboot when target host si not reachable

Change-Id: I2467feac8e2a2bd6406b476c50f7fe64e9f27ca1

tobiko/shell/sh/_reboot.py

@@ -18,17 +18,14 @@ import typing  # noqa
 from oslo_log import log
 
 import tobiko
-from tobiko.shell.sh import _exception
 from tobiko.shell.sh import _uptime
 from tobiko.shell import ssh
 
 
 LOG = log.getLogger(__name__)
 
-hard_reset_method = 'sudo chmod o+w /proc/sysrq-trigger;' \
+hard_reset_method = 'echo b > /proc/sysrq-trigger'
-               'sudo echo b > /proc/sysrq-trigger'
+soft_reset_method = '/sbin/reboot'
-
-soft_reset_method = 'sudo /sbin/reboot'
 
 
 class RebootHostError(tobiko.TobikoException):
@@ -39,10 +36,19 @@ class RebootHostTimeoutError(RebootHostError):
     message = "host {hostname!r} not rebooted after {timeout!s} seconds"
 
 
-def reboot_host(ssh_client: ssh.SSHClientFixture, wait: bool = True,
+def reboot_host(ssh_client: ssh.SSHClientFixture,
-                timeout: tobiko.Seconds = None, method=soft_reset_method):
+                wait: bool = True,
-    reboot = RebootHostOperation(ssh_client=ssh_client, wait=wait,
+                timeout: tobiko.Seconds = None,
-                                 timeout=timeout, method=method)
+                method: str = None,
+                hard: bool = False):
+    if method not in (None, hard_reset_method, soft_reset_method):
+        raise ValueError(f"Unsupported method: '{method}'")
+
+    command = method or (hard and hard_reset_method) or None
+    reboot = RebootHostOperation(ssh_client=ssh_client,
+                                 wait=wait,
+                                 timeout=timeout,
+                                 command=command)
     return tobiko.setup_fixture(reboot)
 
 
@@ -56,6 +62,8 @@ class RebootHostOperation(tobiko.Operation):
     default_wait_interval = 5.
     default_wait_count = 60
 
+    command = soft_reset_method
+
     @property
     def ssh_client(self) -> ssh.SSHClientFixture:
         if self._ssh_client is None:
@@ -64,28 +72,53 @@ class RebootHostOperation(tobiko.Operation):
 
     def __init__(self,
                  ssh_client: typing.Optional[ssh.SSHClientFixture] = None,
-                 wait=True,
+                 wait: bool = True,
                  timeout: tobiko.Seconds = None,
-                 method=soft_reset_method):
+                 command: typing.Optional[str] = None):
         super(RebootHostOperation, self).__init__()
         self._ssh_client = ssh_client
         tobiko.check_valid_type(self.ssh_client, ssh.SSHClientFixture)
         self.wait = bool(wait)
         self.timeout = tobiko.to_seconds(timeout)
-        self.method = method
+        if command is not None:
+            self.command = command
 
     def run_operation(self):
         ssh_client = self.ssh_client
-        with ssh_client:
+        self.is_rebooted = None
-            self.hostname = ssh_client.hostname
+        self.start_time = None
-            LOG.debug(f"Rebooting host '{self.hostname}'... ")
+        for attempt in tobiko.retry(
-            self.is_rebooted = False
+                timeout=self.timeout,
-            self.start_time = tobiko.time()
+                default_timeout=self.default_wait_timeout,
+                default_count=self.default_wait_count,
+                default_interval=self.default_wait_interval):
             try:
-                ssh_client.connect(connection_timeout=self.timeout).\
+                channel = ssh_client.connect(
-                    exec_command(self.method)
+                    connection_timeout=attempt.time_left,
-            except _exception.ShellTimeoutExpired as ex:
+                    retry_count=1)
-                LOG.debug(f"Reboot command timeout expired: {ex}")
+                self.hostname = self.hostname or ssh_client.hostname
+                LOG.info("Executing reboot command on host "
+                         f"'{self.hostname}' (command='{self.command}')... ")
+                self.start_time = tobiko.time()
+                channel.exec_command(f"sudo /bin/sh -c '{self.command}'")
+            except Exception as ex:
+                if attempt.time_left > 0.:
+                    LOG.debug(f"Unable to reboot remote host "
+                              f"(time_left={attempt.time_left}): {ex}")
+                else:
+                    LOG.exception(f"Unable to reboot remote host: {ex}")
+                    raise RebootHostTimeoutError(
+                        hostname=self.hostname or ssh_client.host,
+                        timeout=attempt.timeout) from ex
+            else:
+                self.is_rebooted = False
+                LOG.info(f"Host '{self.hostname}' is rebooting "
+                         f"(command='{self.command}').")
+                break
+            finally:
+                # Ensure we close connection after rebooting command
+                ssh_client.close()
+
         if self.wait:
             self.wait_for_operation()
 

tobiko/tests/functional/shell/test_reboot.py

@@ -18,7 +18,6 @@ from __future__ import absolute_import
 import time
 
 from oslo_log import log
-import paramiko
 import testtools
 
 import tobiko
@@ -31,7 +30,7 @@ LOG = log.getLogger(__name__)
 
 
 class RebootHostStack(stacks.CirrosServerStackFixture):
-    "Server to be rebooted"
+    """Server to be rebooted"""
 
 
 class RebootHostTest(testtools.TestCase):
@@ -58,6 +57,11 @@ class RebootHostTest(testtools.TestCase):
         self.assertIs(ssh_client, reboot.ssh_client)
         self.assertEqual(ssh_client.hostname, reboot.hostname)
         self.assertIs(params.get('wait', True), reboot.wait)
+        hard = params.get('hard', False)
+        command = (params.get('method') or
+                   (hard and sh.hard_reset_method) or
+                   sh.soft_reset_method)
+        self.assertEqual(command, reboot.command)
 
         if not reboot.wait:
             self.assertFalse(reboot.is_rebooted)
@@ -76,6 +80,24 @@ class RebootHostTest(testtools.TestCase):
                   "uptime=%r", uptime_1)
         self.assertGreater(boottime_1, boottime_0)
 
+    def test_reboot_host_with_hard(self):
+        self.test_reboot_host(hard=True)
+
+    def test_reboot_host_with_hard_method(self):
+        self.test_reboot_host(method=sh.hard_reset_method)
+
+    def test_reboot_host_with_soft_method(self):
+        self.test_reboot_host(method=sh.soft_reset_method)
+
+    def test_reboot_host_with_invalid_method(self):
+        self.assertRaises(ValueError,
+                          sh.reboot_host,
+                          ssh_client=self.stack.ssh_client,
+                          method='<invalid-method>')
+
+    def test_reboot_host_with_no_hard(self):
+        self.test_reboot_host(hard=False)
+
     def test_reboot_host_with_wait(self):
         self.test_reboot_host(wait=True)
 
@@ -88,12 +110,12 @@ class RebootHostTest(testtools.TestCase):
 
         ssh_client = self.stack.ssh_client
         self.assert_is_not_connected(ssh_client)
-        errors = (paramiko.ssh_exception.NoValidConnectionsError,
+        self.assertRaises(sh.RebootHostTimeoutError,
-                  paramiko.SSHException)
+                          sh.reboot_host,
-        self.assertRaises(errors, sh.reboot_host, ssh_client=ssh_client,
+                          ssh_client=ssh_client,
                           timeout=5.0)
         self.assert_is_not_connected(ssh_client)
-        server = nova.wait_for_server_status(self.stack.server_id, 'SHUTOFF')
+        server = nova.get_server(self.stack.server_id)
         self.assertEqual('SHUTOFF', server.status)
 
     def assert_is_connected(self, ssh_client):

tox.ini

@@ -141,7 +141,7 @@ passenv =
     *_proxy
 setenv =
     {[testenv]setenv}
-    PYTEST_TIMEOUT = 1200
+    PYTEST_TIMEOUT = 1800
 
 
 [testenv:venv]