From 423c3d105838c7e32c7dda09cb5f65cfe8129bf7 Mon Sep 17 00:00:00 2001
From: Alfredo Moralejo <amoralej@redhat.com>
Date: Tue, 7 Mar 2017 14:49:01 +0100
Subject: [PATCH] Grab additional data about IO to profile service timeouts

We are hitting timeouts in IO bound processes as swift but we
don't have much information about what processes are causing
this IO peaks.

This patch adds iotop and iostat information to the logs in
addition to dstat output.

Change-Id: I4c50d4405a923eb2652fc39f6cea9889597ae44e
---
 copy_logs.sh | 10 ++++++++++
 run_tests.sh | 14 ++++++++++++--
 2 files changed, 22 insertions(+), 2 deletions(-)

diff --git a/copy_logs.sh b/copy_logs.sh
index e0598ac8a..37c346713 100755
--- a/copy_logs.sh
+++ b/copy_logs.sh
@@ -114,6 +114,16 @@ if [ -f /var/log/dstat.log ] ; then
     sudo cp /var/log/dstat.log $LOG_DIR/
 fi
 
+# iostat logs
+if [ -f /var/log/iostat.log ] ; then
+    sudo cp /var/log/iostat.log $LOG_DIR/
+fi
+
+# iotop logs
+if [ -f /var/log/iotop.log ] ; then
+    sudo cp /var/log/iotop.log $LOG_DIR/
+fi
+
 # libvirt
 if [ -d /var/log/libvirt ] ; then
     sudo cp -r /var/log/libvirt $LOG_DIR/
diff --git a/run_tests.sh b/run_tests.sh
index ea1f8f8c1..80416dabc 100755
--- a/run_tests.sh
+++ b/run_tests.sh
@@ -140,9 +140,9 @@ if [ "${MANAGE_HIERA}" = true ]; then
 fi
 
 if uses_debs; then
-    $SUDO apt-get install -y dstat ebtables
+    $SUDO apt-get install -y dstat ebtables iotop sysstat
 elif is_fedora; then
-    $SUDO yum install -y dstat setools setroubleshoot audit
+    $SUDO yum install -y dstat setools setroubleshoot audit iotop sysstat
     $SUDO service auditd start
     # SElinux in permissive mode so later we can catch alerts
     $SUDO selinuxenabled && $SUDO setenforce 0
@@ -154,6 +154,16 @@ if type "dstat" 2>/dev/null; then
     $SUDO dstat -tcmndrylpg --top-cpu-adv --top-io-adv --nocolor | $SUDO tee --append /var/log/dstat.log > /dev/null &
 fi
 
+if type "iostat" 2>/dev/null; then
+    print_header 'Start iostat'
+    $SUDO iostat -x -k -d -t 4 | $SUDO tee --append /var/log/iostat.log > /dev/null &
+fi
+
+if [ -f "/usr/sbin/iotop" ]; then
+    print_header 'Start iotop'
+    $SUDO /usr/sbin/iotop --kilobytes --only --batch --time --delay=2 --processes --quiet | $SUDO tee --append /var/log/iotop.log > /dev/null &
+fi
+
 if [ "${MANAGE_PUPPET_MODULES}" = true ]; then
     $SUDO ./install_modules.sh
 fi