From 379c3a8761ae75104577ef2d42d7cddd1e3a0b2d Mon Sep 17 00:00:00 2001
From: Clark Boylan <clark.boylan@gmail.com>
Date: Tue, 10 Sep 2019 11:04:41 -0700
Subject: [PATCH] Add cleanup playbook to all base jobs

This cleanup playbook is used as a last effort to grab debugging info
from hosts that might othewise be missed if we run out of disk space or
have other problems. Using the raw module we ssh in and get df output as
well as ip link and ip addr output for all jobs in the cleanup phase.

Because cleanup runs after the post playbook this info is only captured
in the executor logs.

Depends-On: https://review.opendev.org/#/c/681552/
Change-Id: Ib5d2cb1dab4da8a9a6d0f3e1721fff395b564044
---
 playbooks/base-minimal/cleanup.yaml | 16 ++++++++++++++++
 playbooks/base/cleanup.yaml         | 16 ++++++++++++++++
 zuul.d/jobs.yaml                    |  4 +++-
 3 files changed, 35 insertions(+), 1 deletion(-)
 create mode 100644 playbooks/base-minimal/cleanup.yaml
 create mode 100644 playbooks/base/cleanup.yaml

diff --git a/playbooks/base-minimal/cleanup.yaml b/playbooks/base-minimal/cleanup.yaml
new file mode 100644
index 0000000..719e6d2
--- /dev/null
+++ b/playbooks/base-minimal/cleanup.yaml
@@ -0,0 +1,16 @@
+- hosts: all
+  tasks:
+    - name: Gather debug info on job fail
+      when: not (zuul_success | bool)
+      block:
+        - name: get df disk usage
+          raw: df
+
+        - name: get df inode usage
+          raw: df -i
+
+        - name: get l2 networking
+          raw: ip link
+
+        - name: get l3 networking
+          raw: ip addr
diff --git a/playbooks/base/cleanup.yaml b/playbooks/base/cleanup.yaml
new file mode 100644
index 0000000..719e6d2
--- /dev/null
+++ b/playbooks/base/cleanup.yaml
@@ -0,0 +1,16 @@
+- hosts: all
+  tasks:
+    - name: Gather debug info on job fail
+      when: not (zuul_success | bool)
+      block:
+        - name: get df disk usage
+          raw: df
+
+        - name: get df inode usage
+          raw: df -i
+
+        - name: get l2 networking
+          raw: ip link
+
+        - name: get l3 networking
+          raw: ip addr
diff --git a/zuul.d/jobs.yaml b/zuul.d/jobs.yaml
index 41d73d9..762bfea 100644
--- a/zuul.d/jobs.yaml
+++ b/zuul.d/jobs.yaml
@@ -53,6 +53,7 @@
     post-run:
       - playbooks/base/post.yaml
       - playbooks/base/post-logs.yaml
+    cleanup-run: playbooks/base/cleanup.yaml
     roles:
       - zuul: zuul/zuul-jobs
     timeout: 1800
@@ -81,10 +82,10 @@
       A job to test changes to the base job without disturbing the
       main job in production.  Not for general use.
     pre-run: playbooks/base-test/pre.yaml
-    cleanup-run: playbooks/base-test/cleanup.yaml
     post-run:
       - playbooks/base-test/post.yaml
       - playbooks/base-test/post-logs.yaml
+    cleanup-run: playbooks/base-test/cleanup.yaml
     roles:
       - zuul: zuul/zuul-jobs
     timeout: 1800
@@ -110,6 +111,7 @@
     post-run:
       - playbooks/base-minimal/post.yaml
       - playbooks/base-minimal/post-logs.yaml
+    cleanup-run: playbooks/base-minimal/cleanup.yaml
     roles:
       - zuul: zuul/zuul-jobs
     timeout: 1800