Add cleanup playbook to all base jobs

This cleanup playbook is used as a last effort to grab debugging info
from hosts that might othewise be missed if we run out of disk space or
have other problems. Using the raw module we ssh in and get df output as
well as ip link and ip addr output for all jobs in the cleanup phase.

Because cleanup runs after the post playbook this info is only captured
in the executor logs.

Depends-On: https://review.opendev.org/#/c/681552/
Change-Id: Ib5d2cb1dab4da8a9a6d0f3e1721fff395b564044

playbooks/base-minimal/cleanup.yaml

@@ -0,0 +1,16 @@
+- hosts: all
+  tasks:
+    - name: Gather debug info on job fail
+      when: not (zuul_success | bool)
+      block:
+        - name: get df disk usage
+          raw: df
+
+        - name: get df inode usage
+          raw: df -i
+
+        - name: get l2 networking
+          raw: ip link
+
+        - name: get l3 networking
+          raw: ip addr

playbooks/base/cleanup.yaml

@@ -0,0 +1,16 @@
+- hosts: all
+  tasks:
+    - name: Gather debug info on job fail
+      when: not (zuul_success | bool)
+      block:
+        - name: get df disk usage
+          raw: df
+
+        - name: get df inode usage
+          raw: df -i
+
+        - name: get l2 networking
+          raw: ip link
+
+        - name: get l3 networking
+          raw: ip addr

zuul.d/jobs.yaml

@@ -53,6 +53,7 @@
     post-run:
       - playbooks/base/post.yaml
       - playbooks/base/post-logs.yaml
+    cleanup-run: playbooks/base/cleanup.yaml
     roles:
       - zuul: zuul/zuul-jobs
     timeout: 1800
@@ -81,10 +82,10 @@
       A job to test changes to the base job without disturbing the
       main job in production.  Not for general use.
     pre-run: playbooks/base-test/pre.yaml
-    cleanup-run: playbooks/base-test/cleanup.yaml
     post-run:
       - playbooks/base-test/post.yaml
       - playbooks/base-test/post-logs.yaml
+    cleanup-run: playbooks/base-test/cleanup.yaml
     roles:
       - zuul: zuul/zuul-jobs
     timeout: 1800
@@ -110,6 +111,7 @@
     post-run:
       - playbooks/base-minimal/post.yaml
       - playbooks/base-minimal/post-logs.yaml
+    cleanup-run: playbooks/base-minimal/cleanup.yaml
     roles:
       - zuul: zuul/zuul-jobs
     timeout: 1800