diff --git a/.rules/ZuulJobsNamespaceLoopVar.py b/.rules/ZuulJobsNamespaceLoopVar.py
index c06c48295..c704e9293 100644
--- a/.rules/ZuulJobsNamespaceLoopVar.py
+++ b/.rules/ZuulJobsNamespaceLoopVar.py
@@ -20,6 +20,25 @@ https://zuul-ci.org/docs/zuul-jobs/policy.html\
         if file.get('type') not in ('tasks', 'handlers'):
             return results
 
+        results.extend(self.handle_play(task))
+        return results
+
+    def handle_play(self, task):
+        results = []
+        if 'block' in task:
+            results.extend(self.handle_playlist(task['block']))
+        else:
+            results.extend(self.handle_task(task))
+        return results
+
+    def handle_playlist(self, playlist):
+        results = []
+        for play in playlist:
+            results.extend(self.handle_play(play))
+        return results
+
+    def handle_task(self, task):
+        results = []
         has_loop = 'loop' in task
         for key in task.keys():
             if key.startswith('with_'):
diff --git a/test-playbooks/ansible-lint-rules/roles/handlers-missing-loopvar-loop/handlers/main.yaml b/test-playbooks/ansible-lint-rules/ZUULJOBS0001/faulty/roles/handlers-missing-loopvar-loop/handlers/main.yaml
similarity index 100%
rename from test-playbooks/ansible-lint-rules/roles/handlers-missing-loopvar-loop/handlers/main.yaml
rename to test-playbooks/ansible-lint-rules/ZUULJOBS0001/faulty/roles/handlers-missing-loopvar-loop/handlers/main.yaml
diff --git a/test-playbooks/ansible-lint-rules/roles/handlers-missing-loopvar-with/handlers/main.yaml b/test-playbooks/ansible-lint-rules/ZUULJOBS0001/faulty/roles/handlers-missing-loopvar-with/handlers/main.yaml
similarity index 100%
rename from test-playbooks/ansible-lint-rules/roles/handlers-missing-loopvar-with/handlers/main.yaml
rename to test-playbooks/ansible-lint-rules/ZUULJOBS0001/faulty/roles/handlers-missing-loopvar-with/handlers/main.yaml
diff --git a/test-playbooks/ansible-lint-rules/ZUULJOBS0001/faulty/roles/task-block-missing-loopvar-loop/tasks/main.yaml b/test-playbooks/ansible-lint-rules/ZUULJOBS0001/faulty/roles/task-block-missing-loopvar-loop/tasks/main.yaml
new file mode 100644
index 000000000..c822edf52
--- /dev/null
+++ b/test-playbooks/ansible-lint-rules/ZUULJOBS0001/faulty/roles/task-block-missing-loopvar-loop/tasks/main.yaml
@@ -0,0 +1,6 @@
+- block:
+    - debug:
+        var: item
+      loop:
+        - 1
+        - 2
diff --git a/test-playbooks/ansible-lint-rules/ZUULJOBS0001/faulty/roles/task-block-missing-loopvar-with/tasks/main.yaml b/test-playbooks/ansible-lint-rules/ZUULJOBS0001/faulty/roles/task-block-missing-loopvar-with/tasks/main.yaml
new file mode 100644
index 000000000..1c730fc02
--- /dev/null
+++ b/test-playbooks/ansible-lint-rules/ZUULJOBS0001/faulty/roles/task-block-missing-loopvar-with/tasks/main.yaml
@@ -0,0 +1,6 @@
+- block:
+    - debug:
+        var: item
+      with_items:
+        - 1
+        - 2
diff --git a/test-playbooks/ansible-lint-rules/ZUULJOBS0001/faulty/roles/task-nested-block-missing-loopvar/tasks/main.yaml b/test-playbooks/ansible-lint-rules/ZUULJOBS0001/faulty/roles/task-nested-block-missing-loopvar/tasks/main.yaml
new file mode 100644
index 000000000..e2a93c0bd
--- /dev/null
+++ b/test-playbooks/ansible-lint-rules/ZUULJOBS0001/faulty/roles/task-nested-block-missing-loopvar/tasks/main.yaml
@@ -0,0 +1,7 @@
+- block:
+    - block:
+        - debug:
+            var: item
+          loop:
+            - 1
+            - 2
diff --git a/test-playbooks/ansible-lint-rules/roles/tasks-include-missing-loopvar-loop/tasks/main.yaml b/test-playbooks/ansible-lint-rules/ZUULJOBS0001/faulty/roles/tasks-include-missing-loopvar-loop/tasks/main.yaml
similarity index 100%
rename from test-playbooks/ansible-lint-rules/roles/tasks-include-missing-loopvar-loop/tasks/main.yaml
rename to test-playbooks/ansible-lint-rules/ZUULJOBS0001/faulty/roles/tasks-include-missing-loopvar-loop/tasks/main.yaml
diff --git a/test-playbooks/ansible-lint-rules/roles/tasks-include-missing-loopvar-with/tasks/main.yaml b/test-playbooks/ansible-lint-rules/ZUULJOBS0001/faulty/roles/tasks-include-missing-loopvar-with/tasks/main.yaml
similarity index 100%
rename from test-playbooks/ansible-lint-rules/roles/tasks-include-missing-loopvar-with/tasks/main.yaml
rename to test-playbooks/ansible-lint-rules/ZUULJOBS0001/faulty/roles/tasks-include-missing-loopvar-with/tasks/main.yaml
diff --git a/test-playbooks/ansible-lint-rules/roles/tasks-missing-loopvar-loop/tasks/main.yaml b/test-playbooks/ansible-lint-rules/ZUULJOBS0001/faulty/roles/tasks-missing-loopvar-loop/tasks/main.yaml
similarity index 100%
rename from test-playbooks/ansible-lint-rules/roles/tasks-missing-loopvar-loop/tasks/main.yaml
rename to test-playbooks/ansible-lint-rules/ZUULJOBS0001/faulty/roles/tasks-missing-loopvar-loop/tasks/main.yaml
diff --git a/test-playbooks/ansible-lint-rules/roles/tasks-missing-loopvar-with/tasks/main.yaml b/test-playbooks/ansible-lint-rules/ZUULJOBS0001/faulty/roles/tasks-missing-loopvar-with/tasks/main.yaml
similarity index 100%
rename from test-playbooks/ansible-lint-rules/roles/tasks-missing-loopvar-with/tasks/main.yaml
rename to test-playbooks/ansible-lint-rules/ZUULJOBS0001/faulty/roles/tasks-missing-loopvar-with/tasks/main.yaml
diff --git a/test-playbooks/ansible-lint-rules/roles/handlers-loopvar-loop/handlers/main.yaml b/test-playbooks/ansible-lint-rules/ZUULJOBS0001/valid/roles/handlers-loopvar-loop/handlers/main.yaml
similarity index 100%
rename from test-playbooks/ansible-lint-rules/roles/handlers-loopvar-loop/handlers/main.yaml
rename to test-playbooks/ansible-lint-rules/ZUULJOBS0001/valid/roles/handlers-loopvar-loop/handlers/main.yaml
diff --git a/test-playbooks/ansible-lint-rules/roles/handlers-loopvar-with/handlers/main.yaml b/test-playbooks/ansible-lint-rules/ZUULJOBS0001/valid/roles/handlers-loopvar-with/handlers/main.yaml
similarity index 100%
rename from test-playbooks/ansible-lint-rules/roles/handlers-loopvar-with/handlers/main.yaml
rename to test-playbooks/ansible-lint-rules/ZUULJOBS0001/valid/roles/handlers-loopvar-with/handlers/main.yaml
diff --git a/test-playbooks/ansible-lint-rules/ZUULJOBS0001/valid/roles/task-block-loopvar-loop/tasks/main.yaml b/test-playbooks/ansible-lint-rules/ZUULJOBS0001/valid/roles/task-block-loopvar-loop/tasks/main.yaml
new file mode 100644
index 000000000..d1e0c1b72
--- /dev/null
+++ b/test-playbooks/ansible-lint-rules/ZUULJOBS0001/valid/roles/task-block-loopvar-loop/tasks/main.yaml
@@ -0,0 +1,8 @@
+- block:
+    - debug:
+        msg: zj_item
+      loop:
+        - 1
+        - 2
+      loop_control:
+        loop_var: zj_item
diff --git a/test-playbooks/ansible-lint-rules/ZUULJOBS0001/valid/roles/task-block-loopvar-with/tasks/main.yaml b/test-playbooks/ansible-lint-rules/ZUULJOBS0001/valid/roles/task-block-loopvar-with/tasks/main.yaml
new file mode 100644
index 000000000..b28fdacdd
--- /dev/null
+++ b/test-playbooks/ansible-lint-rules/ZUULJOBS0001/valid/roles/task-block-loopvar-with/tasks/main.yaml
@@ -0,0 +1,8 @@
+- block:
+    - debug:
+        msg: zj_item
+      with_items:
+        - 1
+        - 2
+      loop_control:
+        loop_var: zj_item
diff --git a/test-playbooks/ansible-lint-rules/ZUULJOBS0001/valid/roles/task-missing-loopvar-loop-skip/tasks/main.yaml b/test-playbooks/ansible-lint-rules/ZUULJOBS0001/valid/roles/task-missing-loopvar-loop-skip/tasks/main.yaml
new file mode 100644
index 000000000..4277bc7da
--- /dev/null
+++ b/test-playbooks/ansible-lint-rules/ZUULJOBS0001/valid/roles/task-missing-loopvar-loop-skip/tasks/main.yaml
@@ -0,0 +1,5 @@
+- debug:  # noqa ZUULJOBS0001
+    var: item
+  loop:
+    - 1
+    - 2
diff --git a/test-playbooks/ansible-lint-rules/roles/tasks-include-loopvar-loop/tasks/main.yaml b/test-playbooks/ansible-lint-rules/ZUULJOBS0001/valid/roles/tasks-include-loopvar-loop/tasks/main.yaml
similarity index 100%
rename from test-playbooks/ansible-lint-rules/roles/tasks-include-loopvar-loop/tasks/main.yaml
rename to test-playbooks/ansible-lint-rules/ZUULJOBS0001/valid/roles/tasks-include-loopvar-loop/tasks/main.yaml
diff --git a/test-playbooks/ansible-lint-rules/roles/tasks-include-loopvar-with/tasks/main.yaml b/test-playbooks/ansible-lint-rules/ZUULJOBS0001/valid/roles/tasks-include-loopvar-with/tasks/main.yaml
similarity index 100%
rename from test-playbooks/ansible-lint-rules/roles/tasks-include-loopvar-with/tasks/main.yaml
rename to test-playbooks/ansible-lint-rules/ZUULJOBS0001/valid/roles/tasks-include-loopvar-with/tasks/main.yaml
diff --git a/test-playbooks/ansible-lint-rules/roles/tasks-loopvar-loop/tasks/main.yaml b/test-playbooks/ansible-lint-rules/ZUULJOBS0001/valid/roles/tasks-loopvar-loop/tasks/main.yaml
similarity index 100%
rename from test-playbooks/ansible-lint-rules/roles/tasks-loopvar-loop/tasks/main.yaml
rename to test-playbooks/ansible-lint-rules/ZUULJOBS0001/valid/roles/tasks-loopvar-loop/tasks/main.yaml
diff --git a/test-playbooks/ansible-lint-rules/roles/tasks-loopvar-with/tasks/main.yaml b/test-playbooks/ansible-lint-rules/ZUULJOBS0001/valid/roles/tasks-loopvar-with/tasks/main.yaml
similarity index 100%
rename from test-playbooks/ansible-lint-rules/roles/tasks-loopvar-with/tasks/main.yaml
rename to test-playbooks/ansible-lint-rules/ZUULJOBS0001/valid/roles/tasks-loopvar-with/tasks/main.yaml
diff --git a/test-playbooks/ansible-lint-rules/run.yaml b/test-playbooks/ansible-lint-rules/run.yaml
index a443bfe92..b2f1d0c08 100644
--- a/test-playbooks/ansible-lint-rules/run.yaml
+++ b/test-playbooks/ansible-lint-rules/run.yaml
@@ -15,28 +15,44 @@
     - name: Make sure ansible-lint is installed
       command: "{{ ansible_lint_tempdir.path }}/bin/ansible-lint --version"
 
-    - name: Make sure missing loopvars fail linting
-      command: "{{ ansible_lint_tempdir.path }}/bin/ansible-lint test-playbooks/ansible-lint-rules/roles/{{ item }}"
+    - name: Get faulty playbooks and roles
+      command: >-
+        find test-playbooks/ansible-lint-rules/
+        -mindepth 4
+        -maxdepth 4
+        -wholename '*ZUULJOBS*/faulty/*/*'
+      args:
+        chdir: "{{ ansible_user_dir}}/{{ zuul.project.src_dir }}"
+      register: faulty_ansible_items
+
+    - name: Get valid playbooks and roles
+      command: >-
+        find test-playbooks/ansible-lint-rules/
+        -mindepth 4
+        -maxdepth 4
+        -wholename '*ZUULJOBS*/valid/roles/*'
+      args:
+        chdir: "{{ ansible_user_dir}}/{{ zuul.project.src_dir }}"
+      register: valid_ansible_items
+
+    - name: Make sure faulty roles fail linting
+      command: >-
+        {{ ansible_lint_tempdir.path }}/bin/ansible-lint
+        -t {{ item | regex_replace('.*/(ZUULJOBS.*?)/.*', '\1') }}
+        {{ item }}
+      args:
+        chdir: "{{ ansible_user_dir }}/{{ zuul.project.src_dir }}"
+      register: ansible_lint
+      failed_when: ansible_lint.rc == 0
+      loop: "{{ faulty_ansible_items.stdout_lines }}"
+
+    - name: Make sure valid roles pass linting
+      command: >-
+        {{ ansible_lint_tempdir.path }}/bin/ansible-lint
+        -t {{ item | regex_replace('.*/(ZUULJOBS.*?)/.*', '\1') }}
+        {{ item }}
       args:
         chdir: "{{ansible_user_dir}}/{{ zuul.project.src_dir }}"
       register: ansible_lint
-      failed_when: ansible_lint.rc == 0
-      loop:
-        - tasks-missing-loopvar-with
-        - tasks-missing-loopvar-loop
-        - tasks-include-missing-loopvar-with
-        - tasks-include-missing-loopvar-loop
-        - handlers-missing-loopvar-with
-        - handlers-missing-loopvar-loop
-
-    - name: Make sure valid configuration passes linting
-      command: "{{ ansible_lint_tempdir.path }}/bin/ansible-lint test-playbooks/ansible-lint-rules/roles/{{ item }}"
-      args:
-        chdir: "{{ansible_user_dir}}/{{ zuul.project.src_dir }}"
-      loop:
-        - tasks-loopvar-with
-        - tasks-loopvar-loop
-        - tasks-include-loopvar-with
-        - tasks-include-loopvar-loop
-        - handlers-loopvar-with
-        - handlers-loopvar-loop
+      failed_when: ansible_lint.rc != 0
+      loop: "{{ valid_ansible_items.stdout_lines }}"