diff --git a/setup.py b/setup.py
index 5343f6d1d..3e9e1fedb 100644
--- a/setup.py
+++ b/setup.py
@@ -25,4 +25,7 @@ except ImportError:
 
 setuptools.setup(
     setup_requires=['pbr>=2.0'],
-    pbr=True)
+    pbr=True,
+    # Override this to avoid trying to autodetect modules for
+    # packaging, as some roles have .py files
+    py_modules=[])
diff --git a/test-playbooks/python/fetch-subunit-output.yaml b/test-playbooks/python/fetch-subunit-output.yaml
index d6c4c98a8..c308241f9 100644
--- a/test-playbooks/python/fetch-subunit-output.yaml
+++ b/test-playbooks/python/fetch-subunit-output.yaml
@@ -25,7 +25,7 @@
           dest: tox.ini
         - content: |
             import setuptools
-            setuptools.setup()
+            setuptools.setup(py_modules=[])
           dest: setup.py
         - content: ""
           dest: setup.cfg
diff --git a/test-playbooks/python/fetch-tox-output.yaml b/test-playbooks/python/fetch-tox-output.yaml
index 4e10f5a97..3c3d3bfb1 100644
--- a/test-playbooks/python/fetch-tox-output.yaml
+++ b/test-playbooks/python/fetch-tox-output.yaml
@@ -19,7 +19,7 @@
           dest: tox.ini
         - content: |
             import setuptools
-            setuptools.setup()
+            setuptools.setup(py_modules=[])
           dest: setup.py
         - content: ""
           dest: setup.cfg