diff --git a/connection/ssh.py b/connection/ssh.py
index 547d431..fff6f40 100644
--- a/connection/ssh.py
+++ b/connection/ssh.py
@@ -62,8 +62,9 @@ class Connection(SSH.Connection):
     def set_host_overrides(self, host, hostvars=None):
         if self._container_check() or self._chroot_check():
             physical_host_addr = hostvars.get('physical_host_addr')
-            if physical_host_addr:
-                self.host = self._play_context.remote_addr = physical_host_addr
+            if not physical_host_addr:
+                physical_host_addr = self.physical_host
+            self.host = self._play_context.remote_addr = physical_host_addr
 
     def exec_command(self, cmd, in_data=None, sudoable=True):
         """run a command on the remote host."""