Fix race condition connecting to docker ondemand_cluster

runhouse/resources/hardware/sky_ssh_runner.py

@@ -264,15 +264,17 @@ def tunnel(self, local_port, remote_port):
             ssh_mode=SshMode.NON_INTERACTIVE, port_forward=[(local_port, remote_port)]
         )
         command = " ".join(base_cmd)
-        logger.debug(f"Running forwarding command: {command}")
+        logger.info(f"Running forwarding command: {command}")
         proc = subprocess.Popen(
             shlex.split(command),
             stdin=subprocess.PIPE,
             stdout=subprocess.PIPE,
             stderr=subprocess.PIPE,
         )
-        # Wait for the ssh connection to start
-        time.sleep(1)
+        # Wait until tunnel is formed, i.e. first line is printed from ssh server ("Welcome to Ubuntu...")
+        # Without this we'll get connection errors from the HTTPClient attempting to connect before
+        # the tunnel is fully formed.
+        proc.stdout.readline()
 
         # Set the tunnel process and ports to be cleaned up later
         self.tunnel_proc = proc