[autoscaler] Flush stdout and stdin when running commands. (#19473)

Flush command stdout/stderr before exiting CommandRunner.run, so that setup command output is less likely to get swallowed.

python/ray/autoscaler/_private/command_runner.py

@@ -443,8 +443,8 @@ class SSHCommandRunner(CommandRunnerInterface):
                 Full command to run. Should include SSH options and other
                 processing that we do.
             with_output (bool):
-                If `with_output` is `True`, command stdout and stderr
+                If `with_output` is `True`, command stdout will be captured and
-                will be captured and returned.
+                returned.
             exit_on_fail (bool):
                 If `exit_on_fail` is `True`, the process will exit
                 if the command fails (exits with a code other than 0).
@@ -465,10 +465,8 @@ class SSHCommandRunner(CommandRunnerInterface):
                     silent=silent,
                     use_login_shells=is_using_login_shells(),
                 )
-            if with_output:
-                return self.process_runner.check_output(final_cmd)
             else:
-                return self.process_runner.check_call(final_cmd)
+                return self.process_runner.check_output(final_cmd)
         except subprocess.CalledProcessError as e:
             joined_cmd = " ".join(final_cmd)
             if not is_using_login_shells():
@@ -488,6 +486,11 @@ class SSHCommandRunner(CommandRunnerInterface):
                 if is_output_redirected():
                     fail_msg += " See above for the output from the failure."
                 raise click.ClickException(fail_msg) from None
+        finally:
+            # Do our best to flush output to terminal.
+            # See https://github.com/ray-project/ray/pull/19473.
+            sys.stdout.flush()
+            sys.stderr.flush()
 
     def run(
         self,

python/ray/autoscaler/_private/subprocess_output_util.py

@@ -325,7 +325,7 @@ def run_cmd_redirected(
         process_runner: Process runner used for executing commands.
         silent (bool): If true, the command output will be silenced completely
                        (redirected to /dev/null), unless verbose logging
-                       is enabled. Use this for runnign utility commands like
+                       is enabled. Use this for running utility commands like
                        rsync.
     """
     if silent and cli_logger.verbosity < 1:

python/ray/autoscaler/_private/updater.py

@@ -436,7 +436,7 @@ class NodeUpdater:
                         _numbered=("[]", 4, NUM_SETUP_STEPS),
                     )
                 with cli_logger.group(
-                    "Initalizing command runner",
+                    "Initializing command runner",
                     # todo: fix command numbering
                     _numbered=("[]", 5, NUM_SETUP_STEPS),
                 ):

python/ray/tests/test_cli_patterns/test_ray_up.txt

@@ -32,7 +32,7 @@ Acquiring an up-to-date head node
   \[3/7\] No worker file mounts to sync
   New status: setting-up
   \[4/7\] Running initialization commands
-  \[5/7\] Initalizing command runner
+  \[5/7\] Initializing command runner
   \[6/7\] Running setup commands
     \(0/4\) echo a
     \(1/4\) echo b

python/ray/tests/test_cli_patterns/test_ray_up_docker.txt

@@ -32,7 +32,7 @@ Acquiring an up-to-date head node
   \[3/7\] No worker file mounts to sync
   New status: setting-up
   \[4/7\] Running initialization commands
-  \[5/7\] Initalizing command runner
+  \[5/7\] Initializing command runner
   \[6/7\] Running setup commands
     \(0/4\) echo a
     \(1/4\) echo b

python/ray/tests/test_cli_patterns/test_ray_up_record.txt

@@ -58,7 +58,7 @@
 .+\.py.*Running `echo init`
 .+\.py.*Full command is `ssh.+`
 .+\.py.*NodeUpdater: i-.+: Initialization commands succeeded \[LogTimer=.+\]
-.+\.py.*\[5/7\] Initalizing command runner
+.+\.py.*\[5/7\] Initializing command runner
 .+\.py.*\[6/7\] Running setup commands
 .+\.py.*\(0/4\) echo a
 .+\.py.*Running `echo a`