Add screen custom log file support (#24461)

A simple way to redirect remote job output to a custom file is extremely useful.
Especially if it does not require any code changes for the user, and especially if it captures all output and not limited only to python logging.

When the PR will be merged, the user will be able to do the following (for example):

ray submit some_cluster_config.yaml example_runnable_script.py --screen --extra-screen-args "-Logfile /gpfs/usr/someone/ray/output_log.txt"
which will, in addition to creating the screen session, also output (continuously) to custom text file.
It allows additional flexibility, for example, the user will be able to choose custom screen session name etc.

python/ray/autoscaler/_private/commands.py

@@ -1021,6 +1021,7 @@ def exec_cluster(
     port_forward: Optional[Port_forward] = None,
     with_output: bool = False,
     _allow_uninitialized_state: bool = False,
+    extra_screen_args: Optional[str] = None,
 ) -> str:
     """Runs a command on the specified cluster.
 
@@ -1030,6 +1031,7 @@ def exec_cluster(
         run_env: whether to run the command on the host or in a container.
             Select between "auto", "host" and "docker"
         screen: whether to run in a screen
+        extra_screen_args: optional custom additional args to screen command
         tmux: whether to run in a tmux session
         stop: whether to stop the cluster after command run
         start: whether to start the cluster if it isn't up
@@ -1098,6 +1100,7 @@ def exec_cluster(
         with_output=with_output,
         run_env=run_env,
         shutdown_after_run=shutdown_after_run,
+        extra_screen_args=extra_screen_args,
     )
     if tmux or screen:
         attach_command_parts = ["ray attach", config_file]
@@ -1124,6 +1127,7 @@ def _exec(
     with_output: bool = False,
     run_env: str = "auto",
     shutdown_after_run: bool = False,
+    extra_screen_args: Optional[str] = None,
 ) -> str:
     if cmd:
         if screen:
@@ -1131,6 +1135,12 @@ def _exec(
                 "screen",
                 "-L",
                 "-dm",
+            ]
+
+            if extra_screen_args is not None and len(extra_screen_args) > 0:
+                wrapped_cmd += [extra_screen_args]
+
+            wrapped_cmd += [
                 "bash",
                 "-c",
                 quote(cmd + "; exec bash"),

python/ray/scripts/scripts.py

@@ -1437,6 +1437,13 @@ def rsync_up(cluster_config_file, source, target, cluster_name, all_nodes):
     default=False,
     help="If True, the usage stats collection will be disabled.",
 )
+@click.option(
+    "--extra-screen-args",
+    default=None,
+    help="if screen is enabled, add the provided args to it. A useful example "
+    "usage scenario is passing --extra-screen-args='-Logfile /full/path/blah_log.txt'"
+    " as it redirects screen output also to a custom file",
+)
 @add_click_logging_options
 def submit(
     cluster_config_file,
@@ -1451,6 +1458,7 @@ def submit(
     args,
     script_args,
     disable_usage_stats,
+    extra_screen_args: Optional[str] = None,
 ):
     """Uploads and runs a script on the specified cluster.
 
@@ -1478,6 +1486,11 @@ def submit(
     assert not (screen and tmux), "Can specify only one of `screen` or `tmux`."
     assert not (script_args and args), "Use -- --arg1 --arg2 for script args."
 
+    if (extra_screen_args is not None) and (not screen):
+        cli_logger.abort(
+            "To use extra_screen_args, it is required to use the --screen flag"
+        )
+
     if args:
         cli_logger.warning(
             "`{}` is deprecated and will be removed in the future.", cf.bold("--args")
@@ -1535,6 +1548,7 @@ def submit(
         override_cluster_name=cluster_name,
         no_config_cache=no_config_cache,
         port_forward=port_forward,
+        extra_screen_args=extra_screen_args,
     )