[tune] Fix logging for dockersyncer (#12196)

doc/source/tune/user-guide.rst

@@ -686,6 +686,7 @@ These are the environment variables Ray Tune currently considers:
   with the parameter values in them)
 * **TUNE_RESULT_DIR**: Directory where Tune trial results are stored. If this
   is not set, ``~/ray_results`` will be used.
+* **TUNE_SYNCER_VERBOSITY**: Amount of command output when using Tune with Docker Syncer. Defaults to 0.
 
 
 There are some environment variables that are mostly relevant for integrated libraries:

python/ray/autoscaler/_private/cli_logger.py

@@ -283,6 +283,7 @@ class _CliLogger():
         self.indent_level = 0
 
         self._verbosity = 0
+        self._verbosity_overriden = False
         self._color_mode = "auto"
         self._log_style = "record"
         self.pretty = False
@@ -340,12 +341,15 @@ class _CliLogger():
 
     @property
     def verbosity(self):
-        if not self.pretty:
+        if self._verbosity_overriden:
+            return self._verbosity
+        elif not self.pretty:
             return 999
         return self._verbosity
 
     def _set_verbosity(self, x):
         self._verbosity = x
+        self._verbosity_overriden = True
 
     def detect_colors(self):
         """Update color output settings.

python/ray/autoscaler/_private/commands.py

@@ -972,6 +972,7 @@ def rsync(config_file: str,
 
         if source and target:
             # print rsync progress for single file rsync
+            if cli_logger.verbosity > 0:
                 cmd_output_util.set_output_redirected(False)
                 set_rsync_silent(False)
             rsync(source, target, is_file_mount)

python/ray/autoscaler/sdk.py

@@ -10,6 +10,7 @@ from ray.autoscaler._private import commands
 from ray.autoscaler._private.event_system import (  # noqa: F401
     CreateClusterEvent,  # noqa: F401
     global_event_system)
+from ray.autoscaler._private.cli_logger import cli_logger
 
 
 def create_or_update_cluster(cluster_config: Union[dict, str],
@@ -206,6 +207,32 @@ def request_resources(num_cpus: Optional[int] = None,
     return commands.request_resources(num_cpus, bundles)
 
 
+def configure_logging(log_style: Optional[str] = None,
+                      color_mode: Optional[str] = None,
+                      verbosity: Optional[int] = None):
+    """Configures logging for cluster command calls.
+
+    Args:
+        log_style (str): If 'pretty', outputs with formatting and color.
+            If 'record', outputs record-style without formatting.
+            'auto' defaults to 'pretty', and disables pretty logging
+            if stdin is *not* a TTY. Defaults to "auto".
+        color_mode (str):
+            Can be "true", "false", or "auto".
+
+            Enables or disables `colorful`.
+
+            If `color_mode` is "auto", is set to `not stdout.isatty()`
+        vebosity (int):
+            Output verbosity (0, 1, 2, 3).
+
+            Low verbosity will disable `verbose` and `very_verbose` messages.
+
+    """
+    cli_logger.configure(
+        log_style=log_style, color_mode=color_mode, verbosity=verbosity)
+
+
 @contextmanager
 def _as_config_file(cluster_config: Union[dict, str]) -> Iterator[str]:
     if isinstance(cluster_config, dict):

python/ray/tune/examples/pbt_example.py

@@ -85,8 +85,15 @@ if __name__ == "__main__":
     parser = argparse.ArgumentParser()
     parser.add_argument(
         "--smoke-test", action="store_true", help="Finish quickly for testing")
+    parser.add_argument(
+        "--cluster",
+        action="store_true",
+        help="Distribute tuning on a cluster")
     args, _ = parser.parse_known_args()
-    if args.smoke_test:
+
+    if args.cluster:
+        ray.init(address="auto")
+    elif args.smoke_test:
         ray.init(num_cpus=2)  # force pausing to happen for test
     else:
         ray.init()

python/ray/tune/integration/docker.py

@@ -2,9 +2,10 @@ import os
 from typing import Optional, Tuple
 
 from ray import services
-from ray.autoscaler.sdk import rsync
+from ray.autoscaler.sdk import rsync, configure_logging
 from ray.tune.syncer import NodeSyncer
 from ray.tune.sync_client import SyncClient
+from ray.tune.utils import env_integer
 
 
 class DockerSyncer(NodeSyncer):
@@ -13,6 +14,9 @@ class DockerSyncer(NodeSyncer):
     without a custom sync client. The sync client defaults to
     ``DockerSyncClient`` instead.
 
+    Set the env var `TUNE_SYNCER_VERBOSITY` to increase verbosity
+    of syncing operations (0, 1, 2, 3). Defaults to 0.
+
     .. note::
         This syncer only works with the Ray cluster launcher.
         If you use your own Docker setup, make sure the nodes can connect
@@ -35,6 +39,9 @@ class DockerSyncer(NodeSyncer):
                  local_dir: str,
                  remote_dir: str,
                  sync_client: Optional[SyncClient] = None):
+        configure_logging(
+            log_style="record",
+            verbosity=env_integer("TUNE_SYNCER_VERBOSITY", 0))
         self.local_ip = services.get_node_ip_address()
         self.worker_ip = None
 

python/ray/tune/ray_trial_executor.py

@@ -398,8 +398,8 @@ class RayTrialExecutor(TrialExecutor):
             trial (Trial): Trial to be reset.
             new_config (dict): New configuration for Trial trainable.
             new_experiment_tag (str): New experiment name for trial.
-            logger_creator (Callable[[Dict], Logger]): A function that
-                instantiates a logger on the actor process.
+            logger_creator (Optional[Callable[[Dict], Logger]]): Function
+                that instantiates a logger on the actor process.
 
         Returns:
             True if `reset_config` is successful else False.

python/ray/tune/trainable.py

@@ -391,7 +391,12 @@ class Trainable:
         self._result_logger.flush()
         self._result_logger.close()
 
+        if logger_creator:
+            logger.debug("Logger reset.")
             self._create_logger(new_config.copy(), logger_creator)
+        else:
+            logger.debug("Did not reset logger. Got: "
+                         f"trainable.reset(logger_creator={logger_creator}).")
 
         stdout_file = new_config.pop(STDOUT_FILE, None)
         stderr_file = new_config.pop(STDERR_FILE, None)