[tune] Test TF2.0, TF1.14, TF1.12 Tensorboard support (#5931)

.travis.yml

@@ -176,7 +176,7 @@ script:
   # ray tune tests
   - if [ $RAY_CI_TUNE_AFFECTED == "1" ]; then ./ci/suppress_output python python/ray/tune/tests/test_dependency.py; fi
   # `cluster_tests.py` runs on Jenkins, not Travis.
-  - if [ $RAY_CI_TUNE_AFFECTED == "1" ]; then python -m pytest -v --durations=10 --timeout=300 --ignore=python/ray/tune/tests/test_cluster.py --ignore=python/ray/tune/tests/test_tune_restore.py --ignore=python/ray/tune/tests/test_actor_reuse.py python/ray/tune/tests; fi
+  - if [ $RAY_CI_TUNE_AFFECTED == "1" ]; then python -m pytest -v --durations=10 --timeout=300 --ignore=python/ray/tune/tests/test_cluster.py --ignore=python/ray/tune/tests/test_logger.py --ignore=python/ray/tune/tests/test_tune_restore.py --ignore=python/ray/tune/tests/test_actor_reuse.py python/ray/tune/tests; fi
 
   # ray serve tests
   - if [ $RAY_CI_SERVE_AFFECTED == "1" ]; then python -c 'import sys;exit(sys.version_info>=(3,5))' || python -m pytest -v --durations=5 --timeout=300 python/ray/experimental/serve/tests; fi

ci/jenkins_tests/run_tune_tests.sh

@@ -43,6 +43,18 @@ $SUPPRESS_OUTPUT docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE}
 $SUPPRESS_OUTPUT docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} $DOCKER_SHA \
     python /ray/python/ray/tune/tests/example.py
 
+$SUPPRESS_OUTPUT docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} $DOCKER_SHA \
+    bash -c 'pip install -U tensorflow && python /ray/python/ray/tune/tests/test_logger.py'
+
+$SUPPRESS_OUTPUT docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} $DOCKER_SHA \
+    bash -c 'pip install -U tensorflow==1.15 && python /ray/python/ray/tune/tests/test_logger.py'
+
+$SUPPRESS_OUTPUT docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} $DOCKER_SHA \
+    bash -c 'pip install -U tensorflow==1.14 && python /ray/python/ray/tune/tests/test_logger.py'
+
+$SUPPRESS_OUTPUT docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} $DOCKER_SHA \
+    bash -c 'pip install -U tensorflow==1.12 && python /ray/python/ray/tune/tests/test_logger.py'
+
 $SUPPRESS_OUTPUT docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} $DOCKER_SHA \
     python /ray/python/ray/tune/tests/tutorial.py
 
@@ -58,12 +70,6 @@ $SUPPRESS_OUTPUT docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE}
     python /ray/python/ray/tune/examples/async_hyperband_example.py \
     --smoke-test
 
-$SUPPRESS_OUTPUT docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} $DOCKER_SHA \
-    bash -c 'pip install tensorflow==2.0.0rc1 && python /ray/python/ray/tune/examples/async_hyperband_example.py --smoke-test'
-
-$SUPPRESS_OUTPUT docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} $DOCKER_SHA \
-    bash -c 'pip install tensorflow==1.15.0rc1 && python /ray/python/ray/tune/examples/async_hyperband_example.py --smoke-test'
-
 $SUPPRESS_OUTPUT docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} $DOCKER_SHA \
     python /ray/python/ray/tune/examples/tf_mnist_example.py --smoke-test
 

docker/examples/Dockerfile

@@ -11,7 +11,7 @@ RUN pip install -U pip
 RUN pip install gym[atari] opencv-python-headless tensorflow lz4 keras pytest-timeout smart_open tensorflow_probability
 RUN pip install -U h5py  # Mutes FutureWarnings
 RUN pip install --upgrade bayesian-optimization
-RUN pip install --upgrade git+git://github.com/hyperopt/hyperopt.git
+RUN pip install --upgrade hyperopt==0.1.2
 RUN pip install ConfigSpace==0.4.10
 RUN pip install --upgrade sigopt nevergrad scikit-optimize hpbandster lightgbm xgboost torch torchvision
 RUN pip install -U tabulate mlflow

docker/tune_test/Dockerfile

@@ -14,7 +14,7 @@ RUN apt-get install -y zlib1g-dev
 RUN conda remove -y --force wrapt
 RUN pip install gym[atari]==0.10.11 opencv-python-headless tensorflow lz4 keras pytest-timeout smart_open torch torchvision
 RUN pip install --upgrade bayesian-optimization
-RUN pip install --upgrade git+git://github.com/hyperopt/hyperopt.git
+RUN pip install --upgrade hyperopt==0.1.2
 RUN pip install ConfigSpace==0.4.10
 RUN pip install --upgrade sigopt nevergrad scikit-optimize hpbandster lightgbm xgboost
 RUN pip install -U mlflow

python/ray/tune/examples/pbt_memnn_example.py

@@ -14,12 +14,16 @@ from tensorflow.keras.layers import LSTM
 from tensorflow.keras.optimizers import RMSprop
 from tensorflow.keras.utils import get_file
 from tensorflow.keras.preprocessing.sequence import pad_sequences
-from ray.tune import Trainable
+
+from filelock import FileLock
+import os
 import argparse
 import tarfile
 import numpy as np
 import re
 
+from ray.tune import Trainable
+
 
 def tokenize(sent):
     """Return the tokens of a sentence including punctuation.
@@ -211,6 +215,7 @@ class MemNNModel(Trainable):
         return model
 
     def _setup(self, config):
+        with FileLock(os.path.expanduser("~/.tune.lock")):
             self.train_stories, self.test_stories = read_data()
         model = self.build_model()
         rmsprop = RMSprop(

python/ray/tune/logger.py

@@ -146,7 +146,7 @@ def tf2_compat_logger(config, logdir, trial=None):
     else:
         import tensorflow as tf
         use_tf2_api = (distutils.version.LooseVersion(tf.__version__) >=
-                       distutils.version.LooseVersion("2.0.0"))
+                       distutils.version.LooseVersion("1.15.0"))
         if use_tf2_api:
             tf = tf.compat.v2  # setting this for TF2.0
             return TF2Logger(config, logdir, trial)
@@ -238,7 +238,7 @@ class TFLogger(Logger):
 
     def _init(self):
         logger.debug("Initializing TFLogger instead of TF2Logger.")
-        self._file_writer = tf.compat.v1.summary.FileWriter(self.logdir)
+        self._file_writer = tf.summary.FileWriter(self.logdir)
 
     def on_result(self, result):
         tmp = result.copy()

python/ray/tune/tests/test_logger.py

@@ -0,0 +1,59 @@
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+from collections import namedtuple
+import unittest
+import tempfile
+import shutil
+
+from ray.tune.logger import tf2_compat_logger, JsonLogger, CSVLogger
+
+Trial = namedtuple("MockTrial", ["evaluated_params", "trial_id"])
+
+
+def result(t, rew):
+    return dict(
+        time_total_s=t,
+        episode_reward_mean=rew,
+        mean_accuracy=rew * 2,
+        training_iteration=int(t))
+
+
+class LoggerSuite(unittest.TestCase):
+    """Test built-in loggers."""
+
+    def setUp(self):
+        self.test_dir = tempfile.mkdtemp()
+
+    def tearDown(self):
+        shutil.rmtree(self.test_dir, ignore_errors=True)
+
+    def testTensorBoardLogger(self):
+        config = {"a": 2, "b": 5}
+        t = Trial(evaluated_params=config, trial_id=5342)
+        logger = tf2_compat_logger(
+            config=config, logdir=self.test_dir, trial=t)
+        logger.on_result(result(2, 4))
+        logger.on_result(result(2, 4))
+        logger.close()
+
+    def testCSV(self):
+        config = {"a": 2, "b": 5}
+        t = Trial(evaluated_params=config, trial_id="csv")
+        logger = CSVLogger(config=config, logdir=self.test_dir, trial=t)
+        logger.on_result(result(2, 4))
+        logger.on_result(result(2, 4))
+        logger.close()
+
+    def testJSON(self):
+        config = {"a": 2, "b": 5}
+        t = Trial(evaluated_params=config, trial_id="json")
+        logger = JsonLogger(config=config, logdir=self.test_dir, trial=t)
+        logger.on_result(result(2, 4))
+        logger.on_result(result(2, 4))
+        logger.close()
+
+
+if __name__ == "__main__":
+    unittest.main(verbosity=2)