diff --git a/.travis.yml b/.travis.yml index 250e76165..0c5695128 100644 --- a/.travis.yml +++ b/.travis.yml @@ -176,7 +176,7 @@ script: # ray tune tests - if [ $RAY_CI_TUNE_AFFECTED == "1" ]; then ./ci/suppress_output python python/ray/tune/tests/test_dependency.py; fi # `cluster_tests.py` runs on Jenkins, not Travis. - - if [ $RAY_CI_TUNE_AFFECTED == "1" ]; then python -m pytest -v --durations=10 --timeout=300 --ignore=python/ray/tune/tests/test_cluster.py --ignore=python/ray/tune/tests/test_tune_restore.py --ignore=python/ray/tune/tests/test_actor_reuse.py python/ray/tune/tests; fi + - if [ $RAY_CI_TUNE_AFFECTED == "1" ]; then python -m pytest -v --durations=10 --timeout=300 --ignore=python/ray/tune/tests/test_cluster.py --ignore=python/ray/tune/tests/test_logger.py --ignore=python/ray/tune/tests/test_tune_restore.py --ignore=python/ray/tune/tests/test_actor_reuse.py python/ray/tune/tests; fi # ray serve tests - if [ $RAY_CI_SERVE_AFFECTED == "1" ]; then python -c 'import sys;exit(sys.version_info>=(3,5))' || python -m pytest -v --durations=5 --timeout=300 python/ray/experimental/serve/tests; fi diff --git a/ci/jenkins_tests/run_tune_tests.sh b/ci/jenkins_tests/run_tune_tests.sh index c6bff09d0..3250a4884 100755 --- a/ci/jenkins_tests/run_tune_tests.sh +++ b/ci/jenkins_tests/run_tune_tests.sh @@ -43,6 +43,18 @@ $SUPPRESS_OUTPUT docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} $SUPPRESS_OUTPUT docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} $DOCKER_SHA \ python /ray/python/ray/tune/tests/example.py +$SUPPRESS_OUTPUT docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} $DOCKER_SHA \ + bash -c 'pip install -U tensorflow && python /ray/python/ray/tune/tests/test_logger.py' + +$SUPPRESS_OUTPUT docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} $DOCKER_SHA \ + bash -c 'pip install -U tensorflow==1.15 && python /ray/python/ray/tune/tests/test_logger.py' + +$SUPPRESS_OUTPUT docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} $DOCKER_SHA \ + bash -c 'pip install -U tensorflow==1.14 && python /ray/python/ray/tune/tests/test_logger.py' + +$SUPPRESS_OUTPUT docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} $DOCKER_SHA \ + bash -c 'pip install -U tensorflow==1.12 && python /ray/python/ray/tune/tests/test_logger.py' + $SUPPRESS_OUTPUT docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} $DOCKER_SHA \ python /ray/python/ray/tune/tests/tutorial.py @@ -58,12 +70,6 @@ $SUPPRESS_OUTPUT docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} python /ray/python/ray/tune/examples/async_hyperband_example.py \ --smoke-test -$SUPPRESS_OUTPUT docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} $DOCKER_SHA \ - bash -c 'pip install tensorflow==2.0.0rc1 && python /ray/python/ray/tune/examples/async_hyperband_example.py --smoke-test' - -$SUPPRESS_OUTPUT docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} $DOCKER_SHA \ - bash -c 'pip install tensorflow==1.15.0rc1 && python /ray/python/ray/tune/examples/async_hyperband_example.py --smoke-test' - $SUPPRESS_OUTPUT docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} $DOCKER_SHA \ python /ray/python/ray/tune/examples/tf_mnist_example.py --smoke-test diff --git a/docker/examples/Dockerfile b/docker/examples/Dockerfile index 959c5a930..a07e10b6e 100644 --- a/docker/examples/Dockerfile +++ b/docker/examples/Dockerfile @@ -11,7 +11,7 @@ RUN pip install -U pip RUN pip install gym[atari] opencv-python-headless tensorflow lz4 keras pytest-timeout smart_open tensorflow_probability RUN pip install -U h5py # Mutes FutureWarnings RUN pip install --upgrade bayesian-optimization -RUN pip install --upgrade git+git://github.com/hyperopt/hyperopt.git +RUN pip install --upgrade hyperopt==0.1.2 RUN pip install ConfigSpace==0.4.10 RUN pip install --upgrade sigopt nevergrad scikit-optimize hpbandster lightgbm xgboost torch torchvision RUN pip install -U tabulate mlflow diff --git a/docker/tune_test/Dockerfile b/docker/tune_test/Dockerfile index 623a35ba1..a6cb26f0f 100644 --- a/docker/tune_test/Dockerfile +++ b/docker/tune_test/Dockerfile @@ -14,7 +14,7 @@ RUN apt-get install -y zlib1g-dev RUN conda remove -y --force wrapt RUN pip install gym[atari]==0.10.11 opencv-python-headless tensorflow lz4 keras pytest-timeout smart_open torch torchvision RUN pip install --upgrade bayesian-optimization -RUN pip install --upgrade git+git://github.com/hyperopt/hyperopt.git +RUN pip install --upgrade hyperopt==0.1.2 RUN pip install ConfigSpace==0.4.10 RUN pip install --upgrade sigopt nevergrad scikit-optimize hpbandster lightgbm xgboost RUN pip install -U mlflow diff --git a/python/ray/tune/examples/pbt_memnn_example.py b/python/ray/tune/examples/pbt_memnn_example.py index f19b22b65..3dddf314f 100644 --- a/python/ray/tune/examples/pbt_memnn_example.py +++ b/python/ray/tune/examples/pbt_memnn_example.py @@ -14,12 +14,16 @@ from tensorflow.keras.layers import LSTM from tensorflow.keras.optimizers import RMSprop from tensorflow.keras.utils import get_file from tensorflow.keras.preprocessing.sequence import pad_sequences -from ray.tune import Trainable + +from filelock import FileLock +import os import argparse import tarfile import numpy as np import re +from ray.tune import Trainable + def tokenize(sent): """Return the tokens of a sentence including punctuation. @@ -211,7 +215,8 @@ class MemNNModel(Trainable): return model def _setup(self, config): - self.train_stories, self.test_stories = read_data() + with FileLock(os.path.expanduser("~/.tune.lock")): + self.train_stories, self.test_stories = read_data() model = self.build_model() rmsprop = RMSprop( lr=self.config.get("lr", 1e-3), rho=self.config.get("rho", 0.9)) diff --git a/python/ray/tune/logger.py b/python/ray/tune/logger.py index 393962532..53a09207e 100644 --- a/python/ray/tune/logger.py +++ b/python/ray/tune/logger.py @@ -146,7 +146,7 @@ def tf2_compat_logger(config, logdir, trial=None): else: import tensorflow as tf use_tf2_api = (distutils.version.LooseVersion(tf.__version__) >= - distutils.version.LooseVersion("2.0.0")) + distutils.version.LooseVersion("1.15.0")) if use_tf2_api: tf = tf.compat.v2 # setting this for TF2.0 return TF2Logger(config, logdir, trial) @@ -238,7 +238,7 @@ class TFLogger(Logger): def _init(self): logger.debug("Initializing TFLogger instead of TF2Logger.") - self._file_writer = tf.compat.v1.summary.FileWriter(self.logdir) + self._file_writer = tf.summary.FileWriter(self.logdir) def on_result(self, result): tmp = result.copy() diff --git a/python/ray/tune/tests/test_logger.py b/python/ray/tune/tests/test_logger.py new file mode 100644 index 000000000..59bc4d288 --- /dev/null +++ b/python/ray/tune/tests/test_logger.py @@ -0,0 +1,59 @@ +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +from collections import namedtuple +import unittest +import tempfile +import shutil + +from ray.tune.logger import tf2_compat_logger, JsonLogger, CSVLogger + +Trial = namedtuple("MockTrial", ["evaluated_params", "trial_id"]) + + +def result(t, rew): + return dict( + time_total_s=t, + episode_reward_mean=rew, + mean_accuracy=rew * 2, + training_iteration=int(t)) + + +class LoggerSuite(unittest.TestCase): + """Test built-in loggers.""" + + def setUp(self): + self.test_dir = tempfile.mkdtemp() + + def tearDown(self): + shutil.rmtree(self.test_dir, ignore_errors=True) + + def testTensorBoardLogger(self): + config = {"a": 2, "b": 5} + t = Trial(evaluated_params=config, trial_id=5342) + logger = tf2_compat_logger( + config=config, logdir=self.test_dir, trial=t) + logger.on_result(result(2, 4)) + logger.on_result(result(2, 4)) + logger.close() + + def testCSV(self): + config = {"a": 2, "b": 5} + t = Trial(evaluated_params=config, trial_id="csv") + logger = CSVLogger(config=config, logdir=self.test_dir, trial=t) + logger.on_result(result(2, 4)) + logger.on_result(result(2, 4)) + logger.close() + + def testJSON(self): + config = {"a": 2, "b": 5} + t = Trial(evaluated_params=config, trial_id="json") + logger = JsonLogger(config=config, logdir=self.test_dir, trial=t) + logger.on_result(result(2, 4)) + logger.on_result(result(2, 4)) + logger.close() + + +if __name__ == "__main__": + unittest.main(verbosity=2)