From d16b6f6a324eb1ac0f7876f9a6941e50f20b642d Mon Sep 17 00:00:00 2001
From: Michael Tu <michaeltu22@gmail.com>
Date: Fri, 24 Aug 2018 15:05:24 -0700
Subject: [PATCH] [tune] Rename 'repeat' to 'num_samples' (#2698)

Deprecates the `repeat` argument and introduces `num_samples`. Also updates docs accordingly.
---
 doc/source/tune-usage.rst                     |  8 ++++----
 python/ray/rllib/train.py                     |  2 +-
 .../tuned_examples/hyperband-cartpole.yaml    |  2 +-
 .../regression_tests/regression_test.py       |  2 +-
 python/ray/tune/config_parser.py              |  2 +-
 .../tune/examples/async_hyperband_example.py  |  2 +-
 python/ray/tune/examples/hyperband_example.py |  2 +-
 python/ray/tune/examples/hyperopt_example.py  |  2 +-
 python/ray/tune/examples/pbt_example.py       |  2 +-
 python/ray/tune/examples/pbt_ppo_example.py   |  2 +-
 .../examples/pbt_tune_cifar10_with_keras.py   |  2 +-
 .../examples/tune_mnist_async_hyperband.py    |  4 ++--
 python/ray/tune/examples/tune_mnist_keras.py  |  2 +-
 .../tune/examples/tune_mnist_ray_hyperband.py |  4 ++--
 python/ray/tune/experiment.py                 | 19 ++++++++++++++++---
 python/ray/tune/suggest/basic_variant.py      |  2 +-
 python/ray/tune/suggest/hyperopt.py           |  2 +-
 python/ray/tune/suggest/suggestion.py         |  2 +-
 python/ray/tune/test/trial_runner_test.py     |  6 +++---
 19 files changed, 41 insertions(+), 28 deletions(-)

diff --git a/doc/source/tune-usage.rst b/doc/source/tune-usage.rst
index b3be494a8..a765ca23e 100644
--- a/doc/source/tune-usage.rst
+++ b/doc/source/tune-usage.rst
@@ -72,7 +72,7 @@ dictionary. Tune will convert the dict into an ``ray.tune.Experiment`` object.
                 "beta": tune.grid_search([1, 2]),
             },
             "trial_resources": { "cpu": 1, "gpu": 0 },
-            "repeat": 10,
+            "num_samples": 10,
             "local_dir": "~/ray_results",
             "upload_dir": "s3://your_bucket/path",
             "checkpoint_freq": 10,
@@ -165,7 +165,7 @@ For more information on variant generation, see `basic_variant.py <https://githu
 Sampling Multiple Times
 ~~~~~~~~~~~~~~~~~~~~~~~
 
-By default, each random variable and grid search point is sampled once. To take multiple random samples or repeat grid search runs, add ``repeat: N`` to the experiment config.
+By default, each random variable and grid search point is sampled once. To take multiple random samples, add ``num_samples: N`` to the experiment config. If `grid_search` is provided as an argument, the grid will be repeated `num_samples` of times.
 
 .. code-block:: python
    :emphasize-lines: 12
@@ -181,11 +181,11 @@ By default, each random variable and grid search point is sampled once. To take
                     tune.grid_search([16, 64, 256]),
                 ],
             },
-            "repeat": 10
+            "num_samples": 10
         }
     })
 
-E.g. in the above, ``"repeat": 10`` repeats the 3x3 grid search 10 times, for a total of 90 trials, each with randomly sampled values of ``alpha`` and ``beta``.
+E.g. in the above, ``"num_samples": 10`` repeats the 3x3 grid search 10 times, for a total of 90 trials, each with randomly sampled values of ``alpha`` and ``beta``.
 
 
 Using GPUs (Resource Allocation)
diff --git a/python/ray/rllib/train.py b/python/ray/rllib/train.py
index ac18025a9..b9e7b72ef 100755
--- a/python/ray/rllib/train.py
+++ b/python/ray/rllib/train.py
@@ -91,7 +91,7 @@ def run(args, parser):
                 "stop": args.stop,
                 "config": dict(args.config, env=args.env),
                 "restore": args.restore,
-                "repeat": args.repeat,
+                "num_samples": args.num_samples,
                 "upload_dir": args.upload_dir,
             }
         }
diff --git a/python/ray/rllib/tuned_examples/hyperband-cartpole.yaml b/python/ray/rllib/tuned_examples/hyperband-cartpole.yaml
index 01e475491..0d6ffb25c 100644
--- a/python/ray/rllib/tuned_examples/hyperband-cartpole.yaml
+++ b/python/ray/rllib/tuned_examples/hyperband-cartpole.yaml
@@ -1,7 +1,7 @@
 cartpole-ppo:
     env: CartPole-v0
     run: PPO
-    repeat: 3
+    num_samples: 3
     stop:
         episode_reward_mean: 200
         time_total_s: 180
diff --git a/python/ray/rllib/tuned_examples/regression_tests/regression_test.py b/python/ray/rllib/tuned_examples/regression_tests/regression_test.py
index 681e7832e..ff994b904 100644
--- a/python/ray/rllib/tuned_examples/regression_tests/regression_test.py
+++ b/python/ray/rllib/tuned_examples/regression_tests/regression_test.py
@@ -22,7 +22,7 @@ def _evaulate_config(filename):
     with open(os.path.join(CONFIG_DIR, filename)) as f:
         experiments = yaml.load(f)
         for _, config in experiments.items():
-            config["repeat"] = 3
+            config["num_samples"] = 3
     ray.init()
     trials = tune.run_experiments(experiments)
     results = defaultdict(list)
diff --git a/python/ray/tune/config_parser.py b/python/ray/tune/config_parser.py
index a49fcb2fb..91d1c63e7 100644
--- a/python/ray/tune/config_parser.py
+++ b/python/ray/tune/config_parser.py
@@ -91,7 +91,7 @@ def make_parser(parser_creator=None, **kwargs):
         "unless you specify them here. For RLlib, you probably want to "
         "leave this alone and use RLlib configs to control parallelism.")
     parser.add_argument(
-        "--repeat",
+        "--num-samples",
         default=1,
         type=int,
         help="Number of times to repeat each trial.")
diff --git a/python/ray/tune/examples/async_hyperband_example.py b/python/ray/tune/examples/async_hyperband_example.py
index 13d27d9f8..2c368b4e3 100644
--- a/python/ray/tune/examples/async_hyperband_example.py
+++ b/python/ray/tune/examples/async_hyperband_example.py
@@ -70,7 +70,7 @@ if __name__ == "__main__":
                 "stop": {
                     "training_iteration": 1 if args.smoke_test else 99999
                 },
-                "repeat": 20,
+                "num_samples": 20,
                 "trial_resources": {
                     "cpu": 1,
                     "gpu": 0
diff --git a/python/ray/tune/examples/hyperband_example.py b/python/ray/tune/examples/hyperband_example.py
index c015b2cd6..94f603e82 100755
--- a/python/ray/tune/examples/hyperband_example.py
+++ b/python/ray/tune/examples/hyperband_example.py
@@ -64,7 +64,7 @@ if __name__ == "__main__":
     exp = Experiment(
         name="hyperband_test",
         run=MyTrainableClass,
-        repeat=20,
+        num_samples=20,
         stop={"training_iteration": 1 if args.smoke_test else 99999},
         config={
             "width": lambda spec: 10 + int(90 * random.random()),
diff --git a/python/ray/tune/examples/hyperopt_example.py b/python/ray/tune/examples/hyperopt_example.py
index 8f5a5c20b..6d61b1321 100644
--- a/python/ray/tune/examples/hyperopt_example.py
+++ b/python/ray/tune/examples/hyperopt_example.py
@@ -46,7 +46,7 @@ if __name__ == '__main__':
     config = {
         "my_exp": {
             "run": "exp",
-            "repeat": 10 if args.smoke_test else 1000,
+            "num_samples": 10 if args.smoke_test else 1000,
             "stop": {
                 "training_iteration": 100
             },
diff --git a/python/ray/tune/examples/pbt_example.py b/python/ray/tune/examples/pbt_example.py
index f0d3f5266..56557ee58 100755
--- a/python/ray/tune/examples/pbt_example.py
+++ b/python/ray/tune/examples/pbt_example.py
@@ -81,7 +81,7 @@ if __name__ == "__main__":
                 "stop": {
                     "training_iteration": 2 if args.smoke_test else 99999
                 },
-                "repeat": 10,
+                "num_samples": 10,
                 "config": {
                     "factor_1": 4.0,
                     "factor_2": 1.0,
diff --git a/python/ray/tune/examples/pbt_ppo_example.py b/python/ray/tune/examples/pbt_ppo_example.py
index b1842bd54..fb659b007 100755
--- a/python/ray/tune/examples/pbt_ppo_example.py
+++ b/python/ray/tune/examples/pbt_ppo_example.py
@@ -50,7 +50,7 @@ if __name__ == "__main__":
             "pbt_humanoid_test": {
                 "run": "PPO",
                 "env": "Humanoid-v1",
-                "repeat": 8,
+                "num_samples": 8,
                 "config": {
                     "kl_coeff": 1.0,
                     "num_workers": 8,
diff --git a/python/ray/tune/examples/pbt_tune_cifar10_with_keras.py b/python/ray/tune/examples/pbt_tune_cifar10_with_keras.py
index 8eba6f301..11163f722 100755
--- a/python/ray/tune/examples/pbt_tune_cifar10_with_keras.py
+++ b/python/ray/tune/examples/pbt_tune_cifar10_with_keras.py
@@ -195,7 +195,7 @@ if __name__ == "__main__":
             "decay": lambda spec: spec.config.lr / 100.0,
             "dropout": grid_search([0.25, 0.5]),
         },
-        "repeat": 4,
+        "num_samples": 4,
     }
 
     if args.smoke_test:
diff --git a/python/ray/tune/examples/tune_mnist_async_hyperband.py b/python/ray/tune/examples/tune_mnist_async_hyperband.py
index 471bcc42c..0f8bddc5e 100755
--- a/python/ray/tune/examples/tune_mnist_async_hyperband.py
+++ b/python/ray/tune/examples/tune_mnist_async_hyperband.py
@@ -220,7 +220,7 @@ if __name__ == '__main__':
 
     mnist_spec = {
         'run': train,
-        'repeat': 10,
+        'num_samples': 10,
         'stop': {
             'mean_accuracy': 0.99,
             'timesteps_total': 600,
@@ -232,7 +232,7 @@ if __name__ == '__main__':
 
     if args.smoke_test:
         mnist_spec['stop']['training_iteration'] = 2
-        mnist_spec['repeat'] = 1
+        mnist_spec['num_samples'] = 1
 
     ray.init()
 
diff --git a/python/ray/tune/examples/tune_mnist_keras.py b/python/ray/tune/examples/tune_mnist_keras.py
index aa16c7046..32cfb371e 100644
--- a/python/ray/tune/examples/tune_mnist_keras.py
+++ b/python/ray/tune/examples/tune_mnist_keras.py
@@ -184,7 +184,7 @@ if __name__ == '__main__':
                     "timesteps_total": 10 if args.smoke_test else 300
                 },
                 "run": "train_mnist",
-                "repeat": 1 if args.smoke_test else 10,
+                "num_samples": 1 if args.smoke_test else 10,
                 "config": {
                     "lr": lambda spec: np.random.uniform(0.001, 0.1),
                     "momentum": lambda spec: np.random.uniform(0.1, 0.9),
diff --git a/python/ray/tune/examples/tune_mnist_ray_hyperband.py b/python/ray/tune/examples/tune_mnist_ray_hyperband.py
index 4eee34195..29939ff24 100755
--- a/python/ray/tune/examples/tune_mnist_ray_hyperband.py
+++ b/python/ray/tune/examples/tune_mnist_ray_hyperband.py
@@ -224,12 +224,12 @@ if __name__ == '__main__':
             'learning_rate': lambda spec: 10**np.random.uniform(-5, -3),
             'activation': grid_search(['relu', 'elu', 'tanh']),
         },
-        "repeat": 10,
+        "num_samples": 10,
     }
 
     if args.smoke_test:
         mnist_spec['stop']['training_iteration'] = 2
-        mnist_spec['repeat'] = 2
+        mnist_spec['num_samples'] = 2
 
     ray.init()
     hyperband = HyperBandScheduler(
diff --git a/python/ray/tune/experiment.py b/python/ray/tune/experiment.py
index 45352702e..80767515d 100644
--- a/python/ray/tune/experiment.py
+++ b/python/ray/tune/experiment.py
@@ -31,7 +31,12 @@ class Experiment(object):
             e.g. ``{"cpu": 64, "gpu": 8}``. Note that GPUs will not be
             assigned unless you specify them here. Defaults to 1 CPU and 0
             GPUs in ``Trainable.default_resource_request()``.
-        repeat (int): Number of times to repeat each trial. Defaults to 1.
+        repeat (int): Deprecated and will be removed in future versions of
+            Ray. Use `num_samples` instead.
+        num_samples (int): Number of times to sample from the
+            hyperparameter space. Defaults to 1. If `grid_search` is
+            provided as an argument, the grid will be repeated
+            `num_samples` of times.
         local_dir (str): Local dir to save training results to.
             Defaults to ``~/ray_results``.
         upload_dir (str): Optional URI to sync training results
@@ -58,7 +63,7 @@ class Experiment(object):
         >>>         "cpu": 1,
         >>>         "gpu": 0
         >>>     },
-        >>>     repeat=10,
+        >>>     num_samples=10,
         >>>     local_dir="~/ray_results",
         >>>     upload_dir="s3://your_bucket/path",
         >>>     checkpoint_freq=10,
@@ -73,6 +78,7 @@ class Experiment(object):
                  config=None,
                  trial_resources=None,
                  repeat=1,
+                 num_samples=1,
                  local_dir=None,
                  upload_dir="",
                  checkpoint_freq=0,
@@ -83,7 +89,7 @@ class Experiment(object):
             "stop": stop or {},
             "config": config or {},
             "trial_resources": trial_resources,
-            "repeat": repeat,
+            "num_samples": num_samples,
             "local_dir": local_dir or DEFAULT_RESULTS_DIR,
             "upload_dir": upload_dir,
             "checkpoint_freq": checkpoint_freq,
@@ -105,6 +111,13 @@ class Experiment(object):
         if "run" not in spec:
             raise TuneError("No trainable specified!")
 
+        if "repeat" in spec:
+            raise DeprecationWarning("The parameter `repeat` is deprecated; \
+                converting to `num_samples`. `repeat` will be removed in \
+                future versions of Ray.")
+            spec["num_samples"] = spec["repeat"]
+            del spec["repeat"]
+
         # Special case the `env` param for RLlib by automatically
         # moving it into the `config` section.
         if "env" in spec:
diff --git a/python/ray/tune/suggest/basic_variant.py b/python/ray/tune/suggest/basic_variant.py
index 4cb652be4..63bdbb7bb 100644
--- a/python/ray/tune/suggest/basic_variant.py
+++ b/python/ray/tune/suggest/basic_variant.py
@@ -65,7 +65,7 @@ class BasicVariantGenerator(SearchAlgorithm):
 
         if "run" not in unresolved_spec:
             raise TuneError("Must specify `run` in {}".format(unresolved_spec))
-        for _ in range(unresolved_spec.get("repeat", 1)):
+        for _ in range(unresolved_spec.get("num_samples", 1)):
             for resolved_vars, spec in generate_variants(unresolved_spec):
                 experiment_tag = str(self._counter)
                 if resolved_vars:
diff --git a/python/ray/tune/suggest/hyperopt.py b/python/ray/tune/suggest/hyperopt.py
index 26291c1ef..45fe9753e 100644
--- a/python/ray/tune/suggest/hyperopt.py
+++ b/python/ray/tune/suggest/hyperopt.py
@@ -39,7 +39,7 @@ class HyperOptSearch(SuggestionAlgorithm):
         >>> config = {
         >>>     "my_exp": {
         >>>         "run": "exp",
-        >>>         "repeat": 10 if args.smoke_test else 1000,
+        >>>         "num_samples": 10 if args.smoke_test else 1000,
         >>>         "stop": {
         >>>             "training_iteration": 100
         >>>         },
diff --git a/python/ray/tune/suggest/suggestion.py b/python/ray/tune/suggest/suggestion.py
index d2c682b22..31ab844cd 100644
--- a/python/ray/tune/suggest/suggestion.py
+++ b/python/ray/tune/suggest/suggestion.py
@@ -81,7 +81,7 @@ class SuggestionAlgorithm(SearchAlgorithm):
         """
         if "run" not in experiment_spec:
             raise TuneError("Must specify `run` in {}".format(experiment_spec))
-        for _ in range(experiment_spec.get("repeat", 1)):
+        for _ in range(experiment_spec.get("num_samples", 1)):
             trial_id = Trial.generate_id()
             while True:
                 suggested_config = self._suggest(trial_id)
diff --git a/python/ray/tune/test/trial_runner_test.py b/python/ray/tune/test/trial_runner_test.py
index e7814a20c..d2930a0b3 100644
--- a/python/ray/tune/test/trial_runner_test.py
+++ b/python/ray/tune/test/trial_runner_test.py
@@ -522,7 +522,7 @@ class VariantGeneratorTest(unittest.TestCase):
     def testParseToTrials(self):
         trials = self.generate_trials({
             "run": "PPO",
-            "repeat": 2,
+            "num_samples": 2,
             "max_failures": 5,
             "config": {
                 "env": "Pong-v0",
@@ -651,7 +651,7 @@ class VariantGeneratorTest(unittest.TestCase):
         """Checks that next_trials() supports throttling."""
         experiment_spec = {
             "run": "PPO",
-            "repeat": 6,
+            "num_samples": 6,
         }
         experiments = [Experiment.from_json("test", experiment_spec)]
 
@@ -1133,7 +1133,7 @@ class TrialRunnerTest(unittest.TestCase):
         ray.init(num_cpus=4, num_gpus=2)
         experiment_spec = {
             "run": "__fake",
-            "repeat": 3,
+            "num_samples": 3,
             "stop": {
                 "training_iteration": 1
             }