[tune] Rename 'repeat' to 'num_samples' (#2698)

Deprecates the `repeat` argument and introduces `num_samples`. Also updates docs accordingly.

doc/source/tune-usage.rst

@@ -72,7 +72,7 @@ dictionary. Tune will convert the dict into an ``ray.tune.Experiment`` object.
                 "beta": tune.grid_search([1, 2]),
             },
             "trial_resources": { "cpu": 1, "gpu": 0 },
-            "repeat": 10,
+            "num_samples": 10,
             "local_dir": "~/ray_results",
             "upload_dir": "s3://your_bucket/path",
             "checkpoint_freq": 10,
@@ -165,7 +165,7 @@ For more information on variant generation, see `basic_variant.py <https://githu
 Sampling Multiple Times
 ~~~~~~~~~~~~~~~~~~~~~~~
 
-By default, each random variable and grid search point is sampled once. To take multiple random samples or repeat grid search runs, add ``repeat: N`` to the experiment config.
+By default, each random variable and grid search point is sampled once. To take multiple random samples, add ``num_samples: N`` to the experiment config. If `grid_search` is provided as an argument, the grid will be repeated `num_samples` of times.
 
 .. code-block:: python
    :emphasize-lines: 12
@@ -181,11 +181,11 @@ By default, each random variable and grid search point is sampled once. To take
                     tune.grid_search([16, 64, 256]),
                 ],
             },
-            "repeat": 10
+            "num_samples": 10
         }
     })
 
-E.g. in the above, ``"repeat": 10`` repeats the 3x3 grid search 10 times, for a total of 90 trials, each with randomly sampled values of ``alpha`` and ``beta``.
+E.g. in the above, ``"num_samples": 10`` repeats the 3x3 grid search 10 times, for a total of 90 trials, each with randomly sampled values of ``alpha`` and ``beta``.
 
 
 Using GPUs (Resource Allocation)

python/ray/rllib/train.py

@@ -91,7 +91,7 @@ def run(args, parser):
                 "stop": args.stop,
                 "config": dict(args.config, env=args.env),
                 "restore": args.restore,
-                "repeat": args.repeat,
+                "num_samples": args.num_samples,
                 "upload_dir": args.upload_dir,
             }
         }

python/ray/rllib/tuned_examples/hyperband-cartpole.yaml

@@ -1,7 +1,7 @@
 cartpole-ppo:
     env: CartPole-v0
     run: PPO
-    repeat: 3
+    num_samples: 3
     stop:
         episode_reward_mean: 200
         time_total_s: 180

python/ray/rllib/tuned_examples/regression_tests/regression_test.py

@@ -22,7 +22,7 @@ def _evaulate_config(filename):
     with open(os.path.join(CONFIG_DIR, filename)) as f:
         experiments = yaml.load(f)
         for _, config in experiments.items():
-            config["repeat"] = 3
+            config["num_samples"] = 3
     ray.init()
     trials = tune.run_experiments(experiments)
     results = defaultdict(list)

python/ray/tune/config_parser.py

@@ -91,7 +91,7 @@ def make_parser(parser_creator=None, **kwargs):
         "unless you specify them here. For RLlib, you probably want to "
         "leave this alone and use RLlib configs to control parallelism.")
     parser.add_argument(
-        "--repeat",
+        "--num-samples",
         default=1,
         type=int,
         help="Number of times to repeat each trial.")

python/ray/tune/examples/async_hyperband_example.py

@@ -70,7 +70,7 @@ if __name__ == "__main__":
                 "stop": {
                     "training_iteration": 1 if args.smoke_test else 99999
                 },
-                "repeat": 20,
+                "num_samples": 20,
                 "trial_resources": {
                     "cpu": 1,
                     "gpu": 0

python/ray/tune/examples/hyperband_example.py

@@ -64,7 +64,7 @@ if __name__ == "__main__":
     exp = Experiment(
         name="hyperband_test",
         run=MyTrainableClass,
-        repeat=20,
+        num_samples=20,
         stop={"training_iteration": 1 if args.smoke_test else 99999},
         config={
             "width": lambda spec: 10 + int(90 * random.random()),

python/ray/tune/examples/hyperopt_example.py

@@ -46,7 +46,7 @@ if __name__ == '__main__':
     config = {
         "my_exp": {
             "run": "exp",
-            "repeat": 10 if args.smoke_test else 1000,
+            "num_samples": 10 if args.smoke_test else 1000,
             "stop": {
                 "training_iteration": 100
             },

python/ray/tune/examples/pbt_example.py

@@ -81,7 +81,7 @@ if __name__ == "__main__":
                 "stop": {
                     "training_iteration": 2 if args.smoke_test else 99999
                 },
-                "repeat": 10,
+                "num_samples": 10,
                 "config": {
                     "factor_1": 4.0,
                     "factor_2": 1.0,

python/ray/tune/examples/pbt_ppo_example.py

@@ -50,7 +50,7 @@ if __name__ == "__main__":
             "pbt_humanoid_test": {
                 "run": "PPO",
                 "env": "Humanoid-v1",
-                "repeat": 8,
+                "num_samples": 8,
                 "config": {
                     "kl_coeff": 1.0,
                     "num_workers": 8,

python/ray/tune/examples/pbt_tune_cifar10_with_keras.py

@@ -195,7 +195,7 @@ if __name__ == "__main__":
             "decay": lambda spec: spec.config.lr / 100.0,
             "dropout": grid_search([0.25, 0.5]),
         },
-        "repeat": 4,
+        "num_samples": 4,
     }
 
     if args.smoke_test:

python/ray/tune/examples/tune_mnist_async_hyperband.py

@@ -220,7 +220,7 @@ if __name__ == '__main__':
 
     mnist_spec = {
         'run': train,
-        'repeat': 10,
+        'num_samples': 10,
         'stop': {
             'mean_accuracy': 0.99,
             'timesteps_total': 600,
@@ -232,7 +232,7 @@ if __name__ == '__main__':
 
     if args.smoke_test:
         mnist_spec['stop']['training_iteration'] = 2
-        mnist_spec['repeat'] = 1
+        mnist_spec['num_samples'] = 1
 
     ray.init()
 

python/ray/tune/examples/tune_mnist_keras.py

@@ -184,7 +184,7 @@ if __name__ == '__main__':
                     "timesteps_total": 10 if args.smoke_test else 300
                 },
                 "run": "train_mnist",
-                "repeat": 1 if args.smoke_test else 10,
+                "num_samples": 1 if args.smoke_test else 10,
                 "config": {
                     "lr": lambda spec: np.random.uniform(0.001, 0.1),
                     "momentum": lambda spec: np.random.uniform(0.1, 0.9),

python/ray/tune/examples/tune_mnist_ray_hyperband.py

@@ -224,12 +224,12 @@ if __name__ == '__main__':
             'learning_rate': lambda spec: 10**np.random.uniform(-5, -3),
             'activation': grid_search(['relu', 'elu', 'tanh']),
         },
-        "repeat": 10,
+        "num_samples": 10,
     }
 
     if args.smoke_test:
         mnist_spec['stop']['training_iteration'] = 2
-        mnist_spec['repeat'] = 2
+        mnist_spec['num_samples'] = 2
 
     ray.init()
     hyperband = HyperBandScheduler(

python/ray/tune/experiment.py

@@ -31,7 +31,12 @@ class Experiment(object):
             e.g. ``{"cpu": 64, "gpu": 8}``. Note that GPUs will not be
             assigned unless you specify them here. Defaults to 1 CPU and 0
             GPUs in ``Trainable.default_resource_request()``.
-        repeat (int): Number of times to repeat each trial. Defaults to 1.
+        repeat (int): Deprecated and will be removed in future versions of
+            Ray. Use `num_samples` instead.
+        num_samples (int): Number of times to sample from the
+            hyperparameter space. Defaults to 1. If `grid_search` is
+            provided as an argument, the grid will be repeated
+            `num_samples` of times.
         local_dir (str): Local dir to save training results to.
             Defaults to ``~/ray_results``.
         upload_dir (str): Optional URI to sync training results
@@ -58,7 +63,7 @@ class Experiment(object):
         >>>         "cpu": 1,
         >>>         "gpu": 0
         >>>     },
-        >>>     repeat=10,
+        >>>     num_samples=10,
         >>>     local_dir="~/ray_results",
         >>>     upload_dir="s3://your_bucket/path",
         >>>     checkpoint_freq=10,
@@ -73,6 +78,7 @@ class Experiment(object):
                  config=None,
                  trial_resources=None,
                  repeat=1,
+                 num_samples=1,
                  local_dir=None,
                  upload_dir="",
                  checkpoint_freq=0,
@@ -83,7 +89,7 @@ class Experiment(object):
             "stop": stop or {},
             "config": config or {},
             "trial_resources": trial_resources,
-            "repeat": repeat,
+            "num_samples": num_samples,
             "local_dir": local_dir or DEFAULT_RESULTS_DIR,
             "upload_dir": upload_dir,
             "checkpoint_freq": checkpoint_freq,
@@ -105,6 +111,13 @@ class Experiment(object):
         if "run" not in spec:
             raise TuneError("No trainable specified!")
 
+        if "repeat" in spec:
+            raise DeprecationWarning("The parameter `repeat` is deprecated; \
+                converting to `num_samples`. `repeat` will be removed in \
+                future versions of Ray.")
+            spec["num_samples"] = spec["repeat"]
+            del spec["repeat"]
+
         # Special case the `env` param for RLlib by automatically
         # moving it into the `config` section.
         if "env" in spec:

python/ray/tune/suggest/basic_variant.py

@@ -65,7 +65,7 @@ class BasicVariantGenerator(SearchAlgorithm):
 
         if "run" not in unresolved_spec:
             raise TuneError("Must specify `run` in {}".format(unresolved_spec))
-        for _ in range(unresolved_spec.get("repeat", 1)):
+        for _ in range(unresolved_spec.get("num_samples", 1)):
             for resolved_vars, spec in generate_variants(unresolved_spec):
                 experiment_tag = str(self._counter)
                 if resolved_vars:

python/ray/tune/suggest/hyperopt.py

@@ -39,7 +39,7 @@ class HyperOptSearch(SuggestionAlgorithm):
         >>> config = {
         >>>     "my_exp": {
         >>>         "run": "exp",
-        >>>         "repeat": 10 if args.smoke_test else 1000,
+        >>>         "num_samples": 10 if args.smoke_test else 1000,
         >>>         "stop": {
         >>>             "training_iteration": 100
         >>>         },

python/ray/tune/suggest/suggestion.py

@@ -81,7 +81,7 @@ class SuggestionAlgorithm(SearchAlgorithm):
         """
         if "run" not in experiment_spec:
             raise TuneError("Must specify `run` in {}".format(experiment_spec))
-        for _ in range(experiment_spec.get("repeat", 1)):
+        for _ in range(experiment_spec.get("num_samples", 1)):
             trial_id = Trial.generate_id()
             while True:
                 suggested_config = self._suggest(trial_id)

python/ray/tune/test/trial_runner_test.py

@@ -522,7 +522,7 @@ class VariantGeneratorTest(unittest.TestCase):
     def testParseToTrials(self):
         trials = self.generate_trials({
             "run": "PPO",
-            "repeat": 2,
+            "num_samples": 2,
             "max_failures": 5,
             "config": {
                 "env": "Pong-v0",
@@ -651,7 +651,7 @@ class VariantGeneratorTest(unittest.TestCase):
         """Checks that next_trials() supports throttling."""
         experiment_spec = {
             "run": "PPO",
-            "repeat": 6,
+            "num_samples": 6,
         }
         experiments = [Experiment.from_json("test", experiment_spec)]
 
@@ -1133,7 +1133,7 @@ class TrialRunnerTest(unittest.TestCase):
         ray.init(num_cpus=4, num_gpus=2)
         experiment_spec = {
             "run": "__fake",
-            "repeat": 3,
+            "num_samples": 3,
             "stop": {
                 "training_iteration": 1
             }