ray/doc/source/tune/examples/pbt_ppo_example.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "3b05af3b",
   "metadata": {},
   "source": [
    "(tune-rllib-example)=\n",
    "\n",
    "# Using RLlib with Tune\n",
    "\n",
    "```{image} /rllib/images/rllib-logo.png\n",
    ":align: center\n",
    ":alt: RLlib Logo\n",
    ":height: 120px\n",
    ":target: https://docs.ray.io\n",
    "```\n",
    "\n",
    "```{contents}\n",
    ":backlinks: none\n",
    ":local: true\n",
    "```\n",
    "\n",
    "## Example\n",
    "\n",
    "Example of using PBT with RLlib.\n",
    "\n",
    "Note that this requires a cluster with at least 8 GPUs in order for all trials\n",
    "to run concurrently, otherwise PBT will round-robin train the trials which\n",
    "is less efficient (or you can set {\"gpu\": 0} to use CPUs for SGD instead).\n",
    "\n",
    "Note that Tune in general does not need 8 GPUs, and this is just a more\n",
    "computationally demanding example."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "19e3c389",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2022-07-22 16:45:04,893\tWARNING deprecation.py:48 -- DeprecationWarning: `ray.rllib.execution.buffers` has been deprecated. Use `ray.rllib.utils.replay_buffers` instead. This will raise an error in the future!\n",
      "2022-07-22 16:45:08,004\tINFO services.py:1483 -- View the Ray dashboard at \u001b[1m\u001b[32mhttp://127.0.0.1:8274\u001b[39m\u001b[22m\n",
      "/Users/kai/coding/ray/python/ray/tune/tune.py:487: UserWarning: Consider boosting PBT performance by enabling `reuse_actors` as well as implementing `reset_config` for Trainable.\n",
      "  \"Consider boosting PBT performance by enabling `reuse_actors` as \"\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "== Status ==<br>Current time: 2022-07-22 16:45:36 (running for 00:00:25.48)<br>Memory usage on this node: 11.3/16.0 GiB<br>PopulationBasedTraining: 0 checkpoints, 0 perturbs<br>Resources requested: 0/16 CPUs, 0/0 GPUs, 0.0/5.53 GiB heap, 0.0/2.0 GiB objects<br>Result logdir: /Users/kai/ray_results/PPO<br>Number of trials: 1/1 (1 ERROR)<br><table>\n",
       "<thead>\n",
       "<tr><th>Trial name                 </th><th>status  </th><th>loc  </th><th style=\"text-align: right;\">  num_sgd_iter</th><th style=\"text-align: right;\">  sgd_minibatch_size</th><th style=\"text-align: right;\">  train_batch_size</th></tr>\n",
       "</thead>\n",
       "<tbody>\n",
       "<tr><td>PPO_Humanoid-v1_45196_00000</td><td>ERROR   </td><td>     </td><td style=\"text-align: right;\">            30</td><td style=\"text-align: right;\">                 128</td><td style=\"text-align: right;\">             20000</td></tr>\n",
       "</tbody>\n",
       "</table><br>Number of errored trials: 1<br><table>\n",
       "<thead>\n",
       "<tr><th>Trial name                 </th><th style=\"text-align: right;\">  # failures</th><th>error file                                                                                                                                          </th></tr>\n",
       "</thead>\n",
       "<tbody>\n",
       "<tr><td>PPO_Humanoid-v1_45196_00000</td><td style=\"text-align: right;\">           1</td><td>/Users/kai/ray_results/PPO/PPO_Humanoid-v1_45196_00000_0_num_sgd_iter=30,sgd_minibatch_size=128,train_batch_size=20000_2022-07-22_16-45-11/error.txt</td></tr>\n",
       "</tbody>\n",
       "</table><br>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2022-07-22 16:45:11,206\tWARNING trial_runner.py:1517 -- You are trying to access _search_alg interface of TrialRunner in TrialScheduler, which is being restricted. If you believe it is reasonable for your scheduler to access this TrialRunner API, please reach out to Ray team on GitHub. A more strict API access pattern would be enforced starting 1.12s.0\n",
      "2022-07-22 16:45:11,640\tINFO plugin_schema_manager.py:52 -- Loading the default runtime env schemas: ['/Users/kai/coding/ray/python/ray/_private/runtime_env/../../runtime_env/schemas/working_dir_schema.json', '/Users/kai/coding/ray/python/ray/_private/runtime_env/../../runtime_env/schemas/pip_schema.json'].\n",
      "\u001b[2m\u001b[36m(PPO pid=53765)\u001b[0m 2022-07-22 16:45:21,449\tINFO algorithm.py:1855 -- Your framework setting is 'tf', meaning you are using static-graph mode. Set framework='tf2' to enable eager execution with tf2.x. You may also then want to set eager_tracing=True in order to reach similar execution speed as with static-graph mode.\n",
      "\u001b[2m\u001b[36m(PPO pid=53765)\u001b[0m 2022-07-22 16:45:21,450\tWARNING ppo.py:359 -- `train_batch_size` (20000) cannot be achieved with your other settings (num_workers=8 num_envs_per_worker=1 rollout_fragment_length=200)! Auto-adjusting `rollout_fragment_length` to 2500.\n",
      "\u001b[2m\u001b[36m(PPO pid=53765)\u001b[0m 2022-07-22 16:45:21,450\tINFO ppo.py:379 -- In multi-agent mode, policies will be optimized sequentially by the multi-GPU optimizer. Consider setting simple_optimizer=True if this doesn't work for you.\n",
      "\u001b[2m\u001b[36m(PPO pid=53765)\u001b[0m 2022-07-22 16:45:21,450\tINFO algorithm.py:343 -- Current log_level is WARN. For more information, set 'log_level': 'INFO' / 'DEBUG' or use the -v and -vv flags.\n",
      "\u001b[2m\u001b[36m(PPO pid=53765)\u001b[0m 2022-07-22 16:45:36,608\tERROR worker.py:754 -- Exception raised in creation task: The actor died because of an error raised in its creation task, \u001b[36mray::PPO.__init__()\u001b[39m (pid=53765, ip=127.0.0.1, repr=PPO)\n",
      "\u001b[2m\u001b[36m(PPO pid=53765)\u001b[0m   File \"/Users/kai/coding/ray/python/ray/rllib/evaluation/worker_set.py\", line 127, in __init__\n",
      "\u001b[2m\u001b[36m(PPO pid=53765)\u001b[0m     validate=trainer_config.get(\"validate_workers_after_construction\"),\n",
      "\u001b[2m\u001b[36m(PPO pid=53765)\u001b[0m   File \"/Users/kai/coding/ray/python/ray/rllib/evaluation/worker_set.py\", line 269, in add_workers\n",
      "\u001b[2m\u001b[36m(PPO pid=53765)\u001b[0m     self.foreach_worker(lambda w: w.assert_healthy())\n",
      "\u001b[2m\u001b[36m(PPO pid=53765)\u001b[0m   File \"/Users/kai/coding/ray/python/ray/rllib/evaluation/worker_set.py\", line 391, in foreach_worker\n",
      "\u001b[2m\u001b[36m(PPO pid=53765)\u001b[0m     remote_results = ray.get([w.apply.remote(func) for w in self.remote_workers()])\n",
      "\u001b[2m\u001b[36m(PPO pid=53765)\u001b[0m ray.exceptions.RayActorError: The actor died because of an error raised in its creation task, \u001b[36mray::RolloutWorker.__init__()\u001b[39m (pid=53786, ip=127.0.0.1, repr=<ray.rllib.evaluation.rollout_worker.RolloutWorker object at 0x1193ddd10>)\n",
      "\u001b[2m\u001b[36m(PPO pid=53765)\u001b[0m KeyError: 'Humanoid-v1'\n",
      "\u001b[2m\u001b[36m(PPO pid=53765)\u001b[0m \n",
      "\u001b[2m\u001b[36m(PPO pid=53765)\u001b[0m During handling of the above exception, another exception occurred:\n",
      "\u001b[2m\u001b[36m(PPO pid=53765)\u001b[0m \n",
      "\u001b[2m\u001b[36m(PPO pid=53765)\u001b[0m \u001b[36mray::RolloutWorker.__init__()\u001b[39m (pid=53786, ip=127.0.0.1, repr=<ray.rllib.evaluation.rollout_worker.RolloutWorker object at 0x1193ddd10>)\n",
      "\u001b[2m\u001b[36m(PPO pid=53765)\u001b[0m   File \"/Users/kai/coding/ray/python/ray/rllib/env/utils.py\", line 50, in _gym_env_creator\n",
      "\u001b[2m\u001b[36m(PPO pid=53765)\u001b[0m     return gym.make(env_descriptor, **env_context)\n",
      "\u001b[2m\u001b[36m(PPO pid=53765)\u001b[0m   File \"/Users/kai/.pyenv/versions/3.7.7/lib/python3.7/site-packages/gym/envs/registration.py\", line 235, in make\n",
      "\u001b[2m\u001b[36m(PPO pid=53765)\u001b[0m     return registry.make(id, **kwargs)\n",
      "\u001b[2m\u001b[36m(PPO pid=53765)\u001b[0m   File \"/Users/kai/.pyenv/versions/3.7.7/lib/python3.7/site-packages/gym/envs/registration.py\", line 128, in make\n",
      "\u001b[2m\u001b[36m(PPO pid=53765)\u001b[0m     spec = self.spec(path)\n",
      "\u001b[2m\u001b[36m(PPO pid=53765)\u001b[0m   File \"/Users/kai/.pyenv/versions/3.7.7/lib/python3.7/site-packages/gym/envs/registration.py\", line 187, in spec\n",
      "\u001b[2m\u001b[36m(PPO pid=53765)\u001b[0m     id, matching_envs\n",
      "\u001b[2m\u001b[36m(PPO pid=53765)\u001b[0m gym.error.DeprecatedEnv: Env Humanoid-v1 not found (valid versions include ['Humanoid-v2', 'Humanoid-v3'])\n",
      "\u001b[2m\u001b[36m(PPO pid=53765)\u001b[0m \n",
      "\u001b[2m\u001b[36m(PPO pid=53765)\u001b[0m During handling of the above exception, another exception occurred:\n",
      "\u001b[2m\u001b[36m(PPO pid=53765)\u001b[0m \n",
      "\u001b[2m\u001b[36m(PPO pid=53765)\u001b[0m \u001b[36mray::RolloutWorker.__init__()\u001b[39m (pid=53786, ip=127.0.0.1, repr=<ray.rllib.evaluation.rollout_worker.RolloutWorker object at 0x1193ddd10>)\n",
      "\u001b[2m\u001b[36m(PPO pid=53765)\u001b[0m   File \"/Users/kai/coding/ray/python/ray/rllib/evaluation/rollout_worker.py\", line 490, in __init__\n",
      "\u001b[2m\u001b[36m(PPO pid=53765)\u001b[0m     self.env = env_creator(copy.deepcopy(self.env_context))\n",
      "\u001b[2m\u001b[36m(PPO pid=53765)\u001b[0m   File \"/Users/kai/coding/ray/python/ray/rllib/env/utils.py\", line 52, in _gym_env_creator\n",
      "\u001b[2m\u001b[36m(PPO pid=53765)\u001b[0m     raise EnvError(ERR_MSG_INVALID_ENV_DESCRIPTOR.format(env_descriptor))\n",
      "\u001b[2m\u001b[36m(PPO pid=53765)\u001b[0m ray.rllib.utils.error.EnvError: The env string you provided ('Humanoid-v1') is:\n",
      "\u001b[2m\u001b[36m(PPO pid=53765)\u001b[0m a) Not a supported/installed environment.\n",
      "\u001b[2m\u001b[36m(PPO pid=53765)\u001b[0m b) Not a tune-registered environment creator.\n",
      "\u001b[2m\u001b[36m(PPO pid=53765)\u001b[0m c) Not a valid env class string.\n",
      "\u001b[2m\u001b[36m(PPO pid=53765)\u001b[0m \n",
      "\u001b[2m\u001b[36m(PPO pid=53765)\u001b[0m Try one of the following:\n",
      "\u001b[2m\u001b[36m(PPO pid=53765)\u001b[0m a) For Atari support: `pip install gym[atari] autorom[accept-rom-license]`.\n",
      "\u001b[2m\u001b[36m(PPO pid=53765)\u001b[0m    For VizDoom support: Install VizDoom\n",
      "\u001b[2m\u001b[36m(PPO pid=53765)\u001b[0m    (https://github.com/mwydmuch/ViZDoom/blob/master/doc/Building.md) and\n",
      "\u001b[2m\u001b[36m(PPO pid=53765)\u001b[0m    `pip install vizdoomgym`.\n",
      "\u001b[2m\u001b[36m(PPO pid=53765)\u001b[0m    For PyBullet support: `pip install pybullet`.\n",
      "\u001b[2m\u001b[36m(PPO pid=53765)\u001b[0m b) To register your custom env, do `from ray import tune;\n",
      "\u001b[2m\u001b[36m(PPO pid=53765)\u001b[0m    tune.register('[name]', lambda cfg: [return env obj from here using cfg])`.\n",
      "\u001b[2m\u001b[36m(PPO pid=53765)\u001b[0m    Then in your config, do `config['env'] = [name]`.\n",
      "\u001b[2m\u001b[36m(PPO pid=53765)\u001b[0m c) Make sure you provide a fully qualified classpath, e.g.:\n",
      "\u001b[2m\u001b[36m(PPO pid=53765)\u001b[0m    `ray.rllib.examples.env.repeat_after_me_env.RepeatAfterMeEnv`\n",
      "\u001b[2m\u001b[36m(PPO pid=53765)\u001b[0m \n",
      "\u001b[2m\u001b[36m(PPO pid=53765)\u001b[0m During handling of the above exception, another exception occurred:\n",
      "\u001b[2m\u001b[36m(PPO pid=53765)\u001b[0m \n",
      "\u001b[2m\u001b[36m(PPO pid=53765)\u001b[0m \u001b[36mray::PPO.__init__()\u001b[39m (pid=53765, ip=127.0.0.1, repr=PPO)\n",
      "\u001b[2m\u001b[36m(PPO pid=53765)\u001b[0m   File \"/Users/kai/coding/ray/python/ray/rllib/algorithms/algorithm.py\", line 299, in __init__\n",
      "\u001b[2m\u001b[36m(PPO pid=53765)\u001b[0m     super().__init__(config=config, logger_creator=logger_creator, **kwargs)\n",
      "\u001b[2m\u001b[36m(PPO pid=53765)\u001b[0m   File \"/Users/kai/coding/ray/python/ray/tune/trainable/trainable.py\", line 157, in __init__\n",
      "\u001b[2m\u001b[36m(PPO pid=53765)\u001b[0m     self.setup(copy.deepcopy(self.config))\n",
      "\u001b[2m\u001b[36m(PPO pid=53765)\u001b[0m   File \"/Users/kai/coding/ray/python/ray/rllib/algorithms/algorithm.py\", line 432, in setup\n",
      "\u001b[2m\u001b[36m(PPO pid=53765)\u001b[0m     raise e.args[0].args[2]\n",
      "\u001b[2m\u001b[36m(PPO pid=53765)\u001b[0m ray.rllib.utils.error.EnvError: The env string you provided ('Humanoid-v1') is:\n",
      "\u001b[2m\u001b[36m(PPO pid=53765)\u001b[0m a) Not a supported/installed environment.\n",
      "\u001b[2m\u001b[36m(PPO pid=53765)\u001b[0m b) Not a tune-registered environment creator.\n",
      "\u001b[2m\u001b[36m(PPO pid=53765)\u001b[0m c) Not a valid env class string.\n",
      "\u001b[2m\u001b[36m(PPO pid=53765)\u001b[0m \n",
      "\u001b[2m\u001b[36m(PPO pid=53765)\u001b[0m Try one of the following:\n",
      "\u001b[2m\u001b[36m(PPO pid=53765)\u001b[0m a) For Atari support: `pip install gym[atari] autorom[accept-rom-license]`.\n",
      "\u001b[2m\u001b[36m(PPO pid=53765)\u001b[0m    For VizDoom support: Install VizDoom\n",
      "\u001b[2m\u001b[36m(PPO pid=53765)\u001b[0m    (https://github.com/mwydmuch/ViZDoom/blob/master/doc/Building.md) and\n",
      "\u001b[2m\u001b[36m(PPO pid=53765)\u001b[0m    `pip install vizdoomgym`.\n",
      "\u001b[2m\u001b[36m(PPO pid=53765)\u001b[0m    For PyBullet support: `pip install pybullet`.\n",
      "\u001b[2m\u001b[36m(PPO pid=53765)\u001b[0m b) To register your custom env, do `from ray import tune;\n",
      "\u001b[2m\u001b[36m(PPO pid=53765)\u001b[0m    tune.register('[name]', lambda cfg: [return env obj from here using cfg])`.\n",
      "\u001b[2m\u001b[36m(PPO pid=53765)\u001b[0m    Then in your config, do `config['env'] = [name]`.\n",
      "\u001b[2m\u001b[36m(PPO pid=53765)\u001b[0m c) Make sure you provide a fully qualified classpath, e.g.:\n",
      "\u001b[2m\u001b[36m(PPO pid=53765)\u001b[0m    `ray.rllib.examples.env.repeat_after_me_env.RepeatAfterMeEnv`\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=53785)\u001b[0m 2022-07-22 16:45:36,570\tERROR worker.py:754 -- Exception raised in creation task: The actor died because of an error raised in its creation task, \u001b[36mray::RolloutWorker.__init__()\u001b[39m (pid=53785, ip=127.0.0.1, repr=<ray.rllib.evaluation.rollout_worker.RolloutWorker object at 0x1b6a39f90>)\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=53785)\u001b[0m KeyError: 'Humanoid-v1'\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=53785)\u001b[0m \n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=53785)\u001b[0m During handling of the above exception, another exception occurred:\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=53785)\u001b[0m \n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=53785)\u001b[0m \u001b[36mray::RolloutWorker.__init__()\u001b[39m (pid=53785, ip=127.0.0.1, repr=<ray.rllib.evaluation.rollout_worker.RolloutWorker object at 0x1b6a39f90>)\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=53785)\u001b[0m   File \"/Users/kai/coding/ray/python/ray/rllib/env/utils.py\", line 50, in _gym_env_creator\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=53785)\u001b[0m     return gym.make(env_descriptor, **env_context)\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=53785)\u001b[0m   File \"/Users/kai/.pyenv/versions/3.7.7/lib/python3.7/site-packages/gym/envs/registration.py\", line 235, in make\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=53785)\u001b[0m     return registry.make(id, **kwargs)\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=53785)\u001b[0m   File \"/Users/kai/.pyenv/versions/3.7.7/lib/python3.7/site-packages/gym/envs/registration.py\", line 128, in make\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=53785)\u001b[0m     spec = self.spec(path)\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=53785)\u001b[0m   File \"/Users/kai/.pyenv/versions/3.7.7/lib/python3.7/site-packages/gym/envs/registration.py\", line 187, in spec\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=53785)\u001b[0m     id, matching_envs\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=53785)\u001b[0m gym.error.DeprecatedEnv: Env Humanoid-v1 not found (valid versions include ['Humanoid-v2', 'Humanoid-v3'])\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=53785)\u001b[0m \n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=53785)\u001b[0m During handling of the above exception, another exception occurred:\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=53785)\u001b[0m \n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=53785)\u001b[0m \u001b[36mray::RolloutWorker.__init__()\u001b[39m (pid=53785, ip=127.0.0.1, repr=<ray.rllib.evaluation.rollout_worker.RolloutWorker object at 0x1b6a39f90>)\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=53785)\u001b[0m   File \"/Users/kai/coding/ray/python/ray/rllib/evaluation/rollout_worker.py\", line 490, in __init__\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=53785)\u001b[0m     self.env = env_creator(copy.deepcopy(self.env_context))\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=53785)\u001b[0m   File \"/Users/kai/coding/ray/python/ray/rllib/env/utils.py\", line 52, in _gym_env_creator\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=53785)\u001b[0m     raise EnvError(ERR_MSG_INVALID_ENV_DESCRIPTOR.format(env_descriptor))\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=53785)\u001b[0m ray.rllib.utils.error.EnvError: The env string you provided ('Humanoid-v1') is:\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=53785)\u001b[0m a) Not a supported/installed environment.\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=53785)\u001b[0m b) Not a tune-registered environment creator.\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=53785)\u001b[0m c) Not a valid env class string.\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=53785)\u001b[0m \n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=53785)\u001b[0m Try one of the following:\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=53785)\u001b[0m a) For Atari support: `pip install gym[atari] autorom[accept-rom-license]`.\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=53785)\u001b[0m    For VizDoom support: Install VizDoom\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=53785)\u001b[0m    (https://github.com/mwydmuch/ViZDoom/blob/master/doc/Building.md) and\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=53785)\u001b[0m    `pip install vizdoomgym`.\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=53785)\u001b[0m    For PyBullet support: `pip install pybullet`.\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=53785)\u001b[0m b) To register your custom env, do `from ray import tune;\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=53785)\u001b[0m    tune.register('[name]', lambda cfg: [return env obj from here using cfg])`.\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=53785)\u001b[0m    Then in your config, do `config['env'] = [name]`.\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=53785)\u001b[0m c) Make sure you provide a fully qualified classpath, e.g.:\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=53785)\u001b[0m    `ray.rllib.examples.env.repeat_after_me_env.RepeatAfterMeEnv`\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=53787)\u001b[0m 2022-07-22 16:45:36,570\tERROR worker.py:754 -- Exception raised in creation task: The actor died because of an error raised in its creation task, \u001b[36mray::RolloutWorker.__init__()\u001b[39m (pid=53787, ip=127.0.0.1, repr=<ray.rllib.evaluation.rollout_worker.RolloutWorker object at 0x1b29b1990>)\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=53787)\u001b[0m KeyError: 'Humanoid-v1'\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=53787)\u001b[0m \n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=53787)\u001b[0m During handling of the above exception, another exception occurred:\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=53787)\u001b[0m \n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=53787)\u001b[0m \u001b[36mray::RolloutWorker.__init__()\u001b[39m (pid=53787, ip=127.0.0.1, repr=<ray.rllib.evaluation.rollout_worker.RolloutWorker object at 0x1b29b1990>)\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=53787)\u001b[0m   File \"/Users/kai/coding/ray/python/ray/rllib/env/utils.py\", line 50, in _gym_env_creator\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=53787)\u001b[0m     return gym.make(env_descriptor, **env_context)\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=53787)\u001b[0m   File \"/Users/kai/.pyenv/versions/3.7.7/lib/python3.7/site-packages/gym/envs/registration.py\", line 235, in make\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=53787)\u001b[0m     return registry.make(id, **kwargs)\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=53787)\u001b[0m   File \"/Users/kai/.pyenv/versions/3.7.7/lib/python3.7/site-packages/gym/envs/registration.py\", line 128, in make\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=53787)\u001b[0m     spec = self.spec(path)\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=53787)\u001b[0m   File \"/Users/kai/.pyenv/versions/3.7.7/lib/python3.7/site-packages/gym/envs/registration.py\", line 187, in spec\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=53787)\u001b[0m     id, matching_envs\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=53787)\u001b[0m gym.error.DeprecatedEnv: Env Humanoid-v1 not found (valid versions include ['Humanoid-v2', 'Humanoid-v3'])\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=53787)\u001b[0m \n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=53787)\u001b[0m During handling of the above exception, another exception occurred:\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=53787)\u001b[0m \n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=53787)\u001b[0m \u001b[36mray::RolloutWorker.__init__()\u001b[39m (pid=53787, ip=127.0.0.1, repr=<ray.rllib.evaluation.rollout_worker.RolloutWorker object at 0x1b29b1990>)\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=53787)\u001b[0m   File \"/Users/kai/coding/ray/python/ray/rllib/evaluation/rollout_worker.py\", line 490, in __init__\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=53787)\u001b[0m     self.env = env_creator(copy.deepcopy(self.env_context))\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=53787)\u001b[0m   File \"/Users/kai/coding/ray/python/ray/rllib/env/utils.py\", line 52, in _gym_env_creator\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=53787)\u001b[0m     raise EnvError(ERR_MSG_INVALID_ENV_DESCRIPTOR.format(env_descriptor))\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=53787)\u001b[0m ray.rllib.utils.error.EnvError: The env string you provided ('Humanoid-v1') is:\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=53787)\u001b[0m a) Not a supported/installed environment.\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=53787)\u001b[0m b) Not a tune-registered environment creator.\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=53787)\u001b[0m c) Not a valid env class string.\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=53787)\u001b[0m \n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=53787)\u001b[0m Try one of the following:\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=53787)\u001b[0m a) For Atari support: `pip install gym[atari] autorom[accept-rom-license]`.\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=53787)\u001b[0m    For VizDoom support: Install VizDoom\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=53787)\u001b[0m    (https://github.com/mwydmuch/ViZDoom/blob/master/doc/Building.md) and\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=53787)\u001b[0m    `pip install vizdoomgym`.\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=53787)\u001b[0m    For PyBullet support: `pip install pybullet`.\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=53787)\u001b[0m b) To register your custom env, do `from ray import tune;\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=53787)\u001b[0m    tune.register('[name]', lambda cfg: [return env obj from here using cfg])`.\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=53787)\u001b[0m    Then in your config, do `config['env'] = [name]`.\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=53787)\u001b[0m c) Make sure you provide a fully qualified classpath, e.g.:\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=53787)\u001b[0m    `ray.rllib.examples.env.repeat_after_me_env.RepeatAfterMeEnv`\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=53791)\u001b[0m 2022-07-22 16:45:36,570\tERROR worker.py:754 -- Exception raised in creation task: The actor died because of an error raised in its creation task, \u001b[36mray::RolloutWorker.__init__()\u001b[39m (pid=53791, ip=127.0.0.1, repr=<ray.rllib.evaluation.rollout_worker.RolloutWorker object at 0x1bb744490>)\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=53791)\u001b[0m KeyError: 'Humanoid-v1'\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=53791)\u001b[0m \n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=53791)\u001b[0m During handling of the above exception, another exception occurred:\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=53791)\u001b[0m \n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=53791)\u001b[0m \u001b[36mray::RolloutWorker.__init__()\u001b[39m (pid=53791, ip=127.0.0.1, repr=<ray.rllib.evaluation.rollout_worker.RolloutWorker object at 0x1bb744490>)\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=53791)\u001b[0m   File \"/Users/kai/coding/ray/python/ray/rllib/env/utils.py\", line 50, in _gym_env_creator\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=53791)\u001b[0m     return gym.make(env_descriptor, **env_context)\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=53791)\u001b[0m   File \"/Users/kai/.pyenv/versions/3.7.7/lib/python3.7/site-packages/gym/envs/registration.py\", line 235, in make\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=53791)\u001b[0m     return registry.make(id, **kwargs)\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=53791)\u001b[0m   File \"/Users/kai/.pyenv/versions/3.7.7/lib/python3.7/site-packages/gym/envs/registration.py\", line 128, in make\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=53791)\u001b[0m     spec = self.spec(path)\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=53791)\u001b[0m   File \"/Users/kai/.pyenv/versions/3.7.7/lib/python3.7/site-packages/gym/envs/registration.py\", line 187, in spec\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=53791)\u001b[0m     id, matching_envs\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=53791)\u001b[0m gym.error.DeprecatedEnv: Env Humanoid-v1 not found (valid versions include ['Humanoid-v2', 'Humanoid-v3'])\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=53791)\u001b[0m \n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=53791)\u001b[0m During handling of the above exception, another exception occurred:\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=53791)\u001b[0m \n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=53791)\u001b[0m \u001b[36mray::RolloutWorker.__init__()\u001b[39m (pid=53791, ip=127.0.0.1, repr=<ray.rllib.evaluation.rollout_worker.RolloutWorker object at 0x1bb744490>)\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=53791)\u001b[0m   File \"/Users/kai/coding/ray/python/ray/rllib/evaluation/rollout_worker.py\", line 490, in __init__\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=53791)\u001b[0m     self.env = env_creator(copy.deepcopy(self.env_context))\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=53791)\u001b[0m   File \"/Users/kai/coding/ray/python/ray/rllib/env/utils.py\", line 52, in _gym_env_creator\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=53791)\u001b[0m     raise EnvError(ERR_MSG_INVALID_ENV_DESCRIPTOR.format(env_descriptor))\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=53791)\u001b[0m ray.rllib.utils.error.EnvError: The env string you provided ('Humanoid-v1') is:\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=53791)\u001b[0m a) Not a supported/installed environment.\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=53791)\u001b[0m b) Not a tune-registered environment creator.\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=53791)\u001b[0m c) Not a valid env class string.\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=53791)\u001b[0m \n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=53791)\u001b[0m Try one of the following:\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=53791)\u001b[0m a) For Atari support: `pip install gym[atari] autorom[accept-rom-license]`.\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=53791)\u001b[0m    For VizDoom support: Install VizDoom\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=53791)\u001b[0m    (https://github.com/mwydmuch/ViZDoom/blob/master/doc/Building.md) and\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=53791)\u001b[0m    `pip install vizdoomgym`.\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=53791)\u001b[0m    For PyBullet support: `pip install pybullet`.\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=53791)\u001b[0m b) To register your custom env, do `from ray import tune;\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=53791)\u001b[0m    tune.register('[name]', lambda cfg: [return env obj from here using cfg])`.\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=53791)\u001b[0m    Then in your config, do `config['env'] = [name]`.\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=53791)\u001b[0m c) Make sure you provide a fully qualified classpath, e.g.:\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=53791)\u001b[0m    `ray.rllib.examples.env.repeat_after_me_env.RepeatAfterMeEnv`\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=53788)\u001b[0m 2022-07-22 16:45:36,570\tERROR worker.py:754 -- Exception raised in creation task: The actor died because of an error raised in its creation task, \u001b[36mray::RolloutWorker.__init__()\u001b[39m (pid=53788, ip=127.0.0.1, repr=<ray.rllib.evaluation.rollout_worker.RolloutWorker object at 0x1bb4f53d0>)\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=53788)\u001b[0m KeyError: 'Humanoid-v1'\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=53788)\u001b[0m \n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=53788)\u001b[0m During handling of the above exception, another exception occurred:\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=53788)\u001b[0m \n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=53788)\u001b[0m \u001b[36mray::RolloutWorker.__init__()\u001b[39m (pid=53788, ip=127.0.0.1, repr=<ray.rllib.evaluation.rollout_worker.RolloutWorker object at 0x1bb4f53d0>)\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=53788)\u001b[0m   File \"/Users/kai/coding/ray/python/ray/rllib/env/utils.py\", line 50, in _gym_env_creator\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=53788)\u001b[0m     return gym.make(env_descriptor, **env_context)\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=53788)\u001b[0m   File \"/Users/kai/.pyenv/versions/3.7.7/lib/python3.7/site-packages/gym/envs/registration.py\", line 235, in make\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=53788)\u001b[0m     return registry.make(id, **kwargs)\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=53788)\u001b[0m   File \"/Users/kai/.pyenv/versions/3.7.7/lib/python3.7/site-packages/gym/envs/registration.py\", line 128, in make\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=53788)\u001b[0m     spec = self.spec(path)\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=53788)\u001b[0m   File \"/Users/kai/.pyenv/versions/3.7.7/lib/python3.7/site-packages/gym/envs/registration.py\", line 187, in spec\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=53788)\u001b[0m     id, matching_envs\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=53788)\u001b[0m gym.error.DeprecatedEnv: Env Humanoid-v1 not found (valid versions include ['Humanoid-v2', 'Humanoid-v3'])\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=53788)\u001b[0m \n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=53788)\u001b[0m During handling of the above exception, another exception occurred:\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=53788)\u001b[0m \n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=53788)\u001b[0m \u001b[36mray::RolloutWorker.__init__()\u001b[39m (pid=53788, ip=127.0.0.1, repr=<ray.rllib.evaluation.rollout_worker.RolloutWorker object at 0x1bb4f53d0>)\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=53788)\u001b[0m   File \"/Users/kai/coding/ray/python/ray/rllib/evaluation/rollout_worker.py\", line 490, in __init__\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=53788)\u001b[0m     self.env = env_creator(copy.deepcopy(self.env_context))\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=53788)\u001b[0m   File \"/Users/kai/coding/ray/python/ray/rllib/env/utils.py\", line 52, in _gym_env_creator\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=53788)\u001b[0m     raise EnvError(ERR_MSG_INVALID_ENV_DESCRIPTOR.format(env_descriptor))\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=53788)\u001b[0m ray.rllib.utils.error.EnvError: The env string you provided ('Humanoid-v1') is:\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=53788)\u001b[0m a) Not a supported/installed environment.\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=53788)\u001b[0m b) Not a tune-registered environment creator.\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=53788)\u001b[0m c) Not a valid env class string.\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=53788)\u001b[0m \n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=53788)\u001b[0m Try one of the following:\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=53788)\u001b[0m a) For Atari support: `pip install gym[atari] autorom[accept-rom-license]`.\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=53788)\u001b[0m    For VizDoom support: Install VizDoom\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=53788)\u001b[0m    (https://github.com/mwydmuch/ViZDoom/blob/master/doc/Building.md) and\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=53788)\u001b[0m    `pip install vizdoomgym`.\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=53788)\u001b[0m    For PyBullet support: `pip install pybullet`.\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=53788)\u001b[0m b) To register your custom env, do `from ray import tune;\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=53788)\u001b[0m    tune.register('[name]', lambda cfg: [return env obj from here using cfg])`.\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=53788)\u001b[0m    Then in your config, do `config['env'] = [name]`.\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=53788)\u001b[0m c) Make sure you provide a fully qualified classpath, e.g.:\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=53788)\u001b[0m    `ray.rllib.examples.env.repeat_after_me_env.RepeatAfterMeEnv`\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=53792)\u001b[0m 2022-07-22 16:45:36,570\tERROR worker.py:754 -- Exception raised in creation task: The actor died because of an error raised in its creation task, \u001b[36mray::RolloutWorker.__init__()\u001b[39m (pid=53792, ip=127.0.0.1, repr=<ray.rllib.evaluation.rollout_worker.RolloutWorker object at 0x117d54a50>)\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=53792)\u001b[0m KeyError: 'Humanoid-v1'\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=53792)\u001b[0m \n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=53792)\u001b[0m During handling of the above exception, another exception occurred:\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=53792)\u001b[0m \n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=53792)\u001b[0m \u001b[36mray::RolloutWorker.__init__()\u001b[39m (pid=53792, ip=127.0.0.1, repr=<ray.rllib.evaluation.rollout_worker.RolloutWorker object at 0x117d54a50>)\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=53792)\u001b[0m   File \"/Users/kai/coding/ray/python/ray/rllib/env/utils.py\", line 50, in _gym_env_creator\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=53792)\u001b[0m     return gym.make(env_descriptor, **env_context)\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=53792)\u001b[0m   File \"/Users/kai/.pyenv/versions/3.7.7/lib/python3.7/site-packages/gym/envs/registration.py\", line 235, in make\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=53792)\u001b[0m     return registry.make(id, **kwargs)\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=53792)\u001b[0m   File \"/Users/kai/.pyenv/versions/3.7.7/lib/python3.7/site-packages/gym/envs/registration.py\", line 128, in make\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=53792)\u001b[0m     spec = self.spec(path)\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=53792)\u001b[0m   File \"/Users/kai/.pyenv/versions/3.7.7/lib/python3.7/site-packages/gym/envs/registration.py\", line 187, in spec\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=53792)\u001b[0m     id, matching_envs\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=53792)\u001b[0m gym.error.DeprecatedEnv: Env Humanoid-v1 not found (valid versions include ['Humanoid-v2', 'Humanoid-v3'])\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=53792)\u001b[0m \n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=53792)\u001b[0m During handling of the above exception, another exception occurred:\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=53792)\u001b[0m \n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=53792)\u001b[0m \u001b[36mray::RolloutWorker.__init__()\u001b[39m (pid=53792, ip=127.0.0.1, repr=<ray.rllib.evaluation.rollout_worker.RolloutWorker object at 0x117d54a50>)\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=53792)\u001b[0m   File \"/Users/kai/coding/ray/python/ray/rllib/evaluation/rollout_worker.py\", line 490, in __init__\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=53792)\u001b[0m     self.env = env_creator(copy.deepcopy(self.env_context))\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=53792)\u001b[0m   File \"/Users/kai/coding/ray/python/ray/rllib/env/utils.py\", line 52, in _gym_env_creator\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=53792)\u001b[0m     raise EnvError(ERR_MSG_INVALID_ENV_DESCRIPTOR.format(env_descriptor))\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=53792)\u001b[0m ray.rllib.utils.error.EnvError: The env string you provided ('Humanoid-v1') is:\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=53792)\u001b[0m a) Not a supported/installed environment.\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=53792)\u001b[0m b) Not a tune-registered environment creator.\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=53792)\u001b[0m c) Not a valid env class string.\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=53792)\u001b[0m \n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=53792)\u001b[0m Try one of the following:\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=53792)\u001b[0m a) For Atari support: `pip install gym[atari] autorom[accept-rom-license]`.\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=53792)\u001b[0m    For VizDoom support: Install VizDoom\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=53792)\u001b[0m    (https://github.com/mwydmuch/ViZDoom/blob/master/doc/Building.md) and\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=53792)\u001b[0m    `pip install vizdoomgym`.\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=53792)\u001b[0m    For PyBullet support: `pip install pybullet`.\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=53792)\u001b[0m b) To register your custom env, do `from ray import tune;\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=53792)\u001b[0m    tune.register('[name]', lambda cfg: [return env obj from here using cfg])`.\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=53792)\u001b[0m    Then in your config, do `config['env'] = [name]`.\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=53792)\u001b[0m c) Make sure you provide a fully qualified classpath, e.g.:\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=53792)\u001b[0m    `ray.rllib.examples.env.repeat_after_me_env.RepeatAfterMeEnv`\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=53786)\u001b[0m 2022-07-22 16:45:36,570\tERROR worker.py:754 -- Exception raised in creation task: The actor died because of an error raised in its creation task, \u001b[36mray::RolloutWorker.__init__()\u001b[39m (pid=53786, ip=127.0.0.1, repr=<ray.rllib.evaluation.rollout_worker.RolloutWorker object at 0x1193ddd10>)\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=53786)\u001b[0m KeyError: 'Humanoid-v1'\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=53786)\u001b[0m \n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=53786)\u001b[0m During handling of the above exception, another exception occurred:\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=53786)\u001b[0m \n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=53786)\u001b[0m \u001b[36mray::RolloutWorker.__init__()\u001b[39m (pid=53786, ip=127.0.0.1, repr=<ray.rllib.evaluation.rollout_worker.RolloutWorker object at 0x1193ddd10>)\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=53786)\u001b[0m   File \"/Users/kai/coding/ray/python/ray/rllib/env/utils.py\", line 50, in _gym_env_creator\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=53786)\u001b[0m     return gym.make(env_descriptor, **env_context)\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=53786)\u001b[0m   File \"/Users/kai/.pyenv/versions/3.7.7/lib/python3.7/site-packages/gym/envs/registration.py\", line 235, in make\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=53786)\u001b[0m     return registry.make(id, **kwargs)\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=53786)\u001b[0m   File \"/Users/kai/.pyenv/versions/3.7.7/lib/python3.7/site-packages/gym/envs/registration.py\", line 128, in make\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=53786)\u001b[0m     spec = self.spec(path)\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=53786)\u001b[0m   File \"/Users/kai/.pyenv/versions/3.7.7/lib/python3.7/site-packages/gym/envs/registration.py\", line 187, in spec\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=53786)\u001b[0m     id, matching_envs\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=53786)\u001b[0m gym.error.DeprecatedEnv: Env Humanoid-v1 not found (valid versions include ['Humanoid-v2', 'Humanoid-v3'])\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=53786)\u001b[0m \n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=53786)\u001b[0m During handling of the above exception, another exception occurred:\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=53786)\u001b[0m \n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=53786)\u001b[0m \u001b[36mray::RolloutWorker.__init__()\u001b[39m (pid=53786, ip=127.0.0.1, repr=<ray.rllib.evaluation.rollout_worker.RolloutWorker object at 0x1193ddd10>)\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=53786)\u001b[0m   File \"/Users/kai/coding/ray/python/ray/rllib/evaluation/rollout_worker.py\", line 490, in __init__\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=53786)\u001b[0m     self.env = env_creator(copy.deepcopy(self.env_context))\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=53786)\u001b[0m   File \"/Users/kai/coding/ray/python/ray/rllib/env/utils.py\", line 52, in _gym_env_creator\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=53786)\u001b[0m     raise EnvError(ERR_MSG_INVALID_ENV_DESCRIPTOR.format(env_descriptor))\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=53786)\u001b[0m ray.rllib.utils.error.EnvError: The env string you provided ('Humanoid-v1') is:\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=53786)\u001b[0m a) Not a supported/installed environment.\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=53786)\u001b[0m b) Not a tune-registered environment creator.\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=53786)\u001b[0m c) Not a valid env class string.\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=53786)\u001b[0m \n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=53786)\u001b[0m Try one of the following:\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=53786)\u001b[0m a) For Atari support: `pip install gym[atari] autorom[accept-rom-license]`.\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=53786)\u001b[0m    For VizDoom support: Install VizDoom\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=53786)\u001b[0m    (https://github.com/mwydmuch/ViZDoom/blob/master/doc/Building.md) and\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=53786)\u001b[0m    `pip install vizdoomgym`.\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=53786)\u001b[0m    For PyBullet support: `pip install pybullet`.\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=53786)\u001b[0m b) To register your custom env, do `from ray import tune;\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=53786)\u001b[0m    tune.register('[name]', lambda cfg: [return env obj from here using cfg])`.\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=53786)\u001b[0m    Then in your config, do `config['env'] = [name]`.\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=53786)\u001b[0m c) Make sure you provide a fully qualified classpath, e.g.:\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=53786)\u001b[0m    `ray.rllib.examples.env.repeat_after_me_env.RepeatAfterMeEnv`\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=53790)\u001b[0m 2022-07-22 16:45:36,570\tERROR worker.py:754 -- Exception raised in creation task: The actor died because of an error raised in its creation task, \u001b[36mray::RolloutWorker.__init__()\u001b[39m (pid=53790, ip=127.0.0.1, repr=<ray.rllib.evaluation.rollout_worker.RolloutWorker object at 0x1b35ba250>)\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=53790)\u001b[0m KeyError: 'Humanoid-v1'\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=53790)\u001b[0m \n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=53790)\u001b[0m During handling of the above exception, another exception occurred:\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=53790)\u001b[0m \n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=53790)\u001b[0m \u001b[36mray::RolloutWorker.__init__()\u001b[39m (pid=53790, ip=127.0.0.1, repr=<ray.rllib.evaluation.rollout_worker.RolloutWorker object at 0x1b35ba250>)\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=53790)\u001b[0m   File \"/Users/kai/coding/ray/python/ray/rllib/env/utils.py\", line 50, in _gym_env_creator\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=53790)\u001b[0m     return gym.make(env_descriptor, **env_context)\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=53790)\u001b[0m   File \"/Users/kai/.pyenv/versions/3.7.7/lib/python3.7/site-packages/gym/envs/registration.py\", line 235, in make\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=53790)\u001b[0m     return registry.make(id, **kwargs)\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=53790)\u001b[0m   File \"/Users/kai/.pyenv/versions/3.7.7/lib/python3.7/site-packages/gym/envs/registration.py\", line 128, in make\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=53790)\u001b[0m     spec = self.spec(path)\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=53790)\u001b[0m   File \"/Users/kai/.pyenv/versions/3.7.7/lib/python3.7/site-packages/gym/envs/registration.py\", line 187, in spec\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=53790)\u001b[0m     id, matching_envs\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=53790)\u001b[0m gym.error.DeprecatedEnv: Env Humanoid-v1 not found (valid versions include ['Humanoid-v2', 'Humanoid-v3'])\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=53790)\u001b[0m \n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=53790)\u001b[0m During handling of the above exception, another exception occurred:\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=53790)\u001b[0m \n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=53790)\u001b[0m \u001b[36mray::RolloutWorker.__init__()\u001b[39m (pid=53790, ip=127.0.0.1, repr=<ray.rllib.evaluation.rollout_worker.RolloutWorker object at 0x1b35ba250>)\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=53790)\u001b[0m   File \"/Users/kai/coding/ray/python/ray/rllib/evaluation/rollout_worker.py\", line 490, in __init__\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=53790)\u001b[0m     self.env = env_creator(copy.deepcopy(self.env_context))\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=53790)\u001b[0m   File \"/Users/kai/coding/ray/python/ray/rllib/env/utils.py\", line 52, in _gym_env_creator\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=53790)\u001b[0m     raise EnvError(ERR_MSG_INVALID_ENV_DESCRIPTOR.format(env_descriptor))\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=53790)\u001b[0m ray.rllib.utils.error.EnvError: The env string you provided ('Humanoid-v1') is:\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=53790)\u001b[0m a) Not a supported/installed environment.\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=53790)\u001b[0m b) Not a tune-registered environment creator.\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=53790)\u001b[0m c) Not a valid env class string.\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=53790)\u001b[0m \n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=53790)\u001b[0m Try one of the following:\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=53790)\u001b[0m a) For Atari support: `pip install gym[atari] autorom[accept-rom-license]`.\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=53790)\u001b[0m    For VizDoom support: Install VizDoom\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=53790)\u001b[0m    (https://github.com/mwydmuch/ViZDoom/blob/master/doc/Building.md) and\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=53790)\u001b[0m    `pip install vizdoomgym`.\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=53790)\u001b[0m    For PyBullet support: `pip install pybullet`.\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=53790)\u001b[0m b) To register your custom env, do `from ray import tune;\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=53790)\u001b[0m    tune.register('[name]', lambda cfg: [return env obj from here using cfg])`.\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=53790)\u001b[0m    Then in your config, do `config['env'] = [name]`.\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=53790)\u001b[0m c) Make sure you provide a fully qualified classpath, e.g.:\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=53790)\u001b[0m    `ray.rllib.examples.env.repeat_after_me_env.RepeatAfterMeEnv`\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=53789)\u001b[0m 2022-07-22 16:45:36,570\tERROR worker.py:754 -- Exception raised in creation task: The actor died because of an error raised in its creation task, \u001b[36mray::RolloutWorker.__init__()\u001b[39m (pid=53789, ip=127.0.0.1, repr=<ray.rllib.evaluation.rollout_worker.RolloutWorker object at 0x1be518cd0>)\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=53789)\u001b[0m KeyError: 'Humanoid-v1'\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=53789)\u001b[0m \n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=53789)\u001b[0m During handling of the above exception, another exception occurred:\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=53789)\u001b[0m \n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=53789)\u001b[0m \u001b[36mray::RolloutWorker.__init__()\u001b[39m (pid=53789, ip=127.0.0.1, repr=<ray.rllib.evaluation.rollout_worker.RolloutWorker object at 0x1be518cd0>)\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=53789)\u001b[0m   File \"/Users/kai/coding/ray/python/ray/rllib/env/utils.py\", line 50, in _gym_env_creator\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=53789)\u001b[0m     return gym.make(env_descriptor, **env_context)\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=53789)\u001b[0m   File \"/Users/kai/.pyenv/versions/3.7.7/lib/python3.7/site-packages/gym/envs/registration.py\", line 235, in make\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=53789)\u001b[0m     return registry.make(id, **kwargs)\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=53789)\u001b[0m   File \"/Users/kai/.pyenv/versions/3.7.7/lib/python3.7/site-packages/gym/envs/registration.py\", line 128, in make\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=53789)\u001b[0m     spec = self.spec(path)\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=53789)\u001b[0m   File \"/Users/kai/.pyenv/versions/3.7.7/lib/python3.7/site-packages/gym/envs/registration.py\", line 187, in spec\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=53789)\u001b[0m     id, matching_envs\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=53789)\u001b[0m gym.error.DeprecatedEnv: Env Humanoid-v1 not found (valid versions include ['Humanoid-v2', 'Humanoid-v3'])\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=53789)\u001b[0m \n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=53789)\u001b[0m During handling of the above exception, another exception occurred:\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=53789)\u001b[0m \n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=53789)\u001b[0m \u001b[36mray::RolloutWorker.__init__()\u001b[39m (pid=53789, ip=127.0.0.1, repr=<ray.rllib.evaluation.rollout_worker.RolloutWorker object at 0x1be518cd0>)\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=53789)\u001b[0m   File \"/Users/kai/coding/ray/python/ray/rllib/evaluation/rollout_worker.py\", line 490, in __init__\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=53789)\u001b[0m     self.env = env_creator(copy.deepcopy(self.env_context))\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=53789)\u001b[0m   File \"/Users/kai/coding/ray/python/ray/rllib/env/utils.py\", line 52, in _gym_env_creator\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=53789)\u001b[0m     raise EnvError(ERR_MSG_INVALID_ENV_DESCRIPTOR.format(env_descriptor))\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=53789)\u001b[0m ray.rllib.utils.error.EnvError: The env string you provided ('Humanoid-v1') is:\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=53789)\u001b[0m a) Not a supported/installed environment.\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=53789)\u001b[0m b) Not a tune-registered environment creator.\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=53789)\u001b[0m c) Not a valid env class string.\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=53789)\u001b[0m \n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=53789)\u001b[0m Try one of the following:\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=53789)\u001b[0m a) For Atari support: `pip install gym[atari] autorom[accept-rom-license]`.\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=53789)\u001b[0m    For VizDoom support: Install VizDoom\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=53789)\u001b[0m    (https://github.com/mwydmuch/ViZDoom/blob/master/doc/Building.md) and\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=53789)\u001b[0m    `pip install vizdoomgym`.\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=53789)\u001b[0m    For PyBullet support: `pip install pybullet`.\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=53789)\u001b[0m b) To register your custom env, do `from ray import tune;\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=53789)\u001b[0m    tune.register('[name]', lambda cfg: [return env obj from here using cfg])`.\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=53789)\u001b[0m    Then in your config, do `config['env'] = [name]`.\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=53789)\u001b[0m c) Make sure you provide a fully qualified classpath, e.g.:\n",
      "\u001b[2m\u001b[36m(RolloutWorker pid=53789)\u001b[0m    `ray.rllib.examples.env.repeat_after_me_env.RepeatAfterMeEnv`\n",
      "2022-07-22 16:45:36,660\tERROR trial_runner.py:921 -- Trial PPO_Humanoid-v1_45196_00000: Error processing event.\n",
      "ray.tune.error._TuneNoNextExecutorEventError: Traceback (most recent call last):\n",
      "  File \"/Users/kai/coding/ray/python/ray/tune/execution/ray_trial_executor.py\", line 989, in get_next_executor_event\n",
      "    future_result = ray.get(ready_future)\n",
      "  File \"/Users/kai/coding/ray/python/ray/_private/client_mode_hook.py\", line 105, in wrapper\n",
      "    return func(*args, **kwargs)\n",
      "  File \"/Users/kai/coding/ray/python/ray/_private/worker.py\", line 2199, in get\n",
      "    raise value\n",
      "  File \"python/ray/_raylet.pyx\", line 812, in ray._raylet.task_execution_handler\n",
      "  File \"python/ray/_raylet.pyx\", line 623, in ray._raylet.execute_task\n",
      "  File \"python/ray/_raylet.pyx\", line 772, in ray._raylet.execute_task\n",
      "ray.exceptions.RayActorError: The actor died because of an error raised in its creation task, \u001b[36mray::PPO.__init__()\u001b[39m (pid=53765, ip=127.0.0.1, repr=PPO)\n",
      "  File \"/Users/kai/coding/ray/python/ray/rllib/evaluation/worker_set.py\", line 127, in __init__\n",
      "    validate=trainer_config.get(\"validate_workers_after_construction\"),\n",
      "  File \"/Users/kai/coding/ray/python/ray/rllib/evaluation/worker_set.py\", line 269, in add_workers\n",
      "    self.foreach_worker(lambda w: w.assert_healthy())\n",
      "  File \"/Users/kai/coding/ray/python/ray/rllib/evaluation/worker_set.py\", line 391, in foreach_worker\n",
      "    remote_results = ray.get([w.apply.remote(func) for w in self.remote_workers()])\n",
      "ray.exceptions.RayActorError: The actor died because of an error raised in its creation task, \u001b[36mray::RolloutWorker.__init__()\u001b[39m (pid=53786, ip=127.0.0.1, repr=<ray.rllib.evaluation.rollout_worker.RolloutWorker object at 0x1193ddd10>)\n",
      "KeyError: 'Humanoid-v1'\n",
      "\n",
      "During handling of the above exception, another exception occurred:\n",
      "\n",
      "\u001b[36mray::RolloutWorker.__init__()\u001b[39m (pid=53786, ip=127.0.0.1, repr=<ray.rllib.evaluation.rollout_worker.RolloutWorker object at 0x1193ddd10>)\n",
      "  File \"/Users/kai/coding/ray/python/ray/rllib/env/utils.py\", line 50, in _gym_env_creator\n",
      "    return gym.make(env_descriptor, **env_context)\n",
      "  File \"/Users/kai/.pyenv/versions/3.7.7/lib/python3.7/site-packages/gym/envs/registration.py\", line 235, in make\n",
      "    return registry.make(id, **kwargs)\n",
      "  File \"/Users/kai/.pyenv/versions/3.7.7/lib/python3.7/site-packages/gym/envs/registration.py\", line 128, in make\n",
      "    spec = self.spec(path)\n",
      "  File \"/Users/kai/.pyenv/versions/3.7.7/lib/python3.7/site-packages/gym/envs/registration.py\", line 187, in spec\n",
      "    id, matching_envs\n",
      "gym.error.DeprecatedEnv: Env Humanoid-v1 not found (valid versions include ['Humanoid-v2', 'Humanoid-v3'])\n",
      "\n",
      "During handling of the above exception, another exception occurred:\n",
      "\n",
      "\u001b[36mray::RolloutWorker.__init__()\u001b[39m (pid=53786, ip=127.0.0.1, repr=<ray.rllib.evaluation.rollout_worker.RolloutWorker object at 0x1193ddd10>)\n",
      "  File \"/Users/kai/coding/ray/python/ray/rllib/evaluation/rollout_worker.py\", line 490, in __init__\n",
      "    self.env = env_creator(copy.deepcopy(self.env_context))\n",
      "  File \"/Users/kai/coding/ray/python/ray/rllib/env/utils.py\", line 52, in _gym_env_creator\n",
      "    raise EnvError(ERR_MSG_INVALID_ENV_DESCRIPTOR.format(env_descriptor))\n",
      "ray.rllib.utils.error.EnvError: The env string you provided ('Humanoid-v1') is:\n",
      "a) Not a supported/installed environment.\n",
      "b) Not a tune-registered environment creator.\n",
      "c) Not a valid env class string.\n",
      "\n",
      "Try one of the following:\n",
      "a) For Atari support: `pip install gym[atari] autorom[accept-rom-license]`.\n",
      "   For VizDoom support: Install VizDoom\n",
      "   (https://github.com/mwydmuch/ViZDoom/blob/master/doc/Building.md) and\n",
      "   `pip install vizdoomgym`.\n",
      "   For PyBullet support: `pip install pybullet`.\n",
      "b) To register your custom env, do `from ray import tune;\n",
      "   tune.register('[name]', lambda cfg: [return env obj from here using cfg])`.\n",
      "   Then in your config, do `config['env'] = [name]`.\n",
      "c) Make sure you provide a fully qualified classpath, e.g.:\n",
      "   `ray.rllib.examples.env.repeat_after_me_env.RepeatAfterMeEnv`\n",
      "\n",
      "During handling of the above exception, another exception occurred:\n",
      "\n",
      "\u001b[36mray::PPO.__init__()\u001b[39m (pid=53765, ip=127.0.0.1, repr=PPO)\n",
      "  File \"/Users/kai/coding/ray/python/ray/rllib/algorithms/algorithm.py\", line 299, in __init__\n",
      "    super().__init__(config=config, logger_creator=logger_creator, **kwargs)\n",
      "  File \"/Users/kai/coding/ray/python/ray/tune/trainable/trainable.py\", line 157, in __init__\n",
      "    self.setup(copy.deepcopy(self.config))\n",
      "  File \"/Users/kai/coding/ray/python/ray/rllib/algorithms/algorithm.py\", line 432, in setup\n",
      "    raise e.args[0].args[2]\n",
      "ray.rllib.utils.error.EnvError: The env string you provided ('Humanoid-v1') is:\n",
      "a) Not a supported/installed environment.\n",
      "b) Not a tune-registered environment creator.\n",
      "c) Not a valid env class string.\n",
      "\n",
      "Try one of the following:\n",
      "a) For Atari support: `pip install gym[atari] autorom[accept-rom-license]`.\n",
      "   For VizDoom support: Install VizDoom\n",
      "   (https://github.com/mwydmuch/ViZDoom/blob/master/doc/Building.md) and\n",
      "   `pip install vizdoomgym`.\n",
      "   For PyBullet support: `pip install pybullet`.\n",
      "b) To register your custom env, do `from ray import tune;\n",
      "   tune.register('[name]', lambda cfg: [return env obj from here using cfg])`.\n",
      "   Then in your config, do `config['env'] = [name]`.\n",
      "c) Make sure you provide a fully qualified classpath, e.g.:\n",
      "   `ray.rllib.examples.env.repeat_after_me_env.RepeatAfterMeEnv`\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Result for PPO_Humanoid-v1_45196_00000:\n",
      "  trial_id: '45196_00000'\n",
      "  \n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2022-07-22 16:45:36,688\tERROR ray_trial_executor.py:104 -- An exception occurred when trying to stop the Ray actor:Traceback (most recent call last):\n",
      "  File \"/Users/kai/coding/ray/python/ray/tune/execution/ray_trial_executor.py\", line 94, in post_stop_cleanup\n",
      "    ray.get(future, timeout=0)\n",
      "  File \"/Users/kai/coding/ray/python/ray/_private/client_mode_hook.py\", line 105, in wrapper\n",
      "    return func(*args, **kwargs)\n",
      "  File \"/Users/kai/coding/ray/python/ray/_private/worker.py\", line 2199, in get\n",
      "    raise value\n",
      "  File \"python/ray/_raylet.pyx\", line 812, in ray._raylet.task_execution_handler\n",
      "  File \"python/ray/_raylet.pyx\", line 623, in ray._raylet.execute_task\n",
      "  File \"python/ray/_raylet.pyx\", line 772, in ray._raylet.execute_task\n",
      "ray.exceptions.RayActorError: The actor died because of an error raised in its creation task, \u001b[36mray::PPO.__init__()\u001b[39m (pid=53765, ip=127.0.0.1, repr=PPO)\n",
      "  File \"/Users/kai/coding/ray/python/ray/rllib/evaluation/worker_set.py\", line 127, in __init__\n",
      "    validate=trainer_config.get(\"validate_workers_after_construction\"),\n",
      "  File \"/Users/kai/coding/ray/python/ray/rllib/evaluation/worker_set.py\", line 269, in add_workers\n",
      "    self.foreach_worker(lambda w: w.assert_healthy())\n",
      "  File \"/Users/kai/coding/ray/python/ray/rllib/evaluation/worker_set.py\", line 391, in foreach_worker\n",
      "    remote_results = ray.get([w.apply.remote(func) for w in self.remote_workers()])\n",
      "ray.exceptions.RayActorError: The actor died because of an error raised in its creation task, \u001b[36mray::RolloutWorker.__init__()\u001b[39m (pid=53786, ip=127.0.0.1, repr=<ray.rllib.evaluation.rollout_worker.RolloutWorker object at 0x1193ddd10>)\n",
      "KeyError: 'Humanoid-v1'\n",
      "\n",
      "During handling of the above exception, another exception occurred:\n",
      "\n",
      "\u001b[36mray::RolloutWorker.__init__()\u001b[39m (pid=53786, ip=127.0.0.1, repr=<ray.rllib.evaluation.rollout_worker.RolloutWorker object at 0x1193ddd10>)\n",
      "  File \"/Users/kai/coding/ray/python/ray/rllib/env/utils.py\", line 50, in _gym_env_creator\n",
      "    return gym.make(env_descriptor, **env_context)\n",
      "  File \"/Users/kai/.pyenv/versions/3.7.7/lib/python3.7/site-packages/gym/envs/registration.py\", line 235, in make\n",
      "    return registry.make(id, **kwargs)\n",
      "  File \"/Users/kai/.pyenv/versions/3.7.7/lib/python3.7/site-packages/gym/envs/registration.py\", line 128, in make\n",
      "    spec = self.spec(path)\n",
      "  File \"/Users/kai/.pyenv/versions/3.7.7/lib/python3.7/site-packages/gym/envs/registration.py\", line 187, in spec\n",
      "    id, matching_envs\n",
      "gym.error.DeprecatedEnv: Env Humanoid-v1 not found (valid versions include ['Humanoid-v2', 'Humanoid-v3'])\n",
      "\n",
      "During handling of the above exception, another exception occurred:\n",
      "\n",
      "\u001b[36mray::RolloutWorker.__init__()\u001b[39m (pid=53786, ip=127.0.0.1, repr=<ray.rllib.evaluation.rollout_worker.RolloutWorker object at 0x1193ddd10>)\n",
      "  File \"/Users/kai/coding/ray/python/ray/rllib/evaluation/rollout_worker.py\", line 490, in __init__\n",
      "    self.env = env_creator(copy.deepcopy(self.env_context))\n",
      "  File \"/Users/kai/coding/ray/python/ray/rllib/env/utils.py\", line 52, in _gym_env_creator\n",
      "    raise EnvError(ERR_MSG_INVALID_ENV_DESCRIPTOR.format(env_descriptor))\n",
      "ray.rllib.utils.error.EnvError: The env string you provided ('Humanoid-v1') is:\n",
      "a) Not a supported/installed environment.\n",
      "b) Not a tune-registered environment creator.\n",
      "c) Not a valid env class string.\n",
      "\n",
      "Try one of the following:\n",
      "a) For Atari support: `pip install gym[atari] autorom[accept-rom-license]`.\n",
      "   For VizDoom support: Install VizDoom\n",
      "   (https://github.com/mwydmuch/ViZDoom/blob/master/doc/Building.md) and\n",
      "   `pip install vizdoomgym`.\n",
      "   For PyBullet support: `pip install pybullet`.\n",
      "b) To register your custom env, do `from ray import tune;\n",
      "   tune.register('[name]', lambda cfg: [return env obj from here using cfg])`.\n",
      "   Then in your config, do `config['env'] = [name]`.\n",
      "c) Make sure you provide a fully qualified classpath, e.g.:\n",
      "   `ray.rllib.examples.env.repeat_after_me_env.RepeatAfterMeEnv`\n",
      "\n",
      "During handling of the above exception, another exception occurred:\n",
      "\n",
      "\u001b[36mray::PPO.__init__()\u001b[39m (pid=53765, ip=127.0.0.1, repr=PPO)\n",
      "  File \"/Users/kai/coding/ray/python/ray/rllib/algorithms/algorithm.py\", line 299, in __init__\n",
      "    super().__init__(config=config, logger_creator=logger_creator, **kwargs)\n",
      "  File \"/Users/kai/coding/ray/python/ray/tune/trainable/trainable.py\", line 157, in __init__\n",
      "    self.setup(copy.deepcopy(self.config))\n",
      "  File \"/Users/kai/coding/ray/python/ray/rllib/algorithms/algorithm.py\", line 432, in setup\n",
      "    raise e.args[0].args[2]\n",
      "ray.rllib.utils.error.EnvError: The env string you provided ('Humanoid-v1') is:\n",
      "a) Not a supported/installed environment.\n",
      "b) Not a tune-registered environment creator.\n",
      "c) Not a valid env class string.\n",
      "\n",
      "Try one of the following:\n",
      "a) For Atari support: `pip install gym[atari] autorom[accept-rom-license]`.\n",
      "   For VizDoom support: Install VizDoom\n",
      "   (https://github.com/mwydmuch/ViZDoom/blob/master/doc/Building.md) and\n",
      "   `pip install vizdoomgym`.\n",
      "   For PyBullet support: `pip install pybullet`.\n",
      "b) To register your custom env, do `from ray import tune;\n",
      "   tune.register('[name]', lambda cfg: [return env obj from here using cfg])`.\n",
      "   Then in your config, do `config['env'] = [name]`.\n",
      "c) Make sure you provide a fully qualified classpath, e.g.:\n",
      "   `ray.rllib.examples.env.repeat_after_me_env.RepeatAfterMeEnv`\n",
      "\n",
      "2022-07-22 16:45:36,795\tERROR tune.py:733 -- Trials did not complete: [PPO_Humanoid-v1_45196_00000]\n",
      "2022-07-22 16:45:36,795\tINFO tune.py:738 -- Total run time: 25.77 seconds (25.48 seconds for the tuning loop).\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "best hyperparameters:  None\n"
     ]
    }
   ],
   "source": [
    "import random\n",
    "\n",
    "from ray import air, tune\n",
    "from ray.tune.schedulers import PopulationBasedTraining\n",
    "\n",
    "if __name__ == \"__main__\":\n",
    "\n",
    "    # Postprocess the perturbed config to ensure it's still valid\n",
    "    def explore(config):\n",
    "        # ensure we collect enough timesteps to do sgd\n",
    "        if config[\"train_batch_size\"] < config[\"sgd_minibatch_size\"] * 2:\n",
    "            config[\"train_batch_size\"] = config[\"sgd_minibatch_size\"] * 2\n",
    "        # ensure we run at least one sgd iter\n",
    "        if config[\"num_sgd_iter\"] < 1:\n",
    "            config[\"num_sgd_iter\"] = 1\n",
    "        return config\n",
    "\n",
    "    pbt = PopulationBasedTraining(\n",
    "        time_attr=\"time_total_s\",\n",
    "        perturbation_interval=120,\n",
    "        resample_probability=0.25,\n",
    "        # Specifies the mutations of these hyperparams\n",
    "        hyperparam_mutations={\n",
    "            \"lambda\": lambda: random.uniform(0.9, 1.0),\n",
    "            \"clip_param\": lambda: random.uniform(0.01, 0.5),\n",
    "            \"lr\": [1e-3, 5e-4, 1e-4, 5e-5, 1e-5],\n",
    "            \"num_sgd_iter\": lambda: random.randint(1, 30),\n",
    "            \"sgd_minibatch_size\": lambda: random.randint(128, 16384),\n",
    "            \"train_batch_size\": lambda: random.randint(2000, 160000),\n",
    "        },\n",
    "        custom_explore_fn=explore,\n",
    "    )\n",
    "    \n",
    "    tuner = tune.Tuner(\n",
    "        \"PPO\",\n",
    "        tune_config=tune.TuneConfig(\n",
    "            metric=\"episode_reward_mean\",\n",
    "            mode=\"max\",\n",
    "            scheduler=pbt,\n",
    "            num_samples=1,\n",
    "        ),\n",
    "        param_space={\n",
    "            \"env\": \"Humanoid-v1\",\n",
    "            \"kl_coeff\": 1.0,\n",
    "            \"num_workers\": 8,\n",
    "            \"num_gpus\": 0, # number of GPUs to use\n",
    "            \"model\": {\"free_log_std\": True},\n",
    "            # These params are tuned from a fixed starting value.\n",
    "            \"lambda\": 0.95,\n",
    "            \"clip_param\": 0.2,\n",
    "            \"lr\": 1e-4,\n",
    "            # These params start off randomly drawn from a set.\n",
    "            \"num_sgd_iter\": tune.choice([10, 20, 30]),\n",
    "            \"sgd_minibatch_size\": tune.choice([128, 512, 2048]),\n",
    "            \"train_batch_size\": tune.choice([10000, 20000, 40000]),\n",
    "        },\n",
    "    )\n",
    "    results = tuner.fit()\n",
    "\n",
    "    print(\"best hyperparameters: \", results.get_best_result().config)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "6fb69a24",
   "metadata": {
    "pycharm": {
     "name": "#%% md\n"
    }
   },
   "source": [
    "## More RLlib Examples\n",
    "\n",
    "- {doc}`/tune/examples/includes/pb2_ppo_example`:\n",
    "  Example of optimizing a distributed RLlib algorithm (PPO) with the PB2 scheduler.\n",
    "  Uses a small population size of 4, so can train on a laptop."
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.7"
  },
  "orphan": true
 },
 "nbformat": 4,
 "nbformat_minor": 5
}