[Testing] Dependabot for RLlib. (#15812)

.buildkite/pipeline.yml

@@ -91,7 +91,7 @@
   conditions: ["RAY_CI_PYTHON_AFFECTED", "RAY_CI_SERVE_AFFECTED", "RAY_CI_RLLIB_AFFECTED", "RAY_CI_TUNE_AFFECTED"]
   commands:
     - cleanup() { if [ "${BUILDKITE_PULL_REQUEST}" = "false" ]; then ./ci/travis/upload_build_info.sh; fi }; trap cleanup EXIT
-    - RLLIB_TESTING=1 TF_VERSION=2.1.0 TFP_VERSION=0.8 TORCH_VERSION=1.6 ./ci/travis/install-dependencies.sh
+    - RLLIB_TESTING=1 ./ci/travis/install-dependencies.sh
     # - bazel test --config=ci $(./scripts/bazel_export_options)
     #   --build_tests_only
     #   --test_tag_filters=flaky
@@ -139,7 +139,7 @@
   conditions: ["RAY_CI_RLLIB_AFFECTED"]
   commands:
     - cleanup() { if [ "${BUILDKITE_PULL_REQUEST}" = "false" ]; then ./ci/travis/upload_build_info.sh; fi }; trap cleanup EXIT
-    - RLLIB_TESTING=1 TF_VERSION=2.1.0 TFP_VERSION=0.8 TORCH_VERSION=1.6 ./ci/travis/install-dependencies.sh
+    - RLLIB_TESTING=1 ./ci/travis/install-dependencies.sh
     - bazel test --config=ci $(./scripts/bazel_export_options)
       --build_tests_only
       --test_tag_filters=learning_tests_tf,-flaky
@@ -148,7 +148,7 @@
   conditions: ["RAY_CI_RLLIB_AFFECTED"]
   commands:
     - cleanup() { if [ "${BUILDKITE_PULL_REQUEST}" = "false" ]; then ./ci/travis/upload_build_info.sh; fi }; trap cleanup EXIT
-    - RLLIB_TESTING=1 TF_VERSION=1.14.0 TFP_VERSION=0.7 TORCH_VERSION=1.6 ./ci/travis/install-dependencies.sh
+    - RLLIB_TESTING=1 TF_VERSION=1.14.0 TFP_VERSION=0.7 ./ci/travis/install-dependencies.sh
     - bazel test --config=ci $(./scripts/bazel_export_options)
       --build_tests_only
       --test_tag_filters=learning_tests_tf,-flaky
@@ -157,7 +157,7 @@
   conditions: ["RAY_CI_RLLIB_AFFECTED"]
   commands:
     - cleanup() { if [ "${BUILDKITE_PULL_REQUEST}" = "false" ]; then ./ci/travis/upload_build_info.sh; fi }; trap cleanup EXIT
-    - RLLIB_TESTING=1 TF_VERSION=2.1.0 TFP_VERSION=0.8 TORCH_VERSION=1.6 ./ci/travis/install-dependencies.sh
+    - RLLIB_TESTING=1 ./ci/travis/install-dependencies.sh
     - bazel test --config=ci $(./scripts/bazel_export_options)
       --build_tests_only
       --test_tag_filters=learning_tests_torch,-flaky
@@ -166,7 +166,7 @@
   conditions: ["RAY_CI_RLLIB_AFFECTED"]
   commands:
     - cleanup() { if [ "${BUILDKITE_PULL_REQUEST}" = "false" ]; then ./ci/travis/upload_build_info.sh; fi }; trap cleanup EXIT
-    - RLLIB_TESTING=1 TF_VERSION=2.1.0 TFP_VERSION=0.8 TORCH_VERSION=1.6 ./ci/travis/install-dependencies.sh
+    - RLLIB_TESTING=1 ./ci/travis/install-dependencies.sh
     - bazel test --config=ci $(./scripts/bazel_export_options)
       --build_tests_only
       --test_tag_filters=quick_train,-flaky
@@ -177,7 +177,7 @@
   conditions: ["RAY_CI_RLLIB_AFFECTED"]
   commands:
     - cleanup() { if [ "${BUILDKITE_PULL_REQUEST}" = "false" ]; then ./ci/travis/upload_build_info.sh; fi }; trap cleanup EXIT
-    - RLLIB_TESTING=1 TF_VERSION=2.1.0 TFP_VERSION=0.8 TORCH_VERSION=1.6 ./ci/travis/install-dependencies.sh
+    - RLLIB_TESTING=1 ./ci/travis/install-dependencies.sh
     # Test everything that does not have any of the "main" labels:
     # "learning_tests|quick_train|examples|tests_dir".
     - bazel test --config=ci $(./scripts/bazel_export_options)
@@ -190,7 +190,7 @@
   conditions: ["RAY_CI_RLLIB_AFFECTED"]
   commands:
     - cleanup() { if [ "${BUILDKITE_PULL_REQUEST}" = "false" ]; then ./ci/travis/upload_build_info.sh; fi }; trap cleanup EXIT
-    - RLLIB_TESTING=1 TF_VERSION=2.1.0 TFP_VERSION=0.8 TORCH_VERSION=1.6 ./ci/travis/install-dependencies.sh
+    - RLLIB_TESTING=1 ./ci/travis/install-dependencies.sh
     - bazel test --config=ci $(./scripts/bazel_export_options) --build_tests_only
       --test_tag_filters=examples_A,examples_B,-flaky --test_env=RAY_USE_MULTIPROCESSING_CPU_COUNT=1 rllib/...
     - bazel test --config=ci $(./scripts/bazel_export_options) --build_tests_only
@@ -200,7 +200,7 @@
   conditions: ["RAY_CI_RLLIB_AFFECTED"]
   commands:
     - cleanup() { if [ "${BUILDKITE_PULL_REQUEST}" = "false" ]; then ./ci/travis/upload_build_info.sh; fi }; trap cleanup EXIT
-    - RLLIB_TESTING=1 TF_VERSION=2.1.0 TFP_VERSION=0.8 TORCH_VERSION=1.6 ./ci/travis/install-dependencies.sh
+    - RLLIB_TESTING=1 ./ci/travis/install-dependencies.sh
     - bazel test --config=ci $(./scripts/bazel_export_options) --build_tests_only
       --test_tag_filters=examples_E,examples_F,examples_G,examples_H,examples_I,examples_J,examples_K,examples_L,examples_M,examples_N,examples_O,examples_P,-flaky --test_env=RAY_USE_MULTIPROCESSING_CPU_COUNT=1
       rllib/...
@@ -211,7 +211,7 @@
   conditions: ["RAY_CI_RLLIB_AFFECTED"]
   commands:
     - cleanup() { if [ "${BUILDKITE_PULL_REQUEST}" = "false" ]; then ./ci/travis/upload_build_info.sh; fi }; trap cleanup EXIT
-    - RLLIB_TESTING=1 TF_VERSION=2.1.0 TFP_VERSION=0.8 TORCH_VERSION=1.6 ./ci/travis/install-dependencies.sh
+    - RLLIB_TESTING=1 ./ci/travis/install-dependencies.sh
     - bazel test --config=ci $(./scripts/bazel_export_options) --build_tests_only
       --test_tag_filters=tests_dir_A,tests_dir_B,tests_dir_C,tests_dir_D,tests_dir_E,tests_dir_F,tests_dir_G,tests_dir_H,tests_dir_I,tests_dir_J,tests_dir_K,tests_dir_L,-flaky --test_env=RAY_USE_MULTIPROCESSING_CPU_COUNT=1
       rllib/...
@@ -219,7 +219,7 @@
   conditions: ["RAY_CI_RLLIB_AFFECTED"]
   commands:
     - cleanup() { if [ "${BUILDKITE_PULL_REQUEST}" = "false" ]; then ./ci/travis/upload_build_info.sh; fi }; trap cleanup EXIT
-    - RLLIB_TESTING=1 TF_VERSION=2.4.0 TFP_VERSION=0.12.2 TORCH_VERSION=1.6 ./ci/travis/install-dependencies.sh
+    - RLLIB_TESTING=1 ./ci/travis/install-dependencies.sh
     - bazel test --config=ci $(./scripts/bazel_export_options) --build_tests_only
       --test_tag_filters=tests_dir_M,tests_dir_N,tests_dir_O,tests_dir_P,tests_dir_Q,tests_dir_R,tests_dir_S,tests_dir_T,tests_dir_U,tests_dir_V,tests_dir_W,tests_dir_X,tests_dir_Y,tests_dir_Z,-flaky --test_env=RAY_USE_MULTIPROCESSING_CPU_COUNT=1
       rllib/...

.github/dependabot.yml

@@ -1,10 +1,12 @@
 version: 2
 updates:
-  # Tune/SGD/Doc requirements
+  # Tune/SGD/Doc requirements.
   - package-ecosystem: "pip"
-    # The requirements base directory currently only contains tune requirements.
-    # If we want to add more requirements here (Core, RLlib, etc.), then we should make subdirectories for each one.
-    directory: "/python/requirements"
+    # The requirements base directory currently only contains Tune and RLlib
+    # requirements.
+    # If we want to add more requirements here (Core, Serve, etc.), then we should
+    # make additional subdirectories for each one.
+    directory: "/python/requirements/tune"
     schedule:
       # Automatic upgrade checks Saturday at 12 AM.
       # Dependabot updates can still be manually triggered via Github at any time.
@@ -12,12 +14,34 @@ updates:
       day: "saturday"
       # 12 AM
       time: "00:00"
-      # Use Pacific Standard Time
+      # Use Pacific Standard Time.
       timezone: "America/Los_Angeles"
     commit-message:
       prefix: "[tune]"
       include: "scope"
-    # Only 3 upgrade PRs open at a time.
-    open-pull-requests-limit: 3
+    # Only 5 upgrade PRs open at a time.
+    open-pull-requests-limit: 5
     reviewers:
       - "ray-project/ray-tune"
+
+  # RLlib requirements.
+  - package-ecosystem: "pip"
+    # The requirements base directory currently only contains Tune and RLlib
+    # requirements.
+    # If we want to add more requirements here (Core, Serve, etc.), then we should
+    # make additional subdirectories for each one.
+    directory: "/python/requirements/rllib"
+    schedule:
+      # Automatic upgrade checks Saturday at 12 AM.
+      # Dependabot updates can still be manually triggered via Github at any time.
+      interval: "weekly"
+      day: "saturday"
+      time: "00:00"
+      timezone: "Europe/Amsterdam"
+    commit-message:
+      prefix: "[RLlib]"
+      include: "scope"
+    # Only n upgrade PRs open at a time.
+    open-pull-requests-limit: 3
+    reviewers:
+      - "ray-project/ray-rllib"

ci/travis/build-docker-images.py

@@ -226,9 +226,7 @@ def build_ray():
 def build_ray_ml():
     root_dir = _get_root_dir()
     requirement_files = glob.glob(
-        f"{_get_root_dir()}/python/requirements*.txt")
-    requirement_files.extend(
-        glob.glob(f"{_get_root_dir()}/python/requirements/*.txt"))
+        f"{_get_root_dir()}/python/**/requirements*.txt", recursive=True)
     for fl in requirement_files:
         shutil.copy(fl, os.path.join(root_dir, "docker/ray-ml/"))
     ray_ml_images = _build_cpu_gpu_images("ray-ml")

ci/travis/install-dependencies.sh

@@ -282,8 +282,8 @@ install_dependencies() {
   pip install --no-clean dm-tree==0.1.5  # --no-clean is due to: https://github.com/deepmind/tree/issues/5
 
   if [ -n "${PYTHON-}" ]; then
-    # Remove this entire section once RLlib and Serve dependencies are fixed.
-    if [ "${DOC_TESTING-}" != 1 ] && [ "${SGD_TESTING-}" != 1 ] && [ "${TUNE_TESTING-}" != 1 ]; then
+    # Remove this entire section once Serve dependencies are fixed.
+    if [ "${DOC_TESTING-}" != 1 ] && [ "${SGD_TESTING-}" != 1 ] && [ "${TUNE_TESTING-}" != 1 ] && [ "${RLLIB_TESTING-}" != 1 ]; then
       # PyTorch is installed first since we are using a "-f" directive to find the wheels.
       # We want to install the CPU version only.
       local torch_url="https://download.pytorch.org/whl/torch_stable.html"
@@ -323,38 +323,41 @@ install_dependencies() {
 
   # Additional RLlib test dependencies.
   if [ "${RLLIB_TESTING-}" = 1 ]; then
-    pip install -r "${WORKSPACE_DIR}"/python/requirements_rllib.txt
+    pip install -r "${WORKSPACE_DIR}"/python/requirements/rllib/requirements_rllib.txt
     # install the following packages for testing on travis only
     pip install 'recsim>=0.2.4'
   fi
 
   # Additional Tune/SGD/Doc test dependencies.
   if [ "${TUNE_TESTING-}" = 1 ] || [ "${SGD_TESTING-}" = 1 ] || [ "${DOC_TESTING-}" = 1 ]; then
-    pip install -r "${WORKSPACE_DIR}"/python/requirements/requirements_tune.txt
+    pip install -r "${WORKSPACE_DIR}"/python/requirements/tune/requirements_tune.txt
     download_mnist
   fi
 
   # For Tune, install upstream dependencies.
   if [ "${TUNE_TESTING-}" = 1 ] ||  [ "${DOC_TESTING-}" = 1 ]; then
-    pip install -r "${WORKSPACE_DIR}"/python/requirements/requirements_upstream.txt
+    pip install -r "${WORKSPACE_DIR}"/python/requirements/tune/requirements_upstream.txt
   fi
 
-  # Remove this entire section once RLlib and Serve dependencies are fixed.
-  if [ "${DOC_TESTING-}" != 1 ] && [ "${SGD_TESTING-}" != 1 ] && [ "${TUNE_TESTING-}" != 1 ]; then
-    # If CI has deemed that a different version of Tensorflow or Torch
+  # Remove this entire section once Serve dependencies are fixed.
+  if [ "${DOC_TESTING-}" != 1 ] && [ "${SGD_TESTING-}" != 1 ] && [ "${TUNE_TESTING-}" != 1 ] && [ "${RLLIB_TESTING-}" != 1 ]; then
+    # If CI has deemed that a different version of Torch
     # should be installed, then upgrade/downgrade to that specific version.
-    if [ -n "${TORCH_VERSION-}" ] || [ -n "${TFP_VERSION-}" ] || [ -n "${TF_VERSION-}" ]; then
+    if [ -n "${TORCH_VERSION-}" ]; then
       case "${TORCH_VERSION-1.8.1}" in
         1.8.1) TORCHVISION_VERSION=0.9.1;;
         1.5) TORCHVISION_VERSION=0.6.0;;
         *) TORCHVISION_VERSION=0.5.0;;
       esac
-      pip install --use-deprecated=legacy-resolver --upgrade tensorflow-probability=="${TFP_VERSION-0.8}" \
-        torch=="${TORCH_VERSION-1.8.1}" torchvision=="${TORCHVISION_VERSION}" \
-        tensorflow=="${TF_VERSION-2.2.0}" gym
+      pip install --use-deprecated=legacy-resolver --upgrade torch=="${TORCH_VERSION-1.8.1}" torchvision=="${TORCHVISION_VERSION}"
     fi
   fi
 
+  # RLlib testing with TF 1.x.
+  if [ "${RLLIB_TESTING-}" = 1 ] && { [ -n "${TF_VERSION-}" ] || [ -n "${TFP_VERSION-}" ]; }; then
+    pip install --upgrade tensorflow-probability=="${TFP_VERSION}" tensorflow=="${TF_VERSION}" gym
+  fi
+
   # Additional Tune dependency for Horovod.
   # This must be run last (i.e., torch cannot be re-installed after this)
   if [ "${INSTALL_HOROVOD-}" = 1 ]; then

python/requirements.txt

@@ -74,7 +74,7 @@ pytest-rerunfailures
 pytest-sugar
 pytest-timeout
 scikit-learn==0.22.2
-tensorflow==2.4.1
+tensorflow
 testfixtures
 werkzeug
 xlrd

python/requirements/rllib/requirements_rllib.txt

@@ -0,0 +1,40 @@
+# Deep learning.
+# --------------
+tensorflow==2.4.1
+tensorflow-probability==0.12.2
+torch==1.8.1;sys_platform=="darwin"
+torchvision==0.9.1;sys_platform=="darwin"
+
+# On non-OSX machines only install CPU version of torch and torchvision
+-f https://download.pytorch.org/whl/torch_stable.html
+torch==1.8.1+cpu;sys_platform!="darwin"
+-f https://download.pytorch.org/whl/torch_stable.html
+torchvision==0.9.1+cpu;sys_platform!="darwin"
+
+# Environment adapters.
+# ---------------------
+# Atari
+atari_py==0.2.6
+gym[atari]==0.18.0
+# Kaggle envs.
+kaggle_environments==1.7.11
+# Unity3D testing
+mlagents==0.26.0
+mlagents_envs==0.26.0
+# For tests on PettingZoo's multi-agent envs.
+pettingzoo==1.8.1
+# For testing in MuJoCo-like envs (in PyBullet).
+pybullet==3.1.7
+# For tests on RecSim and Kaggle envs.
+recsim==0.2.4
+
+# Other.
+# ------
+# For MAML on PyTorch.
+higher==0.2.1
+# For auto-generating an env-rendering Window.
+pyglet==1.5.0
+# For JSON reader/writer.
+smart_open==5.0.0
+# Ray Serve example
+starlette==0.14.2

python/requirements_rllib.txt

@@ -1,35 +0,0 @@
-# Deep learning.
-# --------------
-tensorflow-probability
-# Version requirement to match Tune.
-torch>=1.6.0
-# Version requirement to match Tune.
-torchvision>=0.6.0
-
-# Environment adapters.
-# ---------------------
-# Atari
-atari_py
-gym[atari]
-# Kaggle envs.
-kaggle_environments
-# Unity3D testing
-mlagents
-mlagents_envs
-# For tests on PettingZoo's multi-agent envs.
-pettingzoo>=1.4.0
-# For testing in MuJoCo-like envs (in PyBullet).
-pybullet
-# For tests on RecSim and Kaggle envs.
-recsim
-
-# Other.
-# ------
-# For MAML on PyTorch.
-higher
-# For auto-generating an env-rendering Window.
-pyglet
-# For JSON reader/writer.
-smart_open
-# Ray Serve example
-starlette

rllib/BUILD

@@ -1748,7 +1748,7 @@ py_test(
     name = "examples/cartpole_lstm_impala_tf",
     main = "examples/cartpole_lstm.py",
     tags = ["examples", "examples_C"],
-    size = "large",
+    size = "medium",
     srcs = ["examples/cartpole_lstm.py"],
     args = ["--as-test", "--run=IMPALA", "--stop-reward=40", "--num-cpus=4"]
 )
@@ -1757,7 +1757,7 @@ py_test(
     name = "examples/cartpole_lstm_impala_torch",
     main = "examples/cartpole_lstm.py",
     tags = ["examples", "examples_C"],
-    size = "large",
+    size = "medium",
     srcs = ["examples/cartpole_lstm.py"],
     args = ["--as-test", "--framework=torch", "--run=IMPALA", "--stop-reward=40", "--num-cpus=4"]
 )
@@ -1766,7 +1766,7 @@ py_test(
     name = "examples/cartpole_lstm_ppo_tf",
     main = "examples/cartpole_lstm.py",
     tags = ["examples", "examples_C"],
-    size = "large",
+    size = "medium",
     srcs = ["examples/cartpole_lstm.py"],
     args = ["--as-test", "--framework=tf", "--run=PPO", "--stop-reward=40", "--num-cpus=4"]
 )
@@ -1784,7 +1784,7 @@ py_test(
     name = "examples/cartpole_lstm_ppo_torch",
     main = "examples/cartpole_lstm.py",
     tags = ["examples", "examples_C"],
-    size = "large",
+    size = "medium",
     srcs = ["examples/cartpole_lstm.py"],
     args = ["--as-test", "--framework=torch", "--run=PPO", "--stop-reward=40", "--num-cpus=4"]
 )
@@ -1793,7 +1793,7 @@ py_test(
     name = "examples/cartpole_lstm_ppo_tf_with_prev_a_and_r",
     main = "examples/cartpole_lstm.py",
     tags = ["examples", "examples_C"],
-    size = "large",
+    size = "medium",
     srcs = ["examples/cartpole_lstm.py"],
     args = ["--as-test", "--run=PPO", "--stop-reward=40", "--use-prev-action",  "--use-prev-reward", "--num-cpus=4"]
 )

rllib/examples/cartpole_lstm.py

@@ -9,6 +9,7 @@ parser.add_argument("--run", type=str, default="PPO")
 parser.add_argument("--num-cpus", type=int, default=0)
 parser.add_argument(
     "--framework", choices=["tf2", "tf", "tfe", "torch"], default="tf")
+parser.add_argument("--eager-tracing", action="store_true")
 parser.add_argument("--as-test", action="store_true")
 parser.add_argument("--use-prev-action", action="store_true")
 parser.add_argument("--use-prev-reward", action="store_true")
@@ -52,8 +53,8 @@ if __name__ == "__main__":
                 "lstm_use_prev_reward": args.use_prev_reward,
             },
             "framework": args.framework,
-            # Run with tracing enabled for tfe/tf2.
-            "eager_tracing": args.framework in ["tfe", "tf2"],
+            # Run with tracing enabled for tfe/tf2?
+            "eager_tracing": args.eager_tracing,
         })
 
     stop = {