ray/.buildkite/pipeline.ml.yml

- label: ":airplane: ML tests (ray/ml)"
  conditions: ["RAY_CI_ML_AFFECTED"]
  commands:
    - cleanup() { if [ "${BUILDKITE_PULL_REQUEST}" = "false" ]; then ./ci/build/upload_build_info.sh; fi }; trap cleanup EXIT
    - DATA_PROCESSING_TESTING=1 INSTALL_HOROVOD=1 ./ci/env/install-dependencies.sh
    - bazel test --config=ci $(./ci/run/bazel_export_options) --build_tests_only --test_tag_filters=-gpu,-needs_credentials python/ray/ml/...
    # Only setup credentials in branch builds
    - if [ "$BUILDKITE_PULL_REQUEST" != "false" ]; then exit 0; fi
    - python ./ci/env/setup_credentials.py wandb comet_ml
    - bazel test --config=ci $(./ci/run/bazel_export_options) --build_tests_only --test_tag_filters=-gpu,needs_credentials python/ray/ml/...
    - python ./ci/env/cleanup_test_state.py wandb comet_ml

- label: ":brain: RLlib: Learning discr. actions TF2-static-graph"
  conditions: ["RAY_CI_RLLIB_AFFECTED"]
  commands:
    - cleanup() { if [ "${BUILDKITE_PULL_REQUEST}" = "false" ]; then ./ci/build/upload_build_info.sh; fi }; trap cleanup EXIT
    - RLLIB_TESTING=1 PYTHON=3.7 ./ci/env/install-dependencies.sh
    - bazel test --config=ci $(./ci/run/bazel_export_options)
      --build_tests_only
      --test_tag_filters=learning_tests_discrete,-fake_gpus,-torch_only,-tf2_only,-no_tf_static_graph
      --test_arg=--framework=tf
      rllib/...
- label: ":brain: RLlib: Learning cont. actions TF2-static-graph"
  conditions: ["RAY_CI_RLLIB_AFFECTED"]
  commands:
    - cleanup() { if [ "${BUILDKITE_PULL_REQUEST}" = "false" ]; then ./ci/build/upload_build_info.sh; fi }; trap cleanup EXIT
    - RLLIB_TESTING=1 PYTHON=3.7 ./ci/env/install-dependencies.sh
    - bazel test --config=ci $(./ci/run/bazel_export_options)
      --build_tests_only
      --test_tag_filters=learning_tests_continuous,-fake_gpus,-torch_only,-tf2_only,-no_tf_static_graph
      --test_arg=--framework=tf
      rllib/...
- label: ":brain: RLlib: Learning discr. actions TF2-eager-tracing"
  conditions: ["RAY_CI_RLLIB_AFFECTED"]
  commands:
    - cleanup() { if [ "${BUILDKITE_PULL_REQUEST}" = "false" ]; then ./ci/build/upload_build_info.sh; fi }; trap cleanup EXIT
    - RLLIB_TESTING=1 PYTHON=3.7 ./ci/env/install-dependencies.sh
    - bazel test --config=ci $(./ci/run/bazel_export_options)
      --build_tests_only
      --test_tag_filters=learning_tests_discrete,-fake_gpus,-torch_only,-multi_gpu,-no_tf_eager_tracing
      --test_arg=--framework=tf2
      rllib/...
- label: ":brain: RLlib: Learning cont. actions TF2-eager-tracing"
  conditions: ["RAY_CI_RLLIB_AFFECTED"]
  commands:
    - cleanup() { if [ "${BUILDKITE_PULL_REQUEST}" = "false" ]; then ./ci/build/upload_build_info.sh; fi }; trap cleanup EXIT
    - RLLIB_TESTING=1 PYTHON=3.7 ./ci/env/install-dependencies.sh
    - bazel test --config=ci $(./ci/run/bazel_export_options)
      --build_tests_only
      --test_tag_filters=learning_tests_continuous,-fake_gpus,-torch_only,-multi_gpu,-gpu
      --test_arg=--framework=tf2
      rllib/...

- label: ":brain: RLlib: Learning discr. actions PyTorch"
  conditions: ["RAY_CI_RLLIB_AFFECTED"]
  commands:
    - cleanup() { if [ "${BUILDKITE_PULL_REQUEST}" = "false" ]; then ./ci/build/upload_build_info.sh; fi }; trap cleanup EXIT
    - RLLIB_TESTING=1 PYTHON=3.7 ./ci/env/install-dependencies.sh
    - bazel test --config=ci $(./ci/run/bazel_export_options)
      --build_tests_only
      --test_tag_filters=learning_tests_discrete,-fake_gpus,-tf_only,-tf2_only,-multi_gpu
      --test_arg=--framework=torch
      rllib/...
- label: ":brain: RLlib: Learning cont. actions PyTorch"
  conditions: ["RAY_CI_RLLIB_AFFECTED"]
  commands:
    - cleanup() { if [ "${BUILDKITE_PULL_REQUEST}" = "false" ]; then ./ci/build/upload_build_info.sh; fi }; trap cleanup EXIT
    - RLLIB_TESTING=1 PYTHON=3.7 ./ci/env/install-dependencies.sh
    - bazel test --config=ci $(./ci/run/bazel_export_options)
      --build_tests_only
      --test_tag_filters=learning_tests_continuous,-fake_gpus,-tf_only,-tf2_only,-multi_gpu
      --test_arg=--framework=torch
      rllib/...
- label: ":brain: RLlib: Learning tests w/ 2 fake GPUs TF2-static-graph"
  conditions: ["RAY_CI_RLLIB_DIRECTLY_AFFECTED"]
  commands:
    - cleanup() { if [ "${BUILDKITE_PULL_REQUEST}" = "false" ]; then ./ci/build/upload_build_info.sh; fi }; trap cleanup EXIT
    - RLLIB_TESTING=1 PYTHON=3.7 ./ci/env/install-dependencies.sh
    - bazel test --config=ci $(./ci/run/bazel_export_options)
      --build_tests_only
      --test_tag_filters=fake_gpus,-torch_only,-tf2_only,-no_tf_static_graph,-multi_gpu
      --test_arg=--framework=tf
      rllib/...
# TODO: (sven) tf2 (eager) multi-GPU
- label: ":brain: RLlib: Learning tests w/ 2 fake GPUs PyTorch"
  conditions: ["RAY_CI_RLLIB_DIRECTLY_AFFECTED"]
  commands:
    - cleanup() { if [ "${BUILDKITE_PULL_REQUEST}" = "false" ]; then ./ci/build/upload_build_info.sh; fi }; trap cleanup EXIT
    - RLLIB_TESTING=1 PYTHON=3.7 ./ci/env/install-dependencies.sh
    - bazel test --config=ci $(./ci/run/bazel_export_options)
      --build_tests_only
      --test_tag_filters=fake_gpus,-tf_only,-tf2_only,-multi_gpu
      --test_arg=--framework=torch
      rllib/...

- label: ":brain: RLlib: Memory leak tests TF2-eager-tracing"
  conditions: ["RAY_CI_RLLIB_AFFECTED"]
  commands:
    - cleanup() { if [ "${BUILDKITE_PULL_REQUEST}" = "false" ]; then ./ci/build/upload_build_info.sh; fi }; trap cleanup EXIT
    - RLLIB_TESTING=1 PYTHON=3.7 ./ci/env/install-dependencies.sh
    - bazel test --config=ci $(./ci/run/bazel_export_options)
      --build_tests_only
      --test_tag_filters=memory_leak_tests,-flaky
      --test_arg=--framework=tf2
      rllib/...

- label: ":brain: RLlib: Memory leak tests PyTorch"
  conditions: ["RAY_CI_RLLIB_AFFECTED"]
  commands:
    - cleanup() { if [ "${BUILDKITE_PULL_REQUEST}" = "false" ]; then ./ci/build/upload_build_info.sh; fi }; trap cleanup EXIT
    - RLLIB_TESTING=1 PYTHON=3.7 ./ci/env/install-dependencies.sh
    - bazel test --config=ci $(./ci/run/bazel_export_options)
      --build_tests_only
      --test_tag_filters=memory_leak_tests,-flaky
      --test_arg=--framework=torch
      rllib/...

- label: ":brain: RLlib: Quick Agent train.py runs (TODO: obsolete)"
  conditions: ["RAY_CI_RLLIB_DIRECTLY_AFFECTED"]
  commands:
    - cleanup() { if [ "${BUILDKITE_PULL_REQUEST}" = "false" ]; then ./ci/build/upload_build_info.sh; fi }; trap cleanup EXIT
    - RLLIB_TESTING=1 PYTHON=3.7 ./ci/env/install-dependencies.sh
    - bazel test --config=ci $(./ci/run/bazel_export_options)
      --build_tests_only
      --test_tag_filters=quick_train,-multi_gpu
      --test_env=RAY_USE_MULTIPROCESSING_CPU_COUNT=1
      rllib/...

- label: ":brain: RLlib: Trainer Tests (generic)"
  conditions: ["RAY_CI_RLLIB_DIRECTLY_AFFECTED"]
  commands:
    - cleanup() { if [ "${BUILDKITE_PULL_REQUEST}" = "false" ]; then ./ci/build/upload_build_info.sh; fi }; trap cleanup EXIT
    - RLLIB_TESTING=1 PYTHON=3.7 ./ci/env/install-dependencies.sh
    # Test all tests in the `agents` (soon to be "trainers") dir:
    - bazel test --config=ci $(./ci/run/bazel_export_options)
      --build_tests_only
      --test_tag_filters=trainers_dir_generic,-multi_gpu
      --test_env=RAY_USE_MULTIPROCESSING_CPU_COUNT=1
      rllib/...

- label: ":brain: RLlib: Trainer Tests (specific algos)"
  conditions: ["RAY_CI_RLLIB_DIRECTLY_AFFECTED"]
  commands:
    - cleanup() { if [ "${BUILDKITE_PULL_REQUEST}" = "false" ]; then ./ci/build/upload_build_info.sh; fi }; trap cleanup EXIT
    - RLLIB_TESTING=1 PYTHON=3.7 ./ci/env/install-dependencies.sh
    # Test all tests in the `agents` (soon to be "trainers") dir:
    - bazel test --config=ci $(./ci/run/bazel_export_options)
      --build_tests_only
      --test_tag_filters=trainers_dir,-trainers_dir_generic,-multi_gpu
      --test_env=RAY_USE_MULTIPROCESSING_CPU_COUNT=1
      rllib/...

- label: ":brain: RLlib: Everything else (env-, evaluation-, ... dirs)"
  conditions: ["RAY_CI_RLLIB_DIRECTLY_AFFECTED"]
  commands:
    - cleanup() { if [ "${BUILDKITE_PULL_REQUEST}" = "false" ]; then ./ci/build/upload_build_info.sh; fi }; trap cleanup EXIT
    - RLLIB_TESTING=1 PYTHON=3.7 ./ci/env/install-dependencies.sh
    # Test everything that does not have any of the "main" labels:
    # "learning_tests|quick_train|examples|tests_dir".
    - bazel test --config=ci $(./ci/run/bazel_export_options)
      --build_tests_only
      --test_tag_filters=-learning_tests,-quick_train,-memory_leak_tests,-examples,-tests_dir,-trainers_dir,-documentation,-multi_gpu
      --test_env=RAY_USE_MULTIPROCESSING_CPU_COUNT=1
      rllib/...

- label: ":brain: RLlib: Examples {A..B}"
  conditions: ["RAY_CI_RLLIB_AFFECTED"]
  commands:
    - cleanup() { if [ "${BUILDKITE_PULL_REQUEST}" = "false" ]; then ./ci/build/upload_build_info.sh; fi }; trap cleanup EXIT
    - RLLIB_TESTING=1 PYTHON=3.7 ./ci/env/install-dependencies.sh
    - bazel test --config=ci $(./ci/run/bazel_export_options) --build_tests_only
      --test_tag_filters=examples_A,examples_B,-multi_gpu --test_env=RAY_USE_MULTIPROCESSING_CPU_COUNT=1 rllib/...

- label: ":brain: RLlib: Examples {Ca..Ct}"
  conditions: ["RAY_CI_RLLIB_AFFECTED"]
  commands:
    - cleanup() { if [ "${BUILDKITE_PULL_REQUEST}" = "false" ]; then ./ci/build/upload_build_info.sh; fi }; trap cleanup EXIT
    - RLLIB_TESTING=1 PYTHON=3.7 ./ci/env/install-dependencies.sh
    - bazel test --config=ci $(./ci/run/bazel_export_options) --build_tests_only
      --test_tag_filters=examples_C_AtoT,-multi_gpu --test_env=RAY_USE_MULTIPROCESSING_CPU_COUNT=1 rllib/...
- label: ":brain: RLlib: Examples {Cu..Cz}"
  conditions: ["RAY_CI_RLLIB_AFFECTED"]
  commands:
    - cleanup() { if [ "${BUILDKITE_PULL_REQUEST}" = "false" ]; then ./ci/build/upload_build_info.sh; fi }; trap cleanup EXIT
    - RLLIB_TESTING=1 PYTHON=3.7 ./ci/env/install-dependencies.sh
    - bazel test --config=ci $(./ci/run/bazel_export_options) --build_tests_only
      --test_tag_filters=examples_C_UtoZ,-multi_gpu --test_env=RAY_USE_MULTIPROCESSING_CPU_COUNT=1 rllib/...

- label: ":brain: RLlib: Examples {D..P}"
  conditions: ["RAY_CI_RLLIB_AFFECTED"]
  commands:
    - cleanup() { if [ "${BUILDKITE_PULL_REQUEST}" = "false" ]; then ./ci/build/upload_build_info.sh; fi }; trap cleanup EXIT
    - RLLIB_TESTING=1 PYTHON=3.7 ./ci/env/install-dependencies.sh
    - bazel test --config=ci $(./ci/run/bazel_export_options) --build_tests_only
      --test_tag_filters=examples_D,examples_E,examples_F,examples_G,examples_H,examples_I,examples_J,examples_K,examples_L,examples_M,examples_N,examples_O,examples_P,-multi_gpu --test_env=RAY_USE_MULTIPROCESSING_CPU_COUNT=1
      rllib/...

- label: ":brain: RLlib: Examples {Q..Z}"
  conditions: ["RAY_CI_RLLIB_AFFECTED"]
  commands:
    - cleanup() { if [ "${BUILDKITE_PULL_REQUEST}" = "false" ]; then ./ci/build/upload_build_info.sh; fi }; trap cleanup EXIT
    - RLLIB_TESTING=1 PYTHON=3.7 ./ci/env/install-dependencies.sh
    - bazel test --config=ci $(./ci/run/bazel_export_options) --build_tests_only
      --test_tag_filters=examples_Q,examples_R,examples_S,examples_T,examples_U,examples_V,examples_W,examples_X,examples_Y,examples_Z,-multi_gpu --test_env=RAY_USE_MULTIPROCESSING_CPU_COUNT=1
      rllib/...

- label: ":brain: RLlib: tests/ dir (A..L)"
  conditions: ["RAY_CI_RLLIB_DIRECTLY_AFFECTED"]
  commands:
    - cleanup() { if [ "${BUILDKITE_PULL_REQUEST}" = "false" ]; then ./ci/build/upload_build_info.sh; fi }; trap cleanup EXIT
    - RLLIB_TESTING=1 PYTHON=3.7 ./ci/env/install-dependencies.sh
    - bazel test --config=ci $(./ci/run/bazel_export_options) --build_tests_only
      --test_tag_filters=tests_dir_A,tests_dir_B,tests_dir_C,tests_dir_D,tests_dir_E,tests_dir_F,tests_dir_G,tests_dir_H,tests_dir_I,tests_dir_J,tests_dir_K,tests_dir_L --test_env=RAY_USE_MULTIPROCESSING_CPU_COUNT=1
      rllib/...
- label: ":brain: RLlib: tests/ dir (M..Z (no R))"
  conditions: ["RAY_CI_RLLIB_DIRECTLY_AFFECTED"]
  commands:
    - cleanup() { if [ "${BUILDKITE_PULL_REQUEST}" = "false" ]; then ./ci/build/upload_build_info.sh; fi }; trap cleanup EXIT
    - RLLIB_TESTING=1 PYTHON=3.7 ./ci/env/install-dependencies.sh
    - bazel test --config=ci $(./ci/run/bazel_export_options) --build_tests_only
      --test_tag_filters=tests_dir_M,tests_dir_N,tests_dir_O,tests_dir_P,tests_dir_Q,tests_dir_S,tests_dir_T,tests_dir_U,tests_dir_V,tests_dir_W,tests_dir_X,tests_dir_Y,tests_dir_Z,-multi_gpu --test_env=RAY_USE_MULTIPROCESSING_CPU_COUNT=1
      rllib/...
- label: ":brain: RLlib: tests/ dir (R)"
  conditions: ["RAY_CI_RLLIB_DIRECTLY_AFFECTED"]
  commands:
    - cleanup() { if [ "${BUILDKITE_PULL_REQUEST}" = "false" ]; then ./ci/build/upload_build_info.sh; fi }; trap cleanup EXIT
    - RLLIB_TESTING=1 PYTHON=3.7 ./ci/env/install-dependencies.sh
    - bazel test --config=ci $(./ci/run/bazel_export_options) --build_tests_only
      --test_tag_filters=tests_dir_R,-multi_gpu --test_env=RAY_USE_MULTIPROCESSING_CPU_COUNT=1
      rllib/...

- label: ":brain: RLlib: Documentation code/examples"
  conditions: ["RAY_CI_RLLIB_DIRECTLY_AFFECTED"]
  commands:
    - cleanup() { if [ "${BUILDKITE_PULL_REQUEST}" = "false" ]; then ./ci/build/upload_build_info.sh; fi }; trap cleanup EXIT
    - RLLIB_TESTING=1 PYTHON=3.7 ./ci/env/install-dependencies.sh
    - bazel test --config=ci $(./ci/run/bazel_export_options) --build_tests_only
      --test_tag_filters=documentation --test_env=RAY_USE_MULTIPROCESSING_CPU_COUNT=1
      rllib/...

- label: ":octopus: Tune tests {A-R; no RLlib}"
  conditions: ["RAY_CI_TUNE_AFFECTED"]
  commands:
    - cleanup() { if [ "${BUILDKITE_PULL_REQUEST}" = "false" ]; then ./ci/build/upload_build_info.sh; fi }; trap cleanup EXIT
    - TUNE_TESTING=1 PYTHON=3.7 ./ci/env/install-dependencies.sh
    - bazel test --config=ci $(./ci/run/bazel_export_options) --build_tests_only
      --test_tag_filters=tests_dir_A,tests_dir_B,tests_dir_C,tests_dir_D,tests_dir_E,tests_dir_F,tests_dir_G,tests_dir_H,tests_dir_I,tests_dir_J,tests_dir_K,tests_dir_L,tests_dir_M,tests_dir_N,tests_dir_O,tests_dir_P,tests_dir_Q,tests_dir_R,-example,-py37,-soft_imports,-gpu_only,-rllib
      python/ray/tune/...

- label: ":octopus: Tune tests {S-Z; no RLlib}"
  conditions: ["RAY_CI_TUNE_AFFECTED"]
  commands:
    - cleanup() { if [ "${BUILDKITE_PULL_REQUEST}" = "false" ]; then ./ci/build/upload_build_info.sh; fi }; trap cleanup EXIT
    - TUNE_TESTING=1 PYTHON=3.7 ./ci/env/install-dependencies.sh
    - bazel test --config=ci $(./ci/run/bazel_export_options) --build_tests_only
      --test_tag_filters=tests_dir_S,tests_dir_T,tests_dir_U,tests_dir_V,tests_dir_W,tests_dir_X,tests_dir_Y,tests_dir_Z,-example,-py37,-soft_imports,-gpu_only,-rllib
      python/ray/tune/...


- label: ":octopus: Tune multinode tests"
  conditions: [ "RAY_CI_TUNE_AFFECTED" ]
  commands:
    - LINUX_WHEELS=1 ./ci/ci.sh build
    - mkdir -p ~/.docker/cli-plugins/ && curl -SL https://github.com/docker/compose/releases/download/v2.0.1/docker-compose-linux-x86_64 -o ~/.docker/cli-plugins/docker-compose && chmod +x ~/.docker/cli-plugins/docker-compose
    - pip install -U docker aws_requests_auth boto3
    - python ./ci/build/build-docker-images.py --py-versions py37 --device-types cpu --build-type LOCAL --build-base
    - python ./ci/build/build-multinode-image.py rayproject/ray:nightly-py37-cpu rayproject/ray:multinode-py37
    - bazel test --config=ci $(./ci/run/bazel_export_options) --build_tests_only
      --test_tag_filters=multinode,-example,-flaky,-py37,-soft_imports,-gpu_only,-rllib
      python/ray/tune/...
      --test_env=RAY_HAS_SSH="1"
      --test_env=RAY_DOCKER_IMAGE="rayproject/ray:multinode-py37"
      --test_env=RAY_TEMPDIR="/ray-mount"
      --test_env=RAY_HOSTDIR="/ray"
      --test_env=RAY_TESTHOST="dind-daemon"
      --test_env=DOCKER_HOST=tcp://docker:2376
      --test_env=DOCKER_TLS_VERIFY=1
      --test_env=DOCKER_CERT_PATH=/certs/client
      --test_env=DOCKER_TLS_CERTDIR=/certs

- label: ":octopus: Tune examples {w/o tf/pytorch; no RLlib}"
  conditions: ["RAY_CI_TUNE_AFFECTED"]
  commands:
    - cleanup() { if [ "${BUILDKITE_PULL_REQUEST}" = "false" ]; then ./ci/build/upload_build_info.sh; fi }; trap cleanup EXIT
    - TUNE_TESTING=1 PYTHON=3.7 ./ci/env/install-dependencies.sh
    - bazel test --config=ci $(./ci/run/bazel_export_options) --build_tests_only --test_tag_filters=example,-tf,-pytorch,-py37,-soft_imports,-gpu_only,-rllib python/ray/tune/...

- label: ":octopus: Tune examples {w/ tf/pytorch; no RLlib}"
  conditions: ["RAY_CI_TUNE_AFFECTED"]
  commands:
    - cleanup() { if [ "${BUILDKITE_PULL_REQUEST}" = "false" ]; then ./ci/build/upload_build_info.sh; fi }; trap cleanup EXIT
    - TUNE_TESTING=1 PYTHON=3.7 ./ci/env/install-dependencies.sh
    - bazel test --config=ci $(./ci/run/bazel_export_options) --build_tests_only --test_tag_filters=tf,-pytorch,-py37,-soft_imports,-gpu_only,-rllib python/ray/tune/...
    - bazel test --config=ci $(./ci/run/bazel_export_options) --build_tests_only --test_tag_filters=-tf,pytorch,-py37,-soft_imports,-gpu_only,-rllib python/ray/tune/...

- label: ":octopus: :brain: Tune tests and examples {using RLlib}"
  conditions: ["RAY_CI_TUNE_AFFECTED", "RAY_CI_RLLIB_AFFECTED"]
  commands:
    - cleanup() { if [ "${BUILDKITE_PULL_REQUEST}" = "false" ]; then ./ci/build/upload_build_info.sh; fi }; trap cleanup EXIT
    - TUNE_TESTING=1 PYTHON=3.7 ./ci/env/install-dependencies.sh
    - bazel test --config=ci $(./ci/run/bazel_export_options) --build_tests_only --test_tag_filters=-gpu_only,rllib python/ray/tune/...

- label: ":steam_locomotive: Train tests and examples"
  conditions: ["RAY_CI_TRAIN_AFFECTED"]
  commands:
    - cleanup() { if [ "${BUILDKITE_PULL_REQUEST}" = "false" ]; then ./ci/build/upload_build_info.sh; fi }; trap cleanup EXIT
    - TRAIN_TESTING=1 INSTALL_HOROVOD=1 ./ci/env/install-dependencies.sh
    - bazel test --config=ci $(./ci/run/bazel_export_options) --build_tests_only --test_tag_filters=-gpu_only,-minimal,-tune python/ray/train/...

- label: ":steam_locomotive: :octopus: Train + Tune tests and examples"
  conditions: ["RAY_CI_TRAIN_AFFECTED"]
  commands:
    - cleanup() { if [ "${BUILDKITE_PULL_REQUEST}" = "false" ]; then ./ci/build/upload_build_info.sh; fi }; trap cleanup EXIT
    - TRAIN_TESTING=1 TUNE_TESTING=1 ./ci/env/install-dependencies.sh
    - bazel test --config=ci $(./ci/run/bazel_export_options) --build_tests_only --test_tag_filters=tune,-gpu_only python/ray/train/...

- label: ":octopus: Tune/Modin/Dask tests and examples. Python 3.7"
  conditions: ["RAY_CI_TUNE_AFFECTED"]
  commands:
    - cleanup() { if [ "${BUILDKITE_PULL_REQUEST}" = "false" ]; then ./ci/build/upload_build_info.sh; fi }; trap cleanup EXIT
    - TUNE_TESTING=1 PYTHON=3.7 INSTALL_HOROVOD=1 ./ci/env/install-dependencies.sh
    - bazel test --config=ci $(./ci/run/bazel_export_options) --build_tests_only --test_tag_filters=py37,-client python/ray/tune/...
    - bazel test --config=ci $(./ci/run/bazel_export_options) --build_tests_only --test_tag_filters=-client python/ray/util/xgboost/...
    - bazel test --config=ci $(./ci/run/bazel_export_options) --build_tests_only python/ray/util/horovod/...
    - bazel test --config=ci $(./ci/run/bazel_export_options) --build_tests_only python/ray/util/ray_lightning/...

- label: ":octopus: Ludwig tests and examples. Python 3.7"
  conditions: ["RAY_CI_TUNE_AFFECTED"]
  commands:
    - cleanup() { if [ "${BUILDKITE_PULL_REQUEST}" = "false" ]; then ./ci/build/upload_build_info.sh; fi }; trap cleanup EXIT
    - PYTHON=3.7 INSTALL_LUDWIG=1 INSTALL_HOROVOD=1 ./ci/env/install-dependencies.sh
    - bazel test --config=ci $(./ci/run/bazel_export_options) --build_tests_only python/ray/tests/ludwig/...

- label: ":tropical_fish: ML Libraries w/ Ray Client Examples (Python 3.7)."
  conditions: ["RAY_CI_TUNE_AFFECTED"]
  commands:
    - cleanup() { if [ "${BUILDKITE_PULL_REQUEST}" = "false" ]; then ./ci/build/upload_build_info.sh; fi }; trap cleanup EXIT
    - TUNE_TESTING=1 PYTHON=3.7 INSTALL_HOROVOD=1 ./ci/env/install-dependencies.sh
    - rm -rf ./python/ray/thirdparty_files; rm -rf ./python/ray/pickle5_files; ./ci/ci.sh build
    - bazel test --config=ci $(./ci/run/bazel_export_options) --build_tests_only --test_tag_filters=client --test_env=RAY_CLIENT_MODE=1 python/ray/util/dask/...
    - bazel test --config=ci $(./ci/run/bazel_export_options) --build_tests_only --test_tag_filters=client python/ray/tune/...
    - bazel test --config=ci $(./ci/run/bazel_export_options) --build_tests_only --test_tag_filters=client python/ray/util/xgboost/...

- label: ":potable_water: Modin/Dask tests and examples. Python 3.7"
  conditions: ["RAY_CI_PYTHON_AFFECTED"]
  commands:
    - cleanup() { if [ "${BUILDKITE_PULL_REQUEST}" = "false" ]; then ./ci/build/upload_build_info.sh; fi }; trap cleanup EXIT
    - DATA_PROCESSING_TESTING=1 PYTHON=3.7 ./ci/env/install-dependencies.sh
    - bazel test --config=ci $(./ci/run/bazel_export_options) --build_tests_only python/ray/tests/modin/...
    # Dask tests and examples.
    - bazel test --config=ci $(./ci/run/bazel_export_options) --build_tests_only --test_tag_filters=-client python/ray/util/dask/...

- label: ":potable_water: Dataset tests (Python 3.7)"
  conditions: ["RAY_CI_PYTHON_AFFECTED"]
  commands:
    - cleanup() { if [ "${BUILDKITE_PULL_REQUEST}" = "false" ]; then ./ci/build/upload_build_info.sh; fi }; trap cleanup EXIT
    - DATA_PROCESSING_TESTING=1 PYTHON=3.7 ./ci/env/install-dependencies.sh
    - bazel test --config=ci $(./ci/run/bazel_export_options) --build_tests_only python/ray/data/...

- label: ":potable_water: Workflow tests (Python 3.7)"
  conditions: ["RAY_CI_PYTHON_AFFECTED"]
  commands:
    - cleanup() { if [ "${BUILDKITE_PULL_REQUEST}" = "false" ]; then ./ci/build/upload_build_info.sh; fi }; trap cleanup EXIT
    - DATA_PROCESSING_TESTING=1 PYTHON=3.7 ./ci/env/install-dependencies.sh
    - bazel test --config=ci $(./ci/run/bazel_export_options) --build_tests_only python/ray/workflow/...

- label: ":slot_machine: ML Utils tests"
  conditions: ["RAY_CI_ML_UTILS_AFFECTED"]
  commands:
    - cleanup() { if [ "${BUILDKITE_PULL_REQUEST}" = "false" ]; then ./ci/build/upload_build_info.sh; fi }; trap cleanup EXIT
    - TUNE_TESTING=1 ./ci/env/install-dependencies.sh
    - bazel test --config=ci $(./ci/run/bazel_export_options) --build_tests_only python/ray/util/ml_utils/...

- label: ":book: Doc tests and examples"
  conditions:
    ["RAY_CI_PYTHON_AFFECTED", "RAY_CI_TUNE_AFFECTED", "RAY_CI_DOC_AFFECTED"]
  commands:
    - cleanup() { if [ "${BUILDKITE_PULL_REQUEST}" = "false" ]; then ./ci/build/upload_build_info.sh; fi }; trap cleanup EXIT
    - DOC_TESTING=1 PYTHON=3.7 ./ci/env/install-dependencies.sh
    - bazel test --config=ci $(./ci/run/bazel_export_options) --build_tests_only --test_tag_filters=-tf,-pytorch,-py37,-post_wheel_build doc/...
    - bazel test --config=ci $(./ci/run/bazel_export_options) --build_tests_only --test_tag_filters=tf,-pytorch,-py37,-post_wheel_build doc/...
    - bazel test --config=ci $(./ci/run/bazel_export_options) --build_tests_only --test_tag_filters=-tf,pytorch,-py37,-post_wheel_build doc/...