From 97ec2a03b6d4f30273a6314bbb1434a00213829a Mon Sep 17 00:00:00 2001 From: Kai Fricke Date: Thu, 9 Dec 2021 21:14:10 +0000 Subject: [PATCH] [ci/buildkite] Add ml pipeline to speed up ML/RLLib tests (#20895) ML tests will be built in a separate bootstrap step installing all required dependencies. --- .buildkite/Dockerfile | 6 +- .buildkite/pipeline.ml.yml | 346 +++++++++++++++++++++++++ .buildkite/pipeline.yml | 414 +----------------------------- ci/travis/install-dependencies.sh | 3 + 4 files changed, 355 insertions(+), 414 deletions(-) create mode 100644 .buildkite/pipeline.ml.yml diff --git a/.buildkite/Dockerfile b/.buildkite/Dockerfile index 59ec5a626..b8a053708 100644 --- a/.buildkite/Dockerfile +++ b/.buildkite/Dockerfile @@ -4,13 +4,15 @@ ARG REMOTE_CACHE_URL ARG BUILDKITE_PULL_REQUEST ARG BUILDKITE_COMMIT ARG BUILDKITE_PULL_REQUEST_BASE_BRANCH +ARG PYTHON=3.6 +ARG INSTALL_DEPENDENCIES ENV DEBIAN_FRONTEND=noninteractive ENV TZ=America/Los_Angeles ENV BUILDKITE=true ENV CI=true -ENV PYTHON=3.6 +ENV PYTHON=$PYTHON ENV RAY_USE_RANDOM_PORTS=1 ENV RAY_DEFAULT_BUILD=1 ENV RAY_INSTALL_JAVA=1 @@ -62,6 +64,8 @@ COPY . . RUN ./ci/travis/ci.sh init RUN bash --login -i ./ci/travis/ci.sh build +RUN (if [ "${INSTALL_DEPENDENCIES}" = "ML" ]; then RLLIB_TESTING=1 TRAIN_TESTING=1 TUNE_TESTING=1 bash --login -i ./ci/travis/install-dependencies.sh; fi) + # Run determine test to run RUN bash --login -i -c "python ./ci/travis/determine_tests_to_run.py --output=json > affected_set.json" RUN cat affected_set.json diff --git a/.buildkite/pipeline.ml.yml b/.buildkite/pipeline.ml.yml new file mode 100644 index 000000000..abf3297a0 --- /dev/null +++ b/.buildkite/pipeline.ml.yml @@ -0,0 +1,346 @@ +- label: ":python: (Flaky tests)" + conditions: ["RAY_CI_PYTHON_AFFECTED", "RAY_CI_SERVE_AFFECTED", "RAY_CI_RLLIB_DIRECTLY_AFFECTED", "RAY_CI_TUNE_AFFECTED"] + commands: + - cleanup() { if [ "${BUILDKITE_PULL_REQUEST}" = "false" ]; then ./ci/travis/upload_build_info.sh; fi }; trap cleanup EXIT + - RLLIB_TESTING=1 PYTHON=3.7 ./ci/travis/install-dependencies.sh + # - bazel test --config=ci $(./scripts/bazel_export_options) + # --build_tests_only + # --test_tag_filters=flaky + # -- //:all -rllib/... -core_worker_test + - bazel test --config=ci $(./scripts/bazel_export_options) + --test_tag_filters=-kubernetes,flaky + --test_env=CONDA_EXE + --test_env=CONDA_PYTHON_EXE + --test_env=CONDA_SHLVL + --test_env=CONDA_PREFIX + --test_env=CONDA_DEFAULT_ENV + python/ray/tests/... python/ray/serve/... python/ray/tune/... rllib/... +- label: ":brain: RLlib: Learning discr. actions TF2-static-graph (from rllib/tuned_examples/*.yaml)" + conditions: ["RAY_CI_RLLIB_AFFECTED"] + commands: + - cleanup() { if [ "${BUILDKITE_PULL_REQUEST}" = "false" ]; then ./ci/travis/upload_build_info.sh; fi }; trap cleanup EXIT + - RLLIB_TESTING=1 PYTHON=3.7 ./ci/travis/install-dependencies.sh + - bazel test --config=ci $(./scripts/bazel_export_options) + --build_tests_only + --test_tag_filters=learning_tests_discrete,-fake_gpus,-torch_only,-flaky + --test_arg=--framework=tf + rllib/... +- label: ":brain: RLlib: Learning cont. actions TF2-static-graph (from rllib/tuned_examples/*.yaml)" + conditions: ["RAY_CI_RLLIB_AFFECTED"] + commands: + - cleanup() { if [ "${BUILDKITE_PULL_REQUEST}" = "false" ]; then ./ci/travis/upload_build_info.sh; fi }; trap cleanup EXIT + - RLLIB_TESTING=1 PYTHON=3.7 ./ci/travis/install-dependencies.sh + - bazel test --config=ci $(./scripts/bazel_export_options) + --build_tests_only + --test_tag_filters=learning_tests_continuous,-fake_gpus,-torch_only,-flaky + --test_arg=--framework=tf + rllib/... +- label: ":brain: RLlib: Learning discr. actions TF2-eager-tracing (from rllib/tuned_examples/*.yaml)" + conditions: ["RAY_CI_RLLIB_AFFECTED"] + commands: + - cleanup() { if [ "${BUILDKITE_PULL_REQUEST}" = "false" ]; then ./ci/travis/upload_build_info.sh; fi }; trap cleanup EXIT + - RLLIB_TESTING=1 PYTHON=3.7 ./ci/travis/install-dependencies.sh + - bazel test --config=ci $(./scripts/bazel_export_options) + --build_tests_only + --test_tag_filters=learning_tests_discrete,-fake_gpus,-torch_only,-flaky,-multi_gpu,-no_tf_eager_tracing + --test_arg=--framework=tf2 + rllib/... +- label: ":brain: RLlib: Learning cont. actions TF2-eager-tracing (from rllib/tuned_examples/*.yaml)" + conditions: ["RAY_CI_RLLIB_AFFECTED"] + commands: + - cleanup() { if [ "${BUILDKITE_PULL_REQUEST}" = "false" ]; then ./ci/travis/upload_build_info.sh; fi }; trap cleanup EXIT + - RLLIB_TESTING=1 PYTHON=3.7 ./ci/travis/install-dependencies.sh + - bazel test --config=ci $(./scripts/bazel_export_options) + --build_tests_only + --test_tag_filters=learning_tests_continuous,-fake_gpus,-torch_only,-flaky,-multi_gpu + --test_arg=--framework=tf2 + rllib/... +- label: ":brain: RLlib: Learning discr. actions TF1-static-graph (from rllib/tuned_examples/*.yaml)" + conditions: ["RAY_CI_RLLIB_DIRECTLY_AFFECTED"] + commands: + - cleanup() { if [ "${BUILDKITE_PULL_REQUEST}" = "false" ]; then ./ci/travis/upload_build_info.sh; fi }; trap cleanup EXIT + - RLLIB_TESTING=1 PYTHON=3.7 TF_VERSION=1.14.0 TFP_VERSION=0.7 ./ci/travis/install-dependencies.sh + - bazel test --config=ci $(./scripts/bazel_export_options) + --build_tests_only + --test_tag_filters=learning_tests_discrete,-fake_gpus,-torch_only,-flaky,-multi_gpu + --test_arg=--framework=tf + rllib/... +- label: ":brain: RLlib: Learning cont. actions TF1-static-graph (from rllib/tuned_examples/*.yaml)" + conditions: ["RAY_CI_RLLIB_DIRECTLY_AFFECTED"] + commands: + - cleanup() { if [ "${BUILDKITE_PULL_REQUEST}" = "false" ]; then ./ci/travis/upload_build_info.sh; fi }; trap cleanup EXIT + - RLLIB_TESTING=1 PYTHON=3.7 TF_VERSION=1.14.0 TFP_VERSION=0.7 ./ci/travis/install-dependencies.sh + - bazel test --config=ci $(./scripts/bazel_export_options) + --build_tests_only + --test_tag_filters=learning_tests_continuous,-fake_gpus,-torch_only,-flaky,-multi_gpu + --test_arg=--framework=tf + rllib/... +- label: ":brain: RLlib: Learning discr. actions PyTorch (from rllib/tuned_examples/*.yaml)" + conditions: ["RAY_CI_RLLIB_AFFECTED"] + commands: + - cleanup() { if [ "${BUILDKITE_PULL_REQUEST}" = "false" ]; then ./ci/travis/upload_build_info.sh; fi }; trap cleanup EXIT + - RLLIB_TESTING=1 PYTHON=3.7 ./ci/travis/install-dependencies.sh + - bazel test --config=ci $(./scripts/bazel_export_options) + --build_tests_only + --test_tag_filters=learning_tests_discrete,-fake_gpus,-tf_only,-flaky,-multi_gpu + --test_arg=--framework=torch + rllib/... +- label: ":brain: RLlib: Learning cont. actions PyTorch (from rllib/tuned_examples/*.yaml)" + conditions: ["RAY_CI_RLLIB_AFFECTED"] + commands: + - cleanup() { if [ "${BUILDKITE_PULL_REQUEST}" = "false" ]; then ./ci/travis/upload_build_info.sh; fi }; trap cleanup EXIT + - RLLIB_TESTING=1 PYTHON=3.7 ./ci/travis/install-dependencies.sh + - bazel test --config=ci $(./scripts/bazel_export_options) + --build_tests_only + --test_tag_filters=learning_tests_continuous,-fake_gpus,-tf_only,-flaky,-multi_gpu + --test_arg=--framework=torch + rllib/... +- label: ":brain: RLlib: Learning tests w/ 2 fake GPUs TF2-static-graph (from rllib/tuned_examples/*.yaml)" + conditions: ["RAY_CI_RLLIB_DIRECTLY_AFFECTED"] + commands: + - cleanup() { if [ "${BUILDKITE_PULL_REQUEST}" = "false" ]; then ./ci/travis/upload_build_info.sh; fi }; trap cleanup EXIT + - RLLIB_TESTING=1 PYTHON=3.7 ./ci/travis/install-dependencies.sh + - bazel test --config=ci $(./scripts/bazel_export_options) + --build_tests_only + --test_tag_filters=fake_gpus,-torch_only,-flaky,-multi_gpu + --test_arg=--framework=tf + rllib/... +# TODO: (sven) tf2 (eager) multi-GPU +- label: ":brain: RLlib: Learning tests w/ 2 fake GPUs PyTorch (from rllib/tuned_examples/*.yaml)" + conditions: ["RAY_CI_RLLIB_DIRECTLY_AFFECTED"] + commands: + - cleanup() { if [ "${BUILDKITE_PULL_REQUEST}" = "false" ]; then ./ci/travis/upload_build_info.sh; fi }; trap cleanup EXIT + - RLLIB_TESTING=1 PYTHON=3.7 ./ci/travis/install-dependencies.sh + - bazel test --config=ci $(./scripts/bazel_export_options) + --build_tests_only + --test_tag_filters=fake_gpus,-tf_only,-flaky,-multi_gpu + --test_arg=--framework=torch + rllib/... + +- label: ":brain: RLlib: Quick Agent train.py runs (TODO: obsolete)" + conditions: ["RAY_CI_RLLIB_DIRECTLY_AFFECTED"] + commands: + - cleanup() { if [ "${BUILDKITE_PULL_REQUEST}" = "false" ]; then ./ci/travis/upload_build_info.sh; fi }; trap cleanup EXIT + - RLLIB_TESTING=1 PYTHON=3.7 ./ci/travis/install-dependencies.sh + - bazel test --config=ci $(./scripts/bazel_export_options) + --build_tests_only + --test_tag_filters=quick_train,-flaky,-multi_gpu + --test_env=RAY_USE_MULTIPROCESSING_CPU_COUNT=1 + rllib/... + +- label: ":brain: RLlib: Trainer Tests" + conditions: ["RAY_CI_RLLIB_DIRECTLY_AFFECTED"] + commands: + - cleanup() { if [ "${BUILDKITE_PULL_REQUEST}" = "false" ]; then ./ci/travis/upload_build_info.sh; fi }; trap cleanup EXIT + - RLLIB_TESTING=1 PYTHON=3.7 ./ci/travis/install-dependencies.sh + # Test all tests in the `agents` (soon to be "trainers") dir: + - bazel test --config=ci $(./scripts/bazel_export_options) + --build_tests_only + --test_tag_filters=trainers_dir,-flaky,-multi_gpu + --test_env=RAY_USE_MULTIPROCESSING_CPU_COUNT=1 + rllib/... + +- label: ":brain: RLlib: Everything else (env-, evaluation-, ... dirs)" + conditions: ["RAY_CI_RLLIB_DIRECTLY_AFFECTED"] + commands: + - cleanup() { if [ "${BUILDKITE_PULL_REQUEST}" = "false" ]; then ./ci/travis/upload_build_info.sh; fi }; trap cleanup EXIT + - RLLIB_TESTING=1 PYTHON=3.7 ./ci/travis/install-dependencies.sh + # Test everything that does not have any of the "main" labels: + # "learning_tests|quick_train|examples|tests_dir". + - bazel test --config=ci $(./scripts/bazel_export_options) + --build_tests_only + --test_tag_filters=-learning_tests,-quick_train,-examples,-tests_dir,-trainers_dir,-documentation,-flaky,-multi_gpu + --test_env=RAY_USE_MULTIPROCESSING_CPU_COUNT=1 + rllib/... + +- label: ":brain: RLlib: Examples {A/B}" + conditions: ["RAY_CI_RLLIB_AFFECTED"] + commands: + - cleanup() { if [ "${BUILDKITE_PULL_REQUEST}" = "false" ]; then ./ci/travis/upload_build_info.sh; fi }; trap cleanup EXIT + - RLLIB_TESTING=1 PYTHON=3.7 ./ci/travis/install-dependencies.sh + - bazel test --config=ci $(./scripts/bazel_export_options) --build_tests_only + --test_tag_filters=examples_A,examples_B,-flaky,-multi_gpu --test_env=RAY_USE_MULTIPROCESSING_CPU_COUNT=1 rllib/... + +- label: ":brain: RLlib: Examples {Ca..t}" + conditions: ["RAY_CI_RLLIB_AFFECTED"] + commands: + - cleanup() { if [ "${BUILDKITE_PULL_REQUEST}" = "false" ]; then ./ci/travis/upload_build_info.sh; fi }; trap cleanup EXIT + - RLLIB_TESTING=1 PYTHON=3.7 ./ci/travis/install-dependencies.sh + - bazel test --config=ci $(./scripts/bazel_export_options) --build_tests_only + --test_tag_filters=examples_C_AtoT,-flaky,-multi_gpu --test_env=RAY_USE_MULTIPROCESSING_CPU_COUNT=1 rllib/... +- label: ":brain: RLlib: Examples {Cu..z}" + conditions: ["RAY_CI_RLLIB_AFFECTED"] + commands: + - cleanup() { if [ "${BUILDKITE_PULL_REQUEST}" = "false" ]; then ./ci/travis/upload_build_info.sh; fi }; trap cleanup EXIT + - RLLIB_TESTING=1 PYTHON=3.7 ./ci/travis/install-dependencies.sh + - bazel test --config=ci $(./scripts/bazel_export_options) --build_tests_only + --test_tag_filters=examples_C_UtoZ,-flaky,-multi_gpu --test_env=RAY_USE_MULTIPROCESSING_CPU_COUNT=1 rllib/... + +- label: ":brain: RLlib: Examples {D/P}" + conditions: ["RAY_CI_RLLIB_AFFECTED"] + commands: + - cleanup() { if [ "${BUILDKITE_PULL_REQUEST}" = "false" ]; then ./ci/travis/upload_build_info.sh; fi }; trap cleanup EXIT + - RLLIB_TESTING=1 PYTHON=3.7 ./ci/travis/install-dependencies.sh + - bazel test --config=ci $(./scripts/bazel_export_options) --build_tests_only + --test_tag_filters=examples_D,examples_E,examples_F,examples_G,examples_H,examples_I,examples_J,examples_K,examples_L,examples_M,examples_N,examples_O,examples_P,-flaky,-multi_gpu --test_env=RAY_USE_MULTIPROCESSING_CPU_COUNT=1 + rllib/... + +- label: ":brain: RLlib: Examples {Q/Z}" + conditions: ["RAY_CI_RLLIB_AFFECTED"] + commands: + - cleanup() { if [ "${BUILDKITE_PULL_REQUEST}" = "false" ]; then ./ci/travis/upload_build_info.sh; fi }; trap cleanup EXIT + - RLLIB_TESTING=1 PYTHON=3.7 ./ci/travis/install-dependencies.sh + - bazel test --config=ci $(./scripts/bazel_export_options) --build_tests_only + --test_tag_filters=examples_Q,examples_R,examples_S,examples_T,examples_U,examples_V,examples_W,examples_X,examples_Y,examples_Z,-flaky,-multi_gpu --test_env=RAY_USE_MULTIPROCESSING_CPU_COUNT=1 + rllib/... + +- label: ":brain: RLlib: tests/ dir (A-L)" + conditions: ["RAY_CI_RLLIB_DIRECTLY_AFFECTED"] + commands: + - cleanup() { if [ "${BUILDKITE_PULL_REQUEST}" = "false" ]; then ./ci/travis/upload_build_info.sh; fi }; trap cleanup EXIT + - RLLIB_TESTING=1 PYTHON=3.7 ./ci/travis/install-dependencies.sh + - bazel test --config=ci $(./scripts/bazel_export_options) --build_tests_only + --test_tag_filters=tests_dir_A,tests_dir_B,tests_dir_C,tests_dir_D,tests_dir_E,tests_dir_F,tests_dir_G,tests_dir_H,tests_dir_I,tests_dir_J,tests_dir_K,tests_dir_L,-flaky --test_env=RAY_USE_MULTIPROCESSING_CPU_COUNT=1 + rllib/... +- label: ":brain: RLlib: tests/ dir (M-Z (no R))" + conditions: ["RAY_CI_RLLIB_DIRECTLY_AFFECTED"] + commands: + - cleanup() { if [ "${BUILDKITE_PULL_REQUEST}" = "false" ]; then ./ci/travis/upload_build_info.sh; fi }; trap cleanup EXIT + - RLLIB_TESTING=1 PYTHON=3.7 ./ci/travis/install-dependencies.sh + - bazel test --config=ci $(./scripts/bazel_export_options) --build_tests_only + --test_tag_filters=tests_dir_M,tests_dir_N,tests_dir_O,tests_dir_P,tests_dir_Q,tests_dir_S,tests_dir_T,tests_dir_U,tests_dir_V,tests_dir_W,tests_dir_X,tests_dir_Y,tests_dir_Z,-flaky,-multi_gpu --test_env=RAY_USE_MULTIPROCESSING_CPU_COUNT=1 + rllib/... +- label: ":brain: RLlib: tests/ dir (R)" + conditions: ["RAY_CI_RLLIB_DIRECTLY_AFFECTED"] + commands: + - cleanup() { if [ "${BUILDKITE_PULL_REQUEST}" = "false" ]; then ./ci/travis/upload_build_info.sh; fi }; trap cleanup EXIT + - RLLIB_TESTING=1 PYTHON=3.7 ./ci/travis/install-dependencies.sh + - bazel test --config=ci $(./scripts/bazel_export_options) --build_tests_only + --test_tag_filters=tests_dir_R,-flaky,-multi_gpu --test_env=RAY_USE_MULTIPROCESSING_CPU_COUNT=1 + rllib/... + +- label: ":brain: RLlib: Documentation code/examples" + conditions: ["RAY_CI_RLLIB_DIRECTLY_AFFECTED"] + commands: + - cleanup() { if [ "${BUILDKITE_PULL_REQUEST}" = "false" ]; then ./ci/travis/upload_build_info.sh; fi }; trap cleanup EXIT + - RLLIB_TESTING=1 PYTHON=3.7 ./ci/travis/install-dependencies.sh + - bazel test --config=ci $(./scripts/bazel_export_options) --build_tests_only + --test_tag_filters=documentation,-flaky --test_env=RAY_USE_MULTIPROCESSING_CPU_COUNT=1 + rllib/... + +- label: ":octopus: Tune tests {A-R; no RLlib}" + conditions: ["RAY_CI_TUNE_AFFECTED"] + commands: + - cleanup() { if [ "${BUILDKITE_PULL_REQUEST}" = "false" ]; then ./ci/travis/upload_build_info.sh; fi }; trap cleanup EXIT + - TUNE_TESTING=1 PYTHON=3.7 ./ci/travis/install-dependencies.sh + - bazel test --config=ci $(./scripts/bazel_export_options) --build_tests_only + --test_tag_filters=tests_dir_A,tests_dir_B,tests_dir_C,tests_dir_D,tests_dir_E,tests_dir_F,tests_dir_G,tests_dir_H,tests_dir_I,tests_dir_J,tests_dir_K,tests_dir_L,tests_dir_M,tests_dir_N,tests_dir_O,tests_dir_P,tests_dir_Q,tests_dir_R,-example,-flaky,-py37,-soft_imports,-gpu_only,-rllib + python/ray/tune/... + +- label: ":octopus: Tune tests {S-Z; no RLlib}" + conditions: ["RAY_CI_TUNE_AFFECTED"] + commands: + - cleanup() { if [ "${BUILDKITE_PULL_REQUEST}" = "false" ]; then ./ci/travis/upload_build_info.sh; fi }; trap cleanup EXIT + - TUNE_TESTING=1 PYTHON=3.7 ./ci/travis/install-dependencies.sh + - bazel test --config=ci $(./scripts/bazel_export_options) --build_tests_only + --test_tag_filters=tests_dir_S,tests_dir_T,tests_dir_U,tests_dir_V,tests_dir_W,tests_dir_X,tests_dir_Y,tests_dir_Z,-example,-flaky,-py37,-soft_imports,-gpu_only,-rllib + python/ray/tune/... + +- label: ":octopus: Tune examples {w/o tf/pytorch; no RLlib}" + conditions: ["RAY_CI_TUNE_AFFECTED"] + commands: + - cleanup() { if [ "${BUILDKITE_PULL_REQUEST}" = "false" ]; then ./ci/travis/upload_build_info.sh; fi }; trap cleanup EXIT + - TUNE_TESTING=1 PYTHON=3.7 ./ci/travis/install-dependencies.sh + - bazel test --config=ci $(./scripts/bazel_export_options) --build_tests_only --test_tag_filters=example,-tf,-pytorch,-py37,-flaky,-soft_imports,-gpu_only,-rllib python/ray/tune/... + +- label: ":octopus: Tune examples {w/ tf/pytorch; no RLlib}" + conditions: ["RAY_CI_TUNE_AFFECTED"] + commands: + - cleanup() { if [ "${BUILDKITE_PULL_REQUEST}" = "false" ]; then ./ci/travis/upload_build_info.sh; fi }; trap cleanup EXIT + - TUNE_TESTING=1 PYTHON=3.7 ./ci/travis/install-dependencies.sh + - bazel test --config=ci $(./scripts/bazel_export_options) --build_tests_only --test_tag_filters=tf,-pytorch,-py37,-flaky,-soft_imports,-gpu_only,-rllib python/ray/tune/... + - bazel test --config=ci $(./scripts/bazel_export_options) --build_tests_only --test_tag_filters=-tf,pytorch,-py37,-flaky,-soft_imports,-gpu_only,-rllib python/ray/tune/... + +- label: ":octopus: :brain: Tune tests and examples {using RLlib}" + conditions: ["RAY_CI_TUNE_AFFECTED", "RAY_CI_RLLIB_AFFECTED"] + commands: + - cleanup() { if [ "${BUILDKITE_PULL_REQUEST}" = "false" ]; then ./ci/travis/upload_build_info.sh; fi }; trap cleanup EXIT + - TUNE_TESTING=1 PYTHON=3.7 ./ci/travis/install-dependencies.sh + - bazel test --config=ci $(./scripts/bazel_export_options) --build_tests_only --test_tag_filters=-gpu_only,rllib python/ray/tune/... + +- label: ":steam_locomotive: Train tests and examples" + conditions: ["RAY_CI_TRAIN_AFFECTED"] + commands: + - cleanup() { if [ "${BUILDKITE_PULL_REQUEST}" = "false" ]; then ./ci/travis/upload_build_info.sh; fi }; trap cleanup EXIT + - TRAIN_TESTING=1 INSTALL_HOROVOD=1 ./ci/travis/install-dependencies.sh + - bazel test --config=ci $(./scripts/bazel_export_options) --build_tests_only --test_tag_filters=-gpu_only,-tune python/ray/train/... + +- label: ":steam_locomotive: :octopus: Train + Tune tests and examples" + conditions: ["RAY_CI_TRAIN_AFFECTED"] + commands: + - cleanup() { if [ "${BUILDKITE_PULL_REQUEST}" = "false" ]; then ./ci/travis/upload_build_info.sh; fi }; trap cleanup EXIT + - TRAIN_TESTING=1 TUNE_TESTING=1 ./ci/travis/install-dependencies.sh + - bazel test --config=ci $(./scripts/bazel_export_options) --build_tests_only --test_tag_filters=tune,-gpu_only python/ray/train/... + +- label: ":octopus: SGD tests and examples" + conditions: ["RAY_CI_SGD_AFFECTED"] + commands: + - cleanup() { if [ "${BUILDKITE_PULL_REQUEST}" = "false" ]; then ./ci/travis/upload_build_info.sh; fi }; trap cleanup EXIT + - SGD_TESTING=1 INSTALL_HOROVOD=1 ./ci/travis/install-dependencies.sh + - bazel test --config=ci $(./scripts/bazel_export_options) --build_tests_only --test_tag_filters=tf,-pytorch,-py37,-flaky,-client,-gpu_only python/ray/util/sgd/... + - bazel test --config=ci $(./scripts/bazel_export_options) --build_tests_only --test_tag_filters=-tf,pytorch,-py37,-flaky,-client,-gpu_only python/ray/util/sgd/... + - bazel test --config=ci $(./scripts/bazel_export_options) --build_tests_only --test_tag_filters=client_unit_tests,-gpu_only --test_env=RAY_CLIENT_MODE=1 python/ray/util/sgd/... + +- label: ":octopus: Tune/SGD/Modin/Dask tests and examples. Python 3.7" + conditions: ["RAY_CI_TUNE_AFFECTED", "RAY_CI_SGD_AFFECTED"] + commands: + - cleanup() { if [ "${BUILDKITE_PULL_REQUEST}" = "false" ]; then ./ci/travis/upload_build_info.sh; fi }; trap cleanup EXIT + - TUNE_TESTING=1 PYTHON=3.7 INSTALL_HOROVOD=1 ./ci/travis/install-dependencies.sh + - bazel test --config=ci $(./scripts/bazel_export_options) --build_tests_only --test_tag_filters=py37,-flaky,-client python/ray/tune/... + - bazel test --config=ci $(./scripts/bazel_export_options) --build_tests_only --test_tag_filters=-client python/ray/util/xgboost/... + - bazel test --config=ci $(./scripts/bazel_export_options) --build_tests_only python/ray/util/horovod/... + - bazel test --config=ci $(./scripts/bazel_export_options) --build_tests_only python/ray/util/ray_lightning/... + +- label: ":octopus: Ludwig tests and examples. Python 3.7" + conditions: ["RAY_CI_TUNE_AFFECTED", "RAY_CI_SGD_AFFECTED"] + commands: + - cleanup() { if [ "${BUILDKITE_PULL_REQUEST}" = "false" ]; then ./ci/travis/upload_build_info.sh; fi }; trap cleanup EXIT + - SGD_TESTING=1 PYTHON=3.7 INSTALL_LUDWIG=1 INSTALL_HOROVOD=1 ./ci/travis/install-dependencies.sh + - bazel test --config=ci $(./scripts/bazel_export_options) --build_tests_only python/ray/tests/ludwig/... + +- label: ":tropical_fish: ML Libraries w/ Ray Client Examples (Python 3.7)." + conditions: ["RAY_CI_TUNE_AFFECTED", "RAY_CI_SGD_AFFECTED"] + commands: + - cleanup() { if [ "${BUILDKITE_PULL_REQUEST}" = "false" ]; then ./ci/travis/upload_build_info.sh; fi }; trap cleanup EXIT + - TUNE_TESTING=1 PYTHON=3.7 INSTALL_HOROVOD=1 ./ci/travis/install-dependencies.sh + - rm -rf ./python/ray/thirdparty_files; rm -rf ./python/ray/pickle5_files; ./ci/travis/ci.sh build + - bazel test --config=ci $(./scripts/bazel_export_options) --build_tests_only --test_tag_filters=client --test_env=RAY_CLIENT_MODE=1 python/ray/util/dask/... + - bazel test --config=ci $(./scripts/bazel_export_options) --build_tests_only --test_tag_filters=client,-flaky python/ray/tune/... + - bazel test --config=ci $(./scripts/bazel_export_options) --build_tests_only --test_tag_filters=client,-client_unit_tests python/ray/util/sgd/... + - bazel test --config=ci $(./scripts/bazel_export_options) --build_tests_only --test_tag_filters=client,-flaky python/ray/util/xgboost/... + +- label: ":potable_water: Modin/Dask tests and examples. Python 3.7" + conditions: ["RAY_CI_PYTHON_AFFECTED"] + commands: + - cleanup() { if [ "${BUILDKITE_PULL_REQUEST}" = "false" ]; then ./ci/travis/upload_build_info.sh; fi }; trap cleanup EXIT + - DATA_PROCESSING_TESTING=1 PYTHON=3.7 ./ci/travis/install-dependencies.sh + - bazel test --config=ci $(./scripts/bazel_export_options) --build_tests_only python/ray/tests/modin/... + # Dask tests and examples. + - bazel test --config=ci $(./scripts/bazel_export_options) --build_tests_only --test_tag_filters=-client python/ray/util/dask/... + +- label: ":potable_water: Workflow & Dataset tests (Python 3.7)" + conditions: ["RAY_CI_PYTHON_AFFECTED"] + commands: + - cleanup() { if [ "${BUILDKITE_PULL_REQUEST}" = "false" ]; then ./ci/travis/upload_build_info.sh; fi }; trap cleanup EXIT + - DATA_PROCESSING_TESTING=1 PYTHON=3.7 ./ci/travis/install-dependencies.sh + - bazel test --config=ci $(./scripts/bazel_export_options) --build_tests_only python/ray/workflow/... python/ray/data/... + +- label: ":book: Doc tests and examples" + conditions: + ["RAY_CI_PYTHON_AFFECTED", "RAY_CI_TUNE_AFFECTED", "RAY_CI_DOC_AFFECTED"] + commands: + - cleanup() { if [ "${BUILDKITE_PULL_REQUEST}" = "false" ]; then ./ci/travis/upload_build_info.sh; fi }; trap cleanup EXIT + - DOC_TESTING=1 PYTHON=3.7 ./ci/travis/install-dependencies.sh + - bazel test --config=ci $(./scripts/bazel_export_options) --build_tests_only --test_tag_filters=-tf,-pytorch,-py37,-post_wheel_build doc/... + - bazel test --config=ci $(./scripts/bazel_export_options) --build_tests_only --test_tag_filters=tf,-pytorch,-py37,-post_wheel_build doc/... + - bazel test --config=ci $(./scripts/bazel_export_options) --build_tests_only --test_tag_filters=-tf,pytorch,-py37,-post_wheel_build doc/... diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index 34a005e40..303c5d35a 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -245,25 +245,7 @@ python/ray/tests/test_runtime_env_ray_minimal - bazel test --test_output=streamed --config=ci --test_env=RAY_MINIMAL=1 $(./scripts/bazel_export_options) python/ray/tests/test_serve_ray_minimal -- label: ":python: (Flaky tests)" - conditions: ["RAY_CI_PYTHON_AFFECTED", "RAY_CI_SERVE_AFFECTED", "RAY_CI_RLLIB_DIRECTLY_AFFECTED", "RAY_CI_TUNE_AFFECTED"] - commands: - - cleanup() { if [ "${BUILDKITE_PULL_REQUEST}" = "false" ]; then ./ci/travis/upload_build_info.sh; fi }; trap cleanup EXIT - - RLLIB_TESTING=1 PYTHON=3.7 ./ci/travis/install-dependencies.sh - # Because Python version changed, we need to re-install Ray here - - rm -rf ./python/ray/thirdparty_files; rm -rf ./python/ray/pickle5_files; ./ci/travis/ci.sh build - # - bazel test --config=ci $(./scripts/bazel_export_options) - # --build_tests_only - # --test_tag_filters=flaky - # -- //:all -rllib/... -core_worker_test - - bazel test --config=ci $(./scripts/bazel_export_options) - --test_tag_filters=-kubernetes,flaky - --test_env=CONDA_EXE - --test_env=CONDA_PYTHON_EXE - --test_env=CONDA_SHLVL - --test_env=CONDA_PREFIX - --test_env=CONDA_DEFAULT_ENV - python/ray/tests/... python/ray/serve/... python/ray/tune/... rllib/... + - label: ":python: (Small & Large)" conditions: ["RAY_CI_PYTHON_AFFECTED"] commands: @@ -376,313 +358,6 @@ -//python/ray/tests:test_placement_group_2 -//python/ray/tests:test_placement_group_3 -//python/ray/tests:test_multi_node -- label: ":brain: RLlib: Learning discr. actions TF2-static-graph (from rllib/tuned_examples/*.yaml)" - conditions: ["RAY_CI_RLLIB_AFFECTED"] - commands: - - cleanup() { if [ "${BUILDKITE_PULL_REQUEST}" = "false" ]; then ./ci/travis/upload_build_info.sh; fi }; trap cleanup EXIT - - RLLIB_TESTING=1 PYTHON=3.7 ./ci/travis/install-dependencies.sh - # Because Python version changed, we need to re-install Ray here - - rm -rf ./python/ray/thirdparty_files; rm -rf ./python/ray/pickle5_files; ./ci/travis/ci.sh build - - bazel test --config=ci $(./scripts/bazel_export_options) - --build_tests_only - --test_tag_filters=learning_tests_discrete,-fake_gpus,-torch_only,-flaky - --test_arg=--framework=tf - rllib/... -- label: ":brain: RLlib: Learning cont. actions TF2-static-graph (from rllib/tuned_examples/*.yaml)" - conditions: ["RAY_CI_RLLIB_AFFECTED"] - commands: - - cleanup() { if [ "${BUILDKITE_PULL_REQUEST}" = "false" ]; then ./ci/travis/upload_build_info.sh; fi }; trap cleanup EXIT - - RLLIB_TESTING=1 PYTHON=3.7 ./ci/travis/install-dependencies.sh - # Because Python version changed, we need to re-install Ray here - - rm -rf ./python/ray/thirdparty_files; rm -rf ./python/ray/pickle5_files; ./ci/travis/ci.sh build - - bazel test --config=ci $(./scripts/bazel_export_options) - --build_tests_only - --test_tag_filters=learning_tests_continuous,-fake_gpus,-torch_only,-flaky - --test_arg=--framework=tf - rllib/... -- label: ":brain: RLlib: Learning discr. actions TF2-eager-tracing (from rllib/tuned_examples/*.yaml)" - conditions: ["RAY_CI_RLLIB_AFFECTED"] - commands: - - cleanup() { if [ "${BUILDKITE_PULL_REQUEST}" = "false" ]; then ./ci/travis/upload_build_info.sh; fi }; trap cleanup EXIT - - RLLIB_TESTING=1 PYTHON=3.7 ./ci/travis/install-dependencies.sh - # Because Python version changed, we need to re-install Ray here - - rm -rf ./python/ray/thirdparty_files; rm -rf ./python/ray/pickle5_files; ./ci/travis/ci.sh build - - bazel test --config=ci $(./scripts/bazel_export_options) - --build_tests_only - --test_tag_filters=learning_tests_discrete,-fake_gpus,-torch_only,-flaky,-multi_gpu,-no_tf_eager_tracing - --test_arg=--framework=tf2 - rllib/... -- label: ":brain: RLlib: Learning cont. actions TF2-eager-tracing (from rllib/tuned_examples/*.yaml)" - conditions: ["RAY_CI_RLLIB_AFFECTED"] - commands: - - cleanup() { if [ "${BUILDKITE_PULL_REQUEST}" = "false" ]; then ./ci/travis/upload_build_info.sh; fi }; trap cleanup EXIT - - RLLIB_TESTING=1 PYTHON=3.7 ./ci/travis/install-dependencies.sh - # Because Python version changed, we need to re-install Ray here - - rm -rf ./python/ray/thirdparty_files; rm -rf ./python/ray/pickle5_files; ./ci/travis/ci.sh build - - bazel test --config=ci $(./scripts/bazel_export_options) - --build_tests_only - --test_tag_filters=learning_tests_continuous,-fake_gpus,-torch_only,-flaky,-multi_gpu - --test_arg=--framework=tf2 - rllib/... -- label: ":brain: RLlib: Learning discr. actions TF1-static-graph (from rllib/tuned_examples/*.yaml)" - conditions: ["RAY_CI_RLLIB_DIRECTLY_AFFECTED"] - commands: - - cleanup() { if [ "${BUILDKITE_PULL_REQUEST}" = "false" ]; then ./ci/travis/upload_build_info.sh; fi }; trap cleanup EXIT - - RLLIB_TESTING=1 PYTHON=3.7 TF_VERSION=1.14.0 TFP_VERSION=0.7 ./ci/travis/install-dependencies.sh - # Because Python version changed, we need to re-install Ray here - - rm -rf ./python/ray/thirdparty_files; rm -rf ./python/ray/pickle5_files; ./ci/travis/ci.sh build - - bazel test --config=ci $(./scripts/bazel_export_options) - --build_tests_only - --test_tag_filters=learning_tests_discrete,-fake_gpus,-torch_only,-flaky,-multi_gpu - --test_arg=--framework=tf - rllib/... -- label: ":brain: RLlib: Learning cont. actions TF1-static-graph (from rllib/tuned_examples/*.yaml)" - conditions: ["RAY_CI_RLLIB_DIRECTLY_AFFECTED"] - commands: - - cleanup() { if [ "${BUILDKITE_PULL_REQUEST}" = "false" ]; then ./ci/travis/upload_build_info.sh; fi }; trap cleanup EXIT - - RLLIB_TESTING=1 PYTHON=3.7 TF_VERSION=1.14.0 TFP_VERSION=0.7 ./ci/travis/install-dependencies.sh - # Because Python version changed, we need to re-install Ray here - - rm -rf ./python/ray/thirdparty_files; rm -rf ./python/ray/pickle5_files; ./ci/travis/ci.sh build - - bazel test --config=ci $(./scripts/bazel_export_options) - --build_tests_only - --test_tag_filters=learning_tests_continuous,-fake_gpus,-torch_only,-flaky,-multi_gpu - --test_arg=--framework=tf - rllib/... -- label: ":brain: RLlib: Learning discr. actions PyTorch (from rllib/tuned_examples/*.yaml)" - conditions: ["RAY_CI_RLLIB_AFFECTED"] - commands: - - cleanup() { if [ "${BUILDKITE_PULL_REQUEST}" = "false" ]; then ./ci/travis/upload_build_info.sh; fi }; trap cleanup EXIT - - RLLIB_TESTING=1 PYTHON=3.7 ./ci/travis/install-dependencies.sh - # Because Python version changed, we need to re-install Ray here - - rm -rf ./python/ray/thirdparty_files; rm -rf ./python/ray/pickle5_files; ./ci/travis/ci.sh build - - bazel test --config=ci $(./scripts/bazel_export_options) - --build_tests_only - --test_tag_filters=learning_tests_discrete,-fake_gpus,-tf_only,-flaky,-multi_gpu - --test_arg=--framework=torch - rllib/... -- label: ":brain: RLlib: Learning cont. actions PyTorch (from rllib/tuned_examples/*.yaml)" - conditions: ["RAY_CI_RLLIB_AFFECTED"] - commands: - - cleanup() { if [ "${BUILDKITE_PULL_REQUEST}" = "false" ]; then ./ci/travis/upload_build_info.sh; fi }; trap cleanup EXIT - - RLLIB_TESTING=1 PYTHON=3.7 ./ci/travis/install-dependencies.sh - # Because Python version changed, we need to re-install Ray here - - rm -rf ./python/ray/thirdparty_files; rm -rf ./python/ray/pickle5_files; ./ci/travis/ci.sh build - - bazel test --config=ci $(./scripts/bazel_export_options) - --build_tests_only - --test_tag_filters=learning_tests_continuous,-fake_gpus,-tf_only,-flaky,-multi_gpu - --test_arg=--framework=torch - rllib/... -- label: ":brain: RLlib: Learning tests w/ 2 fake GPUs TF2-static-graph (from rllib/tuned_examples/*.yaml)" - conditions: ["RAY_CI_RLLIB_DIRECTLY_AFFECTED"] - commands: - - cleanup() { if [ "${BUILDKITE_PULL_REQUEST}" = "false" ]; then ./ci/travis/upload_build_info.sh; fi }; trap cleanup EXIT - - RLLIB_TESTING=1 PYTHON=3.7 ./ci/travis/install-dependencies.sh - # Because Python version changed, we need to re-install Ray here - - rm -rf ./python/ray/thirdparty_files; rm -rf ./python/ray/pickle5_files; ./ci/travis/ci.sh build - - bazel test --config=ci $(./scripts/bazel_export_options) - --build_tests_only - --test_tag_filters=fake_gpus,-torch_only,-flaky,-multi_gpu - --test_arg=--framework=tf - rllib/... -# TODO: (sven) tf2 (eager) multi-GPU -- label: ":brain: RLlib: Learning tests w/ 2 fake GPUs PyTorch (from rllib/tuned_examples/*.yaml)" - conditions: ["RAY_CI_RLLIB_DIRECTLY_AFFECTED"] - commands: - - cleanup() { if [ "${BUILDKITE_PULL_REQUEST}" = "false" ]; then ./ci/travis/upload_build_info.sh; fi }; trap cleanup EXIT - - RLLIB_TESTING=1 PYTHON=3.7 ./ci/travis/install-dependencies.sh - # Because Python version changed, we need to re-install Ray here - - rm -rf ./python/ray/thirdparty_files; rm -rf ./python/ray/pickle5_files; ./ci/travis/ci.sh build - - bazel test --config=ci $(./scripts/bazel_export_options) - --build_tests_only - --test_tag_filters=fake_gpus,-tf_only,-flaky,-multi_gpu - --test_arg=--framework=torch - rllib/... - -- label: ":brain: RLlib: Quick Agent train.py runs (TODO: obsolete)" - conditions: ["RAY_CI_RLLIB_DIRECTLY_AFFECTED"] - commands: - - cleanup() { if [ "${BUILDKITE_PULL_REQUEST}" = "false" ]; then ./ci/travis/upload_build_info.sh; fi }; trap cleanup EXIT - - RLLIB_TESTING=1 PYTHON=3.7 ./ci/travis/install-dependencies.sh - # Because Python version changed, we need to re-install Ray here - - rm -rf ./python/ray/thirdparty_files; rm -rf ./python/ray/pickle5_files; ./ci/travis/ci.sh build - - bazel test --config=ci $(./scripts/bazel_export_options) - --build_tests_only - --test_tag_filters=quick_train,-flaky,-multi_gpu - --test_env=RAY_USE_MULTIPROCESSING_CPU_COUNT=1 - rllib/... - -- label: ":brain: RLlib: Trainer Tests" - conditions: ["RAY_CI_RLLIB_DIRECTLY_AFFECTED"] - commands: - - cleanup() { if [ "${BUILDKITE_PULL_REQUEST}" = "false" ]; then ./ci/travis/upload_build_info.sh; fi }; trap cleanup EXIT - - RLLIB_TESTING=1 PYTHON=3.7 ./ci/travis/install-dependencies.sh - # Because Python version changed, we need to re-install Ray here - - rm -rf ./python/ray/thirdparty_files; rm -rf ./python/ray/pickle5_files; ./ci/travis/ci.sh build - # Test all tests in the `agents` (soon to be "trainers") dir: - - bazel test --config=ci $(./scripts/bazel_export_options) - --build_tests_only - --test_tag_filters=trainers_dir,-flaky,-multi_gpu - --test_env=RAY_USE_MULTIPROCESSING_CPU_COUNT=1 - rllib/... - -- label: ":brain: RLlib: Everything else (env-, evaluation-, ... dirs)" - conditions: ["RAY_CI_RLLIB_DIRECTLY_AFFECTED"] - commands: - - cleanup() { if [ "${BUILDKITE_PULL_REQUEST}" = "false" ]; then ./ci/travis/upload_build_info.sh; fi }; trap cleanup EXIT - - RLLIB_TESTING=1 PYTHON=3.7 ./ci/travis/install-dependencies.sh - # Because Python version changed, we need to re-install Ray here - - rm -rf ./python/ray/thirdparty_files; rm -rf ./python/ray/pickle5_files; ./ci/travis/ci.sh build - # Test everything that does not have any of the "main" labels: - # "learning_tests|quick_train|examples|tests_dir". - - bazel test --config=ci $(./scripts/bazel_export_options) - --build_tests_only - --test_tag_filters=-learning_tests,-quick_train,-examples,-tests_dir,-trainers_dir,-documentation,-flaky,-multi_gpu - --test_env=RAY_USE_MULTIPROCESSING_CPU_COUNT=1 - rllib/... - -- label: ":brain: RLlib: Examples {A/B}" - conditions: ["RAY_CI_RLLIB_AFFECTED"] - commands: - - cleanup() { if [ "${BUILDKITE_PULL_REQUEST}" = "false" ]; then ./ci/travis/upload_build_info.sh; fi }; trap cleanup EXIT - - RLLIB_TESTING=1 PYTHON=3.7 ./ci/travis/install-dependencies.sh - # Because Python version changed, we need to re-install Ray here - - rm -rf ./python/ray/thirdparty_files; rm -rf ./python/ray/pickle5_files; ./ci/travis/ci.sh build - - bazel test --config=ci $(./scripts/bazel_export_options) --build_tests_only - --test_tag_filters=examples_A,examples_B,-flaky,-multi_gpu --test_env=RAY_USE_MULTIPROCESSING_CPU_COUNT=1 rllib/... - -- label: ":brain: RLlib: Examples {Ca..t}" - conditions: ["RAY_CI_RLLIB_AFFECTED"] - commands: - - cleanup() { if [ "${BUILDKITE_PULL_REQUEST}" = "false" ]; then ./ci/travis/upload_build_info.sh; fi }; trap cleanup EXIT - - RLLIB_TESTING=1 PYTHON=3.7 ./ci/travis/install-dependencies.sh - # Because Python version changed, we need to re-install Ray here - - rm -rf ./python/ray/thirdparty_files; rm -rf ./python/ray/pickle5_files; ./ci/travis/ci.sh build - - bazel test --config=ci $(./scripts/bazel_export_options) --build_tests_only - --test_tag_filters=examples_C_AtoT,-flaky,-multi_gpu --test_env=RAY_USE_MULTIPROCESSING_CPU_COUNT=1 rllib/... -- label: ":brain: RLlib: Examples {Cu..z}" - conditions: ["RAY_CI_RLLIB_AFFECTED"] - commands: - - cleanup() { if [ "${BUILDKITE_PULL_REQUEST}" = "false" ]; then ./ci/travis/upload_build_info.sh; fi }; trap cleanup EXIT - - RLLIB_TESTING=1 PYTHON=3.7 ./ci/travis/install-dependencies.sh - # Because Python version changed, we need to re-install Ray here - - rm -rf ./python/ray/thirdparty_files; rm -rf ./python/ray/pickle5_files; ./ci/travis/ci.sh build - - bazel test --config=ci $(./scripts/bazel_export_options) --build_tests_only - --test_tag_filters=examples_C_UtoZ,-flaky,-multi_gpu --test_env=RAY_USE_MULTIPROCESSING_CPU_COUNT=1 rllib/... - -- label: ":brain: RLlib: Examples {D/P}" - conditions: ["RAY_CI_RLLIB_AFFECTED"] - commands: - - cleanup() { if [ "${BUILDKITE_PULL_REQUEST}" = "false" ]; then ./ci/travis/upload_build_info.sh; fi }; trap cleanup EXIT - - RLLIB_TESTING=1 PYTHON=3.7 ./ci/travis/install-dependencies.sh - # Because Python version changed, we need to re-install Ray here - - rm -rf ./python/ray/thirdparty_files; rm -rf ./python/ray/pickle5_files; ./ci/travis/ci.sh build - - bazel test --config=ci $(./scripts/bazel_export_options) --build_tests_only - --test_tag_filters=examples_D,examples_E,examples_F,examples_G,examples_H,examples_I,examples_J,examples_K,examples_L,examples_M,examples_N,examples_O,examples_P,-flaky,-multi_gpu --test_env=RAY_USE_MULTIPROCESSING_CPU_COUNT=1 - rllib/... - -- label: ":brain: RLlib: Examples {Q/Z}" - conditions: ["RAY_CI_RLLIB_AFFECTED"] - commands: - - cleanup() { if [ "${BUILDKITE_PULL_REQUEST}" = "false" ]; then ./ci/travis/upload_build_info.sh; fi }; trap cleanup EXIT - - RLLIB_TESTING=1 PYTHON=3.7 ./ci/travis/install-dependencies.sh - # Because Python version changed, we need to re-install Ray here - - rm -rf ./python/ray/thirdparty_files; rm -rf ./python/ray/pickle5_files; ./ci/travis/ci.sh build - - bazel test --config=ci $(./scripts/bazel_export_options) --build_tests_only - --test_tag_filters=examples_Q,examples_R,examples_S,examples_T,examples_U,examples_V,examples_W,examples_X,examples_Y,examples_Z,-flaky,-multi_gpu --test_env=RAY_USE_MULTIPROCESSING_CPU_COUNT=1 - rllib/... - -- label: ":brain: RLlib: tests/ dir (A-L)" - conditions: ["RAY_CI_RLLIB_DIRECTLY_AFFECTED"] - commands: - - cleanup() { if [ "${BUILDKITE_PULL_REQUEST}" = "false" ]; then ./ci/travis/upload_build_info.sh; fi }; trap cleanup EXIT - - RLLIB_TESTING=1 PYTHON=3.7 ./ci/travis/install-dependencies.sh - # Because Python version changed, we need to re-install Ray here - - rm -rf ./python/ray/thirdparty_files; rm -rf ./python/ray/pickle5_files; ./ci/travis/ci.sh build - - bazel test --config=ci $(./scripts/bazel_export_options) --build_tests_only - --test_tag_filters=tests_dir_A,tests_dir_B,tests_dir_C,tests_dir_D,tests_dir_E,tests_dir_F,tests_dir_G,tests_dir_H,tests_dir_I,tests_dir_J,tests_dir_K,tests_dir_L,-flaky --test_env=RAY_USE_MULTIPROCESSING_CPU_COUNT=1 - rllib/... -- label: ":brain: RLlib: tests/ dir (M-Z (no R))" - conditions: ["RAY_CI_RLLIB_DIRECTLY_AFFECTED"] - commands: - - cleanup() { if [ "${BUILDKITE_PULL_REQUEST}" = "false" ]; then ./ci/travis/upload_build_info.sh; fi }; trap cleanup EXIT - - RLLIB_TESTING=1 PYTHON=3.7 ./ci/travis/install-dependencies.sh - # Because Python version changed, we need to re-install Ray here - - rm -rf ./python/ray/thirdparty_files; rm -rf ./python/ray/pickle5_files; ./ci/travis/ci.sh build - - bazel test --config=ci $(./scripts/bazel_export_options) --build_tests_only - --test_tag_filters=tests_dir_M,tests_dir_N,tests_dir_O,tests_dir_P,tests_dir_Q,tests_dir_S,tests_dir_T,tests_dir_U,tests_dir_V,tests_dir_W,tests_dir_X,tests_dir_Y,tests_dir_Z,-flaky,-multi_gpu --test_env=RAY_USE_MULTIPROCESSING_CPU_COUNT=1 - rllib/... -- label: ":brain: RLlib: tests/ dir (R)" - conditions: ["RAY_CI_RLLIB_DIRECTLY_AFFECTED"] - commands: - - cleanup() { if [ "${BUILDKITE_PULL_REQUEST}" = "false" ]; then ./ci/travis/upload_build_info.sh; fi }; trap cleanup EXIT - - RLLIB_TESTING=1 PYTHON=3.7 ./ci/travis/install-dependencies.sh - # Because Python version changed, we need to re-install Ray here - - rm -rf ./python/ray/thirdparty_files; rm -rf ./python/ray/pickle5_files; ./ci/travis/ci.sh build - - bazel test --config=ci $(./scripts/bazel_export_options) --build_tests_only - --test_tag_filters=tests_dir_R,-flaky,-multi_gpu --test_env=RAY_USE_MULTIPROCESSING_CPU_COUNT=1 - rllib/... - -- label: ":brain: RLlib: Documentation code/examples" - conditions: ["RAY_CI_RLLIB_DIRECTLY_AFFECTED"] - commands: - - cleanup() { if [ "${BUILDKITE_PULL_REQUEST}" = "false" ]; then ./ci/travis/upload_build_info.sh; fi }; trap cleanup EXIT - - RLLIB_TESTING=1 PYTHON=3.7 ./ci/travis/install-dependencies.sh - # Because Python version changed, we need to re-install Ray here - - rm -rf ./python/ray/thirdparty_files; rm -rf ./python/ray/pickle5_files; ./ci/travis/ci.sh build - - bazel test --config=ci $(./scripts/bazel_export_options) --build_tests_only - --test_tag_filters=documentation,-flaky --test_env=RAY_USE_MULTIPROCESSING_CPU_COUNT=1 - rllib/... - -- label: ":octopus: Tune tests {A-R; no RLlib}" - conditions: ["RAY_CI_TUNE_AFFECTED"] - commands: - - cleanup() { if [ "${BUILDKITE_PULL_REQUEST}" = "false" ]; then ./ci/travis/upload_build_info.sh; fi }; trap cleanup EXIT - - TUNE_TESTING=1 PYTHON=3.7 ./ci/travis/install-dependencies.sh - # Because Python version changed, we need to re-install Ray here - - rm -rf ./python/ray/thirdparty_files; rm -rf ./python/ray/pickle5_files; ./ci/travis/ci.sh build - - bazel test --config=ci $(./scripts/bazel_export_options) --build_tests_only - --test_tag_filters=tests_dir_A,tests_dir_B,tests_dir_C,tests_dir_D,tests_dir_E,tests_dir_F,tests_dir_G,tests_dir_H,tests_dir_I,tests_dir_J,tests_dir_K,tests_dir_L,tests_dir_M,tests_dir_N,tests_dir_O,tests_dir_P,tests_dir_Q,tests_dir_R,-example,-flaky,-py37,-soft_imports,-gpu_only,-rllib - python/ray/tune/... - -- label: ":octopus: Tune tests {S-Z; no RLlib}" - conditions: ["RAY_CI_TUNE_AFFECTED"] - commands: - - cleanup() { if [ "${BUILDKITE_PULL_REQUEST}" = "false" ]; then ./ci/travis/upload_build_info.sh; fi }; trap cleanup EXIT - - TUNE_TESTING=1 PYTHON=3.7 ./ci/travis/install-dependencies.sh - # Because Python version changed, we need to re-install Ray here - - rm -rf ./python/ray/thirdparty_files; rm -rf ./python/ray/pickle5_files; ./ci/travis/ci.sh build - - bazel test --config=ci $(./scripts/bazel_export_options) --build_tests_only - --test_tag_filters=tests_dir_S,tests_dir_T,tests_dir_U,tests_dir_V,tests_dir_W,tests_dir_X,tests_dir_Y,tests_dir_Z,-example,-flaky,-py37,-soft_imports,-gpu_only,-rllib - python/ray/tune/... - -- label: ":octopus: Tune examples {w/o tf/pytorch; no RLlib}" - conditions: ["RAY_CI_TUNE_AFFECTED"] - commands: - - cleanup() { if [ "${BUILDKITE_PULL_REQUEST}" = "false" ]; then ./ci/travis/upload_build_info.sh; fi }; trap cleanup EXIT - - TUNE_TESTING=1 PYTHON=3.7 ./ci/travis/install-dependencies.sh - # Because Python version changed, we need to re-install Ray here - - rm -rf ./python/ray/thirdparty_files; rm -rf ./python/ray/pickle5_files; ./ci/travis/ci.sh build - - bazel test --config=ci $(./scripts/bazel_export_options) --build_tests_only --test_tag_filters=example,-tf,-pytorch,-py37,-flaky,-soft_imports,-gpu_only,-rllib python/ray/tune/... - -- label: ":octopus: Tune examples {w/ tf/pytorch; no RLlib}" - conditions: ["RAY_CI_TUNE_AFFECTED"] - commands: - - cleanup() { if [ "${BUILDKITE_PULL_REQUEST}" = "false" ]; then ./ci/travis/upload_build_info.sh; fi }; trap cleanup EXIT - - TUNE_TESTING=1 PYTHON=3.7 ./ci/travis/install-dependencies.sh - # Because Python version changed, we need to re-install Ray here - - rm -rf ./python/ray/thirdparty_files; rm -rf ./python/ray/pickle5_files; ./ci/travis/ci.sh build - - bazel test --config=ci $(./scripts/bazel_export_options) --build_tests_only --test_tag_filters=tf,-pytorch,-py37,-flaky,-soft_imports,-gpu_only,-rllib python/ray/tune/... - - bazel test --config=ci $(./scripts/bazel_export_options) --build_tests_only --test_tag_filters=-tf,pytorch,-py37,-flaky,-soft_imports,-gpu_only,-rllib python/ray/tune/... - -- label: ":octopus: :brain: Tune tests and examples {using RLlib}" - conditions: ["RAY_CI_TUNE_AFFECTED", "RAY_CI_RLLIB_AFFECTED"] - commands: - - cleanup() { if [ "${BUILDKITE_PULL_REQUEST}" = "false" ]; then ./ci/travis/upload_build_info.sh; fi }; trap cleanup EXIT - - TUNE_TESTING=1 PYTHON=3.7 ./ci/travis/install-dependencies.sh - # Because Python version changed, we need to re-install Ray here - - rm -rf ./python/ray/thirdparty_files; rm -rf ./python/ray/pickle5_files; ./ci/travis/ci.sh build - - bazel test --config=ci $(./scripts/bazel_export_options) --build_tests_only --test_tag_filters=-gpu_only,rllib python/ray/tune/... - - label: ":octopus: Tune soft imports test" conditions: ["RAY_CI_TUNE_AFFECTED"] commands: @@ -690,90 +365,3 @@ # no TUNE_TESTING=1 on purpose - ./ci/travis/install-dependencies.sh - bazel test --config=ci $(./scripts/bazel_export_options) --build_tests_only --test_tag_filters=soft_imports python/ray/tune/... - -- label: ":steam_locomotive: Train tests and examples" - conditions: ["RAY_CI_TRAIN_AFFECTED"] - commands: - - cleanup() { if [ "${BUILDKITE_PULL_REQUEST}" = "false" ]; then ./ci/travis/upload_build_info.sh; fi }; trap cleanup EXIT - - TRAIN_TESTING=1 INSTALL_HOROVOD=1 ./ci/travis/install-dependencies.sh - - bazel test --config=ci $(./scripts/bazel_export_options) --build_tests_only --test_tag_filters=-gpu_only,-tune python/ray/train/... - -- label: ":steam_locomotive: :octopus: Train + Tune tests and examples" - conditions: ["RAY_CI_TRAIN_AFFECTED"] - commands: - - cleanup() { if [ "${BUILDKITE_PULL_REQUEST}" = "false" ]; then ./ci/travis/upload_build_info.sh; fi }; trap cleanup EXIT - - TRAIN_TESTING=1 TUNE_TESTING=1 ./ci/travis/install-dependencies.sh - - bazel test --config=ci $(./scripts/bazel_export_options) --build_tests_only --test_tag_filters=tune,-gpu_only python/ray/train/... - -- label: ":octopus: SGD tests and examples" - conditions: ["RAY_CI_SGD_AFFECTED"] - commands: - - cleanup() { if [ "${BUILDKITE_PULL_REQUEST}" = "false" ]; then ./ci/travis/upload_build_info.sh; fi }; trap cleanup EXIT - - SGD_TESTING=1 INSTALL_HOROVOD=1 ./ci/travis/install-dependencies.sh - - bazel test --config=ci $(./scripts/bazel_export_options) --build_tests_only --test_tag_filters=tf,-pytorch,-py37,-flaky,-client,-gpu_only python/ray/util/sgd/... - - bazel test --config=ci $(./scripts/bazel_export_options) --build_tests_only --test_tag_filters=-tf,pytorch,-py37,-flaky,-client,-gpu_only python/ray/util/sgd/... - - bazel test --config=ci $(./scripts/bazel_export_options) --build_tests_only --test_tag_filters=client_unit_tests,-gpu_only --test_env=RAY_CLIENT_MODE=1 python/ray/util/sgd/... - -- label: ":octopus: Tune/SGD/Modin/Dask tests and examples. Python 3.7" - conditions: ["RAY_CI_TUNE_AFFECTED", "RAY_CI_SGD_AFFECTED"] - commands: - - cleanup() { if [ "${BUILDKITE_PULL_REQUEST}" = "false" ]; then ./ci/travis/upload_build_info.sh; fi }; trap cleanup EXIT - - TUNE_TESTING=1 PYTHON=3.7 INSTALL_HOROVOD=1 ./ci/travis/install-dependencies.sh - # Because Python version changed, we need to re-install Ray here - - rm -rf ./python/ray/thirdparty_files; rm -rf ./python/ray/pickle5_files; ./ci/travis/ci.sh build - - bazel test --config=ci $(./scripts/bazel_export_options) --build_tests_only --test_tag_filters=py37,-flaky,-client python/ray/tune/... - - bazel test --config=ci $(./scripts/bazel_export_options) --build_tests_only --test_tag_filters=-client python/ray/util/xgboost/... - - bazel test --config=ci $(./scripts/bazel_export_options) --build_tests_only python/ray/util/horovod/... - - bazel test --config=ci $(./scripts/bazel_export_options) --build_tests_only python/ray/util/ray_lightning/... - -- label: ":octopus: Ludwig tests and examples. Python 3.7" - conditions: ["RAY_CI_TUNE_AFFECTED", "RAY_CI_SGD_AFFECTED"] - commands: - - cleanup() { if [ "${BUILDKITE_PULL_REQUEST}" = "false" ]; then ./ci/travis/upload_build_info.sh; fi }; trap cleanup EXIT - - SGD_TESTING=1 PYTHON=3.7 INSTALL_LUDWIG=1 INSTALL_HOROVOD=1 ./ci/travis/install-dependencies.sh - # Because Python version changed, we need to re-install Ray here - - rm -rf ./python/ray/thirdparty_files; rm -rf ./python/ray/pickle5_files; ./ci/travis/ci.sh build - - bazel test --config=ci $(./scripts/bazel_export_options) --build_tests_only python/ray/tests/ludwig/... - -- label: ":tropical_fish: ML Libraries w/ Ray Client Examples (Python 3.7)." - conditions: ["RAY_CI_TUNE_AFFECTED", "RAY_CI_SGD_AFFECTED"] - commands: - - cleanup() { if [ "${BUILDKITE_PULL_REQUEST}" = "false" ]; then ./ci/travis/upload_build_info.sh; fi }; trap cleanup EXIT - - TUNE_TESTING=1 PYTHON=3.7 INSTALL_HOROVOD=1 ./ci/travis/install-dependencies.sh - - rm -rf ./python/ray/thirdparty_files; rm -rf ./python/ray/pickle5_files; ./ci/travis/ci.sh build - - bazel test --config=ci $(./scripts/bazel_export_options) --build_tests_only --test_tag_filters=client --test_env=RAY_CLIENT_MODE=1 python/ray/util/dask/... - - bazel test --config=ci $(./scripts/bazel_export_options) --build_tests_only --test_tag_filters=client,-flaky python/ray/tune/... - - bazel test --config=ci $(./scripts/bazel_export_options) --build_tests_only --test_tag_filters=client,-client_unit_tests python/ray/util/sgd/... - - bazel test --config=ci $(./scripts/bazel_export_options) --build_tests_only --test_tag_filters=client,-flaky python/ray/util/xgboost/... - -- label: ":potable_water: Modin/Dask tests and examples. Python 3.7" - conditions: ["RAY_CI_PYTHON_AFFECTED"] - commands: - - cleanup() { if [ "${BUILDKITE_PULL_REQUEST}" = "false" ]; then ./ci/travis/upload_build_info.sh; fi }; trap cleanup EXIT - - DATA_PROCESSING_TESTING=1 PYTHON=3.7 ./ci/travis/install-dependencies.sh - # Because Python version changed, we need to re-install Ray here - - rm -rf ./python/ray/thirdparty_files; rm -rf ./python/ray/pickle5_files; ./ci/travis/ci.sh build - - bazel test --config=ci $(./scripts/bazel_export_options) --build_tests_only python/ray/tests/modin/... - # Dask tests and examples. - - bazel test --config=ci $(./scripts/bazel_export_options) --build_tests_only --test_tag_filters=-client python/ray/util/dask/... - -- label: ":potable_water: Workflow & Dataset tests (Python 3.7)" - conditions: ["RAY_CI_PYTHON_AFFECTED"] - commands: - - cleanup() { if [ "${BUILDKITE_PULL_REQUEST}" = "false" ]; then ./ci/travis/upload_build_info.sh; fi }; trap cleanup EXIT - - DATA_PROCESSING_TESTING=1 PYTHON=3.7 ./ci/travis/install-dependencies.sh - # Because Python version changed, we need to re-install Ray here - - rm -rf ./python/ray/thirdparty_files; rm -rf ./python/ray/pickle5_files; ./ci/travis/ci.sh build - - bazel test --config=ci $(./scripts/bazel_export_options) --build_tests_only python/ray/workflow/... python/ray/data/... - -- label: ":book: Doc tests and examples" - conditions: - ["RAY_CI_PYTHON_AFFECTED", "RAY_CI_TUNE_AFFECTED", "RAY_CI_DOC_AFFECTED"] - commands: - - cleanup() { if [ "${BUILDKITE_PULL_REQUEST}" = "false" ]; then ./ci/travis/upload_build_info.sh; fi }; trap cleanup EXIT - - DOC_TESTING=1 PYTHON=3.7 ./ci/travis/install-dependencies.sh - # Because Python version changed, we need to re-install Ray here - - rm -rf ./python/ray/thirdparty_files; rm -rf ./python/ray/pickle5_files; ./ci/travis/ci.sh build - - bazel test --config=ci $(./scripts/bazel_export_options) --build_tests_only --test_tag_filters=-tf,-pytorch,-py37,-post_wheel_build doc/... - - bazel test --config=ci $(./scripts/bazel_export_options) --build_tests_only --test_tag_filters=tf,-pytorch,-py37,-post_wheel_build doc/... - - bazel test --config=ci $(./scripts/bazel_export_options) --build_tests_only --test_tag_filters=-tf,pytorch,-py37,-post_wheel_build doc/... diff --git a/ci/travis/install-dependencies.sh b/ci/travis/install-dependencies.sh index b7d0cae62..c5094ca2a 100755 --- a/ci/travis/install-dependencies.sh +++ b/ci/travis/install-dependencies.sh @@ -275,6 +275,9 @@ install_toolchains() { } download_mnist() { + if [ -d "${HOME}/data/MNIST" ]; then + return + fi mkdir -p "${HOME}/data" curl -o "${HOME}/data/mnist.zip" https://ray-ci-mnist.s3-us-west-2.amazonaws.com/mnist.zip unzip "${HOME}/data/mnist.zip" -d "${HOME}/data"