[Test] Shard "Small & Large" tests (#21351)

2025-03-05 10:01:43 -05:00 · 2022-01-05 10:49:14 -08:00 · 2022-01-05 10:49:14 -08:00 · 24da654d90
commit 24da654d90
parent 853d10871c
5 changed files with 66 additions and 53 deletions
--- a/.buildkite/pipeline.macos.yml
+++ b/.buildkite/pipeline.macos.yml
@ -79,15 +79,23 @@ steps:
    - cleanup() { if [ "${BUILDKITE_PULL_REQUEST}" = "false" ]; then ./ci/travis/upload_build_info.sh; fi }; trap cleanup EXIT
    - ./ci/travis/ci.sh test_cpp

- label: ":mac: :apple: Small and Large"
+- label: ":mac: :apple: Small & Client"
  <<: *common
  commands:
    - *prelude_commands
    - bazel test $(./scripts/bazel_export_options) --config=ci
      --test_env=CONDA_EXE --test_env=CONDA_PYTHON_EXE --test_env=CONDA_SHLVL --test_env=CONDA_PREFIX
      --test_env=CONDA_DEFAULT_ENV --test_env=CONDA_PROMPT_MODIFIER --test_env=CI
-      --test_tag_filters=-kubernetes,-medium_size_python_tests_a_to_j,-medium_size_python_tests_k_to_z
-      python/ray/tests/...
+      --test_tag_filters=client_tests,small_size_python_tests
+      -- python/ray/tests/...
+    - *epilogue_commands
+
+- label: ":mac: :apple: Large"
+  <<: *common
+  parallelism: 3
+  commands:
+    - *prelude_commands
+    - . ./ci/travis/ci.sh test_large
    - *epilogue_commands

 - label: ":mac: :apple: Medium A-J"
--- a/.buildkite/pipeline.yml
+++ b/.buildkite/pipeline.yml
@ -246,22 +246,19 @@
    - bazel test --test_output=streamed --config=ci --test_env=RAY_MINIMAL=1 $(./scripts/bazel_export_options)
      python/ray/tests/test_serve_ray_minimal

- label: ":python: (Small & Large)"
+- label: ":python: (Small & Client)"
  conditions: ["RAY_CI_PYTHON_AFFECTED"]
  commands:
    - cleanup() { if [ "${BUILDKITE_PULL_REQUEST}" = "false" ]; then ./ci/travis/upload_build_info.sh; fi }; trap cleanup EXIT
    - bazel test --config=ci $(./scripts/bazel_export_options)
-      --test_tag_filters=-kubernetes,-medium_size_python_tests_a_to_j,-medium_size_python_tests_k_to_z,-client_tests,-post_wheel_build,-worker-container
-      --test_env=CONDA_EXE
-      --test_env=CONDA_PYTHON_EXE
-      --test_env=CONDA_SHLVL
-      --test_env=CONDA_PREFIX
-      --test_env=CONDA_DEFAULT_ENV
-      python/ray/tests/...
-    - bazel test --config=ci $(./scripts/bazel_export_options)
-      --test_tag_filters=-kubernetes,client_tests
-      --test_env=RAY_CLIENT_MODE=1 --test_env=RAY_PROFILING=1
-      python/ray/tests/...
+      --test_tag_filters=client_tests,small_size_python_tests
+      -- python/ray/tests/...
+- label: ":python: (Large)"
+  conditions: ["RAY_CI_PYTHON_AFFECTED"]
+  parallelism: 3
+  commands:
+    - cleanup() { if [ "${BUILDKITE_PULL_REQUEST}" = "false" ]; then ./ci/travis/upload_build_info.sh; fi }; trap cleanup EXIT
+    - . ./ci/travis/ci.sh test_large
 - label: ":python: (Medium A-J)"
  conditions: ["RAY_CI_PYTHON_AFFECTED"]
  commands:
@ -340,31 +337,23 @@
    #  --test_tag_filters=team:serve
    #  --test_env=RAY_gcs_grpc_based_pubsub=1 --test_env=RAY_gcs_storage=memory
    #  release/...
- label: ":redis: HA GCS (Small & Large)"
+- label: ":redis: HA GCS (Small & Client)"
  conditions: ["RAY_CI_PYTHON_AFFECTED"]
  commands:
    - cleanup() { if [ "${BUILDKITE_PULL_REQUEST}" = "false" ]; then ./ci/travis/upload_build_info.sh; fi }; trap cleanup EXIT
    - bazel test --config=ci $(./scripts/bazel_export_options)
-      --test_tag_filters=-kubernetes,-medium_size_python_tests_a_to_j,-medium_size_python_tests_k_to_z,-client_tests,-post_wheel_build,-worker-container
-      --test_env=CONDA_EXE
-      --test_env=CONDA_PYTHON_EXE
-      --test_env=CONDA_SHLVL
-      --test_env=CONDA_PREFIX
-      --test_env=CONDA_DEFAULT_ENV
+      --test_tag_filters=client_tests,small_size_python_tests
      --test_env=RAY_gcs_grpc_based_pubsub=1
      --test_env=RAY_bootstrap_with_gcs=1
      --test_env=RAY_gcs_storage=memory
-      -- //python/ray/tests/...
-      -//python/ray/tests:test_failure_2
-      -//python/ray/tests:test_job
-      -//python/ray/tests:test_autoscaler_drain_node_api
-    - bazel test --config=ci $(./scripts/bazel_export_options)
-      --test_tag_filters=-kubernetes,client_tests
-      --test_env=RAY_CLIENT_MODE=1 --test_env=RAY_PROFILING=1
-      --test_env=RAY_gcs_grpc_based_pubsub=1
-      --test_env=RAY_bootstrap_with_gcs=1
-      --test_env=RAY_gcs_storage=memory
-      -- //python/ray/tests/...
+      -- python/ray/tests/...
+      -//python/ray/tests:test_autoscaler_drain_node_api -//python/ray/tests:test_client_reconnect
+- label: ":redis: HA GCS (Large)"
+  conditions: ["RAY_CI_PYTHON_AFFECTED"]
+  parallelism: 3
+  commands:
+    - cleanup() { if [ "${BUILDKITE_PULL_REQUEST}" = "false" ]; then ./ci/travis/upload_build_info.sh; fi }; trap cleanup EXIT
+    - . ./ci/travis/ci.sh test_large_gcs
 - label: ":redis: HA GCS (Medium A-J)"
  conditions: ["RAY_CI_PYTHON_AFFECTED"]
  commands:
--- a/ci/travis/bazel_export_options
+++ b/ci/travis/bazel_export_options
@ -1,4 +1,4 @@
 #!/usr/bin/env bash

 mkdir -p /tmp/bazel_event_logs
-echo "--build_event_json_file $(mktemp /tmp/bazel_event_logs/bazel_log.XXXXX)"
+echo "--build_event_json_file=$(mktemp /tmp/bazel_event_logs/bazel_log.XXXXX)"
--- a/ci/travis/ci.sh
+++ b/ci/travis/ci.sh
@ -132,6 +132,7 @@ test_core() {
  bazel test --config=ci --build_tests_only $(./scripts/bazel_export_options) -- "${args[@]}"
 }

+# For running Python tests on Windows.
 test_python() {
  local pathsep=":" args=()
  if [ "${OSTYPE}" = msys ]; then
@ -195,6 +196,23 @@ test_python() {
  fi
 }

+# For running large Python tests on Linux and MacOS.
+test_large() {
+  bazel test --config=ci "$(./scripts/bazel_export_options)" --test_env=CONDA_EXE --test_env=CONDA_PYTHON_EXE \
+      --test_env=CONDA_SHLVL --test_env=CONDA_PREFIX --test_env=CONDA_DEFAULT_ENV --test_env=CONDA_PROMPT_MODIFIER \
+      --test_env=CI --test_tag_filters="large_size_python_tests_shard_${BUILDKITE_PARALLEL_JOB}" \
+      -- python/ray/tests/...
+}
+
+# For running large Python tests on Linux and MacOS in GCS HA mode.
+test_large_gcs() {
+  bazel test --config=ci "$(./scripts/bazel_export_options)" --test_env=CONDA_EXE --test_env=CONDA_PYTHON_EXE \
+      --test_env=CONDA_SHLVL --test_env=CONDA_PREFIX --test_env=CONDA_DEFAULT_ENV --test_env=CONDA_PROMPT_MODIFIER \
+      --test_env=CI --test_tag_filters="large_size_python_tests_shard_${BUILDKITE_PARALLEL_JOB}" \
+      --test_env=RAY_gcs_grpc_based_pubsub=1 --test_env=RAY_bootstrap_with_gcs=1 --test_env=RAY_gcs_storage=memory \
+      -- python/ray/tests/... -//python/ray/tests:test_failure_2
+}
+
 test_cpp() {
  # C++ worker example need _GLIBCXX_USE_CXX11_ABI flag, but if we put the flag into .bazelrc, the linux ci can't pass.
  # So only set the flag in c++ worker example. More details: https://github.com/ray-project/ray/pull/18273
--- a/python/ray/tests/BUILD
+++ b/python/ray/tests/BUILD
@ -67,7 +67,7 @@ py_test_module_list(
  ],
  size = "medium",
  extra_srcs = SRCS,
-  tags = ["exclusive", "medium_size_python_tests_a_to_j", "team:serverless"],
+  tags = ["exclusive", "client_tests", "team:serverless"],
  deps = ["//:ray_lib"],
 )

@ -138,7 +138,7 @@ py_test_module_list(
  ],
  size = "small",
  extra_srcs = SRCS,
-  tags = ["exclusive", "team:core"],
+  tags = ["exclusive", "small_size_python_tests", "team:core"],
  deps = ["//:ray_lib"],
 )

@ -160,7 +160,7 @@ py_test_module_list(
  ],
  size = "small",
  extra_srcs = SRCS,
-  tags = ["exclusive", "team:serverless"],
+  tags = ["exclusive", "small_size_python_tests", "team:serverless"],
  deps = ["//:ray_lib"],
 )

@ -177,7 +177,7 @@ py_test_module_list(
  ],
  size = "large",
  extra_srcs = SRCS,
-  tags = ["exclusive", "team:core"],
+  tags = ["exclusive", "large_size_python_tests_shard_0", "team:core"],
  deps = ["//:ray_lib"],
 )

@ -192,7 +192,7 @@ py_test_module_list(
  ],
  size = "large",
  extra_srcs = SRCS,
-  tags = ["exclusive", "team:core"],
+  tags = ["exclusive", "large_size_python_tests_shard_1", "team:core"],
  deps = ["//:ray_lib"],
 )

@ -205,7 +205,7 @@ py_test_module_list(
  ],
  size = "large",
  extra_srcs = SRCS,
-  tags = ["exclusive", "team:serve"],
+  tags = ["exclusive", "large_size_python_tests_shard_2", "team:serve"],
  deps = ["//:ray_lib"],
 )

@ -226,7 +226,7 @@ py_test(
    name = "test_actor_group",
    size = "medium",
    srcs = SRCS + ["test_actor_group.py"],
-    tags = ["exclusive", "team:serve"],
+    tags = ["exclusive", "medium_size_python_tests_a_to_j", "team:serve"],
    deps = ["//:ray_lib"]
 )

@ -236,7 +236,7 @@ py_test(
    name = "test_autoscaler_aws",
    size = "small",
    srcs = SRCS + ["aws/test_autoscaler_aws.py"],
-    tags = ["exclusive", "team:serverless"],
+    tags = ["exclusive", "small_size_python_tests", "team:serverless"],
    deps = ["//:ray_lib"],
 )

@ -244,7 +244,7 @@ py_test(
    name = "test_aws_batch_tag_update",
    size = "small",
    srcs = SRCS + ["aws/test_aws_batch_tag_update.py"],
-    tags = ["exclusive", "team:serverless"],
+    tags = ["exclusive", "small_size_python_tests", "team:serverless"],
    deps = ["//:ray_lib"],
 )

@ -252,7 +252,7 @@ py_test(
    name = "test_gcp_node_provider",
    size = "small",
    srcs = SRCS + ["gcp/test_gcp_node_provider.py"],
-    tags = ["exclusive", "team:serverless"],
+    tags = ["exclusive", "small_size_python_tests", "team:serverless"],
    deps = ["//:ray_lib"],
 )

@ -262,7 +262,7 @@ py_test(
    name = "test_typing",
    size = "small",
    srcs = SRCS + ["test_typing.py"] + glob(["typing_files"]),
-    tags = ["exclusive", "team:core"],
+    tags = ["exclusive", "small_size_python_tests", "team:core"],
    deps = ["//:ray_lib"],
 )

@ -271,7 +271,7 @@ py_test(
  name = "test_tracing",
  size = "medium",
  srcs = SRCS + ["test_tracing.py"],
-  tags = ["exclusive", "team:serve"],
+  tags = ["exclusive", "medium_size_python_tests_k_to_z", "team:serve"],
  deps = ["//:ray_lib"]
 )

@ -279,7 +279,7 @@ py_test(
    name = "test_pydantic_serialization",
    size = "small",
    srcs = SRCS + ["test_pydantic_serialization.py", "pydantic_module.py"],
-    tags = ["exclusive", "team:serve"],
+    tags = ["exclusive", "small_size_python_tests", "team:serve"],
    deps = ["//:ray_lib"],
 )

@ -287,7 +287,7 @@ py_test(
    name = "test_runtime_env_validation",
    size = "small",
    srcs = SRCS + ["test_runtime_env_validation.py"],
-    tags = ["exclusive", "team:serve"],
+    tags = ["exclusive", "small_size_python_tests", "team:serve"],
    deps = ["//:ray_lib"],
 )

@ -295,7 +295,7 @@ py_test(
    name = "test_runtime_env_ray_minimal",
    size = "medium",
    srcs = SRCS + ["test_runtime_env_ray_minimal.py"],
-    tags = ["exclusive", "team:serve"],
+    tags = ["exclusive", "medium_size_python_tests_k_to_z", "team:serve"],
    deps = ["//:ray_lib"],
 )

@ -303,7 +303,7 @@ py_test(
    name = "test_serve_ray_minimal",
    size = "small",
    srcs = SRCS + ["test_serve_ray_minimal.py"],
-    tags = ["exclusive", "team:serve"],
+    tags = ["exclusive", "small_size_python_tests", "team:serve"],
    deps = ["//:ray_lib"],
 )

@ -325,9 +325,7 @@ py_test_module_list(
  size = "medium",
  extra_srcs = SRCS,
  name_suffix = "_client_mode",
-  # TODO(barakmich): py_test will support env in Bazel 4.0.0...
-  # Until then, we can use tags.
-  #env = {"RAY_CLIENT_MODE": "1", "RAY_PROFILING": "1"},
+  env = {"RAY_CLIENT_MODE": "1", "RAY_PROFILING": "1"},
  tags = ["exclusive", "client_tests", "team:serverless"],
  deps = ["//:ray_lib"],
 )