Flaky test builder for tests tagged "flaky" (#15408)

2025-03-05 18:11:42 -05:00 · 2021-04-20 00:19:07 -07:00 · 2021-04-20 00:19:07 -07:00 · a482034916
commit a482034916
parent 7ff27dfe07
4 changed files with 56 additions and 11 deletions
--- a/.buildkite/pipeline.yml
+++ b/.buildkite/pipeline.yml
@ -60,6 +60,7 @@
    - cleanup() { if [ "${BUILDKITE_PULL_REQUEST}" = "false" ]; then ./ci/travis/upload_build_info.sh; fi }; trap cleanup EXIT
    - bazel test --config=ci $(./scripts/bazel_export_options)
      --build_tests_only
+      --test_tag_filters=-flaky
      -- //:all -rllib/... -core_worker_test

 - label: ":cpp: Tests (ASAN)"
@ -69,6 +70,7 @@
      --build_tests_only
      --config=asan-buildkite
      --jobs=2
+      --test_tag_filters=-flaky
      -- //:all -//:core_worker_test

 - label: ":serverless: Dashboard + Serve Tests"
@ -85,12 +87,27 @@
    - bazel test --config=ci $(./scripts/bazel_export_options) python/ray/new_dashboard/...
    - bazel test --config=ci $(./scripts/bazel_export_options) python/ray/serve/...

+- label: ":python: (Flaky tests)"
+  conditions: ["RAY_CI_PYTHON_AFFECTED"]
+  commands:
+    - bazel test --config=ci $(./scripts/bazel_export_options)
+      --build_tests_only
+      --test_tag_filters=flaky
+      -- //:all -rllib/... -core_worker_test
+    - bazel test --config=ci $(./scripts/bazel_export_options)
+      --test_tag_filters=-kubernetes,-jenkins_only,flaky
+      --test_env=CONDA_EXE
+      --test_env=CONDA_PYTHON_EXE
+      --test_env=CONDA_SHLVL
+      --test_env=CONDA_PREFIX
+      --test_env=CONDA_DEFAULT_ENV
+      python/ray/tests/...
 - label: ":python: (Small & Large)"
  conditions: ["RAY_CI_PYTHON_AFFECTED"]
  commands:
    - cleanup() { if [ "${BUILDKITE_PULL_REQUEST}" = "false" ]; then ./ci/travis/upload_build_info.sh; fi }; trap cleanup EXIT
    - bazel test --config=ci $(./scripts/bazel_export_options)
-      --test_tag_filters=-kubernetes,-jenkins_only,-medium_size_python_tests_a_to_j,-medium_size_python_tests_k_to_z
+      --test_tag_filters=-kubernetes,-jenkins_only,-medium_size_python_tests_a_to_j,-medium_size_python_tests_k_to_z,-flaky
      --test_env=CONDA_EXE
      --test_env=CONDA_PYTHON_EXE
      --test_env=CONDA_SHLVL
@ -106,14 +123,14 @@
  commands:
    - cleanup() { if [ "${BUILDKITE_PULL_REQUEST}" = "false" ]; then ./ci/travis/upload_build_info.sh; fi }; trap cleanup EXIT
    - bazel test --config=ci $(./scripts/bazel_export_options)
-      --test_tag_filters=-kubernetes,-jenkins_only,medium_size_python_tests_a_to_j
+      --test_tag_filters=-kubernetes,-jenkins_only,medium_size_python_tests_a_to_j,-flaky
      python/ray/tests/...
 - label: ":python: (Medium K-Z)"
  conditions: ["RAY_CI_PYTHON_AFFECTED"]
  commands:
    - cleanup() { if [ "${BUILDKITE_PULL_REQUEST}" = "false" ]; then ./ci/travis/upload_build_info.sh; fi }; trap cleanup EXIT
    - bazel test --config=ci $(./scripts/bazel_export_options)
-      --test_tag_filters=-kubernetes,-jenkins_only,medium_size_python_tests_k_to_z
+      --test_tag_filters=-kubernetes,-jenkins_only,medium_size_python_tests_k_to_z,-flaky
      python/ray/tests/...

 - label: ":brain: RLlib: Learning tests (from rllib/tuned_examples/*.yaml)"
--- a/.travis.yml
+++ b/.travis.yml
@ -40,7 +40,7 @@ matrix:
        # cc bazel tests (w/o RLlib)
        # NOTE: core_worker_test is out-of-date and should already covered by Python
        # tests.
-        - ./ci/suppress_output bazel test --config=ci $(./scripts/bazel_export_options) --build_tests_only -- //:all -rllib/... -core_worker_test
+        - ./ci/suppress_output bazel test --config=ci $(./scripts/bazel_export_options) --build_tests_only --test_tag_filters=-flaky -- //:all -rllib/... -core_worker_test

        # ray serve tests
        - if [ $RAY_CI_SERVE_AFFECTED == "1" ]; then ./ci/keep_alive bazel test --config=ci $(./scripts/bazel_export_options) --test_tag_filters=-jenkins_only python/ray/serve/...; fi
@ -49,7 +49,7 @@ matrix:
        - if [ "$RAY_CI_DASHBOARD_AFFECTED" == "1" ]; then ./ci/keep_alive bazel test python/ray/new_dashboard/...; fi

        # bazel python tests. This should be run last to keep its logs at the end of travis logs.
-        - if [ $RAY_CI_PYTHON_AFFECTED == "1" ]; then ./ci/keep_alive bazel test --test_env=CONDA_EXE --test_env=CONDA_PYTHON_EXE --test_env=CONDA_SHLVL --test_env=CONDA_PREFIX --test_env=CONDA_DEFAULT_ENV --test_env=CONDA_PROMPT_MODIFIER --config=ci $(./scripts/bazel_export_options) --test_tag_filters=-kubernetes,-jenkins_only,-medium_size_python_tests_a_to_j,-medium_size_python_tests_k_to_z python/ray/tests/...; fi
+        - if [ $RAY_CI_PYTHON_AFFECTED == "1" ]; then ./ci/keep_alive bazel test --test_env=CONDA_EXE --test_env=CONDA_PYTHON_EXE --test_env=CONDA_SHLVL --test_env=CONDA_PREFIX --test_env=CONDA_DEFAULT_ENV --test_env=CONDA_PROMPT_MODIFIER --config=ci $(./scripts/bazel_export_options) --test_tag_filters=-kubernetes,-jenkins_only,-medium_size_python_tests_a_to_j,-medium_size_python_tests_k_to_z,-flaky python/ray/tests/...; fi
        # NO MORE TESTS BELOW, keep them above.

    - os: osx
@ -66,7 +66,7 @@ matrix:
        - . ./ci/travis/ci.sh build
      script:
        # bazel python tests for medium size tests. Used for parallelization.
-        - if [ $RAY_CI_PYTHON_AFFECTED == "1" ]; then ./ci/keep_alive bazel test --config=ci $(./scripts/bazel_export_options) --test_tag_filters=-kubernetes,-jenkins_only,medium_size_python_tests_a_to_j python/ray/tests/...; fi
+        - if [ $RAY_CI_PYTHON_AFFECTED == "1" ]; then ./ci/keep_alive bazel test --config=ci $(./scripts/bazel_export_options) --test_tag_filters=-kubernetes,-jenkins_only,medium_size_python_tests_a_to_j,-flaky python/ray/tests/...; fi

    - os: osx
      osx_image: xcode7
@ -82,7 +82,7 @@ matrix:
        - . ./ci/travis/ci.sh build
      script:
        # bazel python tests for medium size tests. Used for parallelization.
-        - if [ $RAY_CI_PYTHON_AFFECTED == "1" ]; then ./ci/keep_alive bazel test --config=ci $(./scripts/bazel_export_options) --test_tag_filters=-kubernetes,-jenkins_only,medium_size_python_tests_k_to_z python/ray/tests/...; fi
+        - if [ $RAY_CI_PYTHON_AFFECTED == "1" ]; then ./ci/keep_alive bazel test --config=ci $(./scripts/bazel_export_options) --test_tag_filters=-kubernetes,-jenkins_only,medium_size_python_tests_k_to_z,-flaky python/ray/tests/...; fi

    # Build MacOS wheels and MacOS jars
    - os: osx
--- a/BUILD.bazel
+++ b/BUILD.bazel
@ -1370,6 +1370,7 @@ cc_test(
        "//:redis-cli",
        "//:redis-server",
    ],
+    tags = ["flaky"],
    deps = [
        ":gcs_server_lib",
        ":gcs_test_util_lib",
--- a/python/ray/tests/BUILD
+++ b/python/ray/tests/BUILD
@ -12,7 +12,6 @@ py_test_module_list(
  files = [
 #    "test_dynres.py",  # dyn res not implemented
    "test_async.py",
-    "test_actor.py",
    "test_actor_advanced.py",
    "test_actor_failures.py",
    "test_actor_resources.py",
@ -45,6 +44,16 @@ py_test_module_list(
  deps = ["//:ray_lib"],
 )

+py_test_module_list(
+  files = [
+    "test_actor.py",
+  ],
+  size = "medium",
+  extra_srcs = SRCS,
+  tags = ["exclusive", "medium_size_python_tests_a_to_j", "flaky"],
+  deps = ["//:ray_lib"],
+)
+
 py_test_module_list(
  files = [
    "test_memory_scheduling.py",
@ -58,11 +67,9 @@ py_test_module_list(
    "test_multiprocessing.py",
    "test_object_manager.py",
    "test_object_spilling.py",
-    "test_object_spilling_2.py",
    "test_output.py",
    "test_reconstruction.py",
    "test_reference_counting.py",
-    "test_reference_counting_2.py",
    "test_resource_demand_scheduler.py",
    "test_scheduling.py",
    "test_serialization.py",
@ -76,6 +83,17 @@ py_test_module_list(
  deps = ["//:ray_lib"],
 )

+py_test_module_list(
+  files = [
+    "test_object_spilling_2.py",
+    "test_reference_counting_2.py",
+  ],
+  size = "medium",
+  extra_srcs = SRCS,
+  tags = ["exclusive", "medium_size_python_tests_k_to_z", "flaky"],
+  deps = ["//:ray_lib"],
+)
+
 py_test_module_list(
  files = [
    "test_actor_pool.py",
@ -120,7 +138,6 @@ py_test_module_list(
  files = [
    "test_cli.py",
    "test_failure.py",
-    "test_failure_2.py",
    "test_stress_failure.py",
  ],
  size = "large",
@ -129,6 +146,16 @@ py_test_module_list(
  deps = ["//:ray_lib"],
 )

+py_test_module_list(
+  files = [
+    "test_failure_2.py",
+  ],
+  size = "large",
+  extra_srcs = SRCS,
+  tags = ["exclusive", "flaky"],
+  deps = ["//:ray_lib"],
+)
+
 py_test_module_list(
  files = [
    "test_placement_group.py",