[Serve] Add serve failure test to CI (#20392)

.buildkite/pipeline.yml

@@ -222,6 +222,9 @@
     - bazel test --config=ci $(./scripts/bazel_export_options)
       --test_tag_filters=-post_wheel_build
       python/ray/serve/...
+    - bazel test --config=ci $(./scripts/bazel_export_options)
+      --test_tag_filters=team:serve
+      release/...
 
 - label: ":python: Minimal install"
   conditions: ["RAY_CI_PYTHON_AFFECTED"]

release/BUILD

@@ -0,0 +1,21 @@
+load("@rules_python//python:defs.bzl", "py_test")
+
+test_srcs = glob(["**/*.py"])
+
+py_test(
+    name = "serve_failure_smoke_test",
+    size = "medium",
+    srcs = test_srcs,
+    env = {
+        "IS_SMOKE_TEST": "1",
+    },
+    main = "serve_failure.py",
+    tags = [
+        "exclusive",
+        "team:serve",
+    ],
+    deps = [
+        "//:ray_lib",
+        "//python/ray/serve:serve_lib",
+    ],
+)

release/long_running_tests/workloads/serve_failure.py

@@ -24,6 +24,8 @@ NUM_NODES = 4
 # RandomTest setup constants
 CPUS_PER_NODE = 10
 
+IS_SMOKE_TEST = "IS_SMOKE_TEST" in os.environ
+
 
 def update_progress(result):
     """
@@ -54,7 +56,8 @@ ray.init(
     namespace="serve_failure_test",
     address=cluster.address,
     dashboard_host="0.0.0.0",
-    log_to_driver=True)
+    log_to_driver=True,
+)
 serve.start(detached=True)
 
 
@@ -124,7 +127,7 @@ class RandomTest:
         start_time = time.time()
         previous_time = start_time
         while True:
-            for _ in range(100):
+            for _ in range(20):
                 actions, weights = zip(*self.weighted_actions)
                 action_chosen = random.choices(actions, weights=weights)[0]
                 print(f"Executing {action_chosen}")
@@ -146,7 +149,11 @@ class RandomTest:
             previous_time = new_time
             iteration += 1
 
+            if IS_SMOKE_TEST:
+                break
 
+
+tester = RandomTest(max_deployments=NUM_NODES * CPUS_PER_NODE)
 random_killer = RandomKiller.remote()
 random_killer.run.remote()
-RandomTest(max_deployments=NUM_NODES * CPUS_PER_NODE).run()
+tester.run()