[CI] Add bazel py_test checking for Serve (#25509)

ci/ci.sh

@@ -525,7 +525,13 @@ lint_bazel() {
 lint_bazel_pytest() {
   pip install yq
   cd "${WORKSPACE_DIR}"
-  bazel query 'kind(py_test.*, tests(python/...) intersect attr(tags, "\bteam:ml\b", python/...)  except attr(tags, "\bno_main\b", python/...))' --output xml | xq | python scripts/pytest_checker.py 
+  for team in "team:ml" "team:serve"; do
+    # this does the following:
+    # - find all py_test rules in bazel that have the specified team tag EXCEPT ones with "no_main" tag and outputs them as xml
+    # - converts the xml to json
+    # - feeds the json into pytest_checker.py
+    bazel query "kind(py_test.*, tests(python/...) intersect attr(tags, \"\b$team\b\", python/...) except attr(tags, \"\bno_main\b\", python/...))" --output xml | xq | python scripts/pytest_checker.py
+  done
 }
 
 lint_web() {

python/ray/serve/BUILD

@@ -278,7 +278,7 @@ py_test(
     name = "test_logs",
     size = "small",
     srcs = serve_tests_srcs,
-    tags = ["exclusive", "team:serve"],
+    tags = ["exclusive", "team:serve", "no_main"],
     deps = [":serve_lib"],
 )
 
@@ -419,7 +419,7 @@ py_test(
     name = "quickstart_class",
     size = "small",
     srcs = glob(["examples/doc/*.py"]),
-    tags = ["exclusive", "team:serve"],
+    tags = ["exclusive", "team:serve", "no_main"],
     deps = [":serve_lib"]
 )
 
@@ -427,7 +427,7 @@ py_test(
     name = "quickstart_function",
     size = "small",
     srcs = glob(["examples/doc/*.py"]),
-    tags = ["exclusive", "team:serve"],
+    tags = ["exclusive", "team:serve", "no_main"],
     deps = [":serve_lib"]
 )
 
@@ -435,7 +435,7 @@ py_test(
     name = "tutorial_tensorflow",
     size = "small",
     srcs = glob(["examples/doc/*.py"]),
-    tags = ["exclusive", "team:serve"],
+    tags = ["exclusive", "team:serve", "no_main"],
     deps = [":serve_lib"]
 )
 
@@ -443,7 +443,7 @@ py_test(
     name = "tutorial_pytorch",
     size = "small",
     srcs = glob(["examples/doc/*.py"]),
-    tags = ["exclusive", "team:serve"],
+    tags = ["exclusive", "team:serve", "no_main"],
     deps = [":serve_lib"]
 )
 
@@ -451,7 +451,7 @@ py_test(
     name = "tutorial_sklearn",
     size = "small",
     srcs = glob(["examples/doc/*.py"]),
-    tags = ["exclusive", "team:serve"],
+    tags = ["exclusive", "team:serve", "no_main"],
     deps = [":serve_lib"]
 )
 
@@ -462,7 +462,7 @@ py_test(
     main = "test_myst_doc.py",
     args = ["--path", "doc/source/serve/tutorials/rllib.md"],
     data = ["//doc/source/serve/tutorials:markdowns"],
-    tags = ["exclusive", "team:serve"],
+    tags = ["exclusive", "team:serve", "no_main"],
 )
 
 py_test(
@@ -477,7 +477,7 @@ py_test(
     name = "conda_env",
     size = "medium",
     srcs = glob(["examples/doc/*.py"]),
-    tags = ["exclusive", "post_wheel_build", "team:serve"],
+    tags = ["exclusive", "post_wheel_build", "team:serve", "no_main"],
     deps = [":serve_lib"]
 )
 

python/ray/serve/tests/test_http_adapters.py

@@ -87,3 +87,9 @@ async def test_pandas_dataframe():
         MockRequest(_body=raw_json, query_params={"orient": "records"})
     )
     assert parsed_df.equals(df)
+
+
+if __name__ == "__main__":
+    import sys
+
+    sys.exit(pytest.main(["-v", "-s", __file__]))

scripts/pytest_checker.py

@@ -5,6 +5,7 @@ from pathlib import Path
 
 
 def check_file(file_contents: str) -> bool:
+    """Check file for the snippet"""
     return bool(re.search(r"^if __name__ == \"__main__\":", file_contents, re.M))
 
 
@@ -13,18 +14,75 @@ def parse_json(data: str) -> dict:
 
 
 def treat_path(path: str) -> Path:
+    """Treat bazel paths to filesystem paths"""
     path = path[2:].replace(":", "/")
     return Path(path)
 
 
 def get_paths_from_parsed_data(parsed_data: dict) -> list:
+    # Example JSON input:
+    # "rule": [
+    #     {
+    #       "@class": "py_test",
+    #       "@location": "/home/ubuntu/ray/python/ray/tests/BUILD:345:8",
+    #       "@name": "//python/ray/tests:test_tracing",
+    #       "string": [
+    #         {
+    #           "@name": "name",
+    #           "@value": "test_tracing"
+    #         },
+    #       ],
+    #       "list": [
+    #         {
+    #           "@name": "srcs",
+    #           "label": [
+    #             {
+    #               "@value": "//python/ray/tests:aws/conftest.py"
+    #             },
+    #             {
+    #               "@value": "//python/ray/tests:conftest.py"
+    #             },
+    #             {
+    #               "@value": "//python/ray/tests:test_tracing.py"
+    #             }
+    #           ]
+    #         }
+    #       ],
+    #       ... other fields ...
+    #       "label": {
+    #         "@name": "main",
+    #         "@value": "//python/ray/tests:test_runtime_env_working_dir_remote_uri.py"
+    #       },
+    #       ... other fields ...
+    #     }
+    # ]
+    #
+    # We want to get the location of the actual test file.
+    # This can be, in order of priority:
+    #   1. Specified as the "main" label
+    #   2. Specified as the ONLY "srcs" label
+    #   3. Specified as the "srcs" label matching the "name" of the test
+    # https://docs.bazel.build/versions/main/be/python.html#py_test
+
     paths = []
     for rule in parsed_data["query"]["rule"]:
+        name = rule["@name"]
         if "label" in rule and rule["label"]["@name"] == "main":
-            paths.append(treat_path(rule["label"]["@value"]))
+            paths.append((name, treat_path(rule["label"]["@value"])))
         else:
             list_args = {e["@name"]: e for e in rule["list"]}
-            paths.append(treat_path(list_args["srcs"]["label"]["@value"]))
+            label = list_args["srcs"]["label"]
+            if isinstance(label, dict):
+                paths.append((name, treat_path(label["@value"])))
+            else:
+                # list
+                string_name = next(
+                    x["@value"] for x in rule["string"] if x["@name"] == "name"
+                )
+                main_path = next(
+                    x["@value"] for x in label if string_name in x["@value"]
+                )
+                paths.append((name, treat_path(main_path)))
     return paths
 
 
@@ -34,18 +92,27 @@ def main(data: str):
     paths = get_paths_from_parsed_data(parsed_data)
 
     bad_paths = []
-    for path in paths:
-        print(f"Checking file {path}...")
-        with open(path, "r") as f:
-            if not check_file(f.read()):
-                print(f"File {path} is missing the pytest snippet.")
-                bad_paths.append(path)
+    for name, path in paths:
+        # Special case for myst doc checker
+        if "test_myst_doc" in str(path):
+            continue
+
+        print(f"Checking test '{name}' | file '{path}'...")
+        try:
+            with open(path, "r") as f:
+                if not check_file(f.read()):
+                    print(f"File '{path}' is missing the pytest snippet.")
+                    bad_paths.append(path)
+        except FileNotFoundError:
+            print(f"File '{path}' is missing.")
+            bad_paths.append((path, "path is missing!"))
     if bad_paths:
+        formatted_bad_paths = "\n".join([str(x) for x in bad_paths])
         raise RuntimeError(
             'Found py_test files without `if __name__ == "__main__":` snippet:'
-            f" {[str(x) for x in bad_paths]}\n"
+            f"\n{formatted_bad_paths}\n"
             "If this is intentional, please add a `no_main` tag to bazel BUILD "
-            "entry for that file."
+            "entry for those files."
         )