Move serve and asyncio tests to bazel (#6979)

.travis.yml

@@ -143,9 +143,8 @@ script:
   # cc bazel tests
   - ./ci/suppress_output bazel test --build_tests_only --show_progress_rate_limit=100 --test_output=errors //:all
 
-  # ray serve tests TODO(ekl): these should be moved to bazel
-  - if [ $RAY_CI_SERVE_AFFECTED == "1" ]; then SERVE_LOG_DEBUG=1 python -m pytest -v --durations=5 --timeout=300 python/ray/experimental/serve/tests; fi
-  - if [ $RAY_CI_SERVE_AFFECTED == "1" ]; then SERVE_LOG_DEBUG=1 ./ci/suppress_output python python/ray/experimental/serve/examples/echo_full.py; fi
+  # ray serve tests
+  - if [ $RAY_CI_SERVE_AFFECTED == "1" ]; then ./ci/keep_alive bazel test --spawn_strategy=local --flaky_test_attempts=3 --nocache_test_results --test_verbose_timeout_warnings --progress_report_interval=100 --show_progress_rate_limit=100 --show_timestamps --test_output=errors --test_tag_filters=-jenkins_only python/ray/experimental/serve/...; fi
 
   # ray operator tests
   - cd ./deploy/ray-operator/
@@ -155,7 +154,6 @@ script:
 
   # random python tests TODO(ekl): these should be moved to bazel
   - if [ $RAY_CI_PYTHON_AFFECTED == "1" ]; then RAY_FORCE_DIRECT=0 python -m pytest -v --durations=5 --timeout=300 python/ray/experimental/test/async_test.py; fi
-  - if [ $RAY_CI_PYTHON_AFFECTED == "1" ]; then python -m pytest -v --durations=5 --timeout=300 python/ray/tests/py3_test.py; fi
 
   # bazel python tests. This should be run last to keep its logs at the end of travis logs.
   - if [ $RAY_CI_PYTHON_AFFECTED == "1" ]; then ./ci/keep_alive bazel test --spawn_strategy=local --flaky_test_attempts=3 --nocache_test_results --test_verbose_timeout_warnings --progress_report_interval=100 --show_progress_rate_limit=100 --show_timestamps --test_output=errors --test_tag_filters=-jenkins_only python/ray/tests/...; fi

python/ray/experimental/serve/BUILD

@@ -0,0 +1,28 @@
+# This is a dummy test dependency that causes the above tests to be
+# re-run if any of these files changes.
+py_library(
+    name = "serve_lib",
+    srcs = glob(["**/*.py"], exclude=["tests/*.py"]),
+)
+
+# This test aggregates all serve tests and run them in a single session
+# similar to `pytest .`
+# Serve tests need to run in a single session because starting and stopping
+# serve cluster take a large chunk of time. All serve tests use a shared
+# cluster.
+py_test(
+    name = "test_serve",
+    size = "medium",
+    srcs = glob(["tests/*.py"]),
+    tags = ["exclusive"],
+    deps = [":serve_lib"],
+)
+
+# Make sure the example showing in doc is tested
+py_test(
+    name = "echo_full",
+    size = "small",
+    srcs = glob(["examples/*.py"]),
+    tags = ["exclusive"],
+    deps = [":serve_lib"]
+)

python/ray/experimental/serve/tests/test_serve.py

@@ -0,0 +1,15 @@
+import pytest
+from pathlib import Path
+import sys
+
+if __name__ == "__main__":
+    curr_dir = Path(__file__).parent
+    test_paths = curr_dir.rglob("test_*.py")
+    sorted_path = sorted(map(lambda path: str(path.absolute()), test_paths))
+    serve_tests_files = list(sorted_path)
+
+    print("Testing the following files")
+    for test_file in serve_tests_files:
+        print("->", test_file.split("/")[-1])
+
+    sys.exit(pytest.main(["-v", "-s"] + serve_tests_files))

python/ray/tests/BUILD

@@ -359,3 +359,19 @@ py_test(
     tags = ["exclusive"],
     deps = ["//:ray_lib"],
 )
+
+py_test(
+    name = "test_args",
+    size = "small",
+    srcs = ["test_args.py"],
+    tags = ["exclusive"],
+    deps = ["//:ray_lib"],
+)
+
+py_test(
+    name = "test_asyncio",
+    size = "small",
+    srcs = ["test_asyncio.py"],
+    tags = ["exclusive"],
+    deps = ["//:ray_lib"],
+)

python/ray/tests/test_args.py

@@ -0,0 +1,98 @@
+# coding: utf-8
+import pytest
+
+import ray
+import ray.cluster_utils
+import ray.test_utils
+
+
+@pytest.mark.parametrize(
+    "ray_start_regular", [{
+        "local_mode": True
+    }, {
+        "local_mode": False
+    }],
+    indirect=True)
+def test_args_force_positional(ray_start_regular):
+    def force_positional(*, a="hello", b="helxo", **kwargs):
+        return a, b, kwargs
+
+    class TestActor():
+        def force_positional(self, a="hello", b="heo", *args, **kwargs):
+            return a, b, args, kwargs
+
+    def test_function(fn, remote_fn):
+        assert fn(a=1, b=3, c=5) == ray.get(remote_fn.remote(a=1, b=3, c=5))
+        assert fn(a=1) == ray.get(remote_fn.remote(a=1))
+        assert fn(a=1) == ray.get(remote_fn.remote(a=1))
+
+    remote_test_function = ray.remote(test_function)
+
+    remote_force_positional = ray.remote(force_positional)
+    test_function(force_positional, remote_force_positional)
+    ray.get(
+        remote_test_function.remote(force_positional, remote_force_positional))
+
+    remote_actor_class = ray.remote(TestActor)
+    remote_actor = remote_actor_class.remote()
+    actor_method = remote_actor.force_positional
+    local_actor = TestActor()
+    local_method = local_actor.force_positional
+    test_function(local_method, actor_method)
+    ray.get(remote_test_function.remote(local_method, actor_method))
+
+
+@pytest.mark.parametrize(
+    "ray_start_regular", [{
+        "local_mode": False
+    }, {
+        "local_mode": True
+    }],
+    indirect=True)
+def test_args_intertwined(ray_start_regular):
+    def args_intertwined(a, *args, x="hello", **kwargs):
+        return a, args, x, kwargs
+
+    class TestActor():
+        def args_intertwined(self, a, *args, x="hello", **kwargs):
+            return a, args, x, kwargs
+
+        @classmethod
+        def cls_args_intertwined(cls, a, *args, x="hello", **kwargs):
+            return a, args, x, kwargs
+
+    def test_function(fn, remote_fn):
+        assert fn(
+            1, 2, 3, x="hi", y="hello") == ray.get(
+                remote_fn.remote(1, 2, 3, x="hi", y="hello"))
+        assert fn(
+            1, 2, 3, y="1hello") == ray.get(
+                remote_fn.remote(1, 2, 3, y="1hello"))
+        assert fn(1, y="1hello") == ray.get(remote_fn.remote(1, y="1hello"))
+
+    remote_test_function = ray.remote(test_function)
+
+    remote_args_intertwined = ray.remote(args_intertwined)
+    test_function(args_intertwined, remote_args_intertwined)
+    ray.get(
+        remote_test_function.remote(args_intertwined, remote_args_intertwined))
+
+    remote_actor_class = ray.remote(TestActor)
+    remote_actor = remote_actor_class.remote()
+    actor_method = remote_actor.args_intertwined
+    local_actor = TestActor()
+    local_method = local_actor.args_intertwined
+    test_function(local_method, actor_method)
+    ray.get(remote_test_function.remote(local_method, actor_method))
+
+    actor_method = remote_actor.cls_args_intertwined
+    local_actor = TestActor()
+    local_method = local_actor.cls_args_intertwined
+    test_function(local_method, actor_method)
+    ray.get(remote_test_function.remote(local_method, actor_method))
+
+
+if __name__ == "__main__":
+    import pytest
+    import sys
+    sys.exit(pytest.main(["-v", __file__]))

python/ray/tests/test_asyncio.py

@@ -5,94 +5,6 @@ import pytest
 import sys
 
 import ray
-import ray.cluster_utils
-import ray.test_utils
-
-
-@pytest.mark.parametrize(
-    "ray_start_regular", [{
-        "local_mode": True
-    }, {
-        "local_mode": False
-    }],
-    indirect=True)
-def test_args_force_positional(ray_start_regular):
-    def force_positional(*, a="hello", b="helxo", **kwargs):
-        return a, b, kwargs
-
-    class TestActor():
-        def force_positional(self, a="hello", b="heo", *args, **kwargs):
-            return a, b, args, kwargs
-
-    def test_function(fn, remote_fn):
-        assert fn(a=1, b=3, c=5) == ray.get(remote_fn.remote(a=1, b=3, c=5))
-        assert fn(a=1) == ray.get(remote_fn.remote(a=1))
-        assert fn(a=1) == ray.get(remote_fn.remote(a=1))
-
-    remote_test_function = ray.remote(test_function)
-
-    remote_force_positional = ray.remote(force_positional)
-    test_function(force_positional, remote_force_positional)
-    ray.get(
-        remote_test_function.remote(force_positional, remote_force_positional))
-
-    remote_actor_class = ray.remote(TestActor)
-    remote_actor = remote_actor_class.remote()
-    actor_method = remote_actor.force_positional
-    local_actor = TestActor()
-    local_method = local_actor.force_positional
-    test_function(local_method, actor_method)
-    ray.get(remote_test_function.remote(local_method, actor_method))
-
-
-@pytest.mark.parametrize(
-    "ray_start_regular", [{
-        "local_mode": False
-    }, {
-        "local_mode": True
-    }],
-    indirect=True)
-def test_args_intertwined(ray_start_regular):
-    def args_intertwined(a, *args, x="hello", **kwargs):
-        return a, args, x, kwargs
-
-    class TestActor():
-        def args_intertwined(self, a, *args, x="hello", **kwargs):
-            return a, args, x, kwargs
-
-        @classmethod
-        def cls_args_intertwined(cls, a, *args, x="hello", **kwargs):
-            return a, args, x, kwargs
-
-    def test_function(fn, remote_fn):
-        assert fn(
-            1, 2, 3, x="hi", y="hello") == ray.get(
-                remote_fn.remote(1, 2, 3, x="hi", y="hello"))
-        assert fn(
-            1, 2, 3, y="1hello") == ray.get(
-                remote_fn.remote(1, 2, 3, y="1hello"))
-        assert fn(1, y="1hello") == ray.get(remote_fn.remote(1, y="1hello"))
-
-    remote_test_function = ray.remote(test_function)
-
-    remote_args_intertwined = ray.remote(args_intertwined)
-    test_function(args_intertwined, remote_args_intertwined)
-    ray.get(
-        remote_test_function.remote(args_intertwined, remote_args_intertwined))
-
-    remote_actor_class = ray.remote(TestActor)
-    remote_actor = remote_actor_class.remote()
-    actor_method = remote_actor.args_intertwined
-    local_actor = TestActor()
-    local_method = local_actor.args_intertwined
-    test_function(local_method, actor_method)
-    ray.get(remote_test_function.remote(local_method, actor_method))
-
-    actor_method = remote_actor.cls_args_intertwined
-    local_actor = TestActor()
-    local_method = local_actor.cls_args_intertwined
-    test_function(local_method, actor_method)
-    ray.get(remote_test_function.remote(local_method, actor_method))
 
 
 def test_asyncio_actor(ray_start_regular_shared):
@@ -265,3 +177,8 @@ def test_asyncio_actor_async_get(ray_start_regular_shared):
     getter = AsyncGetter.options().remote()
     assert ray.get(getter.get.remote()) == 1
     assert ray.get(getter.plasma_get.remote()) == 2
+
+
+if __name__ == "__main__":
+    import pytest
+    sys.exit(pytest.main(["-v", __file__]))