[Tests] Skip failing windows tests (#13495)

* skip failing windows tests

* skip more

* remove

* updates

ci/travis/ci.sh

@@ -140,17 +140,20 @@ test_python() {
       python/ray/serve/...
       python/ray/tests/...
       -python/ray/serve:test_api # segfault on windows? https://github.com/ray-project/ray/issues/12541
+      -python/ray/tests:test_actor_advanced # timeout
       -python/ray/tests:test_advanced_2
       -python/ray/tests:test_advanced_3  # test_invalid_unicode_in_worker_log() fails on Windows
       -python/ray/tests:test_autoscaler_aws
       -python/ray/tests:test_component_failures
       -python/ray/tests:test_basic_2  # hangs on shared cluster tests
+      -python/ray/tests:test_basic_2_client_mode
       -python/ray/tests:test_cli
       -python/ray/tests:test_failure
       -python/ray/tests:test_global_gc
       -python/ray/tests:test_job
       -python/ray/tests:test_memstat
       -python/ray/tests:test_metrics
+      -python/ray/tests:test_metrics_agent # timeout
       -python/ray/tests:test_multi_node
       -python/ray/tests:test_multi_node_2
       -python/ray/tests:test_multiprocessing  # test_connect_to_ray() fails to connect to raylet

python/ray/tests/test_actor_advanced.py

@@ -1095,4 +1095,5 @@ def test_actor_resource_demand(shutdown_only):
 
 if __name__ == "__main__":
     import pytest
+    # Test suite is timing out. Disable on windows for now.
     sys.exit(pytest.main(["-v", __file__]))

python/ray/tests/test_actor_resources.py

@@ -229,6 +229,7 @@ def test_actor_different_numbers_of_gpus(ray_start_cluster):
     assert ready_ids == []
 
 
+@pytest.mark.skipif(sys.platform == "win32", reason="Failing on Windows.")
 def test_actor_multiple_gpus_from_multiple_tasks(ray_start_cluster):
     cluster = ray_start_cluster
     num_nodes = 5

python/ray/tests/test_basic.py

@@ -267,6 +267,7 @@ def test_background_tasks_with_max_calls(shutdown_only):
         wait_for_pid_to_exit(pid)
 
 
+@pytest.mark.skipif(sys.platform == "win32", reason="Failing on Windows.")
 def test_fair_queueing(shutdown_only):
     ray.init(num_cpus=1, _system_config={"fair_queueing_enabled": 1})
 

python/ray/tests/test_basic_2.py

@@ -651,4 +651,5 @@ def test_get_correct_node_ip():
 
 if __name__ == "__main__":
     import pytest
+    # Skip test_basic_2_client_mode for now- the test suite is breaking.
     sys.exit(pytest.main(["-v", __file__]))

python/ray/tests/test_client.py

@@ -38,6 +38,7 @@ def test_num_clients(shutdown_only):
         server.stop(0)
 
 
+@pytest.mark.skipif(sys.platform == "win32", reason="Failing on Windows.")
 def test_real_ray_fallback(ray_start_regular_shared):
     with ray_start_client_server() as ray:
 
@@ -58,6 +59,7 @@ def test_real_ray_fallback(ray_start_regular_shared):
         assert len(nodes) == 1, nodes
 
 
+@pytest.mark.skipif(sys.platform == "win32", reason="Failing on Windows.")
 def test_nested_function(ray_start_regular_shared):
     with ray_start_client_server() as ray:
 
@@ -72,6 +74,7 @@ def test_nested_function(ray_start_regular_shared):
         assert ray.get(g.remote()) == "OK"
 
 
+@pytest.mark.skipif(sys.platform == "win32", reason="Failing on Windows.")
 def test_put_get(ray_start_regular_shared):
     with ray_start_client_server() as ray:
         objectref = ray.put("hello world")
@@ -81,6 +84,7 @@ def test_put_get(ray_start_regular_shared):
         assert retval == "hello world"
 
 
+@pytest.mark.skipif(sys.platform == "win32", reason="Failing on Windows.")
 def test_wait(ray_start_regular_shared):
     with ray_start_client_server() as ray:
         objectref = ray.put("hello world")
@@ -109,6 +113,7 @@ def test_wait(ray_start_regular_shared):
             ray.wait(["blabla"])
 
 
+@pytest.mark.skipif(sys.platform == "win32", reason="Failing on Windows.")
 def test_remote_functions(ray_start_regular_shared):
     with ray_start_client_server() as ray:
 
@@ -153,6 +158,7 @@ def test_remote_functions(ray_start_regular_shared):
         assert all_vals == [236, 2_432_902_008_176_640_000, 120, 3628800]
 
 
+@pytest.mark.skipif(sys.platform == "win32", reason="Failing on Windows.")
 def test_function_calling_function(ray_start_regular_shared):
     with ray_start_client_server() as ray:
 
@@ -169,6 +175,7 @@ def test_function_calling_function(ray_start_regular_shared):
         assert ray.get(f.remote()) == "OK"
 
 
+@pytest.mark.skipif(sys.platform == "win32", reason="Failing on Windows.")
 def test_basic_actor(ray_start_regular_shared):
     with ray_start_client_server() as ray:
 
@@ -190,6 +197,7 @@ def test_basic_actor(ray_start_regular_shared):
         assert count == 2
 
 
+@pytest.mark.skipif(sys.platform == "win32", reason="Failing on Windows.")
 def test_pass_handles(ray_start_regular_shared):
     """Test that passing client handles to actors and functions to remote actors
     in functions (on the server or raylet side) works transparently to the
@@ -253,6 +261,7 @@ def test_pass_handles(ray_start_regular_shared):
                                                 4)) == local_fact(4)
 
 
+@pytest.mark.skipif(sys.platform == "win32", reason="Failing on Windows.")
 def test_basic_log_stream(ray_start_regular_shared):
     with ray_start_client_server() as ray:
         log_msgs = []
@@ -273,6 +282,7 @@ def test_basic_log_stream(ray_start_regular_shared):
         assert any((msg.find("put") >= 0 for msg in logs_with_id))
 
 
+@pytest.mark.skipif(sys.platform == "win32", reason="Failing on Windows.")
 def test_stdout_log_stream(ray_start_regular_shared):
     with ray_start_client_server() as ray:
         log_msgs = []
@@ -294,6 +304,7 @@ def test_stdout_log_stream(ray_start_regular_shared):
         assert all((msg.find("Hello world") for msg in log_msgs))
 
 
+@pytest.mark.skipif(sys.platform == "win32", reason="Failing on Windows.")
 def test_create_remote_before_start(ray_start_regular_shared):
     """Creates remote objects (as though in a library) before
     starting the client.
@@ -318,6 +329,7 @@ def test_create_remote_before_start(ray_start_regular_shared):
         assert ray.get(a.doit.remote()) == "foo"
 
 
+@pytest.mark.skipif(sys.platform == "win32", reason="Failing on Windows.")
 def test_basic_named_actor(ray_start_regular_shared):
     """Test that ray.get_actor() can create and return a detached actor.
     """
@@ -346,6 +358,7 @@ def test_basic_named_actor(ray_start_regular_shared):
         assert ray.get(new_actor.get.remote()) == 3
 
 
+@pytest.mark.skipif(sys.platform == "win32", reason="Failing on Windows.")
 def test_internal_kv(ray_start_regular_shared):
     with ray_start_client_server() as ray:
         assert ray._internal_kv_initialized()

python/ray/tests/test_metrics_agent.py

@@ -2,6 +2,7 @@ import json
 import pathlib
 import platform
 from pprint import pformat
+import sys
 import time
 from unittest.mock import MagicMock
 
@@ -108,6 +109,7 @@ def _setup_cluster_for_test(ray_start_cluster):
     cluster.shutdown()
 
 
+@pytest.mark.skipif(sys.platform == "win32", reason="Failing on Windows.")
 def test_metrics_export_end_to_end(_setup_cluster_for_test):
     TEST_TIMEOUT_S = 20
 
@@ -301,4 +303,5 @@ def test_metrics_override_shouldnt_warn(ray_start_regular, log_pubsub):
 
 if __name__ == "__main__":
     import sys
+    # Test suite is timing out. Disable on windows for now.
     sys.exit(pytest.main(["-v", __file__]))

python/ray/tests/test_multi_tenancy.py

@@ -42,6 +42,7 @@ def test_initial_workers(shutdown_only):
 # all the PIDs don't overlap. If overlapped, it means that tasks owned by
 # different drivers were scheduled to the same worker process, that is, tasks
 # of different jobs were not correctly isolated during execution.
+@pytest.mark.skipif(sys.platform == "win32", reason="Failing on Windows.")
 def test_multi_drivers(shutdown_only):
     info = ray.init(num_cpus=10)
 

python/ray/tests/test_reconstruction.py

@@ -487,6 +487,7 @@ def test_reconstruction_chain(ray_start_cluster, reconstruction_enabled):
                 raise e.as_instanceof_cause()
 
 
+@pytest.mark.skipif(sys.platform == "win32", reason="Failing on Windows.")
 def test_reconstruction_stress(ray_start_cluster):
     config = {
         "num_heartbeats_timeout": 10,

python/ray/tests/test_reference_counting_2.py

@@ -249,6 +249,7 @@ def test_recursively_pass_returned_object_ref(one_worker_100MiB, use_ray_put,
 # returns the same ObjectRef by calling ray.get() on its submitted task and
 # returning the result. The reference should still exist while the driver has a
 # reference to the final task's ObjectRef.
+@pytest.mark.skipif(sys.platform == "win32", reason="Failing on Windows.")
 @pytest.mark.parametrize("use_ray_put,failure", [(False, False), (False, True),
                                                  (True, False), (True, True)])
 def test_recursively_return_borrowed_object_ref(one_worker_100MiB, use_ray_put,