hiro/ray
1
0
Fork 0
mirror of https://github.com/vale981/ray synced 2025-03-06 02:21:39 -05:00
Code Issues Projects Releases Packages Wiki Activity Actions

[gcs/ha] Fix some tests failed in HA mode (#21587)

Browse source 
This PR fixed and reenabled tests in HA mode

- //python/ray/tests:test_healthcheck
- //python/ray/tests:test_autoscaler_drain_node_api 
- //python/ray/tests:test_ray_debugger
This commit is contained in:
Yi Cheng 2022-01-16 21:53:14 -08:00 • committed by GitHub
parent 5f7224bd51
commit 87d852fc28
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
5 changed files with 29 additions and 65 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

5
.buildkite/pipeline.yml
View file

@ -347,8 +347,7 @@
--test_env=RAY_gcs_grpc_based_pubsub=1
--test_env=RAY_bootstrap_with_gcs=1
--test_env=RAY_gcs_storage=memory
-- python/ray/tests/...
-//python/ray/tests:test_autoscaler_drain_node_api -//python/ray/tests:test_client_reconnect
-- python/ray/tests/... -//python/ray/tests:test_client_reconnect
- label: ":redis: HA GCS (Large)"
conditions: ["RAY_CI_PYTHON_AFFECTED"]
parallelism: 3
@ -365,7 +364,6 @@
--test_env=RAY_bootstrap_with_gcs=1
--test_env=RAY_gcs_storage=memory
-- //python/ray/tests/...
-//python/ray/tests:test_healthcheck
- label: ":redis: HA GCS (Medium K-Z)"
conditions: ["RAY_CI_PYTHON_AFFECTED"]
commands:
@ -376,7 +374,6 @@
--test_env=RAY_bootstrap_with_gcs=1
--test_env=RAY_gcs_storage=memory
-- //python/ray/tests/...
-//python/ray/tests:test_ray_debugger
- label: ":octopus: Tune soft imports test"
conditions: ["RAY_CI_TUNE_AFFECTED"]

10
python/ray/tests/BUILD
View file

@ -54,12 +54,20 @@ py_test_module_list(
py_test_module_list(
files = [
"test_client.py",
],
size = "large",
extra_srcs = SRCS,
tags = ["exclusive", "client_tests", "team:serverless"],
deps = ["//:ray_lib"],
)
py_test_module_list(
files = [
"test_client_builder.py",
"test_client_compat.py",
"test_client_init.py",
"test_client_multi.py",
"test_client_proxy.py",
"test_client_server.py",
"test_client_references.py",
"test_client_warnings.py",
"test_client_library_integration.py",

37
python/ray/tests/test_client_server.py
View file

@ -1,37 +0,0 @@
import pytest
from unittest.mock import patch, Mock
from ray.ray_constants import REDIS_DEFAULT_PASSWORD
import ray.util.client.server.server as client_server
@pytest.mark.parametrize("redis_password", [None, "random_password"])
def test_try_create_redis_client(redis_password):
create_mock = Mock(side_effect=lambda x, y: x)
with patch(
"ray._private.services", create_redis_client=create_mock), patch(
"ray._private.services.find_redis_address",
side_effect=[["address0", "address1"], [], ["address0"]]):
# Two redis addresses found
assert client_server.try_create_redis_client(None,
redis_password) is None
create_mock.assert_not_called()
# No redis addresses found
assert client_server.try_create_redis_client(None,
redis_password) is None
create_mock.assert_not_called()
# Exactly one redis address found
assert client_server.try_create_redis_client(
None, redis_password) == "address0"
create_mock.assert_called_once_with(
"address0", redis_password or REDIS_DEFAULT_PASSWORD)
create_mock.reset_mock()
# Manually specify redis
client_server.try_create_redis_client("address100", redis_password)
create_mock.assert_called_once_with(
"address100", redis_password or REDIS_DEFAULT_PASSWORD)
if __name__ == "__main__":
import sys
sys.exit(pytest.main(["-v", __file__]))

4
python/ray/tests/test_ray_debugger.py
View file

@ -59,6 +59,10 @@ def test_ray_debugger_commands(shutdown_only):
result1 = f.remote()
result2 = f.remote()
wait_for_condition(lambda: len(
ray.experimental.internal_kv._internal_kv_list(
"RAY_PDB_", namespace=ray_constants.KV_NAMESPACE_PDB)) > 0)
# Make sure that calling "continue" in the debugger
# gives back control to the debugger loop:
p = pexpect.spawn("ray debug")

38
python/ray/util/client/server/server.py
View file

@ -35,8 +35,9 @@ from ray.util.client.server.logservicer import LogstreamServicer
from ray.util.client.server.server_stubs import current_server
from ray.ray_constants import env_integer
from ray._private.client_mode_hook import disable_client_hook
from ray._private.services import canonicalize_bootstrap_address
from ray._private.tls_utils import add_port_to_grpc_server
from ray._private.gcs_utils import use_gcs_for_bootstrap
from ray._private.gcs_utils import use_gcs_for_bootstrap, GcsClient
logger = logging.getLogger(__name__)
@ -735,23 +736,20 @@ def create_ray_handler(address, redis_password):
return ray_connect_handler
def try_create_redis_client(redis_address: Optional[str],
redis_password: Optional[str]) -> Optional[Any]:
def try_create_gcs_client(address: Optional[str], redis_password: Optional[str]
) -> Optional[GcsClient]:
"""
Try to create a redis client based on the the command line args or by
Try to create a gcs client based on the the command line args or by
autodetecting a running Ray cluster.
"""
if redis_address is None:
possible = ray._private.services.find_redis_address()
if len(possible) != 1:
return None
redis_address = possible.pop()
if redis_password is None:
redis_password = ray.ray_constants.REDIS_DEFAULT_PASSWORD
return ray._private.services.create_redis_client(redis_address,
redis_password)
address = canonicalize_bootstrap_address(address)
if use_gcs_for_bootstrap():
return GcsClient(address=address)
else:
if redis_password is None:
redis_password = ray.ray_constants.REDIS_DEFAULT_PASSWORD
return GcsClient.connect_to_gcs_by_redis_address(
address, redis_password)
def main():
@ -813,14 +811,8 @@ def main():
try:
if not ray.experimental.internal_kv._internal_kv_initialized():
if use_gcs_for_bootstrap():
gcs_client = ray._private.gcs_utils.GcsClient(
address=args.address)
else:
redis_client = try_create_redis_client(
args.address, args.redis_password)
gcs_client = (ray._private.gcs_utils.GcsClient.
create_from_redis(redis_client))
gcs_client = try_create_gcs_client(args.address,
args.redis_password)
ray.experimental.internal_kv._initialize_internal_kv(
gcs_client)
ray.experimental.internal_kv._internal_kv_put(