From b56db5a22ff332d09ec6bdf2881cdd9a836e7057 Mon Sep 17 00:00:00 2001 From: Simon Mo Date: Mon, 14 Dec 2020 15:09:43 -0800 Subject: [PATCH] [Serve] Wait for actor name to be cleaned up (#12215) --- python/ray/serve/api.py | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/python/ray/serve/api.py b/python/ray/serve/api.py index bd7b7d41a..e23e94e07 100644 --- a/python/ray/serve/api.py +++ b/python/ray/serve/api.py @@ -1,4 +1,5 @@ import atexit +import time from functools import wraps import os from uuid import UUID @@ -81,9 +82,25 @@ class Client: Shuts down all processes and deletes all state associated with the instance. """ - if not self._shutdown: + if (not self._shutdown) and ray.is_initialized(): ray.get(self._controller.shutdown.remote()) ray.kill(self._controller, no_restart=True) + + # Wait for the named actor entry gets removed as well. + started = time.time() + while True: + try: + ray.get_actor(self._controller_name) + if time.time() - started > 5: + logger.warning( + "Waited 5s for Serve to shutdown gracefully but " + "the controller is still not cleaned up. " + "You can ignore this warning if you are shutting " + "down the Ray cluster.") + break + except ValueError: # actor name is removed + break + self._shutdown = True @_ensure_connected