[ci/release] Re-install anyscale package after local env setup (#24373)

The local environment setup of release tests (in client tests) can sometimes update dependencies of the `anyscale` package to an unsupported version. By re-installing the `anyscale` package after local env setup, we make sure that we can connect to the cluster. Note that this may lead to incompatibilities of the test script, however.

release/ml_user_tests/train/driver_setup.sh

@@ -2,4 +2,4 @@
 
 cd "${0%/*}" || exit 1
 
-pip install -U -r ./driver_requirements.txt
+pip install -U -r ./driver_requirements.txt

release/ml_user_tests/tune_rllib/driver_setup.sh

@@ -3,4 +3,3 @@
 cd "${0%/*}" || exit 1
 
 pip install -U -r ./driver_requirements.txt
-pip install -U typing-extensions anyscale

release/ray_release/command_runner/client_runner.py

@@ -55,9 +55,7 @@ class ClientRunner(CommandRunner):
         self.result_output_json = tempfile.mktemp()
 
     def prepare_remote_env(self):
-        # Give 5 seconds of slack time for the cluster to start up properly.
-        # This is to avoid e.g. Ray client connection timeouts.
-        time.sleep(5)
+        pass
 
     def prepare_local_env(self, ray_wheels_url: Optional[str] = None):
         try:

release/ray_release/glue.py

@@ -38,7 +38,11 @@ from ray_release.file_manager.session_controller import SessionControllerFileMan
 from ray_release.logger import logger
 from ray_release.reporter.reporter import Reporter
 from ray_release.result import Result, handle_exception
-from ray_release.util import run_bash_script, get_pip_packages
+from ray_release.util import (
+    run_bash_script,
+    get_pip_packages,
+    reinstall_anyscale_dependencies,
+)
 
 type_str_to_command_runner = {
     "command": SDKRunner,
@@ -167,6 +171,10 @@ def run_release_test(
         command_runner.prepare_local_env(ray_wheels_url)
         command_timeout = test["run"].get("timeout", DEFAULT_COMMAND_TIMEOUT)
 
+        # Re-install anyscale package as local dependencies might have changed
+        # from local env setup
+        reinstall_anyscale_dependencies()
+
         # Print installed pip packages
         buildkite_group(":bulb: Local environment information")
         pip_packages = get_pip_packages()

release/ray_release/tests/test_glue.py

@@ -68,6 +68,8 @@ class MockReturn:
         return object.__getattribute__(self, item)
 
 
+@patch("ray_release.glue.reinstall_anyscale_dependencies", lambda: None)
+@patch("ray_release.glue.get_pip_packages", lambda: ["pip-packages"])
 class GlueTest(unittest.TestCase):
     def writeClusterEnv(self, content: str):
         with open(os.path.join(self.tempdir, "cluster_env.yaml"), "wt") as fp:

release/ray_release/util.py

@@ -110,6 +110,16 @@ def run_bash_script(bash_script: str):
     subprocess.run(f"bash {bash_script}", shell=True, check=True)
 
 
+def reinstall_anyscale_dependencies():
+    logger.info("Re-installing `anyscale` package")
+
+    subprocess.check_output(
+        "pip install -U anyscale",
+        shell=True,
+        text=True,
+    )
+
+
 def get_pip_packages() -> List[str]:
     from pip._internal.operations import freeze