diff --git a/release/ml_user_tests/train/driver_setup.sh b/release/ml_user_tests/train/driver_setup.sh
index 431233835..adee34910 100755
--- a/release/ml_user_tests/train/driver_setup.sh
+++ b/release/ml_user_tests/train/driver_setup.sh
@@ -2,4 +2,4 @@
 
 cd "${0%/*}" || exit 1
 
-pip install -U -r ./driver_requirements.txt
\ No newline at end of file
+pip install -U -r ./driver_requirements.txt
diff --git a/release/ml_user_tests/tune_rllib/driver_setup.sh b/release/ml_user_tests/tune_rllib/driver_setup.sh
index bf726e95a..adee34910 100755
--- a/release/ml_user_tests/tune_rllib/driver_setup.sh
+++ b/release/ml_user_tests/tune_rllib/driver_setup.sh
@@ -3,4 +3,3 @@
 cd "${0%/*}" || exit 1
 
 pip install -U -r ./driver_requirements.txt
-pip install -U typing-extensions anyscale
diff --git a/release/ray_release/command_runner/client_runner.py b/release/ray_release/command_runner/client_runner.py
index b80e0934d..53cade0b7 100644
--- a/release/ray_release/command_runner/client_runner.py
+++ b/release/ray_release/command_runner/client_runner.py
@@ -55,9 +55,7 @@ class ClientRunner(CommandRunner):
         self.result_output_json = tempfile.mktemp()
 
     def prepare_remote_env(self):
-        # Give 5 seconds of slack time for the cluster to start up properly.
-        # This is to avoid e.g. Ray client connection timeouts.
-        time.sleep(5)
+        pass
 
     def prepare_local_env(self, ray_wheels_url: Optional[str] = None):
         try:
diff --git a/release/ray_release/glue.py b/release/ray_release/glue.py
index 023560693..5c0d60cc3 100644
--- a/release/ray_release/glue.py
+++ b/release/ray_release/glue.py
@@ -38,7 +38,11 @@ from ray_release.file_manager.session_controller import SessionControllerFileMan
 from ray_release.logger import logger
 from ray_release.reporter.reporter import Reporter
 from ray_release.result import Result, handle_exception
-from ray_release.util import run_bash_script, get_pip_packages
+from ray_release.util import (
+    run_bash_script,
+    get_pip_packages,
+    reinstall_anyscale_dependencies,
+)
 
 type_str_to_command_runner = {
     "command": SDKRunner,
@@ -167,6 +171,10 @@ def run_release_test(
         command_runner.prepare_local_env(ray_wheels_url)
         command_timeout = test["run"].get("timeout", DEFAULT_COMMAND_TIMEOUT)
 
+        # Re-install anyscale package as local dependencies might have changed
+        # from local env setup
+        reinstall_anyscale_dependencies()
+
         # Print installed pip packages
         buildkite_group(":bulb: Local environment information")
         pip_packages = get_pip_packages()
diff --git a/release/ray_release/tests/test_glue.py b/release/ray_release/tests/test_glue.py
index d839a0197..d89612bbd 100644
--- a/release/ray_release/tests/test_glue.py
+++ b/release/ray_release/tests/test_glue.py
@@ -68,6 +68,8 @@ class MockReturn:
         return object.__getattribute__(self, item)
 
 
+@patch("ray_release.glue.reinstall_anyscale_dependencies", lambda: None)
+@patch("ray_release.glue.get_pip_packages", lambda: ["pip-packages"])
 class GlueTest(unittest.TestCase):
     def writeClusterEnv(self, content: str):
         with open(os.path.join(self.tempdir, "cluster_env.yaml"), "wt") as fp:
diff --git a/release/ray_release/util.py b/release/ray_release/util.py
index 42ce8decb..c835cb61d 100644
--- a/release/ray_release/util.py
+++ b/release/ray_release/util.py
@@ -110,6 +110,16 @@ def run_bash_script(bash_script: str):
     subprocess.run(f"bash {bash_script}", shell=True, check=True)
 
 
+def reinstall_anyscale_dependencies():
+    logger.info("Re-installing `anyscale` package")
+
+    subprocess.check_output(
+        "pip install -U anyscale",
+        shell=True,
+        text=True,
+    )
+
+
 def get_pip_packages() -> List[str]:
     from pip._internal.operations import freeze