mirror of
https://github.com/vale981/ray
synced 2025-03-06 02:21:39 -05:00
331b71ea8d
Adds a unit-tested and restructured ray_release package for running release tests.
Relevant changes in behavior:
Per default, Buildkite will wait for the wheels of the current commit to be available. Alternatively, users can a) specify a different commit hash, b) a wheels URL (which we will also wait for to be available) or c) specify a branch (or user/branch combination), in which case the latest available wheels will be used (e.g. if master is passed, behavior matches old default behavior).
The main subpackages are:
Cluster manager: Creates cluster envs/computes, starts cluster, terminates cluster
Command runner: Runs commands, e.g. as client command or sdk command
File manager: Uploads/downloads files to/from session
Reporter: Reports results (e.g. to database)
Much of the code base is unit tested, but there are probably some pieces missing.
Example build (waited for wheels to be built): https://buildkite.com/ray-project/kf-dev/builds/51#_
Wheel build: https://buildkite.com/ray-project/ray-builders-branch/builds/6023
54 lines
1.8 KiB
Python
54 lines
1.8 KiB
Python
import os
|
|
from typing import Optional
|
|
|
|
from anyscale.controllers.session_controller import SessionController
|
|
|
|
from ray_release.cluster_manager.cluster_manager import ClusterManager
|
|
from ray_release.file_manager.file_manager import FileManager
|
|
from ray_release.logger import logger
|
|
|
|
|
|
class SessionControllerFileManager(FileManager):
|
|
def __init__(
|
|
self,
|
|
cluster_manager: ClusterManager,
|
|
session_controller: Optional[SessionController] = None,
|
|
):
|
|
super(SessionControllerFileManager, self).__init__(cluster_manager)
|
|
self.session_controller = session_controller or SessionController()
|
|
|
|
# Write legacy anyscale project yaml
|
|
with open(os.path.join(os.getcwd(), ".anyscale.yaml"), "wt") as f:
|
|
f.write(f"project_id: {self.cluster_manager.project_id}")
|
|
|
|
def upload(self, source: Optional[str] = None, target: Optional[str] = None):
|
|
logger.info(
|
|
f"Uploading {source or '<cwd>'} to {target or '<cwd>'} "
|
|
f"using SessionController"
|
|
)
|
|
|
|
if source and os.path.isdir(source) and target:
|
|
# Add trailing slashes
|
|
source = os.path.join(source, "")
|
|
target = os.path.join(target, "")
|
|
|
|
self.session_controller.push(
|
|
session_name=self.cluster_manager.cluster_name,
|
|
source=source,
|
|
target=target,
|
|
config=None,
|
|
all_nodes=False,
|
|
no_warning=True,
|
|
)
|
|
|
|
def download(self, source: str, target: str):
|
|
logger.info(
|
|
f"Downloading {source or '<cwd>'} to {target or '<cwd>'} "
|
|
f"using SessionController"
|
|
)
|
|
self.session_controller.pull(
|
|
session_name=self.cluster_manager.cluster_name,
|
|
source=source,
|
|
target=target,
|
|
config=None,
|
|
)
|