mirror of
https://github.com/vale981/ray
synced 2025-03-06 10:31:39 -05:00
331b71ea8d
Adds a unit-tested and restructured ray_release package for running release tests.
Relevant changes in behavior:
Per default, Buildkite will wait for the wheels of the current commit to be available. Alternatively, users can a) specify a different commit hash, b) a wheels URL (which we will also wait for to be available) or c) specify a branch (or user/branch combination), in which case the latest available wheels will be used (e.g. if master is passed, behavior matches old default behavior).
The main subpackages are:
Cluster manager: Creates cluster envs/computes, starts cluster, terminates cluster
Command runner: Runs commands, e.g. as client command or sdk command
File manager: Uploads/downloads files to/from session
Reporter: Reports results (e.g. to database)
Much of the code base is unit tested, but there are probably some pieces missing.
Example build (waited for wheels to be built): https://buildkite.com/ray-project/kf-dev/builds/51#_
Wheel build: https://buildkite.com/ray-project/ray-builders-branch/builds/6023
44 lines
1.6 KiB
Python
44 lines
1.6 KiB
Python
import os
|
|
|
|
import boto3
|
|
from anyscale.authenticate import AuthenticationBlock
|
|
|
|
from ray_release.logger import logger
|
|
|
|
RELEASE_AWS_BUCKET = os.getenv("RELEASE_AWS_BUCKET", "ray-release-automation-results")
|
|
RELEASE_AWS_DB_NAME = os.getenv("RELEASE_AWS_DB_NAME", "ray_ci")
|
|
RELEASE_AWS_DB_TABLE = os.getenv("RELEASE_AWS_DB_TABLE", "release_test_result")
|
|
|
|
RELEASE_AWS_DB_SECRET_ARN = os.getenv(
|
|
"RELEASE_AWS_DB_SECRET_ARN",
|
|
"arn:aws:secretsmanager:us-west-2:029272617770:secret:"
|
|
"rds-db-credentials/cluster-7RB7EYTTBK2EUC3MMTONYRBJLE/ray_ci-MQN2hh",
|
|
)
|
|
RELEASE_AWS_DB_RESOURCE_ARN = os.getenv(
|
|
"RELEASE_AWS_DB_RESOURCE_ARN",
|
|
"arn:aws:rds:us-west-2:029272617770:cluster:ci-reporting",
|
|
)
|
|
RELEASE_AWS_ANYSCALE_SECRET_ARN = os.getenv(
|
|
"RELEASE_AWS_ANYSCALE_SECRET_ARN",
|
|
"arn:aws:secretsmanager:us-west-2:029272617770:secret:"
|
|
"release-automation/"
|
|
"anyscale-token20210505220406333800000001-BcUuKB",
|
|
)
|
|
|
|
|
|
def maybe_fetch_api_token():
|
|
if not os.environ.get("ANYSCALE_CLI_TOKEN"):
|
|
try:
|
|
token, _ = AuthenticationBlock._load_credentials()
|
|
logger.info("Loaded anyscale credentials from local storage.")
|
|
os.environ["ANYSCALE_CLI_TOKEN"] = token
|
|
return
|
|
except Exception:
|
|
pass # Ignore errors
|
|
|
|
logger.info("Missing ANYSCALE_CLI_TOKEN, retrieving from AWS secrets store")
|
|
# NOTE(simon) This should automatically retrieve
|
|
# release-automation@anyscale.com's anyscale token
|
|
os.environ["ANYSCALE_CLI_TOKEN"] = boto3.client(
|
|
"secretsmanager", region_name="us-west-2"
|
|
).get_secret_value(SecretId=RELEASE_AWS_ANYSCALE_SECRET_ARN)["SecretString"]
|