hiro/ray
1
0
Fork 0
mirror of https://github.com/vale981/ray synced 2025-03-05 10:01:43 -05:00
Code Issues Projects Releases Packages Wiki Activity Actions

[ci/release] Allow for preferring smoke tests when filtering (#23887)

Browse source 
What: Adds a setting "prefer_smoke_tests" to the Buildkite settings. With this, user can specify to kick off smoke tests, if available.

Why: The filtering interface of the release testing dialog is a bit complicated at the moment - in order to kick off smoke tests, users have to know with which frequency they are configured to run. Instead users should usually just filter the tests they want to run (using frequency ANY) and optionally specify to run smoke tests, if available.
This commit is contained in:
Kai Fricke 2022-04-14 06:12:27 +01:00 • committed by GitHub
parent 7c1934dc40
commit 6e37a48632
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
6 changed files with 93 additions and 7 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

9
release/ray_release/buildkite/filter.py
View file

@ -10,6 +10,7 @@ def filter_tests(
test_collection: List[Test],
frequency: Frequency,
test_attr_regex_filters: Optional[Dict[str, str]] = None,
prefer_smoke_tests: bool = False,
) -> List[Tuple[Test, bool]]:
if test_attr_regex_filters is None:
test_attr_regex_filters = {}
@ -31,7 +32,13 @@ def filter_tests(
continue
if frequency == Frequency.ANY or frequency == test_frequency:
tests_to_run.append((test, False))
if prefer_smoke_tests and "smoke_test" in test:
# If we prefer smoke tests and a smoke test is available for this test,
# then use the smoke test
smoke_test = True
else:
smoke_test = False
tests_to_run.append((test, smoke_test))
continue
elif "smoke_test" in test:

11
release/ray_release/buildkite/settings.py
View file

@ -19,6 +19,7 @@ class Frequency(enum.Enum):
frequency_str_to_enum = {
"disabled": Frequency.DISABLED,
"any": Frequency.ANY,
"any-smoke": Frequency.ANY,
"multi": Frequency.MULTI,
"nightly": Frequency.NIGHTLY,
"weekly": Frequency.WEEKLY,
@ -123,6 +124,7 @@ def get_pipeline_settings() -> Dict:
def get_default_settings() -> Dict:
settings = {
"frequency": Frequency.ANY,
"prefer_smoke_tests": False,
"test_attr_regex_filters": None,
"ray_wheels": None,
"ray_test_repo": None,
@ -137,6 +139,13 @@ def update_settings_from_environment(settings: Dict) -> Dict:
if "RELEASE_FREQUENCY" in os.environ:
settings["frequency"] = get_frequency(os.environ["RELEASE_FREQUENCY"])
if "RELEASE_PREFER_SMOKE_TESTS" in os.environ:
settings["prefer_smoke_tests"] = bool(
int(os.environ["RELEASE_PREFER_SMOKE_TESTS"])
)
elif os.environ.get("RELEASE_FREQUENCY", "").endswith("-smoke"):
settings["prefer_smoke_tests"] = True
if "RAY_TEST_REPO" in os.environ:
settings["ray_test_repo"] = os.environ["RAY_TEST_REPO"]
settings["ray_test_branch"] = os.environ.get("RAY_TEST_BRANCH", DEFAULT_BRANCH)
@ -173,6 +182,8 @@ def update_settings_from_buildkite(settings: Dict):
release_frequency = get_buildkite_prompt_value("release-frequency")
if release_frequency:
settings["frequency"] = get_frequency(release_frequency)
if release_frequency.endswith("-smoke"):
settings["prefer_smoke_tests"] = True
ray_test_repo_branch = get_buildkite_prompt_value("release-ray-test-repo-branch")
if ray_test_repo_branch:

5
release/ray_release/config.py
View file

@ -10,7 +10,7 @@ import jsonschema
import yaml
from ray_release.anyscale_util import find_cloud_by_name
from ray_release.exception import ReleaseTestConfigError
from ray_release.exception import ReleaseTestConfigError, ReleaseTestCLIError
from ray_release.logger import logger
from ray_release.util import deep_update
@ -134,11 +134,10 @@ def find_test(test_collection: List[Test], test_name: str) -> Optional[Test]:
def as_smoke_test(test: Test) -> Test:
if "smoke_test" not in test:
logger.warning(
raise ReleaseTestCLIError(
f"Requested smoke test, but test with name {test['name']} does "
f"not have any smoke test configuration."
)
return test
smoke_test_config = test.pop("smoke_test")
new_test = deep_update(test, smoke_test_config)

3
release/ray_release/scripts/build_pipeline.py
View file

@ -67,6 +67,7 @@ def main(test_collection_file: Optional[str] = None):
shutil.rmtree(tmpdir, ignore_errors=True)
frequency = settings["frequency"]
prefer_smoke_tests = settings["prefer_smoke_tests"]
test_attr_regex_filters = settings["test_attr_regex_filters"]
ray_wheels = settings["ray_wheels"]
priority = settings["priority"]
@ -74,6 +75,7 @@ def main(test_collection_file: Optional[str] = None):
logger.info(
f"Found the following buildkite pipeline settings:\n\n"
f" frequency = {settings['frequency']}\n"
f" prefer_smoke_tests = {settings['prefer_smoke_tests']}\n"
f" test_attr_regex_filters = {settings['test_attr_regex_filters']}\n"
f" ray_wheels = {settings['ray_wheels']}\n"
f" ray_test_repo = {settings['ray_test_repo']}\n"
@ -86,6 +88,7 @@ def main(test_collection_file: Optional[str] = None):
test_collection,
frequency=frequency,
test_attr_regex_filters=test_attr_regex_filters,
prefer_smoke_tests=prefer_smoke_tests,
)
logger.info(f"Found {len(filtered_tests)} tests to run.")
if len(filtered_tests) == 0:

66
release/ray_release/tests/test_buildkite.py
View file

@ -158,6 +158,23 @@ class BuildkiteSettingsTest(unittest.TestCase):
updated_settings,
{
"frequency": Frequency.NIGHTLY,
"prefer_smoke_tests": False,
"test_attr_regex_filters": {"name": "name_filter"},
"ray_wheels": "custom-wheels",
"ray_test_repo": "https://github.com/user/ray.git",
"ray_test_branch": "sub/branch",
"priority": Priority.MANUAL,
"no_concurrency_limit": False,
},
)
os.environ["RELEASE_FREQUENCY"] = "any-smoke"
update_settings_from_environment(updated_settings)
self.assertDictEqual(
updated_settings,
{
"frequency": Frequency.ANY,
"prefer_smoke_tests": True,
"test_attr_regex_filters": {"name": "name_filter"},
"ray_wheels": "custom-wheels",
"ray_test_repo": "https://github.com/user/ray.git",
@ -307,6 +324,24 @@ class BuildkiteSettingsTest(unittest.TestCase):
updated_settings,
{
"frequency": Frequency.NIGHTLY,
"prefer_smoke_tests": False,
"test_attr_regex_filters": {"name": "name_filter"},
"ray_wheels": "custom-wheels",
"ray_test_repo": "https://github.com/user/ray.git",
"ray_test_branch": "sub/branch",
"priority": Priority.MANUAL,
"no_concurrency_limit": False,
},
)
self.buildkite["release-frequency"] = "any-smoke"
update_settings_from_buildkite(updated_settings)
self.assertDictEqual(
updated_settings,
{
"frequency": Frequency.ANY,
"prefer_smoke_tests": True,
"test_attr_regex_filters": {"name": "name_filter"},
"ray_wheels": "custom-wheels",
"ray_test_repo": "https://github.com/user/ray.git",
@ -363,6 +398,22 @@ class BuildkiteSettingsTest(unittest.TestCase):
],
)
filtered = self._filter_names_smoke(
tests,
frequency=Frequency.ANY,
prefer_smoke_tests=True,
)
self.assertSequenceEqual(
filtered,
[
("test_1", True),
("test_2", True),
("other_1", False),
("other_2", True),
("test_3", False),
],
)
filtered = self._filter_names_smoke(tests, frequency=Frequency.NIGHTLY)
self.assertSequenceEqual(
filtered,
@ -374,6 +425,21 @@ class BuildkiteSettingsTest(unittest.TestCase):
],
)
filtered = self._filter_names_smoke(
tests,
frequency=Frequency.NIGHTLY,
prefer_smoke_tests=True,
)
self.assertSequenceEqual(
filtered,
[
("test_1", True),
("test_2", True),
("other_2", True),
("test_3", False),
],
)
filtered = self._filter_names_smoke(tests, frequency=Frequency.WEEKLY)
self.assertSequenceEqual(filtered, [("test_2", False), ("other_1", False)])

6
release/ray_release/tests/test_wheels.py
View file

@ -19,7 +19,9 @@ from ray_release.wheels import (
class WheelsFinderTest(unittest.TestCase):
def setUp(self) -> None:
pass
for key in os.environ:
if key.startswith("BUILDKITE"):
os.environ.pop(key)
def testGetRayVersion(self):
init_file = os.path.join(
@ -72,8 +74,6 @@ class WheelsFinderTest(unittest.TestCase):
@patch("ray_release.wheels.get_ray_version", lambda *a, **kw: "2.0.0.dev0")
def testFindRayWheelsCommitOnly(self):
os.environ.pop("BUILDKITE_BRANCH")
repo = DEFAULT_REPO
branch = "master"
commit = "1234" * 10