diff --git a/doc/source/installation.rst b/doc/source/installation.rst index 6eb36df3c..e5071b413 100644 --- a/doc/source/installation.rst +++ b/doc/source/installation.rst @@ -65,13 +65,20 @@ You can install the Ray wheels of any particular commit on ``master`` with the f .. code-block:: bash - pip install https://s3-us-west-2.amazonaws.com/ray-wheels/master/{COMMIT_HASH}/ray-{RAY_VERSION}-{PYTHON_VERSION}-{PYTHON_VERSION}m-{OS_VERSION}_intel.whl + pip install https://s3-us-west-2.amazonaws.com/ray-wheels/master/{COMMIT_HASH}/ray-{RAY_VERSION}-{PYTHON_VERSION}-{PYTHON_VERSION}m-{OS_VERSION}.whl For example, here are the Ray 2.0.0.dev0 wheels for Python 3.7, MacOS for commit ``ba6cebe30fab6925e5b2d9e859ad064d53015246``: .. code-block:: bash - pip install https://s3-us-west-2.amazonaws.com/ray-wheels/master/ba6cebe30fab6925e5b2d9e859ad064d53015246/ray-2.0.0.dev0-cp37-cp37m-macosx_10_13_intel.whl + pip install https://s3-us-west-2.amazonaws.com/ray-wheels/master/ba6cebe30fab6925e5b2d9e859ad064d53015246/ray-2.0.0.dev0-cp37-cp37m-macosx_10_15_intel.whl + +There are minor variations to the format of the wheel filename; it's best to match against the format in the URLs listed in the :ref:`Nightlies section `. +Here's a summary of the variations: + +* For Python 3.8 and 3.9, the ``m`` before the OS version should be deleted and the OS version for MacOS should read ``macosx_10_15_x86_64`` instead of ``macosx_10_15_intel``. + +* For MacOS, commits predating August 7, 2021 will have ``macosx_10_13`` in the filename instad of ``macosx_10_15``. .. _ray-install-java: diff --git a/python/ray/_private/utils.py b/python/ray/_private/utils.py index fd208b304..2d60427b8 100644 --- a/python/ray/_private/utils.py +++ b/python/ray/_private/utils.py @@ -1041,27 +1041,26 @@ def get_wheel_filename( `ray --version`. Examples: "2.0.0.dev0" py_version (str): The major and minor Python versions concatenated. Examples: "36", - "37", "38" + "37", "38", "39" Returns: The wheel file name. Examples: ray-2.0.0.dev0-cp38-cp38-manylinux2014_x86_64.whl """ - assert py_version in ["36", "37", "38"], ("py_version must be one of '36'," - " '37', or '38'") + assert py_version in ["36", "37", "38", "39"], py_version os_strings = { "darwin": "macosx_10_15_x86_64" - if py_version == "38" else "macosx_10_15_intel", + if py_version in ["38", "39"] else "macosx_10_15_intel", "linux": "manylinux2014_x86_64", "win32": "win_amd64" } - assert sys_platform in os_strings, ("sys_platform must be one of 'darwin'," - " 'linux', or 'win32'") + assert sys_platform in os_strings, sys_platform - wheel_filename = (f"ray-{ray_version}-cp{py_version}-" - f"cp{py_version}{'m' if py_version != '38' else ''}" - f"-{os_strings[sys_platform]}.whl") + wheel_filename = ( + f"ray-{ray_version}-cp{py_version}-" + f"cp{py_version}{'m' if py_version in ['36', '37'] else ''}" + f"-{os_strings[sys_platform]}.whl") return wheel_filename diff --git a/python/ray/tests/test_runtime_env.py b/python/ray/tests/test_runtime_env.py index f7048dc3b..37a1fe732 100644 --- a/python/ray/tests/test_runtime_env.py +++ b/python/ray/tests/test_runtime_env.py @@ -663,19 +663,19 @@ def test_init(shutdown_only): def test_get_wheel_filename(): ray_version = "2.0.0.dev0" for sys_platform in ["darwin", "linux", "win32"]: - for py_version in ["36", "37", "38"]: + for py_version in ["36", "37", "38", "39"]: filename = get_wheel_filename(sys_platform, ray_version, py_version) prefix = "https://s3-us-west-2.amazonaws.com/ray-wheels/latest/" url = f"{prefix}{filename}" - assert requests.head(url).status_code == 200 + assert requests.head(url).status_code == 200, url def test_get_master_wheel_url(): ray_version = "2.0.0.dev0" test_commit = "58a73821fbfefbf53a19b6c7ffd71e70ccf258c7" for sys_platform in ["darwin", "linux", "win32"]: - for py_version in ["36", "37", "38"]: + for py_version in ["36", "37", "38", "39"]: url = get_master_wheel_url(test_commit, sys_platform, ray_version, py_version) assert requests.head(url).status_code == 200, url @@ -684,7 +684,7 @@ def test_get_master_wheel_url(): def test_get_release_wheel_url(): test_commits = {"1.6.0": "5052fe67d99f1d4bfc81b2a8694dbf2aa807bbdc"} for sys_platform in ["darwin", "linux", "win32"]: - for py_version in ["36", "37", "38"]: + for py_version in ["36", "37", "38", "39"]: for version, commit in test_commits.items(): url = get_release_wheel_url(commit, sys_platform, version, py_version) diff --git a/release/.buildkite/build_pipeline.py b/release/.buildkite/build_pipeline.py index 9aee4ae6f..92ffa1c2e 100644 --- a/release/.buildkite/build_pipeline.py +++ b/release/.buildkite/build_pipeline.py @@ -147,6 +147,7 @@ NIGHTLY_TESTS = { ], "~/ray/release/runtime_env_tests/runtime_env_tests.yaml": [ "rte_many_tasks_actors", + "wheel_urls", ], } diff --git a/release/e2e.py b/release/e2e.py index d43cda61f..51d7a82d7 100644 --- a/release/e2e.py +++ b/release/e2e.py @@ -672,7 +672,7 @@ def install_matching_ray(): return assert "manylinux2014_x86_64" in wheel, wheel if sys.platform == "darwin": - platform = "macosx_10_13_intel" + platform = "macosx_10_15_intel" elif sys.platform == "win32": platform = "win_amd64" else: diff --git a/release/runtime_env_tests/rte_minimal.yaml b/release/runtime_env_tests/rte_minimal.yaml new file mode 100644 index 000000000..cac304a8c --- /dev/null +++ b/release/runtime_env_tests/rte_minimal.yaml @@ -0,0 +1,16 @@ +cloud_id: {{env["ANYSCALE_CLOUD_ID"]}} +region: us-west-2 + +max_workers: 0 + +head_node_type: + name: head_node + instance_type: m5.xlarge + +worker_node_types: + - name: worker_node + instance_type: m5.xlarge + min_workers: 0 + max_workers: 0 + use_spot: false + diff --git a/release/runtime_env_tests/runtime_env_tests.yaml b/release/runtime_env_tests/runtime_env_tests.yaml index ce6c62355..96564fc26 100644 --- a/release/runtime_env_tests/runtime_env_tests.yaml +++ b/release/runtime_env_tests/runtime_env_tests.yaml @@ -7,3 +7,13 @@ timeout: 600 prepare: python wait_cluster.py 4 600 script: python workloads/rte_many_tasks_actors.py + +- name: wheel_urls + cluster: + app_config: app_config.yaml + compute_template: rte_minimal.yaml + + run: + timeout: 600 + prepare: python wait_cluster.py 1 600 + script: python workloads/wheel_urls.py diff --git a/release/runtime_env_tests/workloads/wheel_urls.py b/release/runtime_env_tests/workloads/wheel_urls.py new file mode 100644 index 000000000..feb9c840e --- /dev/null +++ b/release/runtime_env_tests/workloads/wheel_urls.py @@ -0,0 +1,57 @@ +"""Test downloading Ray wheels for currently running commit + +This test runs on a single node and verifies that wheel URLs on all platforms +for the currently running Ray commit are valid. This test is necessary to +catch changes in the format or location of uploaded wheels. A test like this is +is not straightforward to add in pre-merge CI because at pre-merge time, there +is no commit to master yet and no uploaded wheels. + +Runtime environments use these URLs to download the currently running Ray wheel +into isolated conda environments on each worker. + +Test owner: architkulkarni + +Acceptance criteria: Should run through and print "PASSED" +""" + +import ray +import os +import json +import time +import requests + +from ray._private.utils import get_master_wheel_url, get_release_wheel_url + + +def update_progress(result): + result["last_update"] = time.time() + test_output_json = os.environ.get("TEST_OUTPUT_JSON", + "/tmp/release_test_output.json") + with open(test_output_json, "wt") as f: + json.dump(result, f) + + +if __name__ == "__main__": + # Fail if running on a build from source that doesn't have a commit and + # hasn't been uploaded as a wheel to AWS. + assert "RAY_COMMIT_SHA" not in ray.__commit__, ray.__commit__ + + for sys_platform in ["darwin", "linux", "win32"]: + for py_version in ["36", "37", "38", "39"]: + if "dev" in ray.__version__: + url = get_master_wheel_url( + ray_commit=ray.__commit__, + sys_platform=sys_platform, + ray_version=ray.__version__, + py_version=py_version) + else: + url = get_release_wheel_url( + ray_commit=ray.__commit__, + sys_platform=sys_platform, + ray_version=ray.__version__, + py_version=py_version) + assert requests.head(url).status_code == 200, url + print("Successfully tested URL: ", url) + update_progress({"url": url}) + + print("PASSED")