[runtime env] [test] Add nightly test to verify Ray wheel URLs are valid (#17938)

doc/source/installation.rst

@@ -65,13 +65,20 @@ You can install the Ray wheels of any particular commit on ``master`` with the f
 
 .. code-block:: bash
 
-    pip install https://s3-us-west-2.amazonaws.com/ray-wheels/master/{COMMIT_HASH}/ray-{RAY_VERSION}-{PYTHON_VERSION}-{PYTHON_VERSION}m-{OS_VERSION}_intel.whl
+    pip install https://s3-us-west-2.amazonaws.com/ray-wheels/master/{COMMIT_HASH}/ray-{RAY_VERSION}-{PYTHON_VERSION}-{PYTHON_VERSION}m-{OS_VERSION}.whl
 
 For example, here are the Ray 2.0.0.dev0 wheels for Python 3.7, MacOS for commit ``ba6cebe30fab6925e5b2d9e859ad064d53015246``:
 
 .. code-block:: bash
 
-    pip install https://s3-us-west-2.amazonaws.com/ray-wheels/master/ba6cebe30fab6925e5b2d9e859ad064d53015246/ray-2.0.0.dev0-cp37-cp37m-macosx_10_13_intel.whl
+    pip install https://s3-us-west-2.amazonaws.com/ray-wheels/master/ba6cebe30fab6925e5b2d9e859ad064d53015246/ray-2.0.0.dev0-cp37-cp37m-macosx_10_15_intel.whl
+
+There are minor variations to the format of the wheel filename; it's best to match against the format in the URLs listed in the :ref:`Nightlies section <install-nightlies>`.
+Here's a summary of the variations:
+
+* For Python 3.8 and 3.9, the ``m`` before the OS version should be deleted and the OS version for MacOS should read ``macosx_10_15_x86_64`` instead of ``macosx_10_15_intel``.
+
+* For MacOS, commits predating August 7, 2021 will have ``macosx_10_13`` in the filename instad of ``macosx_10_15``.
 
 .. _ray-install-java:
 

python/ray/_private/utils.py

@@ -1041,27 +1041,26 @@ def get_wheel_filename(
             `ray --version`.  Examples: "2.0.0.dev0"
         py_version (str):
             The major and minor Python versions concatenated.  Examples: "36",
-            "37", "38"
+            "37", "38", "39"
     Returns:
         The wheel file name.  Examples:
             ray-2.0.0.dev0-cp38-cp38-manylinux2014_x86_64.whl
     """
-    assert py_version in ["36", "37", "38"], ("py_version must be one of '36',"
-                                              " '37', or '38'")
+    assert py_version in ["36", "37", "38", "39"], py_version
 
     os_strings = {
         "darwin": "macosx_10_15_x86_64"
-        if py_version == "38" else "macosx_10_15_intel",
+        if py_version in ["38", "39"] else "macosx_10_15_intel",
         "linux": "manylinux2014_x86_64",
         "win32": "win_amd64"
     }
 
-    assert sys_platform in os_strings, ("sys_platform must be one of 'darwin',"
-                                        " 'linux', or 'win32'")
+    assert sys_platform in os_strings, sys_platform
 
-    wheel_filename = (f"ray-{ray_version}-cp{py_version}-"
-                      f"cp{py_version}{'m' if py_version != '38' else ''}"
-                      f"-{os_strings[sys_platform]}.whl")
+    wheel_filename = (
+        f"ray-{ray_version}-cp{py_version}-"
+        f"cp{py_version}{'m' if py_version in ['36', '37'] else ''}"
+        f"-{os_strings[sys_platform]}.whl")
 
     return wheel_filename
 

python/ray/tests/test_runtime_env.py

@@ -663,19 +663,19 @@ def test_init(shutdown_only):
 def test_get_wheel_filename():
     ray_version = "2.0.0.dev0"
     for sys_platform in ["darwin", "linux", "win32"]:
-        for py_version in ["36", "37", "38"]:
+        for py_version in ["36", "37", "38", "39"]:
             filename = get_wheel_filename(sys_platform, ray_version,
                                           py_version)
             prefix = "https://s3-us-west-2.amazonaws.com/ray-wheels/latest/"
             url = f"{prefix}{filename}"
-            assert requests.head(url).status_code == 200
+            assert requests.head(url).status_code == 200, url
 
 
 def test_get_master_wheel_url():
     ray_version = "2.0.0.dev0"
     test_commit = "58a73821fbfefbf53a19b6c7ffd71e70ccf258c7"
     for sys_platform in ["darwin", "linux", "win32"]:
-        for py_version in ["36", "37", "38"]:
+        for py_version in ["36", "37", "38", "39"]:
             url = get_master_wheel_url(test_commit, sys_platform, ray_version,
                                        py_version)
             assert requests.head(url).status_code == 200, url
@@ -684,7 +684,7 @@ def test_get_master_wheel_url():
 def test_get_release_wheel_url():
     test_commits = {"1.6.0": "5052fe67d99f1d4bfc81b2a8694dbf2aa807bbdc"}
     for sys_platform in ["darwin", "linux", "win32"]:
-        for py_version in ["36", "37", "38"]:
+        for py_version in ["36", "37", "38", "39"]:
             for version, commit in test_commits.items():
                 url = get_release_wheel_url(commit, sys_platform, version,
                                             py_version)

release/.buildkite/build_pipeline.py

@@ -147,6 +147,7 @@ NIGHTLY_TESTS = {
     ],
     "~/ray/release/runtime_env_tests/runtime_env_tests.yaml": [
         "rte_many_tasks_actors",
+        "wheel_urls",
     ],
 }
 

release/e2e.py

@@ -672,7 +672,7 @@ def install_matching_ray():
         return
     assert "manylinux2014_x86_64" in wheel, wheel
     if sys.platform == "darwin":
-        platform = "macosx_10_13_intel"
+        platform = "macosx_10_15_intel"
     elif sys.platform == "win32":
         platform = "win_amd64"
     else:

release/runtime_env_tests/rte_minimal.yaml

@@ -0,0 +1,16 @@
+cloud_id: {{env["ANYSCALE_CLOUD_ID"]}}
+region: us-west-2
+
+max_workers: 0
+
+head_node_type:
+    name: head_node
+    instance_type: m5.xlarge
+
+worker_node_types:
+    - name: worker_node
+      instance_type: m5.xlarge
+      min_workers: 0
+      max_workers: 0
+      use_spot: false
+

release/runtime_env_tests/runtime_env_tests.yaml

@@ -7,3 +7,13 @@
     timeout: 600
     prepare: python wait_cluster.py 4 600
     script: python workloads/rte_many_tasks_actors.py
+
+- name: wheel_urls
+  cluster:
+    app_config: app_config.yaml
+    compute_template: rte_minimal.yaml
+
+  run:
+    timeout: 600
+    prepare: python wait_cluster.py 1 600
+    script: python workloads/wheel_urls.py

release/runtime_env_tests/workloads/wheel_urls.py

@@ -0,0 +1,57 @@
+"""Test downloading Ray wheels for currently running commit
+
+This test runs on a single node and verifies that wheel URLs on all platforms
+for the currently running Ray commit are valid.  This test is necessary to
+catch changes in the format or location of uploaded wheels. A test like this is
+is not straightforward to add in pre-merge CI because at pre-merge time, there
+is no commit to master yet and no uploaded wheels.
+
+Runtime environments use these URLs to download the currently running Ray wheel
+into isolated conda environments on each worker.
+
+Test owner: architkulkarni
+
+Acceptance criteria: Should run through and print "PASSED"
+"""
+
+import ray
+import os
+import json
+import time
+import requests
+
+from ray._private.utils import get_master_wheel_url, get_release_wheel_url
+
+
+def update_progress(result):
+    result["last_update"] = time.time()
+    test_output_json = os.environ.get("TEST_OUTPUT_JSON",
+                                      "/tmp/release_test_output.json")
+    with open(test_output_json, "wt") as f:
+        json.dump(result, f)
+
+
+if __name__ == "__main__":
+    # Fail if running on a build from source that doesn't have a commit and
+    # hasn't been uploaded as a wheel to AWS.
+    assert "RAY_COMMIT_SHA" not in ray.__commit__, ray.__commit__
+
+    for sys_platform in ["darwin", "linux", "win32"]:
+        for py_version in ["36", "37", "38", "39"]:
+            if "dev" in ray.__version__:
+                url = get_master_wheel_url(
+                    ray_commit=ray.__commit__,
+                    sys_platform=sys_platform,
+                    ray_version=ray.__version__,
+                    py_version=py_version)
+            else:
+                url = get_release_wheel_url(
+                    ray_commit=ray.__commit__,
+                    sys_platform=sys_platform,
+                    ray_version=ray.__version__,
+                    py_version=py_version)
+            assert requests.head(url).status_code == 200, url
+            print("Successfully tested URL: ", url)
+            update_progress({"url": url})
+
+    print("PASSED")