Python 3.8 compatibility (#7754)

.bazelrc

@@ -21,6 +21,9 @@ build --per_file_copt="-\\.(asm|S)$,\\.pb\\.cc$@-w"
 build --per_file_copt="-\\.(asm|S)$,external/.*@-w"
 # Ignore this warning since it's impractical to fix in the relevant headers
 build --per_file_copt="-\\.(asm|S)$,.*/ray/raylet/reconstruction_policy_test\\.cc@-Wno-inconsistent-missing-override"
+# Ignore this warning since CPython and Cython have issue removing deprecated tp_print on MacOS
+build --per_file_copt="-\\.(asm|S)$,.*/_raylet\\.cpp@-Wno-deprecated-declarations"
+build --per_file_copt="-\\.(asm|S)$,.*/_streaming\\.cpp@-Wno-deprecated-declarations"
 # Ignore minor warnings for host tools, which we generally can't control
 build --host_copt="-Wno-builtin-macro-redefined"
 build --host_copt="-Wno-inconsistent-missing-override"

.travis.yml

@@ -148,13 +148,13 @@ matrix:
         # This command should be kept in sync with ray/python/README-building-wheels.md,
         # except the `$MOUNT_BAZEL_CACHE` part.
 
-        - ./ci/suppress_output docker run -e TRAVIS_COMMIT=$TRAVIS_COMMIT --rm -w /ray -v `pwd`:/ray $MOUNT_BAZEL_CACHE -ti rayproject/arrow_linux_x86_64_base:ARROW-5631 /ray/python/build-wheel-manylinux1.sh
+        - ./ci/suppress_output docker run --rm -w /ray -v `pwd`:/ray $MOUNT_BAZEL_CACHE -e TRAVIS_COMMIT=$TRAVIS_COMMIT -ti rayproject/arrow_linux_x86_64_base:python-3.8.0 /ray/python/build-wheel-manylinux1.sh
 
       script:
         - if [ $RAY_CI_LINUX_WHEELS_AFFECTED != "1" ]; then exit; fi
 
         # Explicitly sleep 60 seconds for logs to go through
-        - ./ci/travis/test-wheels.sh || cat /tmp/ray/session_latest/logs/* || (sleep 60 && false)
+        - ./ci/travis/test-wheels.sh || { cat /tmp/ray/session_latest/logs/* && sleep 60 && false; }
       cache: false
 
     # Build MacOS wheels.
@@ -172,7 +172,7 @@ matrix:
         - if [ $RAY_CI_MACOS_WHEELS_AFFECTED != "1" ]; then exit; fi
 
         # Explicitly sleep 60 seconds for logs to go through
-        - ./ci/travis/test-wheels.sh || cat /tmp/ray/session_latest/logs/* || (sleep 60 && false)
+        - ./ci/travis/test-wheels.sh || { cat /tmp/ray/session_latest/logs/* && sleep 60 && false; }
 
     # RLlib: Learning tests (from rllib/tuned_examples/regression_tests/*.yaml).
     - os: linux

bazel/ray_deps_setup.bzl

@@ -178,8 +178,8 @@ def ray_deps_setup():
     auto_http_archive(
         name = "cython",
         build_file = True,
-        url = "https://github.com/cython/cython/archive/49414dbc7ddc2ca2979d6dbe1e44714b10d72e7e.tar.gz",
-        sha256 = "0b697ac90d1e46842c7cbbf5f4a1bde5b7b41037c611167417115337e3756eaa",
+        url = "https://github.com/cython/cython/archive/26cb654dcf4ed1b1858daf16b39fd13406b1ac64.tar.gz",
+        sha256 = "d21e155ac9a455831f81608bb06620e4a1d75012a630faf11f4c25ad10cfc9bb",
     )
 
     auto_http_archive(

build.sh

@@ -6,7 +6,7 @@ set -x
 set -e
 
 # As the supported Python versions change, edit this array:
-SUPPORTED_PYTHONS=( "3.5" "3.6" "3.7" )
+SUPPORTED_PYTHONS=( "3.5" "3.6" "3.7" "3.8" )
 
 ROOT_DIR=$(cd "$(dirname "${BASH_SOURCE:-$0}")"; pwd)
 

ci/travis/test-wheels.sh

@@ -21,41 +21,72 @@ else
   exit 1
 fi
 
-TEST_SCRIPT="$TRAVIS_BUILD_DIR/python/ray/tests/test_microbenchmarks.py"
+TEST_DIR="$TRAVIS_BUILD_DIR/python/ray/tests"
+TEST_SCRIPTS=("$TEST_DIR/test_microbenchmarks.py" "$TEST_DIR/test_basic.py")
 UI_TEST_SCRIPT="$TRAVIS_BUILD_DIR/python/ray/tests/test_webui.py"
 
+function retry {
+  local n=1
+  local max=3
+
+  while true; do
+    "$@" && break || {
+      if [[ $n -lt $max ]]; then
+        ((n++))
+        echo "Command failed. Attempt $n/$max:"
+      else
+        echo "The command has failed after $n attempts."
+        exit 1
+      fi
+    }
+  done
+}
+
 if [[ "$platform" == "linux" ]]; then
-  # Now test Python 3.6.
-
   # Install miniconda.
-  wget --quiet https://repo.continuum.io/miniconda/Miniconda3-4.5.4-Linux-x86_64.sh -O miniconda3.sh
+  PY_WHEEL_VERSIONS=("36" "37" "38")
+  PY_MMS=("3.6.9"
+          "3.7.6"
+          "3.8.2")
+  wget --quiet "https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh" -O miniconda3.sh
   bash miniconda3.sh -b -p "$HOME/miniconda3"
+  export PATH="$HOME/miniconda3/bin:$PATH"
 
-  PYTHON_EXE=$HOME/miniconda3/bin/python
-  PIP_CMD=$HOME/miniconda3/bin/pip
+  for ((i=0; i<${#PY_MMS[@]}; ++i)); do
+    PY_MM="${PY_MMS[i]}"
+    PY_WHEEL_VERSION="${PY_WHEEL_VERSIONS[i]}"
 
-  # Find the right wheel by grepping for the Python version.
-  PYTHON_WHEEL=$(find "$ROOT_DIR/../../.whl" -type f -maxdepth 1 -print | grep -m1 '36')
+    conda install -y python="${PY_MM}"
 
-  # Install the wheel.
-  $PIP_CMD install -q "$PYTHON_WHEEL"
+    PYTHON_EXE="$HOME/miniconda3/bin/python"
+    PIP_CMD="$HOME/miniconda3/bin/pip"
 
-  # Check that ray.__commit__ was set properly.
-  $PYTHON_EXE -u -c "import ray; print(ray.__commit__)" | grep $TRAVIS_COMMIT || (echo "ray.__commit__ not set properly!" && exit 1)
+    # Find the right wheel by grepping for the Python version.
+    PYTHON_WHEEL=$(find "$ROOT_DIR/../../.whl" -type f -maxdepth 1 -print | grep -m1 "$PY_WHEEL_VERSION")
 
-  # Install the dependencies to run the tests.
-  $PIP_CMD install -q aiohttp google grpcio pytest requests
+    # Install the wheel.
+    "$PIP_CMD" install -q "$PYTHON_WHEEL"
 
-  # Run a simple test script to make sure that the wheel works.
-  INSTALLED_RAY_DIRECTORY=$(dirname "$($PYTHON_EXE -u -c "import ray; print(ray.__file__)" | tail -n1)")
-  $PYTHON_EXE "$TEST_SCRIPT"
+    # Check that ray.__commit__ was set properly.
+    "$PYTHON_EXE" -u -c "import ray; print(ray.__commit__)" | grep "$TRAVIS_COMMIT" || (echo "ray.__commit__ not set properly!" && exit 1)
 
-  # Run the UI test to make sure that the packaged UI works.
-  $PYTHON_EXE "$UI_TEST_SCRIPT"
+    # Install the dependencies to run the tests.
+    "$PIP_CMD" install -q aiohttp google grpcio pytest requests
+
+    # Run a simple test script to make sure that the wheel works.
+    INSTALLED_RAY_DIRECTORY=$(dirname "$($PYTHON_EXE -u -c "import ray; print(ray.__file__)" | tail -n1)")
+
+    for SCRIPT in "${TEST_SCRIPTS[@]}"; do
+        retry "$PYTHON_EXE" "$SCRIPT"
+    done
+
+    # Run the UI test to make sure that the packaged UI works.
+    retry "$PYTHON_EXE" "$UI_TEST_SCRIPT"
+  done
 
   # Check that the other wheels are present.
   NUMBER_OF_WHEELS=$(ls -1q "$ROOT_DIR"/../../.whl/*.whl | wc -l)
-  if [[ "$NUMBER_OF_WHEELS" != "3" ]]; then
+  if [[ "$NUMBER_OF_WHEELS" != "4" ]]; then
     echo "Wrong number of wheels found."
     ls -l "$ROOT_DIR/../.whl/"
     exit 2
@@ -63,37 +94,38 @@ if [[ "$platform" == "linux" ]]; then
 
 elif [[ "$platform" == "macosx" ]]; then
   MACPYTHON_PY_PREFIX=/Library/Frameworks/Python.framework/Versions
+  PY_WHEEL_VERSIONS=("35" "36" "37" "38")
   PY_MMS=("3.5"
           "3.6"
-          "3.7")
-  # This array is just used to find the right wheel.
-  PY_WHEEL_VERSIONS=("35"
-                     "36"
-                     "37")
+          "3.7"
+          "3.8")
 
   for ((i=0; i<${#PY_MMS[@]}; ++i)); do
-    PY_MM=${PY_MMS[i]}
-    PY_WHEEL_VERSION=${PY_WHEEL_VERSIONS[i]}
+    PY_MM="${PY_MMS[i]}"
 
-    PYTHON_EXE=$MACPYTHON_PY_PREFIX/$PY_MM/bin/python$PY_MM
+    PY_WHEEL_VERSION="${PY_WHEEL_VERSIONS[i]}"
+
+    PYTHON_EXE="$MACPYTHON_PY_PREFIX/$PY_MM/bin/python$PY_MM"
     PIP_CMD="$(dirname "$PYTHON_EXE")/pip$PY_MM"
 
     # Find the appropriate wheel by grepping for the Python version.
     PYTHON_WHEEL=$(find "$ROOT_DIR/../../.whl" -type f -maxdepth 1 -print | grep -m1 "$PY_WHEEL_VERSION")
 
     # Install the wheel.
-    $PIP_CMD install -q "$PYTHON_WHEEL"
+    "$PIP_CMD" install -q "$PYTHON_WHEEL"
 
     # Install the dependencies to run the tests.
-    $PIP_CMD install -q aiohttp google grpcio pytest requests
+    "$PIP_CMD" install -q aiohttp google grpcio pytest requests
 
     # Run a simple test script to make sure that the wheel works.
     INSTALLED_RAY_DIRECTORY=$(dirname "$($PYTHON_EXE -u -c "import ray; print(ray.__file__)" | tail -n1)")
-    $PYTHON_EXE "$TEST_SCRIPT"
+    for SCRIPT in "${TEST_SCRIPTS[@]}"; do
+      retry "$PYTHON_EXE" "$SCRIPT"
+    done
 
     if (( $(echo "$PY_MM >= 3.0" | bc) )); then
       # Run the UI test to make sure that the packaged UI works.
-      $PYTHON_EXE "$UI_TEST_SCRIPT"
+      retry "$PYTHON_EXE" "$UI_TEST_SCRIPT"
     fi
 
   done

python/build-wheel-macos.sh

@@ -15,17 +15,21 @@ DOWNLOAD_DIR=python_downloads
 
 PY_VERSIONS=("3.5.3"
              "3.6.1"
-             "3.7.0")
+             "3.7.0"
+             "3.8.2")
 PY_INSTS=("python-3.5.3-macosx10.6.pkg"
           "python-3.6.1-macosx10.6.pkg"
-          "python-3.7.0-macosx10.6.pkg")
+          "python-3.7.0-macosx10.6.pkg"
+          "python-3.8.2-macosx10.9.pkg")
 PY_MMS=("3.5"
         "3.6"
-        "3.7")
+        "3.7"
+        "3.8")
 
 # The minimum supported numpy version is 1.14, see
 # https://issues.apache.org/jira/browse/ARROW-3141
 NUMPY_VERSIONS=("1.14.5"
+                "1.14.5"
                 "1.14.5"
                 "1.14.5")
 
@@ -76,7 +80,7 @@ for ((i=0; i<${#PY_VERSIONS[@]}; ++i)); do
     $PIP_CMD install -q setuptools_scm==3.1.0
     # Fix the numpy version because this will be the oldest numpy version we can
     # support.
-    $PIP_CMD install -q numpy==$NUMPY_VERSION cython==0.29.0
+    $PIP_CMD install -q numpy==$NUMPY_VERSION cython==0.29.15
     # Install wheel to avoid the error "invalid command 'bdist_wheel'".
     $PIP_CMD install -q wheel
     # Set the commit SHA in __init__.py.

python/build-wheel-manylinux1.sh

@@ -13,11 +13,13 @@ chmod +x /usr/bin/nproc
 
 PYTHONS=("cp35-cp35m"
          "cp36-cp36m"
-         "cp37-cp37m")
+         "cp37-cp37m"
+         "cp38-cp38")
 
 # The minimum supported numpy version is 1.14, see
 # https://issues.apache.org/jira/browse/ARROW-3141
 NUMPY_VERSIONS=("1.14.5"
+                "1.14.5"
                 "1.14.5"
                 "1.14.5")
 
@@ -55,7 +57,7 @@ for ((i=0; i<${#PYTHONS[@]}; ++i)); do
   pushd python
     # Fix the numpy version because this will be the oldest numpy version we can
     # support.
-    /opt/python/${PYTHON}/bin/pip install -q numpy==${NUMPY_VERSION} cython==0.29.0
+    /opt/python/${PYTHON}/bin/pip install -q numpy==${NUMPY_VERSION} cython==0.29.15
     # Set the commit SHA in __init__.py.
     if [ -n "$TRAVIS_COMMIT" ]; then
       sed -i.bak "s/{{RAY_COMMIT_SHA}}/$TRAVIS_COMMIT/g" ray/__init__.py && rm ray/__init__.py.bak

python/ray/cloudpickle/cloudpickle_fast.py

@@ -417,7 +417,7 @@ def _numpy_frombuffer(buffer, dtype, shape, order):
     array = _frombuffer(buffer, dtype, shape, order)
     # Unfortunately, numpy does not follow the standard, so we still
     # have to set the readonly flag for it here.
-    array.setflags(write=not buffer.readonly)
+    array.setflags(write=isinstance(buffer, bytearray) or not buffer.readonly)
     return array
 
 

python/setup.py

@@ -191,7 +191,7 @@ setup(
     # The BinaryDistribution argument triggers build_ext.
     distclass=BinaryDistribution,
     install_requires=requires,
-    setup_requires=["cython >= 0.29"],
+    setup_requires=["cython >= 0.29.14"],
     extras_require=extras,
     entry_points={
         "console_scripts": [