diff --git a/build.sh b/build.sh
index 8c075f641..04d3b81a4 100755
--- a/build.sh
+++ b/build.sh
@@ -121,43 +121,44 @@ fi
 # Now we build everything.
 BUILD_DIR="$ROOT_DIR/build/"
 if [ ! -d "${BUILD_DIR}" ]; then
-mkdir -p "${BUILD_DIR}"
+  mkdir -p "${BUILD_DIR}"
 fi
 
 pushd "$BUILD_DIR"
 
 
-need_pickle5_backport=0
-PYTHON_REVISION="$("$PYTHON_EXECUTABLE" -s -c "import sys; print('{}.{}.{}'.format(*sys.version_info))")"
-case "${PYTHON_REVISION}.0" in
-3\.[5-7].*|3.8.[0-1].*) need_pickle5_backport=1;;
-esac
-
-if [ 0 -ne "${need_pickle5_backport}" ]; then
-WORK_DIR="$(mktemp -d)"
-pushd "${WORK_DIR}"
-git clone https://github.com/suquark/pickle5-backport
-pushd pickle5-backport
-  git checkout 8ffe41ceba9d5e2ce8a98190f6b3d2f3325e5a72
-  CC=gcc "$PYTHON_EXECUTABLE" setup.py --quiet bdist_wheel
-  unzip -q -o dist/*.whl -d "$ROOT_DIR/python/ray/pickle5_files"
-popd
-popd
-fi
-
-
-if [ -z "$SKIP_THIRDPARTY_INSTALL" ]; then
-    CC=gcc "$PYTHON_EXECUTABLE" -m pip install -q psutil setproctitle \
-            --target="$ROOT_DIR/python/ray/thirdparty_files"
-fi
-
-export PYTHON3_BIN_PATH="$PYTHON_EXECUTABLE"
-
 if [ "$RAY_BUILD_JAVA" == "YES" ]; then
   "$BAZEL_EXECUTABLE" build //java:ray_java_pkg --verbose_failures
 fi
 
 if [ "$RAY_BUILD_PYTHON" == "YES" ]; then
+  pickle5_available=0
+  pickle5_path="$ROOT_DIR/python/ray/pickle5_files"
+  # Check if the current Python alrady has pickle5 (either comes with newer Python versions, or has been installed by us before).
+  if PYTHONPATH="$pickle5_path:$PYTHONPATH" "$PYTHON_EXECUTABLE" -s -c "import pickle5" 2>/dev/null; then
+    pickle5_available=1
+  fi
+  if [ 1 -ne "${pickle5_available}" ]; then
+    # Install pickle5-backport.
+    TEMP_DIR="$(mktemp -d)"
+    pushd "$TEMP_DIR"
+    wget --quiet -O pickle5-backport.zip https://github.com/suquark/pickle5-backport/archive/8ffe41ceba9d5e2ce8a98190f6b3d2f3325e5a72.zip
+    unzip pickle5-backport.zip
+    pushd pickle5-backport-8ffe41ceba9d5e2ce8a98190f6b3d2f3325e5a72
+      CC=gcc "$PYTHON_EXECUTABLE" setup.py --quiet bdist_wheel
+      unzip -q -o dist/*.whl -d "$pickle5_path"
+    popd
+    popd
+    rm -rf "$TEMP_DIR"
+  fi
+
+  if [ -z "$SKIP_THIRDPARTY_INSTALL" ]; then
+      CC=gcc "$PYTHON_EXECUTABLE" -m pip install -q psutil setproctitle \
+              --target="$ROOT_DIR/python/ray/thirdparty_files"
+  fi
+
+  export PYTHON3_BIN_PATH="$PYTHON_EXECUTABLE"
+
   "$BAZEL_EXECUTABLE" build //:ray_pkg --verbose_failures
 fi