[Buildkite] Migrate macOS wheel builds (#16913)

.buildkite/pipeline.macos.yml

@@ -9,8 +9,6 @@ common: &common
     RAY_DEFAULT_BUILD: "1"
     LC_ALL: en_US.UTF-8
     LANG: en_US.UTF-8
-    # TODO(simon): Can't figure out how get the JDK working yet.
-    # RAY_INSTALL_JAVA: "1"
 
 prelude_commands: &prelude_commands |-
   ./ci/travis/upload_build_info.sh
@@ -26,6 +24,41 @@ epilogue_commands: &epilogue_commands |-
   bazel clean
 
 steps:
+- label: ":mac: :apple: Wheels and Jars"
+  <<: *common
+  commands:
+    # Cleanup environments
+    - ./ci/travis/upload_build_info.sh
+    - (which bazel && bazel clean) || true
+    # TODO(simon): make sure to change both PR and wheel builds
+    # Special setup for jar builds (will be installed to the machine instead)
+    # - brew remove --force java & brew uninstall --force java & rm -rf /usr/local/Homebrew/Library/Taps/homebrew/homebrew-cask
+    # - brew install --cask adoptopenjdk/openjdk/adoptopenjdk8
+    - diskutil list external physical
+    - export JAVA_HOME=/Library/Java/JavaVirtualMachines/adoptopenjdk-8.jdk/Contents/Home
+    - java -version
+    # Build wheels
+    - export MAC_WHEELS=1
+    - export MAC_JARS=1
+    - export RAY_INSTALL_JAVA=1
+    - . ./ci/travis/ci.sh init && source ~/.zshrc
+    - . ./ci/travis/ci.sh build
+    # Test wheels
+    - . ./ci/travis/ci.sh test_wheels
+    # Build jars
+    - bash ./java/build-jar-multiplatform.sh darwin
+    # Upload the wheels and jars
+    # We don't want to push on PRs, in fact, the copy_files will fail because unauthenticated.
+    - if [ "$BUILDKITE_PULL_REQUEST" != "false" ]; then exit 0; fi
+    - pip install -q docker aws_requests_auth boto3
+    # Upload to branch directory.
+    - python .buildkite/copy_files.py --destination branch_wheels --path ./.whl
+    - python .buildkite/copy_files.py --destination branch_jars --path ./.jar/darwin
+    # Upload to latest directory.
+    - if [ "$BUILDKITE_BRANCH" == "master" ]; then python .buildkite/copy_files.py --destination wheels --path ./.whl; fi
+    - if [ "$BUILDKITE_BRANCH" == "master" ]; then python .buildkite/copy_files.py --destination jars --path ./.jar/darwin; fi
+
+
 - label: ":mac: :apple: Ray C++ and Libraries"
   <<: *common
   commands:
@@ -79,15 +112,3 @@ steps:
       --test_env=CI
       python/ray/tests/... python/ray/serve/...
     - *epilogue_commands
-
-    # The follow is running in Travis for now.
-    # - bazel test --test_env=CONDA_EXE --test_env=CONDA_PYTHON_EXE --test_env=CONDA_SHLVL --test_env=CONDA_PREFIX --test_env=CONDA_DEFAULT_ENV --test_env=CONDA_PROMPT_MODIFIER --config=ci $(./scripts/bazel_export_options) --test_tag_filters=-kubernetes,-jenkins_only,-medium_size_python_tests_a_to_j,-medium_size_python_tests_k_to_z,-flaky python/ray/tests/...
-    # - bazel test --config=ci $(./scripts/bazel_export_options) --test_tag_filters=-kubernetes,-jenkins_only,medium_size_python_tests_k_to_z,-flaky python/ray/tests/...
-    # - bazel test --config=ci $(./scripts/bazel_export_options)
-    #       --test_tag_filters=-kubernetes,-jenkins_only,flaky
-    #       --test_env=CONDA_EXE
-    #       --test_env=CONDA_PYTHON_EXE
-    #       --test_env=CONDA_SHLVL
-    #       --test_env=CONDA_PREFIX
-    #       --test_env=CONDA_DEFAULT_ENV
-    #       python/ray/tests/... python/ray/serve/... python/ray/tune/... rllib/...

.travis.yml

@@ -1,114 +0,0 @@
-language: generic
-# Use Ubuntu 18.04
-dist: bionic
-
-git:
-  clone: false  # Clone manually to work around Travis issues like https://github.com/travis-ci/travis-ci/issues/6337
-  depth: false  # Shallow clones can prevent diff against base branch
-  quiet: true
-
-branches:
-  except:
-    - /dependabot.*/
-
-before_install:
-  - unset -f cd  # Travis defines this on Mac for RVM, but it breaks the Mac build
-  - |
-    git clone -q -n "https://github.com/${TRAVIS_REPO_SLUG}.git" "${TRAVIS_REPO_SLUG}"
-    cd -- "${TRAVIS_REPO_SLUG}"
-    to_fetch=("${TRAVIS_COMMIT}")
-    if [ false != "${TRAVIS_PULL_REQUEST-}" ]; then to_fetch+=("+refs/pull/${TRAVIS_PULL_REQUEST}/merge:"); fi
-    git fetch -q -- origin "${to_fetch[@]}"
-    git checkout -qf "${TRAVIS_COMMIT}" --
-    python -u ci/remote-watch.py --skip_repo=ray-project/ray &
-
-matrix:
-  include:
-    # Build MacOS wheels and MacOS jars
-    - os: osx
-      osx_image: xcode7
-      env:
-        - MAC_WHEELS=1 MAC_JARS=1
-        - PYTHONWARNINGS=ignore
-        - RAY_INSTALL_JAVA=1
-      install:
-        - . ./ci/travis/ci.sh init RAY_CI_MACOS_WHEELS_AFFECTED,RAY_CI_JAVA_AFFECTED,RAY_CI_STREAMING_JAVA_AFFECTED
-      before_script:
-        - ./ci/keep_alive brew remove --force java & brew uninstall --force java & rm -rf /usr/local/Homebrew/Library/Taps/homebrew/homebrew-cask
-        - ./ci/keep_alive brew install --cask adoptopenjdk/openjdk/adoptopenjdk8
-        - export JAVA_HOME=/Library/Java/JavaVirtualMachines/adoptopenjdk-8.jdk/Contents/Home
-        - java -version
-        - ./ci/keep_alive bash ./ci/travis/ci.sh build
-      script:
-        - . ./ci/travis/ci.sh test_wheels
-        - bash ./java/build-jar-multiplatform.sh darwin
-
-
-after_script:
-  - if [ "${TRAVIS_PULL_REQUEST}" = "false" ]; then ./ci/travis/upload_build_info.sh; fi
-
-deploy:
-  - provider: s3
-    edge: true # This supposedly opts in to deploy v2.
-    access_key_id: AKIAU6DMUCJUFL3EX3SM
-    secret_access_key:
-      secure: J1sX71fKFPQhgWzColllxfzcF877ScBZ1cIl71krZ6SO0LKnwsCScpQck5eZOyQo/Iverwye0iKtE87qNsiRi3+V2D9iulSr18T09j7+FjPKfxAmXmjfrNafoMXTDQroSJblCri5vl+DysISPqImJkWTNaYhGJ9QakoSd5djnAopLNWj6PCR3S50baS49+nB5nSIY3jMhtUzlaBdniFPFC81Cxyuafr4pv6McGRfR/dK+ZnPhdGtMnVeIJXB+ooZKQ26mDJKBPka4jm3u1Oa72b/Atu2RO3MwxTg79LTrMxXKh2OcCqhtD2Z3lz1OltvNSunCuwY8AejCJsfSLbM9mGDoz+xhNUWmYNy48YFf+61OY8PXi8S/9Q817yb3GpLbb2l/P+KMgq9eSEiELIOwuYsDxPX5TuAg6dx0wCNgDEBJoThSQjYl6MgJrLrs7p+JBxp3giedHiy0TLa5hCVKTj3euONAXDArYnnT+DvUIOkaeTk5DClRZbZ0sUXhLy//HuT5WJvjFBJJZ0u0f4RLVb5D7DI4uMZr7+yJPDR2AXCyW9YMaBEbmEYbPaKi283jlEyn7R33+AZlnXv0THHwZ4xvjKKG3/fBSXsOUmv5wmUveEqVGDj1mKPGj9NF8iA5qMm2AaZuJpEEBVBZtSlTZt6ZG7rzAJZGNL52t7xuMo=
-    bucket: ray-wheels
-    acl: public_read
-    region: us-west-2
-    local_dir: .whl
-    upload-dir: "$TRAVIS_BRANCH/$TRAVIS_COMMIT"
-    skip_cleanup: true
-    on:
-      repo: ray-project/ray
-      all_branches: true
-      condition: $MAC_WHEELS = 1
-
-  - provider: s3
-    edge: true # This supposedly opts in to deploy v2.
-    access_key_id: AKIAU6DMUCJUFL3EX3SM
-    secret_access_key:
-      secure: J1sX71fKFPQhgWzColllxfzcF877ScBZ1cIl71krZ6SO0LKnwsCScpQck5eZOyQo/Iverwye0iKtE87qNsiRi3+V2D9iulSr18T09j7+FjPKfxAmXmjfrNafoMXTDQroSJblCri5vl+DysISPqImJkWTNaYhGJ9QakoSd5djnAopLNWj6PCR3S50baS49+nB5nSIY3jMhtUzlaBdniFPFC81Cxyuafr4pv6McGRfR/dK+ZnPhdGtMnVeIJXB+ooZKQ26mDJKBPka4jm3u1Oa72b/Atu2RO3MwxTg79LTrMxXKh2OcCqhtD2Z3lz1OltvNSunCuwY8AejCJsfSLbM9mGDoz+xhNUWmYNy48YFf+61OY8PXi8S/9Q817yb3GpLbb2l/P+KMgq9eSEiELIOwuYsDxPX5TuAg6dx0wCNgDEBJoThSQjYl6MgJrLrs7p+JBxp3giedHiy0TLa5hCVKTj3euONAXDArYnnT+DvUIOkaeTk5DClRZbZ0sUXhLy//HuT5WJvjFBJJZ0u0f4RLVb5D7DI4uMZr7+yJPDR2AXCyW9YMaBEbmEYbPaKi283jlEyn7R33+AZlnXv0THHwZ4xvjKKG3/fBSXsOUmv5wmUveEqVGDj1mKPGj9NF8iA5qMm2AaZuJpEEBVBZtSlTZt6ZG7rzAJZGNL52t7xuMo=
-    bucket: ray-wheels
-    acl: public_read
-    region: us-west-2
-    local_dir: .whl
-    upload-dir: latest
-    skip_cleanup: true
-    on:
-      branch: master
-      repo: ray-project/ray
-      condition: $MAC_WHEELS = 1
-
-  # Upload jars so that we can debug locally for every commit
-  - provider: s3
-    edge: true # This supposedly opts in to deploy v2.
-    access_key_id: AKIAU6DMUCJUFL3EX3SM
-    secret_access_key:
-      secure: J1sX71fKFPQhgWzColllxfzcF877ScBZ1cIl71krZ6SO0LKnwsCScpQck5eZOyQo/Iverwye0iKtE87qNsiRi3+V2D9iulSr18T09j7+FjPKfxAmXmjfrNafoMXTDQroSJblCri5vl+DysISPqImJkWTNaYhGJ9QakoSd5djnAopLNWj6PCR3S50baS49+nB5nSIY3jMhtUzlaBdniFPFC81Cxyuafr4pv6McGRfR/dK+ZnPhdGtMnVeIJXB+ooZKQ26mDJKBPka4jm3u1Oa72b/Atu2RO3MwxTg79LTrMxXKh2OcCqhtD2Z3lz1OltvNSunCuwY8AejCJsfSLbM9mGDoz+xhNUWmYNy48YFf+61OY8PXi8S/9Q817yb3GpLbb2l/P+KMgq9eSEiELIOwuYsDxPX5TuAg6dx0wCNgDEBJoThSQjYl6MgJrLrs7p+JBxp3giedHiy0TLa5hCVKTj3euONAXDArYnnT+DvUIOkaeTk5DClRZbZ0sUXhLy//HuT5WJvjFBJJZ0u0f4RLVb5D7DI4uMZr7+yJPDR2AXCyW9YMaBEbmEYbPaKi283jlEyn7R33+AZlnXv0THHwZ4xvjKKG3/fBSXsOUmv5wmUveEqVGDj1mKPGj9NF8iA5qMm2AaZuJpEEBVBZtSlTZt6ZG7rzAJZGNL52t7xuMo=
-    bucket: ray-wheels
-    acl: public_read
-    region: us-west-2
-    local_dir: .jar
-    upload-dir: "jars/$TRAVIS_BRANCH/$TRAVIS_COMMIT"
-    skip_cleanup: true
-    on:
-      repo: ray-project/ray
-      all_branches: true
-      condition: $MAC_JARS = 1
-
-  - provider: s3
-    edge: true # This supposedly opts in to deploy v2.
-    access_key_id: AKIAU6DMUCJUFL3EX3SM
-    secret_access_key:
-      secure: J1sX71fKFPQhgWzColllxfzcF877ScBZ1cIl71krZ6SO0LKnwsCScpQck5eZOyQo/Iverwye0iKtE87qNsiRi3+V2D9iulSr18T09j7+FjPKfxAmXmjfrNafoMXTDQroSJblCri5vl+DysISPqImJkWTNaYhGJ9QakoSd5djnAopLNWj6PCR3S50baS49+nB5nSIY3jMhtUzlaBdniFPFC81Cxyuafr4pv6McGRfR/dK+ZnPhdGtMnVeIJXB+ooZKQ26mDJKBPka4jm3u1Oa72b/Atu2RO3MwxTg79LTrMxXKh2OcCqhtD2Z3lz1OltvNSunCuwY8AejCJsfSLbM9mGDoz+xhNUWmYNy48YFf+61OY8PXi8S/9Q817yb3GpLbb2l/P+KMgq9eSEiELIOwuYsDxPX5TuAg6dx0wCNgDEBJoThSQjYl6MgJrLrs7p+JBxp3giedHiy0TLa5hCVKTj3euONAXDArYnnT+DvUIOkaeTk5DClRZbZ0sUXhLy//HuT5WJvjFBJJZ0u0f4RLVb5D7DI4uMZr7+yJPDR2AXCyW9YMaBEbmEYbPaKi283jlEyn7R33+AZlnXv0THHwZ4xvjKKG3/fBSXsOUmv5wmUveEqVGDj1mKPGj9NF8iA5qMm2AaZuJpEEBVBZtSlTZt6ZG7rzAJZGNL52t7xuMo=
-    bucket: ray-wheels
-    acl: public_read
-    region: us-west-2
-    local_dir: .jar
-    upload-dir: "jars/latest"
-    skip_cleanup: true
-    on:
-      repo: ray-project/ray
-      branch: master
-      condition: $MAC_JARS = 1

ci/travis/ci.sh

@@ -359,7 +359,7 @@ build_wheels() {
       ;;
     darwin*)
       # This command should be kept in sync with ray/python/README-building-wheels.md.
-      suppress_output "${WORKSPACE_DIR}"/python/build-wheel-macos.sh
+      "${WORKSPACE_DIR}"/python/build-wheel-macos.sh
       ;;
     msys*)
       keep_alive "${WORKSPACE_DIR}"/python/build-wheel-windows.sh

ci/travis/test-wheels.sh

@@ -20,6 +20,9 @@ BUILD_DIR="${TRAVIS_BUILD_DIR-}"
 if [ -z "${BUILD_DIR}" ]; then
   BUILD_DIR="${GITHUB_WORKSPACE}"
 fi
+if [ -n "${BUILDKITE-}" ]; then
+  BUILD_DIR="${ROOT_DIR}/../.."
+fi
 TEST_DIR="${BUILD_DIR}/python/ray/tests"
 TEST_SCRIPTS=("$TEST_DIR/test_microbenchmarks.py" "$TEST_DIR/test_basic.py")
 DASHBOARD_TEST_SCRIPT="${BUILD_DIR}/python/ray/tests/test_dashboard.py"

python/build-wheel-macos.sh

@@ -61,17 +61,25 @@ for ((i=0; i<${#PY_VERSIONS[@]}; ++i)); do
   git clean -f -f -x -d -e .whl -e $DOWNLOAD_DIR -e python/ray/new_dashboard/client -e dashboard/client
 
   # Install Python.
-  INST_PATH=python_downloads/$PY_INST
+  # In Buildkite, the Python packages are installed on the machien before the build has ran.
-  curl $MACPYTHON_URL/"$PY_VERSION"/"$PY_INST" > "$INST_PATH"
-  sudo installer -pkg "$INST_PATH" -target /
-
   PYTHON_EXE=$MACPYTHON_PY_PREFIX/$PY_MM/bin/python$PY_MM
   PIP_CMD="$(dirname "$PYTHON_EXE")/pip$PY_MM"
 
-  pushd /tmp
+  if [ -z "${BUILDKITE}" ]; then
-    # Install latest version of pip to avoid brownouts.
+    INST_PATH=python_downloads/$PY_INST
-    curl https://bootstrap.pypa.io/get-pip.py | $PYTHON_EXE
+    curl $MACPYTHON_URL/"$PY_VERSION"/"$PY_INST" > "$INST_PATH"
-  popd
+    sudo installer -pkg "$INST_PATH" -target /
+    installer -pkg "$INST_PATH" -target /
+
+    pushd /tmp
+      # Install latest version of pip to avoid brownouts.
+      curl https://bootstrap.pypa.io/get-pip.py | $PYTHON_EXE
+    popd
+  fi
+
+  if [ -z "${TRAVIS_COMMIT}" ]; then
+    TRAVIS_COMMIT=${BUILDKITE_COMMIT}
+  fi
 
   pushd python
     # Setuptools on CentOS is too old to install arrow 0.9.0, therefore we upgrade.