hiro/ray
1
0
Fork 0
mirror of https://github.com/vale981/ray synced 2025-03-05 10:01:43 -05:00
Code Issues Projects Releases Packages Wiki Activity Actions

[Buildkite] Migrate macOS wheel builds (#16913)

Browse source
This commit is contained in:
Simon Mo 2021-08-07 21:54:34 -07:00 • committed by GitHub
parent 654718902f
commit c315596ed2
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
5 changed files with 55 additions and 137 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

49
.buildkite/pipeline.macos.yml
View file

@ -9,8 +9,6 @@ common: &common
RAY_DEFAULT_BUILD: "1"
LC_ALL: en_US.UTF-8
LANG: en_US.UTF-8
# TODO(simon): Can't figure out how get the JDK working yet.
# RAY_INSTALL_JAVA: "1"
prelude_commands: &prelude_commands |-
./ci/travis/upload_build_info.sh
@ -26,6 +24,41 @@ epilogue_commands: &epilogue_commands |-
bazel clean
steps:
- label: ":mac: :apple: Wheels and Jars"
<<: *common
commands:
# Cleanup environments
- ./ci/travis/upload_build_info.sh
- (which bazel && bazel clean) || true
# TODO(simon): make sure to change both PR and wheel builds
# Special setup for jar builds (will be installed to the machine instead)
# - brew remove --force java & brew uninstall --force java & rm -rf /usr/local/Homebrew/Library/Taps/homebrew/homebrew-cask
# - brew install --cask adoptopenjdk/openjdk/adoptopenjdk8
- diskutil list external physical
- export JAVA_HOME=/Library/Java/JavaVirtualMachines/adoptopenjdk-8.jdk/Contents/Home
- java -version
# Build wheels
- export MAC_WHEELS=1
- export MAC_JARS=1
- export RAY_INSTALL_JAVA=1
- . ./ci/travis/ci.sh init && source ~/.zshrc
- . ./ci/travis/ci.sh build
# Test wheels
- . ./ci/travis/ci.sh test_wheels
# Build jars
- bash ./java/build-jar-multiplatform.sh darwin
# Upload the wheels and jars
# We don't want to push on PRs, in fact, the copy_files will fail because unauthenticated.
- if [ "$BUILDKITE_PULL_REQUEST" != "false" ]; then exit 0; fi
- pip install -q docker aws_requests_auth boto3
# Upload to branch directory.
- python .buildkite/copy_files.py --destination branch_wheels --path ./.whl
- python .buildkite/copy_files.py --destination branch_jars --path ./.jar/darwin
# Upload to latest directory.
- if [ "$BUILDKITE_BRANCH" == "master" ]; then python .buildkite/copy_files.py --destination wheels --path ./.whl; fi
- if [ "$BUILDKITE_BRANCH" == "master" ]; then python .buildkite/copy_files.py --destination jars --path ./.jar/darwin; fi
- label: ":mac: :apple: Ray C++ and Libraries"
<<: *common
commands:
@ -79,15 +112,3 @@ steps:
--test_env=CI
python/ray/tests/... python/ray/serve/...
- *epilogue_commands
# The follow is running in Travis for now.
# - bazel test --test_env=CONDA_EXE --test_env=CONDA_PYTHON_EXE --test_env=CONDA_SHLVL --test_env=CONDA_PREFIX --test_env=CONDA_DEFAULT_ENV --test_env=CONDA_PROMPT_MODIFIER --config=ci $(./scripts/bazel_export_options) --test_tag_filters=-kubernetes,-jenkins_only,-medium_size_python_tests_a_to_j,-medium_size_python_tests_k_to_z,-flaky python/ray/tests/...
# - bazel test --config=ci $(./scripts/bazel_export_options) --test_tag_filters=-kubernetes,-jenkins_only,medium_size_python_tests_k_to_z,-flaky python/ray/tests/...
# - bazel test --config=ci $(./scripts/bazel_export_options)
# --test_tag_filters=-kubernetes,-jenkins_only,flaky
# --test_env=CONDA_EXE
# --test_env=CONDA_PYTHON_EXE
# --test_env=CONDA_SHLVL
# --test_env=CONDA_PREFIX
# --test_env=CONDA_DEFAULT_ENV
# python/ray/tests/... python/ray/serve/... python/ray/tune/... rllib/...

114
.travis.yml
View file

@ -1,114 +0,0 @@
language: generic
# Use Ubuntu 18.04
dist: bionic
git:
clone: false # Clone manually to work around Travis issues like https://github.com/travis-ci/travis-ci/issues/6337
depth: false # Shallow clones can prevent diff against base branch
quiet: true
branches:
except:
- /dependabot.*/
before_install:
- unset -f cd # Travis defines this on Mac for RVM, but it breaks the Mac build
- |
git clone -q -n "https://github.com/${TRAVIS_REPO_SLUG}.git" "${TRAVIS_REPO_SLUG}"
cd -- "${TRAVIS_REPO_SLUG}"
to_fetch=("${TRAVIS_COMMIT}")
if [ false != "${TRAVIS_PULL_REQUEST-}" ]; then to_fetch+=("+refs/pull/${TRAVIS_PULL_REQUEST}/merge:"); fi
git fetch -q -- origin "${to_fetch[@]}"
git checkout -qf "${TRAVIS_COMMIT}" --
python -u ci/remote-watch.py --skip_repo=ray-project/ray &
matrix:
include:
# Build MacOS wheels and MacOS jars
- os: osx
osx_image: xcode7
env:
- MAC_WHEELS=1 MAC_JARS=1
- PYTHONWARNINGS=ignore
- RAY_INSTALL_JAVA=1
install:
- . ./ci/travis/ci.sh init RAY_CI_MACOS_WHEELS_AFFECTED,RAY_CI_JAVA_AFFECTED,RAY_CI_STREAMING_JAVA_AFFECTED
before_script:
- ./ci/keep_alive brew remove --force java & brew uninstall --force java & rm -rf /usr/local/Homebrew/Library/Taps/homebrew/homebrew-cask
- ./ci/keep_alive brew install --cask adoptopenjdk/openjdk/adoptopenjdk8
- export JAVA_HOME=/Library/Java/JavaVirtualMachines/adoptopenjdk-8.jdk/Contents/Home
- java -version
- ./ci/keep_alive bash ./ci/travis/ci.sh build
script:
- . ./ci/travis/ci.sh test_wheels
- bash ./java/build-jar-multiplatform.sh darwin
after_script:
- if [ "${TRAVIS_PULL_REQUEST}" = "false" ]; then ./ci/travis/upload_build_info.sh; fi
deploy:
- provider: s3
edge: true # This supposedly opts in to deploy v2.
access_key_id: AKIAU6DMUCJUFL3EX3SM
secret_access_key:
secure: J1sX71fKFPQhgWzColllxfzcF877ScBZ1cIl71krZ6SO0LKnwsCScpQck5eZOyQo/Iverwye0iKtE87qNsiRi3+V2D9iulSr18T09j7+FjPKfxAmXmjfrNafoMXTDQroSJblCri5vl+DysISPqImJkWTNaYhGJ9QakoSd5djnAopLNWj6PCR3S50baS49+nB5nSIY3jMhtUzlaBdniFPFC81Cxyuafr4pv6McGRfR/dK+ZnPhdGtMnVeIJXB+ooZKQ26mDJKBPka4jm3u1Oa72b/Atu2RO3MwxTg79LTrMxXKh2OcCqhtD2Z3lz1OltvNSunCuwY8AejCJsfSLbM9mGDoz+xhNUWmYNy48YFf+61OY8PXi8S/9Q817yb3GpLbb2l/P+KMgq9eSEiELIOwuYsDxPX5TuAg6dx0wCNgDEBJoThSQjYl6MgJrLrs7p+JBxp3giedHiy0TLa5hCVKTj3euONAXDArYnnT+DvUIOkaeTk5DClRZbZ0sUXhLy//HuT5WJvjFBJJZ0u0f4RLVb5D7DI4uMZr7+yJPDR2AXCyW9YMaBEbmEYbPaKi283jlEyn7R33+AZlnXv0THHwZ4xvjKKG3/fBSXsOUmv5wmUveEqVGDj1mKPGj9NF8iA5qMm2AaZuJpEEBVBZtSlTZt6ZG7rzAJZGNL52t7xuMo=
bucket: ray-wheels
acl: public_read
region: us-west-2
local_dir: .whl
upload-dir: "$TRAVIS_BRANCH/$TRAVIS_COMMIT"
skip_cleanup: true
on:
repo: ray-project/ray
all_branches: true
condition: $MAC_WHEELS = 1
- provider: s3
edge: true # This supposedly opts in to deploy v2.
access_key_id: AKIAU6DMUCJUFL3EX3SM
secret_access_key:
secure: J1sX71fKFPQhgWzColllxfzcF877ScBZ1cIl71krZ6SO0LKnwsCScpQck5eZOyQo/Iverwye0iKtE87qNsiRi3+V2D9iulSr18T09j7+FjPKfxAmXmjfrNafoMXTDQroSJblCri5vl+DysISPqImJkWTNaYhGJ9QakoSd5djnAopLNWj6PCR3S50baS49+nB5nSIY3jMhtUzlaBdniFPFC81Cxyuafr4pv6McGRfR/dK+ZnPhdGtMnVeIJXB+ooZKQ26mDJKBPka4jm3u1Oa72b/Atu2RO3MwxTg79LTrMxXKh2OcCqhtD2Z3lz1OltvNSunCuwY8AejCJsfSLbM9mGDoz+xhNUWmYNy48YFf+61OY8PXi8S/9Q817yb3GpLbb2l/P+KMgq9eSEiELIOwuYsDxPX5TuAg6dx0wCNgDEBJoThSQjYl6MgJrLrs7p+JBxp3giedHiy0TLa5hCVKTj3euONAXDArYnnT+DvUIOkaeTk5DClRZbZ0sUXhLy//HuT5WJvjFBJJZ0u0f4RLVb5D7DI4uMZr7+yJPDR2AXCyW9YMaBEbmEYbPaKi283jlEyn7R33+AZlnXv0THHwZ4xvjKKG3/fBSXsOUmv5wmUveEqVGDj1mKPGj9NF8iA5qMm2AaZuJpEEBVBZtSlTZt6ZG7rzAJZGNL52t7xuMo=
bucket: ray-wheels
acl: public_read
region: us-west-2
local_dir: .whl
upload-dir: latest
skip_cleanup: true
on:
branch: master
repo: ray-project/ray
condition: $MAC_WHEELS = 1
# Upload jars so that we can debug locally for every commit
- provider: s3
edge: true # This supposedly opts in to deploy v2.
access_key_id: AKIAU6DMUCJUFL3EX3SM
secret_access_key:
secure: J1sX71fKFPQhgWzColllxfzcF877ScBZ1cIl71krZ6SO0LKnwsCScpQck5eZOyQo/Iverwye0iKtE87qNsiRi3+V2D9iulSr18T09j7+FjPKfxAmXmjfrNafoMXTDQroSJblCri5vl+DysISPqImJkWTNaYhGJ9QakoSd5djnAopLNWj6PCR3S50baS49+nB5nSIY3jMhtUzlaBdniFPFC81Cxyuafr4pv6McGRfR/dK+ZnPhdGtMnVeIJXB+ooZKQ26mDJKBPka4jm3u1Oa72b/Atu2RO3MwxTg79LTrMxXKh2OcCqhtD2Z3lz1OltvNSunCuwY8AejCJsfSLbM9mGDoz+xhNUWmYNy48YFf+61OY8PXi8S/9Q817yb3GpLbb2l/P+KMgq9eSEiELIOwuYsDxPX5TuAg6dx0wCNgDEBJoThSQjYl6MgJrLrs7p+JBxp3giedHiy0TLa5hCVKTj3euONAXDArYnnT+DvUIOkaeTk5DClRZbZ0sUXhLy//HuT5WJvjFBJJZ0u0f4RLVb5D7DI4uMZr7+yJPDR2AXCyW9YMaBEbmEYbPaKi283jlEyn7R33+AZlnXv0THHwZ4xvjKKG3/fBSXsOUmv5wmUveEqVGDj1mKPGj9NF8iA5qMm2AaZuJpEEBVBZtSlTZt6ZG7rzAJZGNL52t7xuMo=
bucket: ray-wheels
acl: public_read
region: us-west-2
local_dir: .jar
upload-dir: "jars/$TRAVIS_BRANCH/$TRAVIS_COMMIT"
skip_cleanup: true
on:
repo: ray-project/ray
all_branches: true
condition: $MAC_JARS = 1
- provider: s3
edge: true # This supposedly opts in to deploy v2.
access_key_id: AKIAU6DMUCJUFL3EX3SM
secret_access_key:
secure: J1sX71fKFPQhgWzColllxfzcF877ScBZ1cIl71krZ6SO0LKnwsCScpQck5eZOyQo/Iverwye0iKtE87qNsiRi3+V2D9iulSr18T09j7+FjPKfxAmXmjfrNafoMXTDQroSJblCri5vl+DysISPqImJkWTNaYhGJ9QakoSd5djnAopLNWj6PCR3S50baS49+nB5nSIY3jMhtUzlaBdniFPFC81Cxyuafr4pv6McGRfR/dK+ZnPhdGtMnVeIJXB+ooZKQ26mDJKBPka4jm3u1Oa72b/Atu2RO3MwxTg79LTrMxXKh2OcCqhtD2Z3lz1OltvNSunCuwY8AejCJsfSLbM9mGDoz+xhNUWmYNy48YFf+61OY8PXi8S/9Q817yb3GpLbb2l/P+KMgq9eSEiELIOwuYsDxPX5TuAg6dx0wCNgDEBJoThSQjYl6MgJrLrs7p+JBxp3giedHiy0TLa5hCVKTj3euONAXDArYnnT+DvUIOkaeTk5DClRZbZ0sUXhLy//HuT5WJvjFBJJZ0u0f4RLVb5D7DI4uMZr7+yJPDR2AXCyW9YMaBEbmEYbPaKi283jlEyn7R33+AZlnXv0THHwZ4xvjKKG3/fBSXsOUmv5wmUveEqVGDj1mKPGj9NF8iA5qMm2AaZuJpEEBVBZtSlTZt6ZG7rzAJZGNL52t7xuMo=
bucket: ray-wheels
acl: public_read
region: us-west-2
local_dir: .jar
upload-dir: "jars/latest"
skip_cleanup: true
on:
repo: ray-project/ray
branch: master
condition: $MAC_JARS = 1

2
ci/travis/ci.sh
View file

@ -359,7 +359,7 @@ build_wheels() {
;;
darwin*)
# This command should be kept in sync with ray/python/README-building-wheels.md.
suppress_output "${WORKSPACE_DIR}"/python/build-wheel-macos.sh
"${WORKSPACE_DIR}"/python/build-wheel-macos.sh
;;
msys*)
keep_alive "${WORKSPACE_DIR}"/python/build-wheel-windows.sh

3
ci/travis/test-wheels.sh
View file

@ -20,6 +20,9 @@ BUILD_DIR="${TRAVIS_BUILD_DIR-}"
if [ -z "${BUILD_DIR}" ]; then
BUILD_DIR="${GITHUB_WORKSPACE}"
fi
if [ -n "${BUILDKITE-}" ]; then
BUILD_DIR="${ROOT_DIR}/../.."
fi
TEST_DIR="${BUILD_DIR}/python/ray/tests"
TEST_SCRIPTS=("$TEST_DIR/test_microbenchmarks.py" "$TEST_DIR/test_basic.py")
DASHBOARD_TEST_SCRIPT="${BUILD_DIR}/python/ray/tests/test_dashboard.py"

24
python/build-wheel-macos.sh
View file

@ -61,17 +61,25 @@ for ((i=0; i<${#PY_VERSIONS[@]}; ++i)); do
git clean -f -f -x -d -e .whl -e $DOWNLOAD_DIR -e python/ray/new_dashboard/client -e dashboard/client
# Install Python.
INST_PATH=python_downloads/$PY_INST
curl $MACPYTHON_URL/"$PY_VERSION"/"$PY_INST" > "$INST_PATH"
sudo installer -pkg "$INST_PATH" -target /
# In Buildkite, the Python packages are installed on the machien before the build has ran.
PYTHON_EXE=$MACPYTHON_PY_PREFIX/$PY_MM/bin/python$PY_MM
PIP_CMD="$(dirname "$PYTHON_EXE")/pip$PY_MM"
pushd /tmp
# Install latest version of pip to avoid brownouts.
curl https://bootstrap.pypa.io/get-pip.py | $PYTHON_EXE
popd
if [ -z "${BUILDKITE}" ]; then
INST_PATH=python_downloads/$PY_INST
curl $MACPYTHON_URL/"$PY_VERSION"/"$PY_INST" > "$INST_PATH"
sudo installer -pkg "$INST_PATH" -target /
installer -pkg "$INST_PATH" -target /
pushd /tmp
# Install latest version of pip to avoid brownouts.
curl https://bootstrap.pypa.io/get-pip.py | $PYTHON_EXE
popd
fi
if [ -z "${TRAVIS_COMMIT}" ]; then
TRAVIS_COMMIT=${BUILDKITE_COMMIT}
fi
pushd python
# Setuptools on CentOS is too old to install arrow 0.9.0, therefore we upgrade.