diff --git a/.buildkite/pipeline.macos.yml b/.buildkite/pipeline.macos.yml index e2afb4477..f0e354270 100644 --- a/.buildkite/pipeline.macos.yml +++ b/.buildkite/pipeline.macos.yml @@ -9,8 +9,6 @@ common: &common RAY_DEFAULT_BUILD: "1" LC_ALL: en_US.UTF-8 LANG: en_US.UTF-8 - # TODO(simon): Can't figure out how get the JDK working yet. - # RAY_INSTALL_JAVA: "1" prelude_commands: &prelude_commands |- ./ci/travis/upload_build_info.sh @@ -26,6 +24,41 @@ epilogue_commands: &epilogue_commands |- bazel clean steps: +- label: ":mac: :apple: Wheels and Jars" + <<: *common + commands: + # Cleanup environments + - ./ci/travis/upload_build_info.sh + - (which bazel && bazel clean) || true + # TODO(simon): make sure to change both PR and wheel builds + # Special setup for jar builds (will be installed to the machine instead) + # - brew remove --force java & brew uninstall --force java & rm -rf /usr/local/Homebrew/Library/Taps/homebrew/homebrew-cask + # - brew install --cask adoptopenjdk/openjdk/adoptopenjdk8 + - diskutil list external physical + - export JAVA_HOME=/Library/Java/JavaVirtualMachines/adoptopenjdk-8.jdk/Contents/Home + - java -version + # Build wheels + - export MAC_WHEELS=1 + - export MAC_JARS=1 + - export RAY_INSTALL_JAVA=1 + - . ./ci/travis/ci.sh init && source ~/.zshrc + - . ./ci/travis/ci.sh build + # Test wheels + - . ./ci/travis/ci.sh test_wheels + # Build jars + - bash ./java/build-jar-multiplatform.sh darwin + # Upload the wheels and jars + # We don't want to push on PRs, in fact, the copy_files will fail because unauthenticated. + - if [ "$BUILDKITE_PULL_REQUEST" != "false" ]; then exit 0; fi + - pip install -q docker aws_requests_auth boto3 + # Upload to branch directory. + - python .buildkite/copy_files.py --destination branch_wheels --path ./.whl + - python .buildkite/copy_files.py --destination branch_jars --path ./.jar/darwin + # Upload to latest directory. + - if [ "$BUILDKITE_BRANCH" == "master" ]; then python .buildkite/copy_files.py --destination wheels --path ./.whl; fi + - if [ "$BUILDKITE_BRANCH" == "master" ]; then python .buildkite/copy_files.py --destination jars --path ./.jar/darwin; fi + + - label: ":mac: :apple: Ray C++ and Libraries" <<: *common commands: @@ -79,15 +112,3 @@ steps: --test_env=CI python/ray/tests/... python/ray/serve/... - *epilogue_commands - - # The follow is running in Travis for now. - # - bazel test --test_env=CONDA_EXE --test_env=CONDA_PYTHON_EXE --test_env=CONDA_SHLVL --test_env=CONDA_PREFIX --test_env=CONDA_DEFAULT_ENV --test_env=CONDA_PROMPT_MODIFIER --config=ci $(./scripts/bazel_export_options) --test_tag_filters=-kubernetes,-jenkins_only,-medium_size_python_tests_a_to_j,-medium_size_python_tests_k_to_z,-flaky python/ray/tests/... - # - bazel test --config=ci $(./scripts/bazel_export_options) --test_tag_filters=-kubernetes,-jenkins_only,medium_size_python_tests_k_to_z,-flaky python/ray/tests/... - # - bazel test --config=ci $(./scripts/bazel_export_options) - # --test_tag_filters=-kubernetes,-jenkins_only,flaky - # --test_env=CONDA_EXE - # --test_env=CONDA_PYTHON_EXE - # --test_env=CONDA_SHLVL - # --test_env=CONDA_PREFIX - # --test_env=CONDA_DEFAULT_ENV - # python/ray/tests/... python/ray/serve/... python/ray/tune/... rllib/... \ No newline at end of file diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index 7be90fd76..000000000 --- a/.travis.yml +++ /dev/null @@ -1,114 +0,0 @@ -language: generic -# Use Ubuntu 18.04 -dist: bionic - -git: - clone: false # Clone manually to work around Travis issues like https://github.com/travis-ci/travis-ci/issues/6337 - depth: false # Shallow clones can prevent diff against base branch - quiet: true - -branches: - except: - - /dependabot.*/ - -before_install: - - unset -f cd # Travis defines this on Mac for RVM, but it breaks the Mac build - - | - git clone -q -n "https://github.com/${TRAVIS_REPO_SLUG}.git" "${TRAVIS_REPO_SLUG}" - cd -- "${TRAVIS_REPO_SLUG}" - to_fetch=("${TRAVIS_COMMIT}") - if [ false != "${TRAVIS_PULL_REQUEST-}" ]; then to_fetch+=("+refs/pull/${TRAVIS_PULL_REQUEST}/merge:"); fi - git fetch -q -- origin "${to_fetch[@]}" - git checkout -qf "${TRAVIS_COMMIT}" -- - python -u ci/remote-watch.py --skip_repo=ray-project/ray & - -matrix: - include: - # Build MacOS wheels and MacOS jars - - os: osx - osx_image: xcode7 - env: - - MAC_WHEELS=1 MAC_JARS=1 - - PYTHONWARNINGS=ignore - - RAY_INSTALL_JAVA=1 - install: - - . ./ci/travis/ci.sh init RAY_CI_MACOS_WHEELS_AFFECTED,RAY_CI_JAVA_AFFECTED,RAY_CI_STREAMING_JAVA_AFFECTED - before_script: - - ./ci/keep_alive brew remove --force java & brew uninstall --force java & rm -rf /usr/local/Homebrew/Library/Taps/homebrew/homebrew-cask - - ./ci/keep_alive brew install --cask adoptopenjdk/openjdk/adoptopenjdk8 - - export JAVA_HOME=/Library/Java/JavaVirtualMachines/adoptopenjdk-8.jdk/Contents/Home - - java -version - - ./ci/keep_alive bash ./ci/travis/ci.sh build - script: - - . ./ci/travis/ci.sh test_wheels - - bash ./java/build-jar-multiplatform.sh darwin - - -after_script: - - if [ "${TRAVIS_PULL_REQUEST}" = "false" ]; then ./ci/travis/upload_build_info.sh; fi - -deploy: - - provider: s3 - edge: true # This supposedly opts in to deploy v2. - access_key_id: AKIAU6DMUCJUFL3EX3SM - secret_access_key: - secure: J1sX71fKFPQhgWzColllxfzcF877ScBZ1cIl71krZ6SO0LKnwsCScpQck5eZOyQo/Iverwye0iKtE87qNsiRi3+V2D9iulSr18T09j7+FjPKfxAmXmjfrNafoMXTDQroSJblCri5vl+DysISPqImJkWTNaYhGJ9QakoSd5djnAopLNWj6PCR3S50baS49+nB5nSIY3jMhtUzlaBdniFPFC81Cxyuafr4pv6McGRfR/dK+ZnPhdGtMnVeIJXB+ooZKQ26mDJKBPka4jm3u1Oa72b/Atu2RO3MwxTg79LTrMxXKh2OcCqhtD2Z3lz1OltvNSunCuwY8AejCJsfSLbM9mGDoz+xhNUWmYNy48YFf+61OY8PXi8S/9Q817yb3GpLbb2l/P+KMgq9eSEiELIOwuYsDxPX5TuAg6dx0wCNgDEBJoThSQjYl6MgJrLrs7p+JBxp3giedHiy0TLa5hCVKTj3euONAXDArYnnT+DvUIOkaeTk5DClRZbZ0sUXhLy//HuT5WJvjFBJJZ0u0f4RLVb5D7DI4uMZr7+yJPDR2AXCyW9YMaBEbmEYbPaKi283jlEyn7R33+AZlnXv0THHwZ4xvjKKG3/fBSXsOUmv5wmUveEqVGDj1mKPGj9NF8iA5qMm2AaZuJpEEBVBZtSlTZt6ZG7rzAJZGNL52t7xuMo= - bucket: ray-wheels - acl: public_read - region: us-west-2 - local_dir: .whl - upload-dir: "$TRAVIS_BRANCH/$TRAVIS_COMMIT" - skip_cleanup: true - on: - repo: ray-project/ray - all_branches: true - condition: $MAC_WHEELS = 1 - - - provider: s3 - edge: true # This supposedly opts in to deploy v2. - access_key_id: AKIAU6DMUCJUFL3EX3SM - secret_access_key: - secure: J1sX71fKFPQhgWzColllxfzcF877ScBZ1cIl71krZ6SO0LKnwsCScpQck5eZOyQo/Iverwye0iKtE87qNsiRi3+V2D9iulSr18T09j7+FjPKfxAmXmjfrNafoMXTDQroSJblCri5vl+DysISPqImJkWTNaYhGJ9QakoSd5djnAopLNWj6PCR3S50baS49+nB5nSIY3jMhtUzlaBdniFPFC81Cxyuafr4pv6McGRfR/dK+ZnPhdGtMnVeIJXB+ooZKQ26mDJKBPka4jm3u1Oa72b/Atu2RO3MwxTg79LTrMxXKh2OcCqhtD2Z3lz1OltvNSunCuwY8AejCJsfSLbM9mGDoz+xhNUWmYNy48YFf+61OY8PXi8S/9Q817yb3GpLbb2l/P+KMgq9eSEiELIOwuYsDxPX5TuAg6dx0wCNgDEBJoThSQjYl6MgJrLrs7p+JBxp3giedHiy0TLa5hCVKTj3euONAXDArYnnT+DvUIOkaeTk5DClRZbZ0sUXhLy//HuT5WJvjFBJJZ0u0f4RLVb5D7DI4uMZr7+yJPDR2AXCyW9YMaBEbmEYbPaKi283jlEyn7R33+AZlnXv0THHwZ4xvjKKG3/fBSXsOUmv5wmUveEqVGDj1mKPGj9NF8iA5qMm2AaZuJpEEBVBZtSlTZt6ZG7rzAJZGNL52t7xuMo= - bucket: ray-wheels - acl: public_read - region: us-west-2 - local_dir: .whl - upload-dir: latest - skip_cleanup: true - on: - branch: master - repo: ray-project/ray - condition: $MAC_WHEELS = 1 - - # Upload jars so that we can debug locally for every commit - - provider: s3 - edge: true # This supposedly opts in to deploy v2. - access_key_id: AKIAU6DMUCJUFL3EX3SM - secret_access_key: - secure: J1sX71fKFPQhgWzColllxfzcF877ScBZ1cIl71krZ6SO0LKnwsCScpQck5eZOyQo/Iverwye0iKtE87qNsiRi3+V2D9iulSr18T09j7+FjPKfxAmXmjfrNafoMXTDQroSJblCri5vl+DysISPqImJkWTNaYhGJ9QakoSd5djnAopLNWj6PCR3S50baS49+nB5nSIY3jMhtUzlaBdniFPFC81Cxyuafr4pv6McGRfR/dK+ZnPhdGtMnVeIJXB+ooZKQ26mDJKBPka4jm3u1Oa72b/Atu2RO3MwxTg79LTrMxXKh2OcCqhtD2Z3lz1OltvNSunCuwY8AejCJsfSLbM9mGDoz+xhNUWmYNy48YFf+61OY8PXi8S/9Q817yb3GpLbb2l/P+KMgq9eSEiELIOwuYsDxPX5TuAg6dx0wCNgDEBJoThSQjYl6MgJrLrs7p+JBxp3giedHiy0TLa5hCVKTj3euONAXDArYnnT+DvUIOkaeTk5DClRZbZ0sUXhLy//HuT5WJvjFBJJZ0u0f4RLVb5D7DI4uMZr7+yJPDR2AXCyW9YMaBEbmEYbPaKi283jlEyn7R33+AZlnXv0THHwZ4xvjKKG3/fBSXsOUmv5wmUveEqVGDj1mKPGj9NF8iA5qMm2AaZuJpEEBVBZtSlTZt6ZG7rzAJZGNL52t7xuMo= - bucket: ray-wheels - acl: public_read - region: us-west-2 - local_dir: .jar - upload-dir: "jars/$TRAVIS_BRANCH/$TRAVIS_COMMIT" - skip_cleanup: true - on: - repo: ray-project/ray - all_branches: true - condition: $MAC_JARS = 1 - - - provider: s3 - edge: true # This supposedly opts in to deploy v2. - access_key_id: AKIAU6DMUCJUFL3EX3SM - secret_access_key: - secure: J1sX71fKFPQhgWzColllxfzcF877ScBZ1cIl71krZ6SO0LKnwsCScpQck5eZOyQo/Iverwye0iKtE87qNsiRi3+V2D9iulSr18T09j7+FjPKfxAmXmjfrNafoMXTDQroSJblCri5vl+DysISPqImJkWTNaYhGJ9QakoSd5djnAopLNWj6PCR3S50baS49+nB5nSIY3jMhtUzlaBdniFPFC81Cxyuafr4pv6McGRfR/dK+ZnPhdGtMnVeIJXB+ooZKQ26mDJKBPka4jm3u1Oa72b/Atu2RO3MwxTg79LTrMxXKh2OcCqhtD2Z3lz1OltvNSunCuwY8AejCJsfSLbM9mGDoz+xhNUWmYNy48YFf+61OY8PXi8S/9Q817yb3GpLbb2l/P+KMgq9eSEiELIOwuYsDxPX5TuAg6dx0wCNgDEBJoThSQjYl6MgJrLrs7p+JBxp3giedHiy0TLa5hCVKTj3euONAXDArYnnT+DvUIOkaeTk5DClRZbZ0sUXhLy//HuT5WJvjFBJJZ0u0f4RLVb5D7DI4uMZr7+yJPDR2AXCyW9YMaBEbmEYbPaKi283jlEyn7R33+AZlnXv0THHwZ4xvjKKG3/fBSXsOUmv5wmUveEqVGDj1mKPGj9NF8iA5qMm2AaZuJpEEBVBZtSlTZt6ZG7rzAJZGNL52t7xuMo= - bucket: ray-wheels - acl: public_read - region: us-west-2 - local_dir: .jar - upload-dir: "jars/latest" - skip_cleanup: true - on: - repo: ray-project/ray - branch: master - condition: $MAC_JARS = 1 diff --git a/ci/travis/ci.sh b/ci/travis/ci.sh index 8ff463e5e..10f46876c 100755 --- a/ci/travis/ci.sh +++ b/ci/travis/ci.sh @@ -359,7 +359,7 @@ build_wheels() { ;; darwin*) # This command should be kept in sync with ray/python/README-building-wheels.md. - suppress_output "${WORKSPACE_DIR}"/python/build-wheel-macos.sh + "${WORKSPACE_DIR}"/python/build-wheel-macos.sh ;; msys*) keep_alive "${WORKSPACE_DIR}"/python/build-wheel-windows.sh diff --git a/ci/travis/test-wheels.sh b/ci/travis/test-wheels.sh index a37a2f9df..b90963330 100755 --- a/ci/travis/test-wheels.sh +++ b/ci/travis/test-wheels.sh @@ -20,6 +20,9 @@ BUILD_DIR="${TRAVIS_BUILD_DIR-}" if [ -z "${BUILD_DIR}" ]; then BUILD_DIR="${GITHUB_WORKSPACE}" fi +if [ -n "${BUILDKITE-}" ]; then + BUILD_DIR="${ROOT_DIR}/../.." +fi TEST_DIR="${BUILD_DIR}/python/ray/tests" TEST_SCRIPTS=("$TEST_DIR/test_microbenchmarks.py" "$TEST_DIR/test_basic.py") DASHBOARD_TEST_SCRIPT="${BUILD_DIR}/python/ray/tests/test_dashboard.py" diff --git a/python/build-wheel-macos.sh b/python/build-wheel-macos.sh index 12f30128b..21dab72f5 100755 --- a/python/build-wheel-macos.sh +++ b/python/build-wheel-macos.sh @@ -61,17 +61,25 @@ for ((i=0; i<${#PY_VERSIONS[@]}; ++i)); do git clean -f -f -x -d -e .whl -e $DOWNLOAD_DIR -e python/ray/new_dashboard/client -e dashboard/client # Install Python. - INST_PATH=python_downloads/$PY_INST - curl $MACPYTHON_URL/"$PY_VERSION"/"$PY_INST" > "$INST_PATH" - sudo installer -pkg "$INST_PATH" -target / - + # In Buildkite, the Python packages are installed on the machien before the build has ran. PYTHON_EXE=$MACPYTHON_PY_PREFIX/$PY_MM/bin/python$PY_MM PIP_CMD="$(dirname "$PYTHON_EXE")/pip$PY_MM" - pushd /tmp - # Install latest version of pip to avoid brownouts. - curl https://bootstrap.pypa.io/get-pip.py | $PYTHON_EXE - popd + if [ -z "${BUILDKITE}" ]; then + INST_PATH=python_downloads/$PY_INST + curl $MACPYTHON_URL/"$PY_VERSION"/"$PY_INST" > "$INST_PATH" + sudo installer -pkg "$INST_PATH" -target / + installer -pkg "$INST_PATH" -target / + + pushd /tmp + # Install latest version of pip to avoid brownouts. + curl https://bootstrap.pypa.io/get-pip.py | $PYTHON_EXE + popd + fi + + if [ -z "${TRAVIS_COMMIT}" ]; then + TRAVIS_COMMIT=${BUILDKITE_COMMIT} + fi pushd python # Setuptools on CentOS is too old to install arrow 0.9.0, therefore we upgrade.