mirror of
https://github.com/vale981/ray
synced 2025-03-05 18:11:42 -05:00
8ec558dcb9
Since ray supports Redis as a storage backend, we should ensure the code path with Redis as storage is still being covered e2e. The tests don't run for a while after we switch to memory mode by default. This PR tries to fix this and make it run with every commit. In the future, if we support more and more storage backends, this should be revised to be more efficient and selective. But now I think the cost should be ok. This PR is part of GCS HA testing-related work.
121 lines
4.7 KiB
YAML
121 lines
4.7 KiB
YAML
common: &common
|
|
artifact_paths:
|
|
- "/tmp/bazel_event_logs/*"
|
|
env:
|
|
BUILDKITE: "true"
|
|
CI: "true"
|
|
PYTHON: "3.6"
|
|
RAY_USE_RANDOM_PORTS: "1"
|
|
RAY_DEFAULT_BUILD: "1"
|
|
LC_ALL: en_US.UTF-8
|
|
LANG: en_US.UTF-8
|
|
|
|
prelude_commands: &prelude_commands |-
|
|
./ci/build/upload_build_info.sh
|
|
(which bazel && bazel clean) || true;
|
|
. ./ci/ci.sh init && source ~/.zshrc
|
|
. ./ci/ci.sh build
|
|
./ci/env/install-dependencies.sh
|
|
|
|
epilogue_commands: &epilogue_commands |-
|
|
# Cleanup runtime environment to save storage
|
|
rm -rf /tmp/ray || true
|
|
# Cleanup local caches (this shouldn't clean up global disk cache)
|
|
bazel clean
|
|
|
|
steps:
|
|
- label: ":mac: :apple: Wheels and Jars"
|
|
<<: *common
|
|
conditions: ["RAY_CI_MACOS_WHEELS_AFFECTED", "RAY_CI_PYTHON_DEPENDENCIES_AFFECTED"]
|
|
commands:
|
|
# Cleanup environments
|
|
- ./ci/build/upload_build_info.sh
|
|
- (which bazel && bazel clean) || true
|
|
# TODO(simon): make sure to change both PR and wheel builds
|
|
# Special setup for jar builds (will be installed to the machine instead)
|
|
# - brew remove --force java & brew uninstall --force java & rm -rf /usr/local/Homebrew/Library/Taps/homebrew/homebrew-cask
|
|
# - brew install --cask adoptopenjdk/openjdk/adoptopenjdk8
|
|
- diskutil list external physical
|
|
- export JAVA_HOME=/Library/Java/JavaVirtualMachines/adoptopenjdk-8.jdk/Contents/Home
|
|
- java -version
|
|
# Build wheels
|
|
- export UPLOAD_WHEELS_AS_ARTIFACTS=1
|
|
- export MAC_WHEELS=1
|
|
- export MAC_JARS=1
|
|
- export RAY_INSTALL_JAVA=1
|
|
- . ./ci/ci.sh init && source ~/.zshrc
|
|
- . ./ci/ci.sh build
|
|
# Test wheels
|
|
- . ./ci/ci.sh test_wheels
|
|
# Build jars
|
|
- bash ./java/build-jar-multiplatform.sh darwin
|
|
# Upload the wheels and jars
|
|
# We don't want to push on PRs, in fact, the copy_files will fail because unauthenticated.
|
|
- if [ "$BUILDKITE_PULL_REQUEST" != "false" ]; then exit 0; fi
|
|
- pip install -q docker aws_requests_auth boto3
|
|
# Upload to branch directory.
|
|
- python .buildkite/copy_files.py --destination branch_wheels --path ./.whl
|
|
- python .buildkite/copy_files.py --destination branch_jars --path ./.jar/darwin
|
|
# Upload to latest directory.
|
|
- if [ "$BUILDKITE_BRANCH" = "master" ]; then python .buildkite/copy_files.py --destination wheels --path ./.whl; fi
|
|
- if [ "$BUILDKITE_BRANCH" = "master" ]; then python .buildkite/copy_files.py --destination jars --path ./.jar/darwin; fi
|
|
|
|
|
|
- label: ":mac: :apple: Ray C++, Java and Libraries"
|
|
<<: *common
|
|
conditions: ["RAY_CI_SERVE_AFFECTED", "RAY_CI_CORE_CPP_AFFECTED", "RAY_CI_CPP_AFFECTED", "RAY_CI_JAVA_AFFECTED", "RAY_CI_PYTHON_AFFECTED", "RAY_CI_DASHBOARD_AFFECTED"]
|
|
commands:
|
|
- export RAY_INSTALL_JAVA=1
|
|
- *prelude_commands
|
|
- TORCH_VERSION=1.6 ./ci/env/install-dependencies.sh
|
|
# Use --dynamic_mode=off until MacOS CI runs on Big Sur or newer. Otherwise there are problems with running tests
|
|
# with dynamic linking.
|
|
- bazel test --config=ci --dynamic_mode=off --test_env=CI $(./ci/run/bazel_export_options) --build_tests_only --test_tag_filters=-post_wheel_build --
|
|
//:all python/ray/serve/... python/ray/dashboard/... -rllib/... -core_worker_test
|
|
# clang-format is needed by java/test.sh
|
|
- pip install clang-format==12.0.1
|
|
- ./java/test.sh
|
|
- ./ci/ci.sh test_cpp
|
|
- *epilogue_commands
|
|
|
|
|
|
- label: ":mac: :apple: Small & Client"
|
|
<<: *common
|
|
conditions: ["RAY_CI_CORE_CPP_AFFECTED", "RAY_CI_PYTHON_AFFECTED"]
|
|
commands:
|
|
- *prelude_commands
|
|
- bazel test $(./ci/run/bazel_export_options) --config=ci
|
|
--test_env=CONDA_EXE --test_env=CONDA_PYTHON_EXE --test_env=CONDA_SHLVL --test_env=CONDA_PREFIX
|
|
--test_env=CONDA_DEFAULT_ENV --test_env=CONDA_PROMPT_MODIFIER --test_env=CI
|
|
--test_tag_filters=client_tests,small_size_python_tests
|
|
-- python/ray/tests/...
|
|
- *epilogue_commands
|
|
|
|
- label: ":mac: :apple: Large"
|
|
<<: *common
|
|
parallelism: 3
|
|
conditions: ["RAY_CI_CORE_CPP_AFFECTED", "RAY_CI_PYTHON_AFFECTED"]
|
|
commands:
|
|
- *prelude_commands
|
|
- . ./ci/ci.sh test_large
|
|
- *epilogue_commands
|
|
|
|
- label: ":mac: :apple: Medium A-J"
|
|
<<: *common
|
|
conditions: ["RAY_CI_CORE_CPP_AFFECTED", "RAY_CI_PYTHON_AFFECTED"]
|
|
commands:
|
|
- *prelude_commands
|
|
- bazel test --config=ci $(./ci/run/bazel_export_options) --test_env=CI
|
|
--test_tag_filters=-kubernetes,medium_size_python_tests_a_to_j
|
|
python/ray/tests/...
|
|
- *epilogue_commands
|
|
|
|
- label: ":mac: :apple: Medium K-Z"
|
|
<<: *common
|
|
conditions: ["RAY_CI_CORE_CPP_AFFECTED", "RAY_CI_PYTHON_AFFECTED"]
|
|
commands:
|
|
- *prelude_commands
|
|
- bazel test --config=ci $(./ci/run/bazel_export_options) --test_env=CI
|
|
--test_tag_filters=-kubernetes,medium_size_python_tests_k_to_z
|
|
python/ray/tests/...
|
|
- *epilogue_commands
|