hiro/ray
1
0
Fork 0
mirror of https://github.com/vale981/ray synced 2025-03-06 02:21:39 -05:00
Code Issues Projects Releases Packages Wiki Activity Actions

[ci] Only upload wheels to S3 once (#24072)

Browse source 
Currently all jobs that build wheels put them into the artifacts directory and upload them. This leads to the wheels being overwritten on S3 multiple times. This is not a huge problem as ingress is free, but in order to have a single point of reference, it might be beneficial to limit the wheels uploading to a single Buildkite job. Recently, this has led to interference with stale artifact directories.

The downside here is that if the "Wheels & Jars" build fails randomly, the wheels will not be available on S3 - previously they've been also uploaded by several other jobs.
This commit is contained in:
Kai Fricke 2022-04-25 21:19:11 +01:00 • committed by GitHub
parent cc08c01ade
commit b86d420a3c
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
3 changed files with 12 additions and 5 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
.buildkite/pipeline.macos.yml
View file

@ -38,6 +38,7 @@ steps:
- export JAVA_HOME=/Library/Java/JavaVirtualMachines/adoptopenjdk-8.jdk/Contents/Home - export JAVA_HOME=/Library/Java/JavaVirtualMachines/adoptopenjdk-8.jdk/Contents/Home
- java -version - java -version
# Build wheels # Build wheels
- export UPLOAD_WHEELS_AS_ARTIFACTS=1
- export MAC_WHEELS=1 - export MAC_WHEELS=1
- export MAC_JARS=1 - export MAC_JARS=1
- export RAY_INSTALL_JAVA=1 - export RAY_INSTALL_JAVA=1

2
.buildkite/pipeline.yml
View file

@ -6,7 +6,7 @@
] ]
commands: commands:
# Build the wheels and jars # Build the wheels and jars
- LINUX_WHEELS=1 LINUX_JARS=1 ./ci/ci.sh build - UPLOAD_WHEELS_AS_ARTIFACTS=1 LINUX_WHEELS=1 LINUX_JARS=1 ./ci/ci.sh build
- bash ./java/build-jar-multiplatform.sh linux - bash ./java/build-jar-multiplatform.sh linux
# Upload the wheels and jars # Upload the wheels and jars
# We don't want to push on PRs, in fact, the copy_files will fail because unauthenticated. # We don't want to push on PRs, in fact, the copy_files will fail because unauthenticated.

6
ci/ci.sh
View file

@ -430,8 +430,11 @@ build_wheels() {
# Sync the directory to buildkite artifacts # Sync the directory to buildkite artifacts
rm -rf /artifact-mount/.whl || true rm -rf /artifact-mount/.whl || true
if [ "${UPLOAD_WHEELS_AS_ARTIFACTS-}" = "1" ]; then
cp -r .whl /artifact-mount/.whl cp -r .whl /artifact-mount/.whl
chmod -R 777 /artifact-mount/.whl chmod -R 777 /artifact-mount/.whl
fi
validate_wheels_commit_str validate_wheels_commit_str
fi fi
@ -441,8 +444,11 @@ build_wheels() {
"${WORKSPACE_DIR}"/python/build-wheel-macos.sh "${WORKSPACE_DIR}"/python/build-wheel-macos.sh
mkdir -p /tmp/artifacts/.whl mkdir -p /tmp/artifacts/.whl
rm -rf /tmp/artifacts/.whl || true rm -rf /tmp/artifacts/.whl || true
if [ "${UPLOAD_WHEELS_AS_ARTIFACTS-}" = "1" ]; then
cp -r .whl /tmp/artifacts/.whl cp -r .whl /tmp/artifacts/.whl
chmod -R 777 /tmp/artifacts/.whl chmod -R 777 /tmp/artifacts/.whl
fi
validate_wheels_commit_str validate_wheels_commit_str
;; ;;