mirror of
https://github.com/vale981/ray
synced 2025-03-05 10:01:43 -05:00
[CI] Migrate Windows Wheels to Buildkite (#21388)
This commit is contained in:
Simon Mo
GitHub
parent
76b91efd9b
commit
f16b422062
4 changed files with 44 additions and 157 deletions
|
|
@ -50,6 +50,7 @@ def handle_docker_login(resp):
|
|||
|
||||
|
||||
def gather_paths(dir_path) -> List[str]:
|
||||
dir_path = dir_path.replace("/", os.path.sep)
|
||||
assert os.path.exists(dir_path)
|
||||
if os.path.isdir(dir_path):
|
||||
paths = [os.path.join(dir_path, f) for f in os.listdir(dir_path)]
|
||||
|
|
|
|||
|
|
@ -20,8 +20,8 @@ prelude_commands: &prelude_commands |-
|
|||
upload_wheels_if_needed: &upload_wheels_if_needed |-
|
||||
if [ "$BUILDKITE_PULL_REQUEST" != "false" ]; then exit 0; fi
|
||||
pip install -q docker aws_requests_auth boto3
|
||||
python .buildkite/copy_files.py --destination branch_wheels --path ./.whl
|
||||
if [ "$BUILDKITE_BRANCH" = "master" ]; then python .buildkite/copy_files.py --destination wheels --path ./.whl; fi
|
||||
python .buildkite/copy_files.py --destination branch_wheels --path python/dist
|
||||
if [ "$BUILDKITE_BRANCH" = "master" ]; then python .buildkite/copy_files.py --destination wheels --path python/dist; fi
|
||||
|
||||
|
||||
steps:
|
||||
|
|
@ -35,12 +35,10 @@ steps:
|
|||
# The next command will be sharded into $parallelism shards.
|
||||
- . ./ci/travis/ci.sh test_python
|
||||
|
||||
# Doesn't quite work.
|
||||
# /c/Install/ray/ci/travis/../../python/build-wheel-windows.sh: line 75: RUNNER_TOOL_CACHE: unbound variable
|
||||
# - label: ":windows: Wheels"
|
||||
# commands:
|
||||
# - *prelude_commands
|
||||
# - export WINDOWS_WHEELS="1"
|
||||
# - . ./ci/travis/ci.sh init
|
||||
# - . ./ci/travis/ci.sh build
|
||||
# - *upload_wheels_if_needed
|
||||
- label: ":windows: Wheels"
|
||||
commands:
|
||||
- *prelude_commands
|
||||
- export WINDOWS_WHEELS="1"
|
||||
- . ./ci/travis/ci.sh init
|
||||
- . ./ci/travis/ci.sh build
|
||||
- *upload_wheels_if_needed
|
||||
|
|
|
|||
127
.github/workflows/main.yml
vendored
127
.github/workflows/main.yml
vendored
|
|
@ -1,127 +0,0 @@
|
|||
name: CI
|
||||
|
||||
on:
|
||||
push:
|
||||
branches-ignore:
|
||||
# Don't run CI for Dependabot branch pushes.
|
||||
- "dependabot/**"
|
||||
pull_request:
|
||||
|
||||
env:
|
||||
# Git GITHUB_... variables are useful for translating Travis environment variables
|
||||
GITHUB_BASE_SHA: ${{ github.event.pull_request.base.sha }}
|
||||
GITHUB_HEAD_SHA: ${{ github.event.pull_request.head.sha }}
|
||||
GITHUB_PULL_REQUEST: ${{ github.event.number }}
|
||||
|
||||
jobs:
|
||||
build:
|
||||
name: ${{ matrix.name }}
|
||||
runs-on: ${{ matrix.os }}
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
# Useful info: https://help.github.com/en/articles/workflow-syntax-for-github-actions
|
||||
include:
|
||||
- name: windows-msvc
|
||||
os: windows-2019
|
||||
python-version: 3.8
|
||||
# Can be 'msvc' or 'clang-cl'
|
||||
config: msvc-cl
|
||||
env:
|
||||
BAZEL_CONFIG: ${{ matrix.config }}
|
||||
PYTHON: ${{ matrix.python-version }}
|
||||
TRAVIS_COMMIT: ${{ github.sha }}
|
||||
#TRAVIS_PULL_REQUEST is defined in the scripts to account for "false"
|
||||
steps:
|
||||
- name: Pre-checkout system configuration
|
||||
# We add -l to process .bashrc, but need -e -o pipefail for consistency with GitHub Actions's default behavior.
|
||||
shell: bash -e -o pipefail -l {0}
|
||||
run: |
|
||||
# Note: only configure settings here that are required _before_ the repository is cloned (such as Git settings).
|
||||
# Different machines behave very differently with Bash initialization -- make them more uniform
|
||||
rm -f -- ~/.bash_profile ~/.profile ~/.bashrc
|
||||
printf '\n%s\n' 'if [ -n "${BASH_VERSION}" ] && [ -f ~/.bashrc ]; then . ~/.bashrc; fi' >> ~/.profile
|
||||
git config --global core.symlinks true && git config --global core.autocrlf false
|
||||
if command -v dpkg > /dev/null; then sudo dpkg-reconfigure debconf -f noninteractive -p high; fi
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@v1
|
||||
- name: Configure AWS Credentials
|
||||
continue-on-error: true
|
||||
if: github.repository == 'ray-project/ray' && github.event_name != 'pull_request'
|
||||
uses: aws-actions/configure-aws-credentials@v1
|
||||
with:
|
||||
aws-access-key-id: ${{ secrets.GHA_AWS_ACCESS_KEY_ID }}
|
||||
aws-secret-access-key: ${{ secrets.GHA_AWS_SECRET_ACCESS_KEY }}
|
||||
aws-region: us-west-2
|
||||
- name: Prepare caching
|
||||
id: info
|
||||
shell: bash
|
||||
run: |
|
||||
case "${OSTYPE}" in
|
||||
linux*) pip_cache=~/.cache/pip;;
|
||||
darwin*) pip_cache=~/Library/Caches/pip;;
|
||||
msys) pip_cache="${LOCALAPPDATA}\pip\Cache";;
|
||||
esac
|
||||
cat <<-EOF
|
||||
::set-output name=git_tree_hash::$(git cat-file -p HEAD | sed -n "s/^tree //p")
|
||||
::set-output name=pip_cache::${pip_cache}
|
||||
EOF
|
||||
- name: Cache (pip)
|
||||
uses: actions/cache@v1.1.2
|
||||
continue-on-error: true
|
||||
if: steps.info.outputs.pip_cache
|
||||
with:
|
||||
path: ${{ steps.info.outputs.pip_cache }}
|
||||
key: |
|
||||
pip-${{ runner.os }}-${{ steps.info.outputs.git_tree_hash }}
|
||||
restore-keys: |
|
||||
pip-${{ runner.os }}-
|
||||
pip-
|
||||
- name: Build
|
||||
shell: bash -x -e -o pipefail -l {0}
|
||||
env:
|
||||
BAZEL_CACHE_CREDENTIAL_B64: ${{ secrets.BAZEL_CACHE_CREDENTIAL_B64 }}
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
RAY_DEFAULT_BUILD: 1
|
||||
WINDOWS_WHEELS: 1
|
||||
BAZEL_LOG_BUCKET_ACCESS_KEY: ${{ secrets.BAZEL_LOG_BUCKET_ACCESS_KEY }}
|
||||
TRAVIS_BRANCH: ${{ github.ref }}
|
||||
TRAVIS_COMMIT: ${{ github.sha }}
|
||||
TRAVIS_JOB_ID: ${{ github.run_id }}
|
||||
run: |
|
||||
function clean_up() {
|
||||
echo "Performing cleanup"
|
||||
if [ "${GITHUB_EVENT_NAME}" != "pull_request" ]; then ./ci/travis/upload_build_info.sh; fi
|
||||
}
|
||||
trap clean_up EXIT
|
||||
|
||||
python -u ci/remote-watch.py --skip_repo=ray-project/ray &
|
||||
. ./ci/travis/ci.sh init
|
||||
. ./ci/travis/ci.sh build
|
||||
. ./ci/travis/ci.sh upload_wheels || true
|
||||
# - name: Run Tests
|
||||
# shell: bash -x -e -o pipefail -l {0}
|
||||
# env:
|
||||
# BAZEL_CACHE_CREDENTIAL_B64: ${{ secrets.BAZEL_CACHE_CREDENTIAL_B64 }}
|
||||
# GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
# RAY_DEFAULT_BUILD: 1
|
||||
# WINDOWS_WHEELS: 1
|
||||
# BAZEL_LOG_BUCKET_ACCESS_KEY: ${{ secrets.BAZEL_LOG_BUCKET_ACCESS_KEY }}
|
||||
# TRAVIS_BRANCH: ${{ github.ref }}
|
||||
# TRAVIS_COMMIT: ${{ github.sha }}
|
||||
# TRAVIS_JOB_ID: ${{ github.run_id }}
|
||||
# run: |
|
||||
# function clean_up() {
|
||||
# echo "Performing cleanup"
|
||||
# if [ "${GITHUB_EVENT_NAME}" != "pull_request" ]; then ./ci/travis/upload_build_info.sh; fi
|
||||
# }
|
||||
# trap clean_up EXIT
|
||||
|
||||
# python -u ci/remote-watch.py --skip_repo=ray-project/ray &
|
||||
# . ./ci/travis/ci.sh test_core
|
||||
# . ./ci/travis/ci.sh test_python
|
||||
# - name: Run Clang Include-What-You-Use
|
||||
# continue-on-error: true
|
||||
# if: runner.os == 'Linux'
|
||||
# shell: bash -e -o pipefail -l {0}
|
||||
# run: ci/travis/iwyu.sh process "//:*"
|
||||
|
|
@ -6,7 +6,6 @@ ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE:-$0}")"; pwd)"
|
|||
WORKSPACE_DIR="${ROOT_DIR}/.."
|
||||
|
||||
PY_VERSIONS=($(python -s -c "import runpy, sys; runpy.run_path(sys.argv.pop(), run_name='__api__')" python_versions "${ROOT_DIR}"/setup.py | tr -d "\r"))
|
||||
PY_SCRIPT_SUBDIR=Scripts # 'bin' for UNIX, 'Scripts' for Windows
|
||||
|
||||
bazel_preclean() {
|
||||
"${WORKSPACE_DIR}"/ci/travis/bazel.py preclean "mnemonic(\"Genrule\", deps(//:*))"
|
||||
|
|
@ -30,14 +29,41 @@ is_python_version() {
|
|||
return "${result}"
|
||||
}
|
||||
|
||||
refreshenv() {
|
||||
# https://gist.github.com/jayvdb/1daf8c60e20d64024f51ec333f5ce806
|
||||
powershell -NonInteractive - <<\EOF
|
||||
Import-Module "$env:ChocolateyInstall\helpers\chocolateyProfile.psm1"
|
||||
|
||||
Update-SessionEnvironment
|
||||
|
||||
# Round brackets in variable names cause problems with bash
|
||||
Get-ChildItem env:* | %{
|
||||
if (!($_.Name.Contains('('))) {
|
||||
$value = $_.Value
|
||||
if ($_.Name -eq 'PATH') {
|
||||
$value = $value -replace ';',':'
|
||||
}
|
||||
Write-Output ("export " + $_.Name + "='" + $value + "'")
|
||||
}
|
||||
} | Out-File -Encoding ascii $env:TEMP\refreshenv.sh
|
||||
|
||||
EOF
|
||||
|
||||
source "$TEMP/refreshenv.sh"
|
||||
}
|
||||
|
||||
install_ray() {
|
||||
# TODO(mehrdadn): This function should be unified with the one in ci/travis/ci.sh.
|
||||
(
|
||||
pip install wheel
|
||||
|
||||
pushd dashboard/client
|
||||
yarn
|
||||
yarn build
|
||||
choco install nodejs -y
|
||||
refreshenv
|
||||
# https://stackoverflow.com/questions/69692842/error-message-error0308010cdigital-envelope-routinesunsupported
|
||||
export NODE_OPTIONS=--openssl-legacy-provider
|
||||
npm install
|
||||
npm run build
|
||||
popd
|
||||
|
||||
cd "${WORKSPACE_DIR}"/python
|
||||
|
|
@ -55,15 +81,8 @@ build_wheel_windows() {
|
|||
local ray_uninstall_status=0
|
||||
uninstall_ray || ray_uninstall_status=1
|
||||
|
||||
local pyversion pyversions=()
|
||||
for pyversion in "${PY_VERSIONS[@]}"; do
|
||||
if [ "${pyversion}" = "${PYTHON-}" ]; then continue; fi # we'll build ${PYTHON} last
|
||||
pyversions+=("${pyversion}")
|
||||
done
|
||||
pyversions+=("${PYTHON-}") # build this last so any subsequent steps use the right version
|
||||
|
||||
local local_dir="python/dist"
|
||||
for pyversion in "${pyversions[@]}"; do
|
||||
for pyversion in "${PY_VERSIONS[@]}"; do
|
||||
if [ -z "${pyversion}" ]; then continue; fi
|
||||
bazel_preclean
|
||||
git clean -q -f -f -x -d -e "${local_dir}" -e python/ray/dashboard/client
|
||||
|
|
@ -72,11 +91,7 @@ build_wheel_windows() {
|
|||
# Start a subshell to prevent PATH and cd from affecting our shell environment
|
||||
(
|
||||
if ! is_python_version "${pyversion}"; then
|
||||
local pydirs=("${RUNNER_TOOL_CACHE}/Python/${pyversion}".*/x64)
|
||||
local pydir="${pydirs[-1]}"
|
||||
pydir="$(cygpath -u "${pydir}")" # Translate Windows path
|
||||
test -d "${pydir}"
|
||||
export PATH="${pydir}:${pydir}/${PY_SCRIPT_SUBDIR}:${PATH}"
|
||||
conda install -y python="${pyversion}"
|
||||
fi
|
||||
if ! is_python_version "${pyversion}"; then
|
||||
echo "Expected pip for Python ${pyversion} but found Python $(get_python_version) with $(pip --version); exiting..." 1>&2
|
||||
|
|
@ -87,10 +102,10 @@ build_wheel_windows() {
|
|||
install_ray
|
||||
cd "${WORKSPACE_DIR}"/python
|
||||
# Set the commit SHA in __init__.py.
|
||||
if [ -n "$TRAVIS_COMMIT" ]; then
|
||||
sed -i.bak "s/{{RAY_COMMIT_SHA}}/$TRAVIS_COMMIT/g" ray/__init__.py && rm ray/__init__.py.bak
|
||||
if [ -n "$BUILDKITE_COMMIT" ]; then
|
||||
sed -i.bak "s/{{RAY_COMMIT_SHA}}/$BUILDKITE_COMMIT/g" ray/__init__.py && rm ray/__init__.py.bak
|
||||
else
|
||||
echo "TRAVIS_COMMIT variable not set - required to populated ray.__commit__."
|
||||
echo "BUILDKITE_COMMIT variable not set - required to populated ray.__commit__."
|
||||
exit 1
|
||||
fi
|
||||
# build ray wheel
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue