diff --git a/.buildkite/copy_files.py b/.buildkite/copy_files.py index d374de086..7096eef0b 100644 --- a/.buildkite/copy_files.py +++ b/.buildkite/copy_files.py @@ -50,6 +50,7 @@ def handle_docker_login(resp): def gather_paths(dir_path) -> List[str]: + dir_path = dir_path.replace("/", os.path.sep) assert os.path.exists(dir_path) if os.path.isdir(dir_path): paths = [os.path.join(dir_path, f) for f in os.listdir(dir_path)] diff --git a/.buildkite/pipeline.windows.yml b/.buildkite/pipeline.windows.yml index 19fee07e0..327654485 100644 --- a/.buildkite/pipeline.windows.yml +++ b/.buildkite/pipeline.windows.yml @@ -20,8 +20,8 @@ prelude_commands: &prelude_commands |- upload_wheels_if_needed: &upload_wheels_if_needed |- if [ "$BUILDKITE_PULL_REQUEST" != "false" ]; then exit 0; fi pip install -q docker aws_requests_auth boto3 - python .buildkite/copy_files.py --destination branch_wheels --path ./.whl - if [ "$BUILDKITE_BRANCH" = "master" ]; then python .buildkite/copy_files.py --destination wheels --path ./.whl; fi + python .buildkite/copy_files.py --destination branch_wheels --path python/dist + if [ "$BUILDKITE_BRANCH" = "master" ]; then python .buildkite/copy_files.py --destination wheels --path python/dist; fi steps: @@ -35,12 +35,10 @@ steps: # The next command will be sharded into $parallelism shards. - . ./ci/travis/ci.sh test_python -# Doesn't quite work. -# /c/Install/ray/ci/travis/../../python/build-wheel-windows.sh: line 75: RUNNER_TOOL_CACHE: unbound variable -# - label: ":windows: Wheels" -# commands: -# - *prelude_commands -# - export WINDOWS_WHEELS="1" -# - . ./ci/travis/ci.sh init -# - . ./ci/travis/ci.sh build -# - *upload_wheels_if_needed +- label: ":windows: Wheels" + commands: + - *prelude_commands + - export WINDOWS_WHEELS="1" + - . ./ci/travis/ci.sh init + - . ./ci/travis/ci.sh build + - *upload_wheels_if_needed diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml deleted file mode 100644 index 0b06dbc5d..000000000 --- a/.github/workflows/main.yml +++ /dev/null @@ -1,127 +0,0 @@ -name: CI - -on: - push: - branches-ignore: - # Don't run CI for Dependabot branch pushes. - - "dependabot/**" - pull_request: - -env: - # Git GITHUB_... variables are useful for translating Travis environment variables - GITHUB_BASE_SHA: ${{ github.event.pull_request.base.sha }} - GITHUB_HEAD_SHA: ${{ github.event.pull_request.head.sha }} - GITHUB_PULL_REQUEST: ${{ github.event.number }} - -jobs: - build: - name: ${{ matrix.name }} - runs-on: ${{ matrix.os }} - strategy: - fail-fast: false - matrix: - # Useful info: https://help.github.com/en/articles/workflow-syntax-for-github-actions - include: - - name: windows-msvc - os: windows-2019 - python-version: 3.8 - # Can be 'msvc' or 'clang-cl' - config: msvc-cl - env: - BAZEL_CONFIG: ${{ matrix.config }} - PYTHON: ${{ matrix.python-version }} - TRAVIS_COMMIT: ${{ github.sha }} - #TRAVIS_PULL_REQUEST is defined in the scripts to account for "false" - steps: - - name: Pre-checkout system configuration - # We add -l to process .bashrc, but need -e -o pipefail for consistency with GitHub Actions's default behavior. - shell: bash -e -o pipefail -l {0} - run: | - # Note: only configure settings here that are required _before_ the repository is cloned (such as Git settings). - # Different machines behave very differently with Bash initialization -- make them more uniform - rm -f -- ~/.bash_profile ~/.profile ~/.bashrc - printf '\n%s\n' 'if [ -n "${BASH_VERSION}" ] && [ -f ~/.bashrc ]; then . ~/.bashrc; fi' >> ~/.profile - git config --global core.symlinks true && git config --global core.autocrlf false - if command -v dpkg > /dev/null; then sudo dpkg-reconfigure debconf -f noninteractive -p high; fi - - name: Checkout repository - uses: actions/checkout@v1 - - name: Configure AWS Credentials - continue-on-error: true - if: github.repository == 'ray-project/ray' && github.event_name != 'pull_request' - uses: aws-actions/configure-aws-credentials@v1 - with: - aws-access-key-id: ${{ secrets.GHA_AWS_ACCESS_KEY_ID }} - aws-secret-access-key: ${{ secrets.GHA_AWS_SECRET_ACCESS_KEY }} - aws-region: us-west-2 - - name: Prepare caching - id: info - shell: bash - run: | - case "${OSTYPE}" in - linux*) pip_cache=~/.cache/pip;; - darwin*) pip_cache=~/Library/Caches/pip;; - msys) pip_cache="${LOCALAPPDATA}\pip\Cache";; - esac - cat <<-EOF - ::set-output name=git_tree_hash::$(git cat-file -p HEAD | sed -n "s/^tree //p") - ::set-output name=pip_cache::${pip_cache} - EOF - - name: Cache (pip) - uses: actions/cache@v1.1.2 - continue-on-error: true - if: steps.info.outputs.pip_cache - with: - path: ${{ steps.info.outputs.pip_cache }} - key: | - pip-${{ runner.os }}-${{ steps.info.outputs.git_tree_hash }} - restore-keys: | - pip-${{ runner.os }}- - pip- - - name: Build - shell: bash -x -e -o pipefail -l {0} - env: - BAZEL_CACHE_CREDENTIAL_B64: ${{ secrets.BAZEL_CACHE_CREDENTIAL_B64 }} - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - RAY_DEFAULT_BUILD: 1 - WINDOWS_WHEELS: 1 - BAZEL_LOG_BUCKET_ACCESS_KEY: ${{ secrets.BAZEL_LOG_BUCKET_ACCESS_KEY }} - TRAVIS_BRANCH: ${{ github.ref }} - TRAVIS_COMMIT: ${{ github.sha }} - TRAVIS_JOB_ID: ${{ github.run_id }} - run: | - function clean_up() { - echo "Performing cleanup" - if [ "${GITHUB_EVENT_NAME}" != "pull_request" ]; then ./ci/travis/upload_build_info.sh; fi - } - trap clean_up EXIT - - python -u ci/remote-watch.py --skip_repo=ray-project/ray & - . ./ci/travis/ci.sh init - . ./ci/travis/ci.sh build - . ./ci/travis/ci.sh upload_wheels || true - # - name: Run Tests - # shell: bash -x -e -o pipefail -l {0} - # env: - # BAZEL_CACHE_CREDENTIAL_B64: ${{ secrets.BAZEL_CACHE_CREDENTIAL_B64 }} - # GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - # RAY_DEFAULT_BUILD: 1 - # WINDOWS_WHEELS: 1 - # BAZEL_LOG_BUCKET_ACCESS_KEY: ${{ secrets.BAZEL_LOG_BUCKET_ACCESS_KEY }} - # TRAVIS_BRANCH: ${{ github.ref }} - # TRAVIS_COMMIT: ${{ github.sha }} - # TRAVIS_JOB_ID: ${{ github.run_id }} - # run: | - # function clean_up() { - # echo "Performing cleanup" - # if [ "${GITHUB_EVENT_NAME}" != "pull_request" ]; then ./ci/travis/upload_build_info.sh; fi - # } - # trap clean_up EXIT - - # python -u ci/remote-watch.py --skip_repo=ray-project/ray & - # . ./ci/travis/ci.sh test_core - # . ./ci/travis/ci.sh test_python - # - name: Run Clang Include-What-You-Use - # continue-on-error: true - # if: runner.os == 'Linux' - # shell: bash -e -o pipefail -l {0} - # run: ci/travis/iwyu.sh process "//:*" diff --git a/python/build-wheel-windows.sh b/python/build-wheel-windows.sh index ad9f79163..4e24fc79d 100755 --- a/python/build-wheel-windows.sh +++ b/python/build-wheel-windows.sh @@ -6,7 +6,6 @@ ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE:-$0}")"; pwd)" WORKSPACE_DIR="${ROOT_DIR}/.." PY_VERSIONS=($(python -s -c "import runpy, sys; runpy.run_path(sys.argv.pop(), run_name='__api__')" python_versions "${ROOT_DIR}"/setup.py | tr -d "\r")) -PY_SCRIPT_SUBDIR=Scripts # 'bin' for UNIX, 'Scripts' for Windows bazel_preclean() { "${WORKSPACE_DIR}"/ci/travis/bazel.py preclean "mnemonic(\"Genrule\", deps(//:*))" @@ -30,14 +29,41 @@ is_python_version() { return "${result}" } +refreshenv() { + # https://gist.github.com/jayvdb/1daf8c60e20d64024f51ec333f5ce806 + powershell -NonInteractive - <<\EOF +Import-Module "$env:ChocolateyInstall\helpers\chocolateyProfile.psm1" + +Update-SessionEnvironment + +# Round brackets in variable names cause problems with bash +Get-ChildItem env:* | %{ + if (!($_.Name.Contains('('))) { + $value = $_.Value + if ($_.Name -eq 'PATH') { + $value = $value -replace ';',':' + } + Write-Output ("export " + $_.Name + "='" + $value + "'") + } +} | Out-File -Encoding ascii $env:TEMP\refreshenv.sh + +EOF + + source "$TEMP/refreshenv.sh" +} + install_ray() { # TODO(mehrdadn): This function should be unified with the one in ci/travis/ci.sh. ( pip install wheel pushd dashboard/client - yarn - yarn build + choco install nodejs -y + refreshenv + # https://stackoverflow.com/questions/69692842/error-message-error0308010cdigital-envelope-routinesunsupported + export NODE_OPTIONS=--openssl-legacy-provider + npm install + npm run build popd cd "${WORKSPACE_DIR}"/python @@ -55,15 +81,8 @@ build_wheel_windows() { local ray_uninstall_status=0 uninstall_ray || ray_uninstall_status=1 - local pyversion pyversions=() - for pyversion in "${PY_VERSIONS[@]}"; do - if [ "${pyversion}" = "${PYTHON-}" ]; then continue; fi # we'll build ${PYTHON} last - pyversions+=("${pyversion}") - done - pyversions+=("${PYTHON-}") # build this last so any subsequent steps use the right version - local local_dir="python/dist" - for pyversion in "${pyversions[@]}"; do + for pyversion in "${PY_VERSIONS[@]}"; do if [ -z "${pyversion}" ]; then continue; fi bazel_preclean git clean -q -f -f -x -d -e "${local_dir}" -e python/ray/dashboard/client @@ -72,11 +91,7 @@ build_wheel_windows() { # Start a subshell to prevent PATH and cd from affecting our shell environment ( if ! is_python_version "${pyversion}"; then - local pydirs=("${RUNNER_TOOL_CACHE}/Python/${pyversion}".*/x64) - local pydir="${pydirs[-1]}" - pydir="$(cygpath -u "${pydir}")" # Translate Windows path - test -d "${pydir}" - export PATH="${pydir}:${pydir}/${PY_SCRIPT_SUBDIR}:${PATH}" + conda install -y python="${pyversion}" fi if ! is_python_version "${pyversion}"; then echo "Expected pip for Python ${pyversion} but found Python $(get_python_version) with $(pip --version); exiting..." 1>&2 @@ -87,10 +102,10 @@ build_wheel_windows() { install_ray cd "${WORKSPACE_DIR}"/python # Set the commit SHA in __init__.py. - if [ -n "$TRAVIS_COMMIT" ]; then - sed -i.bak "s/{{RAY_COMMIT_SHA}}/$TRAVIS_COMMIT/g" ray/__init__.py && rm ray/__init__.py.bak + if [ -n "$BUILDKITE_COMMIT" ]; then + sed -i.bak "s/{{RAY_COMMIT_SHA}}/$BUILDKITE_COMMIT/g" ray/__init__.py && rm ray/__init__.py.bak else - echo "TRAVIS_COMMIT variable not set - required to populated ray.__commit__." + echo "BUILDKITE_COMMIT variable not set - required to populated ray.__commit__." exit 1 fi # build ray wheel