ray/.travis.yml

language: generic
# Use Ubuntu 18.04
dist: bionic

git:
  clone: false  # Clone manually to work around Travis issues like https://github.com/travis-ci/travis-ci/issues/6337
  depth: false  # Shallow clones can prevent diff against base branch
  quiet: true

branches:
  except:
    - /dependabot.*/

before_install:
  - unset -f cd  # Travis defines this on Mac for RVM, but it breaks the Mac build
  - |
    git clone -q -n "https://github.com/${TRAVIS_REPO_SLUG}.git" "${TRAVIS_REPO_SLUG}"
    cd -- "${TRAVIS_REPO_SLUG}"
    to_fetch=("${TRAVIS_COMMIT}")
    if [ false != "${TRAVIS_PULL_REQUEST-}" ]; then to_fetch+=("+refs/pull/${TRAVIS_PULL_REQUEST}/merge:"); fi
    git fetch -q -- origin "${to_fetch[@]}"
    git checkout -qf "${TRAVIS_COMMIT}" --
    python -u ci/remote-watch.py --skip_repo=ray-project/ray &

matrix:
  include:
    - os: osx
      osx_image: xcode7
      env:
        - PYTHON=3.6 SMALL_AND_LARGE_TESTS=1
        - PYTHONWARNINGS=ignore
        - RAY_DEFAULT_BUILD=1
        - RAY_CYTHON_EXAMPLES=1
        - RAY_USE_RANDOM_PORTS=1
      install:
        - . ./ci/travis/ci.sh init RAY_CI_SERVE_AFFECTED,RAY_CI_TUNE_AFFECTED,RAY_CI_PYTHON_AFFECTED,RAY_CI_DASHBOARD_AFFECTED
      before_script:
        - . ./ci/travis/ci.sh build
      script:
        # cc bazel tests (w/o RLlib)
        # NOTE: core_worker_test is out-of-date and should already covered by Python
        # tests.
        - ./ci/suppress_output bazel test --config=ci $(./scripts/bazel_export_options) --build_tests_only --test_tag_filters=-flaky -- //:all -rllib/... -core_worker_test

        # ray serve tests
        - if [ $RAY_CI_SERVE_AFFECTED == "1" ]; then ./ci/keep_alive bazel test --config=ci $(./scripts/bazel_export_options) --test_tag_filters=-jenkins_only,-flaky python/ray/serve/...; fi

        # ray new dashboard tests
        - if [ "$RAY_CI_DASHBOARD_AFFECTED" == "1" ]; then ./ci/keep_alive bazel test python/ray/new_dashboard/...; fi

        # bazel python tests. This should be run last to keep its logs at the end of travis logs.
        - if [ $RAY_CI_PYTHON_AFFECTED == "1" ]; then ./ci/keep_alive bazel test --test_env=CONDA_EXE --test_env=CONDA_PYTHON_EXE --test_env=CONDA_SHLVL --test_env=CONDA_PREFIX --test_env=CONDA_DEFAULT_ENV --test_env=CONDA_PROMPT_MODIFIER --config=ci $(./scripts/bazel_export_options) --test_tag_filters=-kubernetes,-jenkins_only,-medium_size_python_tests_a_to_j,-medium_size_python_tests_k_to_z,-flaky python/ray/tests/...; fi
        # NO MORE TESTS BELOW, keep them above.

    - os: osx
      osx_image: xcode7
      env:
        - PYTHON=3.6 MEDIUM_TESTS_A_TO_J=1
        - PYTHONWARNINGS=ignore
        - RAY_DEFAULT_BUILD=1
        - RAY_CYTHON_EXAMPLES=1
        - RAY_USE_RANDOM_PORTS=1
      install:
        - . ./ci/travis/ci.sh init RAY_CI_SERVE_AFFECTED,RAY_CI_TUNE_AFFECTED,RAY_CI_PYTHON_AFFECTED,RAY_CI_DASHBOARD_AFFECTED
      before_script:
        - . ./ci/travis/ci.sh build
      script:
        # bazel python tests for medium size tests. Used for parallelization.
        - if [ $RAY_CI_PYTHON_AFFECTED == "1" ]; then ./ci/keep_alive bazel test --config=ci $(./scripts/bazel_export_options) --test_tag_filters=-kubernetes,-jenkins_only,medium_size_python_tests_a_to_j,-flaky python/ray/tests/...; fi

    - os: osx
      osx_image: xcode7
      env:
        - PYTHON=3.6 MEDIUM_TESTS_K_TO_Z=1
        - PYTHONWARNINGS=ignore
        - RAY_DEFAULT_BUILD=1
        - RAY_CYTHON_EXAMPLES=1
        - RAY_USE_RANDOM_PORTS=1
      install:
        - . ./ci/travis/ci.sh init RAY_CI_SERVE_AFFECTED,RAY_CI_TUNE_AFFECTED,RAY_CI_PYTHON_AFFECTED,RAY_CI_DASHBOARD_AFFECTED
      before_script:
        - . ./ci/travis/ci.sh build
      script:
        # bazel python tests for medium size tests. Used for parallelization.
        - if [ $RAY_CI_PYTHON_AFFECTED == "1" ]; then ./ci/keep_alive bazel test --config=ci $(./scripts/bazel_export_options) --test_tag_filters=-kubernetes,-jenkins_only,medium_size_python_tests_k_to_z,-flaky python/ray/tests/...; fi

    - os: osx
      osx_image: xcode7
      env:
        - PYTHON=3.6 FLAKY=1
        - PYTHONWARNINGS=ignore
        - RAY_DEFAULT_BUILD=1
        - RAY_CYTHON_EXAMPLES=1
        - RAY_USE_RANDOM_PORTS=1
        - RLLIB_TESTING=1 TF_VERSION=2.1.0 TFP_VERSION=0.8 TORCH_VERSION=1.6
      install:
        - . ./ci/travis/ci.sh init RAY_CI_SERVE_AFFECTED,RAY_CI_TUNE_AFFECTED,RAY_CI_RLLIB_AFFECTED,RAY_CI_PYTHON_AFFECTED
      before_script:
        - . ./ci/travis/ci.sh build
      script:
        - bazel test --config=ci $(./scripts/bazel_export_options)
          --build_tests_only
          --test_tag_filters=flaky
          -- //:all -rllib/... -core_worker_test
        - bazel test --config=ci $(./scripts/bazel_export_options)
          --test_tag_filters=-kubernetes,-jenkins_only,flaky
          --test_env=CONDA_EXE
          --test_env=CONDA_PYTHON_EXE
          --test_env=CONDA_SHLVL
          --test_env=CONDA_PREFIX
          --test_env=CONDA_DEFAULT_ENV
          python/ray/tests/... python/ray/serve/... python/ray/tune/... rllib/...

    # Build MacOS wheels and MacOS jars
    - os: osx
      osx_image: xcode7
      env:
        - MAC_WHEELS=1 MAC_JARS=1
        - PYTHONWARNINGS=ignore
        - RAY_INSTALL_JAVA=1
      install:
        - . ./ci/travis/ci.sh init RAY_CI_MACOS_WHEELS_AFFECTED,RAY_CI_JAVA_AFFECTED,RAY_CI_STREAMING_JAVA_AFFECTED
      before_script:
        - ./ci/keep_alive brew remove --force java & brew uninstall --force java & rm -rf /usr/local/Homebrew/Library/Taps/homebrew/homebrew-cask
        - ./ci/keep_alive brew install --cask adoptopenjdk/openjdk/adoptopenjdk8
        - export JAVA_HOME=/Library/Java/JavaVirtualMachines/adoptopenjdk-8.jdk/Contents/Home
        - java -version
        - ./ci/keep_alive bash ./ci/travis/ci.sh build
      script:
        - . ./ci/travis/ci.sh test_wheels
        - bash ./java/build-jar-multiplatform.sh darwin

    # Build Linux wheels and jars.
    - os: linux
      env:
        # - PYTHON=3.6
        - LINUX_WHEELS=1 LINUX_JARS=1 DOCKER_BUILD_PY37=1
        - PYTHONWARNINGS=ignore
        - RAY_INSTALL_JAVA=1
      language: java
      jdk: openjdk8
      install:
        - . ./ci/travis/ci.sh init RAY_CI_LINUX_WHEELS_AFFECTED,RAY_CI_JAVA_AFFECTED,RAY_CI_STREAMING_JAVA_AFFECTED
      before_script:
        - . ./ci/travis/ci.sh build
      script:
        - . ./ci/travis/ci.sh test_wheels
        - export PATH="$HOME/miniconda3/bin:$PATH"
        - python -m pip install docker
        - if [[ "$TRAVIS_PULL_REQUEST" != "false" ]]; then python $TRAVIS_BUILD_DIR/ci/travis/build-docker-images.py --py-versions PY37 --build-type PR --build-base; fi
        - bash ./java/build-jar-multiplatform.sh linux
      cache: false

    # Build Py36 Docker Image
    - os: linux
      env:
        - LINUX_WHEELS=1 DOCKER_BUILD_PY36=1
        - PYTHONWARNINGS=ignore
      language: java
      jdk: openjdk8
      install:
        - . ./ci/travis/ci.sh init RAY_CI_LINUX_WHEELS_AFFECTED
      before_script:
        - . ./ci/travis/ci.sh build
      script:
        - wget --quiet "https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh" -O miniconda3.sh
        - bash miniconda3.sh -b -p "$HOME/miniconda3"
        - export PATH="$HOME/miniconda3/bin:$PATH"
        - conda install -y python=3.7.6
        - python -m pip install docker
        - if [[ "$TRAVIS_PULL_REQUEST" != "false" ]]; then python $TRAVIS_BUILD_DIR/ci/travis/build-docker-images.py --py-versions PY36 --build-type PR --build-base; fi
      cache: false

    # Build Py38 Docker Image
    - os: linux
      env:
        - LINUX_WHEELS=1 DOCKER_BUILD_PY38=1
        - PYTHONWARNINGS=ignore
      language: java
      jdk: openjdk8
      install:
        - . ./ci/travis/ci.sh init RAY_CI_LINUX_WHEELS_AFFECTED
      before_script:
        - . ./ci/travis/ci.sh build
      script:
        - wget --quiet "https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh" -O miniconda3.sh
        - bash miniconda3.sh -b -p "$HOME/miniconda3"
        - export PATH="$HOME/miniconda3/bin:$PATH"
        - conda install -y python=3.7.6
        - python -m pip install docker
        - if [[ "$TRAVIS_PULL_REQUEST" != "false" ]]; then python $TRAVIS_BUILD_DIR/ci/travis/build-docker-images.py --py-versions PY38 --build-type PR --build-base; fi
      cache: false

    # Build and deploy multi-platform jars.
    - os: linux
      env:
        - MULTIPLATFORM_JARS=1
        - PYTHONWARNINGS=ignore
        - RAY_INSTALL_JAVA=1
        - secure: "Un2SGOCdD/RiqbO47vtkwn5dPbGbwHi/TKunyNJLKcGILwJc0sZL9uf8pkffyYGbt7ejnYwV9tPgoAlFxcuJYgbmkt84AxDF8oskJmFKYjkxOtEFkqskhTb0u9/usjq23OXrmGN4NqvzLEdbf7Z3wyMxXpzgDKPUgDAFUfB2Ya8OapXuPdt/5KDlDBS0Bj9sKqI+0keYovfRY8dO2/Vd0Ojqkmz1PWHTQP0TrC1X+juciDdoRnU1rO8mxhQW4HKICexwAr1rsAqALpuDlfyhG7I+aicVjK3uiQuviGJOoI813f00YlTOAoXW2YPpblExp0uoTQN2zaYvpp1zUx7V/rVaAoXmFI7ELQ+nr8oAJbImWe1bCNO3UsxpzGamGlTIx7hAiJ0CwFU6qW/+NKWIlhH1iBxUZHw/F8Ixsqtdwx3yhR/rj86sdItAM9CkK6URVdWk2U4VimyDzzJpUWWFsjUZ2V8jNxYPrWg52ciC4k7tp32qrKYrBa+mJduE68/xjFeeZdYUxzg1AT3Lh3rA0ebMIELT7OBz6DRjUmRgO6+VJIRfbPszHEG2X+aPHtSj+Fsc4AacVtiIUUpXMeUx9nxM7oD1lXr8I59vq5+6EhohKBmc4DkpkWLnsCMTnJ1HQ37zKxRSE5jDwyNTJ8x2l8bbK/pqTRZPWUeKL3Is8NE="
        - secure: "IrNncc7RpiSnU/3fB72bG/Vyt/w50uTIVMIdgy64eNw0TgDf15/0IuwzCZDrjTAwq2meGAYFq1KNb7W538UDn+zJs6YIrcEF+2c53alCRASrafFEgAQmbvTpFtxAcHfTFK3c3neVXGiCeOSn64ymV5vQKjLCGvlEpmA10EUmdIE3wIgYRYMTzpzV9A4HmJbAgkBslhIw17BottFbfrL9Z4LlBJjOSP/t9cbGn8QVuM7V4dR+lo5j1Ns8XaVCkNvVqQdJEsp8xeMMn3pvFDxBBEn4zQ2xEGN4QvKquiACaVZLTnNCTkan5pRbswy25hOTnt/zNovJK8TBNuyWTBJNJo+f6vkuQXNR9iRkOQ+FAInjvEVYUWstuc6+opp8yV9cMdOsOMQRX/V6qo5pE2uERU0Fr6WtAhSvZDcGinpfoYkbMXxPdI8E7gmCJKrQ91hVU1ExokztbNesy7YWic/GZPzoBGMSPnNJ8chaydvmeq74P6F0cGI1LU/Tea7Ewf9MbJjWiVZ2C9G1Po3a6vu0ElE1NSIotBZFfpp2P8IeqP5/pZ84E93yRpT4Pt2kBsEjUqiNqLduXDPJnoBHHRzuNovT7sHJoZZAaBipuiNdj4ZQLgA+1WVysaHIDlzlcOMAxNNRMuT6+GATEvtsK8llh0qEzKShbOST54DDp1jST4w="
      language: java
      jdk: openjdk8
      install:
        - . ./ci/travis/ci.sh init RAY_CI_JAVA_AFFECTED,RAY_CI_STREAMING_JAVA_AFFECTED
      script:
        - bash ./java/build-jar-multiplatform.sh multiplatform
        - bash ./java/build-jar-multiplatform.sh deploy
      cache: false
  allow_failures:
    - os: osx
      osx_image: xcode7
      env:
        - PYTHON=3.6 FLAKY=1
        - PYTHONWARNINGS=ignore
        - RAY_DEFAULT_BUILD=1
        - RAY_CYTHON_EXAMPLES=1
        - RAY_USE_RANDOM_PORTS=1
        - RLLIB_TESTING=1 TF_VERSION=2.1.0 TFP_VERSION=0.8 TORCH_VERSION=1.6

after_script:
  - if [ "${TRAVIS_PULL_REQUEST}" = "false" ]; then ./ci/travis/upload_build_info.sh; fi

deploy:
  - provider: s3
    edge: true # This supposedly opts in to deploy v2.
    access_key_id: AKIAU6DMUCJUFL3EX3SM
    secret_access_key:
      secure: J1sX71fKFPQhgWzColllxfzcF877ScBZ1cIl71krZ6SO0LKnwsCScpQck5eZOyQo/Iverwye0iKtE87qNsiRi3+V2D9iulSr18T09j7+FjPKfxAmXmjfrNafoMXTDQroSJblCri5vl+DysISPqImJkWTNaYhGJ9QakoSd5djnAopLNWj6PCR3S50baS49+nB5nSIY3jMhtUzlaBdniFPFC81Cxyuafr4pv6McGRfR/dK+ZnPhdGtMnVeIJXB+ooZKQ26mDJKBPka4jm3u1Oa72b/Atu2RO3MwxTg79LTrMxXKh2OcCqhtD2Z3lz1OltvNSunCuwY8AejCJsfSLbM9mGDoz+xhNUWmYNy48YFf+61OY8PXi8S/9Q817yb3GpLbb2l/P+KMgq9eSEiELIOwuYsDxPX5TuAg6dx0wCNgDEBJoThSQjYl6MgJrLrs7p+JBxp3giedHiy0TLa5hCVKTj3euONAXDArYnnT+DvUIOkaeTk5DClRZbZ0sUXhLy//HuT5WJvjFBJJZ0u0f4RLVb5D7DI4uMZr7+yJPDR2AXCyW9YMaBEbmEYbPaKi283jlEyn7R33+AZlnXv0THHwZ4xvjKKG3/fBSXsOUmv5wmUveEqVGDj1mKPGj9NF8iA5qMm2AaZuJpEEBVBZtSlTZt6ZG7rzAJZGNL52t7xuMo=
    bucket: ray-wheels
    acl: public_read
    region: us-west-2
    local_dir: .whl
    upload-dir: "$TRAVIS_BRANCH/$TRAVIS_COMMIT"
    skip_cleanup: true
    on:
      repo: ray-project/ray
      all_branches: true
      condition: ($LINUX_WHEELS = 1 && $DOCKER_BUILD_PY37=1) || $MAC_WHEELS = 1

  - provider: s3
    edge: true # This supposedly opts in to deploy v2.
    access_key_id: AKIAU6DMUCJUFL3EX3SM
    secret_access_key:
      secure: J1sX71fKFPQhgWzColllxfzcF877ScBZ1cIl71krZ6SO0LKnwsCScpQck5eZOyQo/Iverwye0iKtE87qNsiRi3+V2D9iulSr18T09j7+FjPKfxAmXmjfrNafoMXTDQroSJblCri5vl+DysISPqImJkWTNaYhGJ9QakoSd5djnAopLNWj6PCR3S50baS49+nB5nSIY3jMhtUzlaBdniFPFC81Cxyuafr4pv6McGRfR/dK+ZnPhdGtMnVeIJXB+ooZKQ26mDJKBPka4jm3u1Oa72b/Atu2RO3MwxTg79LTrMxXKh2OcCqhtD2Z3lz1OltvNSunCuwY8AejCJsfSLbM9mGDoz+xhNUWmYNy48YFf+61OY8PXi8S/9Q817yb3GpLbb2l/P+KMgq9eSEiELIOwuYsDxPX5TuAg6dx0wCNgDEBJoThSQjYl6MgJrLrs7p+JBxp3giedHiy0TLa5hCVKTj3euONAXDArYnnT+DvUIOkaeTk5DClRZbZ0sUXhLy//HuT5WJvjFBJJZ0u0f4RLVb5D7DI4uMZr7+yJPDR2AXCyW9YMaBEbmEYbPaKi283jlEyn7R33+AZlnXv0THHwZ4xvjKKG3/fBSXsOUmv5wmUveEqVGDj1mKPGj9NF8iA5qMm2AaZuJpEEBVBZtSlTZt6ZG7rzAJZGNL52t7xuMo=
    bucket: ray-wheels
    acl: public_read
    region: us-west-2
    local_dir: .whl
    upload-dir: latest
    skip_cleanup: true
    on:
      branch: master
      repo: ray-project/ray
      condition: ($LINUX_WHEELS = 1 && $DOCKER_BUILD_PY37=1) || $MAC_WHEELS = 1

  - provider: script
    edge: true # This supposedly opts in to deploy v2.
    script: export PATH="$HOME/miniconda3/bin:$PATH"; ./ci/keep_alive python $TRAVIS_BUILD_DIR/ci/travis/build-docker-images.py --py-versions PY37 --build-type MERGE --build-base
    skip_cleanup: true
    on:
      repo: ray-project/ray
      all_branches: true
      condition: $LINUX_WHEELS = 1 && $DOCKER_BUILD_PY37 = 1

  # Upload jars so that we can debug locally for every commit
  - provider: s3
    edge: true # This supposedly opts in to deploy v2.
    access_key_id: AKIAU6DMUCJUFL3EX3SM
    secret_access_key:
      secure: J1sX71fKFPQhgWzColllxfzcF877ScBZ1cIl71krZ6SO0LKnwsCScpQck5eZOyQo/Iverwye0iKtE87qNsiRi3+V2D9iulSr18T09j7+FjPKfxAmXmjfrNafoMXTDQroSJblCri5vl+DysISPqImJkWTNaYhGJ9QakoSd5djnAopLNWj6PCR3S50baS49+nB5nSIY3jMhtUzlaBdniFPFC81Cxyuafr4pv6McGRfR/dK+ZnPhdGtMnVeIJXB+ooZKQ26mDJKBPka4jm3u1Oa72b/Atu2RO3MwxTg79LTrMxXKh2OcCqhtD2Z3lz1OltvNSunCuwY8AejCJsfSLbM9mGDoz+xhNUWmYNy48YFf+61OY8PXi8S/9Q817yb3GpLbb2l/P+KMgq9eSEiELIOwuYsDxPX5TuAg6dx0wCNgDEBJoThSQjYl6MgJrLrs7p+JBxp3giedHiy0TLa5hCVKTj3euONAXDArYnnT+DvUIOkaeTk5DClRZbZ0sUXhLy//HuT5WJvjFBJJZ0u0f4RLVb5D7DI4uMZr7+yJPDR2AXCyW9YMaBEbmEYbPaKi283jlEyn7R33+AZlnXv0THHwZ4xvjKKG3/fBSXsOUmv5wmUveEqVGDj1mKPGj9NF8iA5qMm2AaZuJpEEBVBZtSlTZt6ZG7rzAJZGNL52t7xuMo=
    bucket: ray-wheels
    acl: public_read
    region: us-west-2
    local_dir: .jar
    upload-dir: "jars/$TRAVIS_BRANCH/$TRAVIS_COMMIT"
    skip_cleanup: true
    on:
      repo: ray-project/ray
      all_branches: true
      condition: $MULTIPLATFORM_JARS = 1 || $MAC_JARS = 1 || $LINUX_JARS = 1

  - provider: s3
    edge: true # This supposedly opts in to deploy v2.
    access_key_id: AKIAU6DMUCJUFL3EX3SM
    secret_access_key:
      secure: J1sX71fKFPQhgWzColllxfzcF877ScBZ1cIl71krZ6SO0LKnwsCScpQck5eZOyQo/Iverwye0iKtE87qNsiRi3+V2D9iulSr18T09j7+FjPKfxAmXmjfrNafoMXTDQroSJblCri5vl+DysISPqImJkWTNaYhGJ9QakoSd5djnAopLNWj6PCR3S50baS49+nB5nSIY3jMhtUzlaBdniFPFC81Cxyuafr4pv6McGRfR/dK+ZnPhdGtMnVeIJXB+ooZKQ26mDJKBPka4jm3u1Oa72b/Atu2RO3MwxTg79LTrMxXKh2OcCqhtD2Z3lz1OltvNSunCuwY8AejCJsfSLbM9mGDoz+xhNUWmYNy48YFf+61OY8PXi8S/9Q817yb3GpLbb2l/P+KMgq9eSEiELIOwuYsDxPX5TuAg6dx0wCNgDEBJoThSQjYl6MgJrLrs7p+JBxp3giedHiy0TLa5hCVKTj3euONAXDArYnnT+DvUIOkaeTk5DClRZbZ0sUXhLy//HuT5WJvjFBJJZ0u0f4RLVb5D7DI4uMZr7+yJPDR2AXCyW9YMaBEbmEYbPaKi283jlEyn7R33+AZlnXv0THHwZ4xvjKKG3/fBSXsOUmv5wmUveEqVGDj1mKPGj9NF8iA5qMm2AaZuJpEEBVBZtSlTZt6ZG7rzAJZGNL52t7xuMo=
    bucket: ray-wheels
    acl: public_read
    region: us-west-2
    local_dir: .jar
    upload-dir: "jars/latest"
    skip_cleanup: true
    on:
      repo: ray-project/ray
      branch: master
      condition: $MULTIPLATFORM_JARS = 1 || $MAC_JARS = 1 || $LINUX_JARS = 1

  - provider: script
    edge: true # This supposedly opts in to deploy v2.
    script: export PATH="$HOME/miniconda3/bin:$PATH"; ./ci/keep_alive python $TRAVIS_BUILD_DIR/ci/travis/build-docker-images.py --py-versions PY36 --build-type MERGE --build-base
    skip_cleanup: true
    on:
      repo: ray-project/ray
      all_branches: true
      condition: $LINUX_WHEELS = 1 && $DOCKER_BUILD_PY36 = 1

  - provider: script
    edge: true # This supposedly opts in to deploy v2.
    script: export PATH="$HOME/miniconda3/bin:$PATH"; ./ci/keep_alive python $TRAVIS_BUILD_DIR/ci/travis/build-docker-images.py --py-versions PY38 --build-type MERGE --build-base
    skip_cleanup: true
    on:
      repo: ray-project/ray
      all_branches: true
      condition: $LINUX_WHEELS = 1 && $DOCKER_BUILD_PY38 = 1