ray/.travis.yml

language: generic
# Use Ubuntu 16.04
dist: xenial

matrix:
  include:
    - os: linux
      env:
        - PYTHON=3.6
        - PYTHONWARNINGS=ignore
        - RAY_DEFAULT_BUILD=1
        - RAY_CYTHON_EXAMPLES=1
      before_install:
        - . ./ci/travis/ci.sh init RAY_CI_SERVE_AFFECTED,RAY_CI_TUNE_AFFECTED,RAY_CI_PYTHON_AFFECTED
      install:
        - . ./ci/travis/ci.sh build

    - os: osx
      osx_image: xcode7
      env:
        - PYTHON=3.6
        - PYTHONWARNINGS=ignore
        - RAY_DEFAULT_BUILD=1
        - RAY_CYTHON_EXAMPLES=1
      before_install:
        - . ./ci/travis/ci.sh init RAY_CI_SERVE_AFFECTED,RAY_CI_TUNE_AFFECTED,RAY_CI_PYTHON_AFFECTED
      install:
        - . ./ci/travis/ci.sh build

    - os: linux
      env:
        - JDK='Oracle JDK 8'
        - PYTHON=3.6 PYTHONWARNINGS=ignore
        - RAY_INSTALL_JAVA=1
      before_install:
        - . ./ci/travis/ci.sh init RAY_CI_JAVA_AFFECTED
      install:
        - . ./ci/travis/ci.sh build
      script:
        - . ./ci/travis/ci.sh preload
        - ./java/test.sh

    - os: linux
      env:
        - TESTSUITE=streaming
        - JDK='Oracle JDK 8'
        - RAY_INSTALL_JAVA=1
        - PYTHON=3.6 PYTHONWARNINGS=ignore
      before_install:
        - . ./ci/travis/ci.sh init RAY_CI_STREAMING_PYTHON_AFFECTED,RAY_CI_STREAMING_JAVA_AFFECTED
      install:
        - . ./ci/travis/ci.sh build
      script:
        - . ./ci/travis/ci.sh preload
        # Streaming cpp test.
        - if [ $RAY_CI_STREAMING_CPP_AFFECTED == "1" ]; then bazel test --config=ci --test_output=all //streaming:all && bash streaming/src/test/run_streaming_queue_test.sh; fi
        - if [ $RAY_CI_STREAMING_PYTHON_AFFECTED == "1" ]; then python -m pytest -v --durations=5 --timeout=300 streaming/python/tests/; fi
        - if [ $RAY_CI_STREAMING_JAVA_AFFECTED == "1" ]; then ./streaming/java/test.sh; fi

    - os: linux
      env:
        - TESTSUITE=gcs_service_disabled
        - JDK='Oracle JDK 8'
        - PYTHON=3.6 PYTHONWARNINGS=ignore
        - RAY_INSTALL_JAVA=1
        - RAY_GCS_SERVICE_ENABLED=false
        - RAY_CYTHON_EXAMPLES=1
      before_install:
        - . ./ci/travis/ci.sh init RAY_CI_ONLY_RLLIB_AFFECTED
      install:
        - . ./ci/travis/ci.sh build
      script:
        - . ./ci/travis/ci.sh preload
        - ./ci/suppress_output bash src/ray/test/run_core_worker_tests.sh
        - ./ci/suppress_output bash streaming/src/test/run_streaming_queue_test.sh
        - ./java/test.sh

    - os: linux
      env:
        - TESTSUITE=gcs_service_disabled_python_testcase
        - JDK='Oracle JDK 8'
        - PYTHON=3.6 PYTHONWARNINGS=ignore
        - RAY_INSTALL_JAVA=1
        - RAY_GCS_SERVICE_ENABLED=false
        - RAY_CYTHON_EXAMPLES=1
      before_install:
        - . ./ci/travis/ci.sh init RAY_CI_ONLY_RLLIB_AFFECTED
      install:
        - . ./ci/travis/ci.sh build
      script:
        - . ./ci/travis/ci.sh preload
        - ./ci/keep_alive bazel test --config=ci --test_tag_filters=-jenkins_only python/ray/tests/...

    - os: linux
      env:
        - LINT=1
        - PYTHONWARNINGS=ignore
      before_install:
        - . ./ci/travis/ci.sh init
      install:
        - . ./ci/travis/ci.sh build
      script:
        - . ./ci/travis/ci.sh preload
        - ./ci/travis/check-git-clang-format-output.sh
        # Try generating Sphinx documentation. To do this, we need to install
        # Ray first.
        - cd doc
        # readthedocs has an antiquated build env.
        # This is a best effort to reproduce it locally to avoid doc build failures
        # and hidden errors.
        - pip install -r requirements-rtd.txt
        - pip install -r requirements-doc.txt
        - pip install yapf==0.23.0
        - sphinx-build -q -W -E -T -b html source _build/html
        - cd ..
        # Run Python linting, ignore dict vs {} (C408), others are defaults
        - flake8 --inline-quotes '"' --no-avoid-escape --exclude=python/ray/core/generated/,streaming/python/generated,doc/source/conf.py,python/ray/cloudpickle/,python/ray/thirdparty_files --ignore=C408,E121,E123,E126,E226,E24,E704,W503,W504,W605
        - ./ci/travis/format.sh --all
        # Make sure that the README is formatted properly.
        - cd python
        - python setup.py check --restructuredtext --strict --metadata
        - cd ..
        # Run Bazel linter Buildifier.
        - wget -q https://dl.google.com/go/go1.14.2.linux-amd64.tar.gz
        - tar -xf go1.14.2.linux-amd64.tar.gz
        - mkdir $HOME/go_dir
        - export GOROOT=`pwd`/go
        - export GOPATH="$HOME/go_dir"
        - export PATH="$GOROOT/bin:$GOPATH/bin:$PATH"
        - go get github.com/bazelbuild/buildtools/buildifier
        - ./ci/travis/bazel-format.sh
        # Run TypeScript and HTML linting.
        - pushd python/ray/dashboard/client
        - source "$HOME/.nvm/nvm.sh"
        - nvm use node
        - node_modules/.bin/eslint --max-warnings 0 $(find src -name "*.ts" -or -name "*.tsx")
        - node_modules/.bin/prettier --check $(find src -name "*.ts" -or -name "*.tsx")
        - node_modules/.bin/prettier --check public/index.html
        - popd

    # Build Linux wheels.
    - os: linux
      env:
        - LINUX_WHEELS=1
        - PYTHONWARNINGS=ignore
        - RAY_INSTALL_JAVA=1
      before_install:
        - . ./ci/travis/ci.sh init RAY_CI_LINUX_WHEELS_AFFECTED
      install:
        - . ./ci/travis/ci.sh build
      script:
        - . ./ci/travis/ci.sh preload
        # Explicitly sleep 60 seconds for logs to go through
        - ./ci/travis/test-wheels.sh || { cat /tmp/ray/session_latest/logs/* && sleep 60 && false; }
      cache: false

    # Build MacOS wheels.
    - os: osx
      osx_image: xcode7
      env:
        - MAC_WHEELS=1
        - PYTHONWARNINGS=ignore
        - RAY_INSTALL_JAVA=1
      before_install:
        - . ./ci/travis/ci.sh init RAY_CI_MACOS_WHEELS_AFFECTED
      install:
        - . ./ci/travis/ci.sh build
      script:
        - . ./ci/travis/ci.sh preload
        # Explicitly sleep 60 seconds for logs to go through
        - ./ci/travis/test-wheels.sh || { cat /tmp/ray/session_latest/logs/* && sleep 60 && false; }

    # RLlib: Learning tests (from rllib/tuned_examples/regression_tests/*.yaml).
    - os: linux
      env:
        - RLLIB_TESTING=1
        - RLLIB_REGRESSION_TESTS=1
        - TF_VERSION=2.0.0b1
        - TFP_VERSION=0.8
        - TORCH_VERSION=1.4
        - PYTHON=3.6
        - PYTHONWARNINGS=ignore
      before_install:
        - . ./ci/travis/ci.sh init RAY_CI_RLLIB_AFFECTED
      install:
        - . ./ci/travis/ci.sh build
      script:
        - . ./ci/travis/ci.sh preload
        - travis_wait 90 bazel test --config=ci --test_output=streamed --build_tests_only --test_tag_filters=learning_tests rllib/...

    # RLlib: Learning tests with tf=1.x (from rllib/tuned_examples/regression_tests/*.yaml).
    # Requested by Edi (MS): Test all learning capabilities with tf1.x
    - os: linux
      env:
        - RLLIB_TESTING=1
        - RLLIB_REGRESSION_TESTS_TF1X=1
        - TF_VERSION=1.14.0
        - TFP_VERSION=0.7
        - TORCH_VERSION=1.4
        - PYTHON=3.6
        - PYTHONWARNINGS=ignore
      before_install:
        - . ./ci/travis/ci.sh init RAY_CI_RLLIB_FULL_AFFECTED
      install:
        - . ./ci/travis/ci.sh build
      script:
        - . ./ci/travis/ci.sh preload
        - travis_wait 90 bazel test --config=ci --test_output=streamed --build_tests_only --test_tag_filters=learning_tests rllib/...

    # RLlib: Quick Agent train.py runs (compilation & running, no(!) learning).
    # Agent single tests (compilation, loss-funcs, etc..).
    - os: linux
      env:
        - RLLIB_TESTING=1
        - RLLIB_QUICK_TRAIN_AND_MISC_TESTS=1
        - PYTHON=3.6
        - TF_VERSION=2.0.0b1
        - TFP_VERSION=0.8
        - TORCH_VERSION=1.4
        - PYTHONWARNINGS=ignore
      before_install:
        - . ./ci/travis/ci.sh init RAY_CI_RLLIB_FULL_AFFECTED
      install:
        - . ./ci/travis/ci.sh build
      script:
        - . ./ci/travis/ci.sh preload
        - travis_wait 60 bazel test --config=ci --build_tests_only --test_tag_filters=quick_train rllib/...
        # Test everything that does not have any of the "main" labels:
        # "learning_tests|quick_train|examples|tests_dir".
        - ./ci/keep_alive bazel test --config=ci --build_tests_only --test_tag_filters=-learning_tests,-quick_train,-examples,-tests_dir rllib/...

    # RLlib: Everything in rllib/examples/ directory.
    - os: linux
      env:
        - RLLIB_TESTING=1
        - RLLIB_EXAMPLE_DIR_TESTS=1
        - PYTHON=3.6
        - TF_VERSION=2.0.0b1
        - TFP_VERSION=0.8
        - TORCH_VERSION=1.4
        - PYTHONWARNINGS=ignore
      before_install:
        - . ./ci/travis/ci.sh init RAY_CI_RLLIB_FULL_AFFECTED
      install:
        - . ./ci/travis/ci.sh build
      script:
        - . ./ci/travis/ci.sh preload
        - ./ci/keep_alive bazel test --config=ci --build_tests_only --test_tag_filters=examples_A,examples_B rllib/...
        - ./ci/keep_alive bazel test --config=ci --build_tests_only --test_tag_filters=examples_C rllib/...
        - ./ci/keep_alive bazel test --config=ci --build_tests_only --test_tag_filters=examples_E,examples_L,examples_M,examples_P rllib/...
        - ./ci/keep_alive bazel test --config=ci --build_tests_only --test_tag_filters=examples_U,examples_R,examples_S,examples_T rllib/...

    # RLlib: tests_dir: Everything in rllib/tests/ directory (A-I).
    - os: linux
      env:
        - RLLIB_TESTING=1
        - RLLIB_TESTS_DIR_TESTS_A_TO_I=1
        - PYTHON=3.6
        - TF_VERSION=2.0.0b1
        - TFP_VERSION=0.8
        - TORCH_VERSION=1.4
        - PYTHONWARNINGS=ignore
      before_install:
        - . ./ci/travis/ci.sh init RAY_CI_RLLIB_FULL_AFFECTED
      install:
        - . ./ci/travis/ci.sh build
      script:
        - . ./ci/travis/ci.sh preload
        - ./ci/keep_alive bazel test --config=ci --build_tests_only --test_tag_filters=tests_dir_A,tests_dir_B,tests_dir_C,tests_dir_D,tests_dir_E,tests_dir_F,tests_dir_G,tests_dir_H,tests_dir_I rllib/...

    # RLlib: tests_dir: Everything in rllib/tests/ directory (J-Z).
    - os: linux
      env:
        - RLLIB_TESTING=1
        - RLLIB_TESTS_DIR_TESTS_J_TO_Z=1
        - PYTHON=3.6
        - TF_VERSION=2.0.0b1
        - TFP_VERSION=0.8
        - TORCH_VERSION=1.4
        - PYTHONWARNINGS=ignore
      before_install:
        - . ./ci/travis/ci.sh init RAY_CI_RLLIB_FULL_AFFECTED
      install:
        - . ./ci/travis/ci.sh build
      script:
        - . ./ci/travis/ci.sh preload
        - ./ci/keep_alive bazel test --config=ci --build_tests_only --test_tag_filters=tests_dir_J,tests_dir_K,tests_dir_L,tests_dir_M,tests_dir_N,tests_dir_O,tests_dir_P,tests_dir_Q,tests_dir_R,tests_dir_S,tests_dir_T,tests_dir_U,tests_dir_V,tests_dir_W,tests_dir_X,tests_dir_Y,tests_dir_Z rllib/...

    # Cpp worker test
    - os: linux
      env:
        - TESTSUITE=cpp_worker
        - PYTHON=3.6
      before_install:
        - . ./ci/travis/ci.sh init
      install:
        - . ./ci/travis/ci.sh build
      script:
        - . ./ci/travis/ci.sh preload
        - bazel test --config=ci //cpp:all --build_tests_only --test_output=streamed

script:
  - . ./ci/travis/ci.sh preload
  # raylet integration tests
  - ./ci/suppress_output bash src/ray/test/run_core_worker_tests.sh
  - ./ci/suppress_output bash src/ray/test/run_object_manager_tests.sh

  # cc bazel tests (w/o RLlib)
  - ./ci/suppress_output bazel test --config=ci --build_tests_only --show_progress_rate_limit=100 --test_output=errors -- //:all -rllib/...

  # ray serve tests
  - if [ $RAY_CI_SERVE_AFFECTED == "1" ]; then ./ci/keep_alive bazel test --config=ci --test_tag_filters=-jenkins_only python/ray/serve/...; fi

  # ray operator tests
  - cd ./deploy/ray-operator/
  - CC=gcc ../../ci/suppress_output go build
  - CC=gcc ../../ci/suppress_output go test ./...
  - cd ../..

  # random python tests TODO(ekl): these should be moved to bazel
  - if [ $RAY_CI_PYTHON_AFFECTED == "1" ]; then python -m pytest -v --durations=5 --timeout=300 python/ray/experimental/test/async_test.py; fi

  # bazel python tests. This should be run last to keep its logs at the end of travis logs.
  - if [ $RAY_CI_PYTHON_AFFECTED == "1" ]; then ./ci/keep_alive bazel test --config=ci --test_tag_filters=-jenkins_only python/ray/tests/...; fi
  - if [ $RAY_CI_TUNE_AFFECTED == "1" ]; then ./ci/keep_alive bazel test --config=ci --test_tag_filters=-jenkins_only python/ray/tune/...; fi
  # NO MORE TESTS BELOW, keep them above.

deploy:
  - provider: s3
    edge: true # This supposedly opts in to deploy v2.
    access_key_id: AKIAU6DMUCJUFL3EX3SM
    secret_access_key:
      secure: J1sX71fKFPQhgWzColllxfzcF877ScBZ1cIl71krZ6SO0LKnwsCScpQck5eZOyQo/Iverwye0iKtE87qNsiRi3+V2D9iulSr18T09j7+FjPKfxAmXmjfrNafoMXTDQroSJblCri5vl+DysISPqImJkWTNaYhGJ9QakoSd5djnAopLNWj6PCR3S50baS49+nB5nSIY3jMhtUzlaBdniFPFC81Cxyuafr4pv6McGRfR/dK+ZnPhdGtMnVeIJXB+ooZKQ26mDJKBPka4jm3u1Oa72b/Atu2RO3MwxTg79LTrMxXKh2OcCqhtD2Z3lz1OltvNSunCuwY8AejCJsfSLbM9mGDoz+xhNUWmYNy48YFf+61OY8PXi8S/9Q817yb3GpLbb2l/P+KMgq9eSEiELIOwuYsDxPX5TuAg6dx0wCNgDEBJoThSQjYl6MgJrLrs7p+JBxp3giedHiy0TLa5hCVKTj3euONAXDArYnnT+DvUIOkaeTk5DClRZbZ0sUXhLy//HuT5WJvjFBJJZ0u0f4RLVb5D7DI4uMZr7+yJPDR2AXCyW9YMaBEbmEYbPaKi283jlEyn7R33+AZlnXv0THHwZ4xvjKKG3/fBSXsOUmv5wmUveEqVGDj1mKPGj9NF8iA5qMm2AaZuJpEEBVBZtSlTZt6ZG7rzAJZGNL52t7xuMo=
    bucket: ray-wheels
    acl: public_read
    region: us-west-2
    local_dir: .whl
    upload-dir: "$TRAVIS_BRANCH/$TRAVIS_COMMIT"
    skip_cleanup: true
    on:
      repo: ray-project/ray
      all_branches: true
      condition: $LINUX_WHEELS = 1 || $MAC_WHEELS = 1

  - provider: s3
    edge: true # This supposedly opts in to deploy v2.
    access_key_id: AKIAU6DMUCJUFL3EX3SM
    secret_access_key:
      secure: J1sX71fKFPQhgWzColllxfzcF877ScBZ1cIl71krZ6SO0LKnwsCScpQck5eZOyQo/Iverwye0iKtE87qNsiRi3+V2D9iulSr18T09j7+FjPKfxAmXmjfrNafoMXTDQroSJblCri5vl+DysISPqImJkWTNaYhGJ9QakoSd5djnAopLNWj6PCR3S50baS49+nB5nSIY3jMhtUzlaBdniFPFC81Cxyuafr4pv6McGRfR/dK+ZnPhdGtMnVeIJXB+ooZKQ26mDJKBPka4jm3u1Oa72b/Atu2RO3MwxTg79LTrMxXKh2OcCqhtD2Z3lz1OltvNSunCuwY8AejCJsfSLbM9mGDoz+xhNUWmYNy48YFf+61OY8PXi8S/9Q817yb3GpLbb2l/P+KMgq9eSEiELIOwuYsDxPX5TuAg6dx0wCNgDEBJoThSQjYl6MgJrLrs7p+JBxp3giedHiy0TLa5hCVKTj3euONAXDArYnnT+DvUIOkaeTk5DClRZbZ0sUXhLy//HuT5WJvjFBJJZ0u0f4RLVb5D7DI4uMZr7+yJPDR2AXCyW9YMaBEbmEYbPaKi283jlEyn7R33+AZlnXv0THHwZ4xvjKKG3/fBSXsOUmv5wmUveEqVGDj1mKPGj9NF8iA5qMm2AaZuJpEEBVBZtSlTZt6ZG7rzAJZGNL52t7xuMo=
    bucket: ray-wheels
    acl: public_read
    region: us-west-2
    local_dir: .whl
    upload-dir: latest
    skip_cleanup: true
    on:
      branch: master
      repo: ray-project/ray
      condition: $LINUX_WHEELS = 1 || $MAC_WHEELS = 1

  - provider: script
    edge: true # This supposedly opts in to deploy v2.
    script: bash $TRAVIS_BUILD_DIR/ci/travis/build-autoscaler-images.sh || true
    skip_cleanup: true
    on:
      repo: ray-project/ray
      all_branches: true
      condition: $LINUX_WHEELS = 1 || $MAC_WHEELS = 1