diff --git a/.bazelrc b/.bazelrc index 727e2101a..4539a1f38 100644 --- a/.bazelrc +++ b/.bazelrc @@ -62,7 +62,8 @@ build:tsan --copt -g build:tsan --copt -fno-omit-frame-pointer build:tsan --linkopt -fsanitize=thread -# Memory sanitizer configuration: +# Memory sanitizer configuration +# If you change these make sure you also change ci/asan_tests/bazelrc build:asan --strip=never build:asan --copt -g build:asan --copt -fsanitize=address diff --git a/ci/asan_tests/Makefile b/ci/asan_tests/Makefile new file mode 100644 index 000000000..09f2ca389 --- /dev/null +++ b/ci/asan_tests/Makefile @@ -0,0 +1,28 @@ +SHELL=/bin/bash + +echo: + @echo "Available commands: setup, run, recompile" + +setup: + echo "Setting up the environment" + pip install -r ray-project/requirements.txt + pip install -U pytest + + echo "Installing cython example" + cd "${HOME}"/ray/doc/examples/cython; python setup.py install --user + + echo "Settting up the shell" + cp bazelrc "${HOME}"/.bazelrc # Setup cache + echo "LD_PRELOAD=/usr/lib/gcc/x86_64-linux-gnu/7/libasan.so" >> "${HOME}"/.bashrc + echo "ASAN_OPTIONS=detect_leaks=0" >> "${HOME}"/.bashrc + + echo "Compiling ray" + cd "${HOME}"/ray; git fetch; git pull origin master + cd "${HOME}"/ray/python; pip install -e . --verbose || true + +run: + ./run_asan_tests.sh + +recompile: + cd "${HOME}"/ray; git fetch; git checkout "${git_sha}" + cd "${HOME}"/ray/python; pip install -e . --verbose || true diff --git a/ci/asan_tests/bazelrc b/ci/asan_tests/bazelrc new file mode 100644 index 000000000..b19a66b4d --- /dev/null +++ b/ci/asan_tests/bazelrc @@ -0,0 +1,30 @@ +# This bazelrc specifies configuration for the release asan test. +# It exists because we want everything to be "default" configuration +# instead of "--config=ci" or "--config=asan". + +# Cache +build --remote_cache=https://storage.googleapis.com/ray-bazel-cache +build --disk_cache=/tmp/bazel-cache +build --repository_cache=/tmp/bazel-cache +build --remote_upload_local_results=false + +# Asan +build --strip=never +build --copt -g +build --copt -fsanitize=address +build --copt -fno-sanitize=vptr +build --copt -DADDRESS_SANITIZER +build --copt -fno-omit-frame-pointer +build --linkopt -fsanitize=address +build --linkopt -fno-sanitize=vptr +test --action_env=ASAN_OPTIONS=detect_leaks=0 +test --action_env=LD_PRELOAD=/usr/lib/gcc/x86_64-linux-gnu/7/libasan.so + +# Travis test config +test --flaky_test_attempts=3 +test --nocache_test_results +test --show_timestamps +test --spawn_strategy=local +test --test_output=errors +test --test_verbose_timeout_warnings +test --jobs 1 diff --git a/ci/asan_tests/ray-project/cluster.yaml b/ci/asan_tests/ray-project/cluster.yaml new file mode 100644 index 000000000..952a14921 --- /dev/null +++ b/ci/asan_tests/ray-project/cluster.yaml @@ -0,0 +1,49 @@ +# This file is generated by `ray project create`. + +# A unique identifier for the head node and workers of this cluster. +cluster_name: asan_python_tests + +# The maximum number of workers nodes to launch in addition to the head +# node. This takes precedence over min_workers. min_workers defaults to 0. +max_workers: 0 + +# Cloud-provider specific configuration. +provider: + type: aws + region: us-west-2 + availability_zone: us-west-2a + +head_node: + InstanceType: m5.4xlarge + ImageId: ami-0d1cd67c26f5fca19 # Default Ubuntu 18.04 AMI. + + # Set primary volume to 50 GiB + BlockDeviceMappings: + - DeviceName: /dev/sda1 + Ebs: + VolumeSize: 50 + +# How Ray will authenticate with newly launched nodes. +auth: + ssh_user: ubuntu + +setup_commands: + # Install basics. + - sudo apt-get update -q + - sudo apt-get install -y -q build-essential curl unzip + # Install Anaconda. + - wget -q https://repo.continuum.io/archive/Anaconda3-5.0.1-Linux-x86_64.sh + - bash Anaconda3-5.0.1-Linux-x86_64.sh -b -p $HOME/anaconda3 || true + - echo 'export PATH="$HOME/anaconda3/bin:$PATH"' >> ~/.bashrc + - pip install -U pip || true + # Clone Ray. + - cd $HOME; rm -rf $HOME/ray; git clone https://github.com/ray-project/ray || true + # Install Node.js in order to build the dashboard. + - curl -sL https://deb.nodesource.com/setup_12.x | sudo -E bash + - sudo apt-get install -y nodejs || true + - cd ray/python/ray/dashboard/client; npm ci; npm run build + # Set up Ray dependencies. + - which bazel || bash -c "cd $HOME/ray; ci/travis/install-bazel.sh" + +# Command to start ray on the head node. You don't need to change this. +head_start_ray_commands: [] diff --git a/ci/asan_tests/ray-project/project.yaml b/ci/asan_tests/ray-project/project.yaml new file mode 100644 index 000000000..1a7dcfedc --- /dev/null +++ b/ci/asan_tests/ray-project/project.yaml @@ -0,0 +1,38 @@ +# This file is generated by `ray project create`. + +name: asan_python_tests + +# description: A short description of the project. +# The URL of the repo this project is part of. +# repo: ... + +cluster: + config: ray-project/cluster.yaml + +# environment: + # dockerfile: The dockerfile to be built and ran the commands with. + # dockerimage: The docker image to be used to run the project in, e.g. ubuntu:18.04. + # requirements: ray-project/requirements.txt + +commands: + - name: Setup + command: make setup + - name: Run + command: make run + - name: Recompile + command: | + git_sha={{git_sha}} make recompile + params: + - name: git_sha # Ray version string. + default: "master" + +# Pathnames for files and directories that should be saved +# in a snapshot but that should not be synced with a# session. Pathnames can be relative to the project +# directory or absolute. Generally, this should be files +# that were created by an active session, such as +# application checkpoints and logs. +output_files: [ + # For example, uncomment this to save the logs from the + # last ray job. + # "/tmp/ray/session_latest", +] diff --git a/ci/asan_tests/ray-project/requirements.txt b/ci/asan_tests/ray-project/requirements.txt new file mode 100644 index 000000000..7fc33e88e --- /dev/null +++ b/ci/asan_tests/ray-project/requirements.txt @@ -0,0 +1,37 @@ +aiohttp +blist +boto3 +cython==0.29.0 +dataclasses +dm-tree +feather-format +flask +grpcio +gym +kubernetes +lxml +networkx +numba +opencv-python-headless +openpyxl +pandas==0.24.2 +Pillow +prometheus_client +py-spy +pygments +pytest +pytest-asyncio +pytest-rerunfailures +pytest-sugar +pytest-timeout +pyyaml +requests +scikit-learn==0.22.2 +scipy==1.4.1 +tabulate +tensorflow==2.0.1 +torch +torchvision +uvicorn +werkzeug +xlrd \ No newline at end of file diff --git a/ci/asan_tests/run_asan_tests.sh b/ci/asan_tests/run_asan_tests.sh new file mode 100755 index 000000000..eba288c1d --- /dev/null +++ b/ci/asan_tests/run_asan_tests.sh @@ -0,0 +1,15 @@ +set -e +set -x + +export LD_PRELOAD=/usr/lib/gcc/x86_64-linux-gnu/7/libasan.so +export ASAN_OPTIONS=detect_leaks=0 + +cd $HOME/ray +# async plasma test +python -m pytest -v --durations=5 --timeout=300 python/ray/experimental/test/async_test.py + +# Ray tests +bazel test --test_tag_filters=-jenkins_only python/ray/serve/... +bazel test --test_tag_filters=-jenkins_only python/ray/dashboard/... +bazel test --test_tag_filters=-jenkins_only python/ray/tests/... +bazel test --test_tag_filters=-jenkins_only python/ray/tune/...