From bb8d3c5cd0ab629f76e8d0d1768fc0ec62f77183 Mon Sep 17 00:00:00 2001 From: Eric Liang Date: Thu, 21 May 2020 15:11:03 -0700 Subject: [PATCH] ASAN build for ray core tests (#8431) --- .bazelrc | 15 +++++++++++++++ .travis.yml | 15 ++++++++++++++- build.sh | 4 ++-- ci/travis/ci.sh | 2 +- src/ray/common/scheduling/scheduling_test.cc | 8 ++++---- .../test/direct_actor_transport_test.cc | 7 ++++++- .../transport/direct_actor_transport.h | 2 +- 7 files changed, 43 insertions(+), 10 deletions(-) diff --git a/.bazelrc b/.bazelrc index 127a4aebd..0d759b222 100644 --- a/.bazelrc +++ b/.bazelrc @@ -53,6 +53,21 @@ build:tsan --copt -g build:tsan --copt -fno-omit-frame-pointer build:tsan --linkopt -fsanitize=thread +# Memory sanitizer configuration: +build:asan --strip=never +build:asan --copt -g +build:asan --copt -fsanitize=address +build:asan --copt -fno-sanitize=vptr +build:asan --copt -DADDRESS_SANITIZER +build:asan --copt -fno-omit-frame-pointer +build:asan --linkopt -fsanitize=address +build:asan --linkopt -fno-sanitize=vptr +test:asan --action_env=ASAN_OPTIONS=detect_leaks=0 +# This LD_PRELOAD is set for Travis. You will need to change it for local debugging. +test:asan --action_env=LD_PRELOAD=/usr/lib/x86_64-linux-gnu/libasan.so.2 +# For example, for Ubuntu 18.04 libasan can be found here: +# test:asan --action_env=LD_PRELOAD=/usr/lib/gcc/x86_64-linux-gnu/7/libasan.so + test:ci --flaky_test_attempts=3 test:ci --nocache_test_results test:ci --progress_report_interval=100 diff --git a/.travis.yml b/.travis.yml index 0a6c777ea..fe82398ee 100644 --- a/.travis.yml +++ b/.travis.yml @@ -31,6 +31,19 @@ matrix: before_script: - . ./ci/travis/ci.sh build + - os: linux + env: + - PYTHON=3.6 ENABLE_ASAN=--config=asan + - PYTHONWARNINGS=ignore + - RAY_DEFAULT_BUILD=1 + install: + - . ./ci/travis/ci.sh init + before_script: + - . ./ci/travis/ci.sh build + script: + # Run all C++ unit tests with ASAN enabled. ASAN adds too much overhead to run Python tests. + - bazel test $ENABLE_ASAN --config=ci --build_tests_only -- //:all + - os: osx osx_image: xcode7 env: @@ -287,7 +300,7 @@ script: - ./ci/suppress_output bash src/ray/test/run_object_manager_tests.sh # cc bazel tests (w/o RLlib) - - ./ci/suppress_output bazel test --config=ci --build_tests_only --show_progress_rate_limit=100 --test_output=errors -- //:all -rllib/... + - ./ci/suppress_output bazel test --config=ci --build_tests_only -- //:all -rllib/... # ray serve tests - if [ $RAY_CI_SERVE_AFFECTED == "1" ]; then ./ci/keep_alive bazel test --config=ci --test_tag_filters=-jenkins_only python/ray/serve/...; fi diff --git a/build.sh b/build.sh index 37f5ad8db..954d66117 100755 --- a/build.sh +++ b/build.sh @@ -128,7 +128,7 @@ pushd "$BUILD_DIR" if [ "$RAY_BUILD_JAVA" == "YES" ]; then - "$BAZEL_EXECUTABLE" build //java:ray_java_pkg --verbose_failures + "$BAZEL_EXECUTABLE" build ${ENABLE_ASAN-} //java:ray_java_pkg --verbose_failures fi if [ "$RAY_BUILD_PYTHON" == "YES" ]; then @@ -160,7 +160,7 @@ if [ "$RAY_BUILD_PYTHON" == "YES" ]; then export PYTHON3_BIN_PATH="$PYTHON_EXECUTABLE" - "$BAZEL_EXECUTABLE" build //:ray_pkg --verbose_failures + "$BAZEL_EXECUTABLE" build ${ENABLE_ASAN-} //:ray_pkg --verbose_failures fi popd diff --git a/ci/travis/ci.sh b/ci/travis/ci.sh index 76b0ee01a..bbc3011ef 100755 --- a/ci/travis/ci.sh +++ b/ci/travis/ci.sh @@ -411,7 +411,7 @@ init() { build() { if ! need_wheels; then - bazel build -k "//:*" # Do a full build first to ensure everything passes + bazel build ${ENABLE_ASAN-} -k "//:*" # Do a full build first to ensure everything passes install_ray if [ "${LINT-}" = 1 ]; then # Try generating Sphinx documentation. To do this, we need to install Ray first. diff --git a/src/ray/common/scheduling/scheduling_test.cc b/src/ray/common/scheduling/scheduling_test.cc index 47105ed06..14a1f94d5 100644 --- a/src/ray/common/scheduling/scheduling_test.cc +++ b/src/ray/common/scheduling/scheduling_test.cc @@ -640,7 +640,7 @@ TEST_F(SchedulingTest, TaskResourceInstancesTest) { TaskRequest task_req; vector pred_demands = {3. /* CPU */, 2. /* MEM */, 1.5 /* GPU */}; - vector pred_soft = {false}; + vector pred_soft = {false, false, false}; initTaskRequest(task_req, pred_demands, pred_soft, EmptyIntVector, EmptyFixedPointVector, EmptyBoolVector, EmptyIntVector); @@ -725,7 +725,7 @@ TEST_F(SchedulingTest, TaskResourceInstancesTest) { TaskRequest task_req; vector pred_demands = {3. /* CPU */, 2. /* MEM */, 1.5 /* GPU */}; - vector pred_soft = {false}; + vector pred_soft = {false, false, false}; vector cust_demands{3, 2}; vector cust_soft{false, false}; initTaskRequest(task_req, pred_demands, pred_soft, cust_ids, cust_demands, cust_soft, @@ -755,7 +755,7 @@ TEST_F(SchedulingTest, TaskResourceInstancesTest) { TaskRequest task_req; vector pred_demands = {3. /* CPU */, 2. /* MEM */, 1.5 /* GPU */}; - vector pred_soft = {false}; + vector pred_soft = {false, false, false}; vector cust_demands{3, 10}; vector cust_soft{false, false}; initTaskRequest(task_req, pred_demands, pred_soft, cust_ids, cust_demands, cust_soft, @@ -782,7 +782,7 @@ TEST_F(SchedulingTest, TaskResourceInstancesTest) { TaskRequest task_req; vector pred_demands = {3. /* CPU */, 2. /* MEM */, 1.5 /* GPU */}; - vector pred_soft = {false}; + vector pred_soft = {false, false, false}; vector cust_demands{3, 10}; vector cust_soft{false, true}; initTaskRequest(task_req, pred_demands, pred_soft, cust_ids, cust_demands, cust_soft, diff --git a/src/ray/core_worker/test/direct_actor_transport_test.cc b/src/ray/core_worker/test/direct_actor_transport_test.cc index c7610e503..fb43ba07e 100644 --- a/src/ray/core_worker/test/direct_actor_transport_test.cc +++ b/src/ray/core_worker/test/direct_actor_transport_test.cc @@ -369,7 +369,12 @@ class DirectActorReceiverTest : public ::testing::Test { void StartIOService() { main_io_service_.run(); } - void StopIOService() { main_io_service_.stop(); } + void StopIOService() { + // We must delete the receiver before stopping the IO service, since it + // contains timers referencing the service. + receiver_.reset(); + main_io_service_.stop(); + } std::unique_ptr receiver_; diff --git a/src/ray/core_worker/transport/direct_actor_transport.h b/src/ray/core_worker/transport/direct_actor_transport.h index 80eb46630..7df2bf373 100644 --- a/src/ray/core_worker/transport/direct_actor_transport.h +++ b/src/ray/core_worker/transport/direct_actor_transport.h @@ -411,8 +411,8 @@ class SchedulingQueue { while (!pending_tasks_.empty()) { auto head = pending_tasks_.begin(); head->second.Cancel(); - pending_tasks_.erase(head); next_seq_no_ = std::max(next_seq_no_, head->first + 1); + pending_tasks_.erase(head); } }