hiro/ray
1
0
Fork 0
mirror of https://github.com/vale981/ray synced 2025-03-06 02:21:39 -05:00
Code Issues Projects Releases Packages Wiki Activity Actions

[Tests] Make nightly test work + Remove work stealing logs (#18300)

Browse source 
* make tests work

* .
This commit is contained in:
SangBin Cho 2021-09-14 09:52:58 -07:00 • committed by GitHub
parent 644f7bd7fa
commit 51d94ebee0
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
3 changed files with 3 additions and 10 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
release/.buildkite/build_pipeline.py
View file

@ -61,7 +61,9 @@ CORE_NIGHTLY_TESTS = {
"shuffle_1tb_1000_partition", "shuffle_1tb_1000_partition",
"non_streaming_shuffle_1tb_1000_partition", "non_streaming_shuffle_1tb_1000_partition",
"shuffle_1tb_5000_partitions", "shuffle_1tb_5000_partitions",
"non_streaming_shuffle_1tb_5000_partitions", # TODO(sang): It doesn't even work without spilling
# as it hits the scalability limit.
# "non_streaming_shuffle_1tb_5000_partitions",
"decision_tree_autoscaling", "decision_tree_autoscaling",
"autoscaling_shuffle_1tb_1000_partitions", "autoscaling_shuffle_1tb_1000_partitions",
SmokeTest("stress_test_many_tasks"), SmokeTest("stress_test_many_tasks"),

1
src/ray/core_worker/core_worker.cc
View file

@ -2444,7 +2444,6 @@ void CoreWorker::HandlePushTask(const rpc::PushTaskRequest &request,
void CoreWorker::HandleStealTasks(const rpc::StealTasksRequest &request, void CoreWorker::HandleStealTasks(const rpc::StealTasksRequest &request,
rpc::StealTasksReply *reply, rpc::StealTasksReply *reply,
rpc::SendReplyCallback send_reply_callback) { rpc::SendReplyCallback send_reply_callback) {
RAY_LOG(DEBUG) << "Entering CoreWorker::HandleStealWork!";
direct_task_receiver_->HandleStealTasks(request, reply, send_reply_callback); direct_task_receiver_->HandleStealTasks(request, reply, send_reply_callback);
} }

8
src/ray/core_worker/transport/direct_task_transport.cc
View file

@ -215,7 +215,6 @@ bool CoreWorkerDirectTaskSubmitter::FindOptimalVictimForStealing(
const auto &victim_entry = worker_to_lease_entry_[victim_addr]; const auto &victim_entry = worker_to_lease_entry_[victim_addr];
// Double check that the victim has the correct SchedulingKey // Double check that the victim has the correct SchedulingKey
RAY_CHECK(victim_entry.scheduling_key == scheduling_key); RAY_CHECK(victim_entry.scheduling_key == scheduling_key);
RAY_LOG(DEBUG) << "Victim is worker " << victim_addr.worker_id << " and has " RAY_LOG(DEBUG) << "Victim is worker " << victim_addr.worker_id << " and has "
<< victim_entry.tasks_in_flight << " tasks in flight, " << victim_entry.tasks_in_flight << " tasks in flight, "
<< " among which we estimate that " << victim_entry.tasks_in_flight / 2 << " among which we estimate that " << victim_entry.tasks_in_flight / 2
@ -248,14 +247,9 @@ void CoreWorkerDirectTaskSubmitter::StealTasksOrReturnWorker(
return; return;
} }
RAY_LOG(DEBUG) << "Beginning to steal work now! Thief is worker: "
<< thief_addr.worker_id;
// Search for a suitable victim // Search for a suitable victim
rpc::Address victim_raw_addr; rpc::Address victim_raw_addr;
if (!FindOptimalVictimForStealing(scheduling_key, thief_addr, &victim_raw_addr)) { if (!FindOptimalVictimForStealing(scheduling_key, thief_addr, &victim_raw_addr)) {
RAY_LOG(DEBUG) << "Could not find a suitable victim for stealing! Returning worker "
<< thief_addr.worker_id;
// If stealing was enabled, we can now cancel any pending new workeer lease request, // If stealing was enabled, we can now cancel any pending new workeer lease request,
// because stealing is now possible this time. // because stealing is now possible this time.
if (max_tasks_in_flight_per_worker_ > 1) { if (max_tasks_in_flight_per_worker_ > 1) {
@ -342,8 +336,6 @@ void CoreWorkerDirectTaskSubmitter::OnWorkerIdle(
// Return the worker only if there are no tasks in flight // Return the worker only if there are no tasks in flight
if (lease_entry.tasks_in_flight == 0) { if (lease_entry.tasks_in_flight == 0) {
RAY_LOG(DEBUG)
<< "Number of tasks in flight == 0, calling StealTasksOrReturnWorker!";
StealTasksOrReturnWorker(addr, was_error, scheduling_key, assigned_resources); StealTasksOrReturnWorker(addr, was_error, scheduling_key, assigned_resources);
} }
} else { } else {