hiro/ray
1
0
Fork 0
mirror of https://github.com/vale981/ray synced 2025-03-06 02:21:39 -05:00
Code Issues Projects Releases Packages Wiki Activity Actions

[Plasma store] Improve the OOM logging message. (#16051)

Browse source
This commit is contained in:
SangBin Cho 2021-05-27 10:09:58 -07:00 • committed by GitHub
parent 5d0b302121
commit d0dc9abdfc
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
2 changed files with 12 additions and 5 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

5
python/ray/_raylet.pyx
View file

@ -719,11 +719,6 @@ cdef int64_t restore_spilled_objects_handler(
"An unexpected internal error occurred while the IO worker " "An unexpected internal error occurred while the IO worker "
"was restoring spilled objects.") "was restoring spilled objects.")
logger.exception(exception_str) logger.exception(exception_str)
ray._private.utils.push_error_to_driver(
ray.worker.global_worker,
"restore_spilled_objects_error",
traceback.format_exc() + exception_str,
job_id=None)
return bytes_restored return bytes_restored

12
src/ray/object_manager/plasma/store.cc
View file

@ -954,6 +954,18 @@ void PlasmaStore::ReplyToCreateClient(const std::shared_ptr<Client> &client,
PlasmaObject result = {}; PlasmaObject result = {};
PlasmaError error; PlasmaError error;
bool finished = create_request_queue_.GetRequestResult(req_id, &result, &error); bool finished = create_request_queue_.GetRequestResult(req_id, &result, &error);
if (error == PlasmaError::OutOfMemory) {
// Logs are suppressed because there is only one OOM error per 10 seconds.
RAY_LOG(INFO) << "Out of memory error is reported to the client for an object id "
<< object_id << ". Object store current usage "
<< (PlasmaAllocator::Allocated() / 1e9) << " / "
<< (PlasmaAllocator::GetFootprintLimit() / 1e9)
<< " GB. Pinned unevictable objects after spilling: "
<< num_bytes_in_use_ / 1024 / 1024
<< " MB. Unsealed objects: " << num_bytes_unsealed_ / 1024 / 1024
<< " MB. Object size: "
<< (result.data_size + result.metadata_size) / 1024 / 1024 << " MB.";
}
if (finished) { if (finished) {
RAY_LOG(DEBUG) << "Finishing create object " << object_id << " request ID " << req_id; RAY_LOG(DEBUG) << "Finishing create object " << object_id << " request ID " << req_id;
if (SendCreateReply(client, object_id, result, error).ok() && if (SendCreateReply(client, object_id, result, error).ok() &&