From a29b157e2e37a970be02595d7b05d5dd50d21781 Mon Sep 17 00:00:00 2001
From: Chen Shen <scv119@gmail.com>
Date: Thu, 26 Aug 2021 00:03:29 -0700
Subject: [PATCH] [core] better error message for lost objects (#18068)

---
 python/ray/external_storage.py         | 7 ++++++-
 src/ray/object_manager/plasma/store.cc | 2 +-
 2 files changed, 7 insertions(+), 2 deletions(-)

diff --git a/python/ray/external_storage.py b/python/ray/external_storage.py
index 691439387..0b2417d97 100644
--- a/python/ray/external_storage.py
+++ b/python/ray/external_storage.py
@@ -126,7 +126,12 @@ class ExternalStorage(metaclass=abc.ABCMeta):
             address_len = len(owner_address)
             metadata_len = len(metadata)
             if buf is None:
-                raise ValueError(f"object ref {ref.hex()} does not exist.")
+                error = f"object ref {ref.hex()} does not exist."
+                # empty data and 1 byte metadata, this object is
+                # marked as failed.
+                if metadata_len == 1:
+                    error += " This is probably since its owner has failed."
+                raise ValueError(error)
             buf_len = len(buf)
             payload = address_len.to_bytes(8, byteorder="little") + \
                 metadata_len.to_bytes(8, byteorder="little") + \
diff --git a/src/ray/object_manager/plasma/store.cc b/src/ray/object_manager/plasma/store.cc
index 9f022f125..b246dc066 100644
--- a/src/ray/object_manager/plasma/store.cc
+++ b/src/ray/object_manager/plasma/store.cc
@@ -710,7 +710,7 @@ bool PlasmaStore::IsObjectSpillable(const ObjectID &object_id) {
     // Object already evicted or deleted.
     return false;
   }
-  return entry->GetRefCount() == 1;
+  return entry->Sealed() && entry->GetRefCount() == 1;
 }
 
 void PlasmaStore::PrintDebugDump() const {