From dcf41d868c99fe9863627876dab19e6548c1c766 Mon Sep 17 00:00:00 2001
From: Dmitri Gekhtman <62982571+DmitriGekhtman@users.noreply.github.com>
Date: Sun, 28 Mar 2021 17:52:12 -0400
Subject: [PATCH] [autoscaler][Kubernetes] Fix non_terminated_nodes consistency
 (#14976)

* Verify pod termination

* deletion-timestamp

* get rid of extra constant
---
 python/ray/autoscaler/_private/kubernetes/node_provider.py | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/python/ray/autoscaler/_private/kubernetes/node_provider.py b/python/ray/autoscaler/_private/kubernetes/node_provider.py
index 3780c5bcf..d82f47931 100644
--- a/python/ray/autoscaler/_private/kubernetes/node_provider.py
+++ b/python/ray/autoscaler/_private/kubernetes/node_provider.py
@@ -51,7 +51,12 @@ class KubernetesNodeProvider(NodeProvider):
             field_selector=field_selector,
             label_selector=label_selector)
 
-        return [pod.metadata.name for pod in pod_list.items]
+        # Don't return pods marked for deletion,
+        # i.e. pods with non-null metadata.DeletionTimestamp.
+        return [
+            pod.metadata.name for pod in pod_list.items
+            if pod.metadata.deletion_timestamp is None
+        ]
 
     def is_running(self, node_id):
         pod = core_api().read_namespaced_pod(node_id, self.namespace)