disable gc

python/ray/data/_internal/block_batching.py

@@ -183,8 +183,8 @@ def _sliding_window(iterable: Iterable, n: int, clear_block_after_read: bool = F
         yield tuple(window)
     for elem in it:
         block_ref = window.popleft()
-        if clear_block_after_read:
-            ray._private.internal_api.free(block_ref, local_only=False)
+        # if clear_block_after_read:
+        #     ray._private.internal_api.free(block_ref, local_only=False)
         window.append(elem)
         yield tuple(window)
 

python/ray/data/_internal/split.py

@@ -175,8 +175,8 @@ def _split_all_blocks(
     # We make a copy for the blocks that have been splitted, so the input blocks
     # can be cleared if they are owned by consumer (consumer-owned blocks will
     # only be consumed by the owner).
-    if block_list._owned_by_consumer:
-        ray._private.internal_api.free(blocks_splitted, local_only=False)
+    # if block_list._owned_by_consumer:
+    #     ray._private.internal_api.free(blocks_splitted, local_only=False)
 
     return itertools.chain.from_iterable(all_blocks_split_results)
 

python/ray/data/dataset.py

@@ -2243,7 +2243,6 @@ class Dataset(Generic[T]):
             stats,
             prefetch_blocks=prefetch_blocks,
             batch_size=batch_size,
-            clear_block_after_read=True,
             batch_format=batch_format,
             drop_last=drop_last,
             shuffle_buffer_min_size=local_shuffle_buffer_size,