hiro/ray
1
0
Fork 0
mirror of https://github.com/vale981/ray synced 2025-03-05 18:11:42 -05:00
Code Issues Projects Releases Packages Wiki Activity Actions

disable gc

Browse source
This commit is contained in:
jianoaix 2022-07-26 20:41:23 +00:00
parent d169b73519
commit 34dc9c1cdd
3 changed files with 4 additions and 5 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
python/ray/data/_internal/block_batching.py
View file

@ -183,8 +183,8 @@ def _sliding_window(iterable: Iterable, n: int, clear_block_after_read: bool = F
yield tuple(window) yield tuple(window)
for elem in it: for elem in it:
block_ref = window.popleft() block_ref = window.popleft()
if clear_block_after_read: # if clear_block_after_read:
ray._private.internal_api.free(block_ref, local_only=False) # ray._private.internal_api.free(block_ref, local_only=False)
window.append(elem) window.append(elem)
yield tuple(window) yield tuple(window)

4
python/ray/data/_internal/split.py
View file

@ -175,8 +175,8 @@ def _split_all_blocks(
# We make a copy for the blocks that have been splitted, so the input blocks # We make a copy for the blocks that have been splitted, so the input blocks
# can be cleared if they are owned by consumer (consumer-owned blocks will # can be cleared if they are owned by consumer (consumer-owned blocks will
# only be consumed by the owner). # only be consumed by the owner).
if block_list._owned_by_consumer: # if block_list._owned_by_consumer:
ray._private.internal_api.free(blocks_splitted, local_only=False) # ray._private.internal_api.free(blocks_splitted, local_only=False)
return itertools.chain.from_iterable(all_blocks_split_results) return itertools.chain.from_iterable(all_blocks_split_results)

1
python/ray/data/dataset.py
View file

@ -2243,7 +2243,6 @@ class Dataset(Generic[T]):
stats, stats,
prefetch_blocks=prefetch_blocks, prefetch_blocks=prefetch_blocks,
batch_size=batch_size, batch_size=batch_size,
clear_block_after_read=True,
batch_format=batch_format, batch_format=batch_format,
drop_last=drop_last, drop_last=drop_last,
shuffle_buffer_min_size=local_shuffle_buffer_size, shuffle_buffer_min_size=local_shuffle_buffer_size,