hiro/ray
1
0
Fork 0
mirror of https://github.com/vale981/ray synced 2025-03-06 10:31:39 -05:00
Code Issues Projects Releases Packages Wiki Activity Actions

Work around Arrow bug by increasing metadata size. (#415)

Browse source
This commit is contained in:
Robert Nishihara 2016-09-07 18:46:04 -07:00 • committed by Philipp Moritz
parent 11a8914684
commit d264713ceb
2 changed files with 3 additions and 3 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
lib/python/ray/worker.py
View file

@ -348,7 +348,7 @@ class Worker(object):
# in the following line, the "8" is for storing the metadata size,
# the len(schema) is for storing the metadata and the 8192 is for storing
# the metadata in the batch (see INITIAL_METADATA_SIZE in arrow)
size = size + 8 + len(schema) + 4096
size = size + 8 + len(schema) + 4096 * 4
buff, segmentid = raylib.allocate_buffer(self.handle, objectid, size)
# write the metadata length
np.frombuffer(buff, dtype="int64", count=1)[0] = len(schema)
@ -371,7 +371,7 @@ class Worker(object):
"""
assert raylib.is_arrow(self.handle, objectid), "All objects should be serialized using Arrow."
buff, segmentid, metadata_offset = raylib.get_buffer(self.handle, objectid)
metadata_size = np.frombuffer(buff, dtype="int64", count=1)[0]
metadata_size = int(np.frombuffer(buff, dtype="int64", count=1)[0])
metadata = np.frombuffer(buff, dtype="byte", offset=8, count=metadata_size)
data = np.frombuffer(buff, dtype="byte")[8 + metadata_size:]
serialized = libnumbuf.read_from_buffer(memoryview(data), bytearray(metadata), metadata_offset)

2
thirdparty/arrow vendored

@ -1 +1 @@
Subproject commit c781ef8564714bb5149e8b5a311673f9162232ac
Subproject commit 4622f4c5fbec31f4d2ce49c90793b37c75a4a838