hiro/ray
1
0
Fork 0
mirror of https://github.com/vale981/ray synced 2025-03-06 18:41:40 -05:00
Code Issues Projects Releases Packages Wiki Activity Actions

Update arrow to use updated pandas serializer. (#1582)

Browse source
This commit is contained in:
Robert Nishihara 2018-02-22 11:10:52 -08:00 • committed by Philipp Moritz
parent 330159d8bd
commit f4b1881fec
2 changed files with 5 additions and 4 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
python/ray/worker.py
View file

@ -1038,12 +1038,10 @@ def _initialize_serialization(worker=global_worker):
This defines a custom serializer for object IDs and also tells ray to This defines a custom serializer for object IDs and also tells ray to
serialize several exception classes that we define for error handling. serialize several exception classes that we define for error handling.
""" """
worker.serialization_context = pyarrow.SerializationContext() worker.serialization_context = pyarrow.default_serialization_context()
# Tell the serialization context to use the cloudpickle version that we # Tell the serialization context to use the cloudpickle version that we
# ship with Ray. # ship with Ray.
worker.serialization_context.set_pickle(pickle.dumps, pickle.loads) worker.serialization_context.set_pickle(pickle.dumps, pickle.loads)
pyarrow.register_default_serialization_handlers(
worker.serialization_context)
pyarrow.register_torch_serialization_handlers(worker.serialization_context) pyarrow.register_torch_serialization_handlers(worker.serialization_context)
# Define a custom serializer and deserializer for handling Object IDs. # Define a custom serializer and deserializer for handling Object IDs.

5
src/thirdparty/download_thirdparty.sh vendored
View file

@ -12,7 +12,10 @@ if [ ! -d $TP_DIR/arrow ]; then
fi fi
pushd $TP_DIR/arrow pushd $TP_DIR/arrow
git fetch origin master git fetch origin master
git checkout e26f3dad3675288564ef0c0330a5c9afcac652f1 # The PR for this commit is https://github.com/apache/arrow/pull/1581. We
# include the link here to make it easier to find the right commit because
# Arrow often rewrites git history and invalidates certain commits.
git checkout 46aa99e9843ac0148357bb36a9235cfd48903e73
popd popd
if [ ! -d $TP_DIR/parquet-cpp ]; then if [ ! -d $TP_DIR/parquet-cpp ]; then