From f4b1881fece4c94c84f126ee2b1fe882fcd42281 Mon Sep 17 00:00:00 2001 From: Robert Nishihara Date: Thu, 22 Feb 2018 11:10:52 -0800 Subject: [PATCH] Update arrow to use updated pandas serializer. (#1582) --- python/ray/worker.py | 4 +--- src/thirdparty/download_thirdparty.sh | 5 ++++- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/python/ray/worker.py b/python/ray/worker.py index cd68e3db8..49c9db09f 100644 --- a/python/ray/worker.py +++ b/python/ray/worker.py @@ -1038,12 +1038,10 @@ def _initialize_serialization(worker=global_worker): This defines a custom serializer for object IDs and also tells ray to serialize several exception classes that we define for error handling. """ - worker.serialization_context = pyarrow.SerializationContext() + worker.serialization_context = pyarrow.default_serialization_context() # Tell the serialization context to use the cloudpickle version that we # ship with Ray. worker.serialization_context.set_pickle(pickle.dumps, pickle.loads) - pyarrow.register_default_serialization_handlers( - worker.serialization_context) pyarrow.register_torch_serialization_handlers(worker.serialization_context) # Define a custom serializer and deserializer for handling Object IDs. diff --git a/src/thirdparty/download_thirdparty.sh b/src/thirdparty/download_thirdparty.sh index 5031f7147..96c41e93e 100755 --- a/src/thirdparty/download_thirdparty.sh +++ b/src/thirdparty/download_thirdparty.sh @@ -12,7 +12,10 @@ if [ ! -d $TP_DIR/arrow ]; then fi pushd $TP_DIR/arrow git fetch origin master - git checkout e26f3dad3675288564ef0c0330a5c9afcac652f1 + # The PR for this commit is https://github.com/apache/arrow/pull/1581. We + # include the link here to make it easier to find the right commit because + # Arrow often rewrites git history and invalidates certain commits. + git checkout 46aa99e9843ac0148357bb36a9235cfd48903e73 popd if [ ! -d $TP_DIR/parquet-cpp ]; then