Update arrow to use updated pandas serializer. (#1582)

python/ray/worker.py

@@ -1038,12 +1038,10 @@ def _initialize_serialization(worker=global_worker):
     This defines a custom serializer for object IDs and also tells ray to
     serialize several exception classes that we define for error handling.
     """
-    worker.serialization_context = pyarrow.SerializationContext()
+    worker.serialization_context = pyarrow.default_serialization_context()
     # Tell the serialization context to use the cloudpickle version that we
     # ship with Ray.
     worker.serialization_context.set_pickle(pickle.dumps, pickle.loads)
-    pyarrow.register_default_serialization_handlers(
-        worker.serialization_context)
     pyarrow.register_torch_serialization_handlers(worker.serialization_context)
 
     # Define a custom serializer and deserializer for handling Object IDs.

src/thirdparty/download_thirdparty.sh

@@ -12,7 +12,10 @@ if [ ! -d $TP_DIR/arrow ]; then
 fi
 pushd $TP_DIR/arrow
   git fetch origin master
-  git checkout e26f3dad3675288564ef0c0330a5c9afcac652f1
+  # The PR for this commit is https://github.com/apache/arrow/pull/1581. We
+  # include the link here to make it easier to find the right commit because
+  # Arrow often rewrites git history and invalidates certain commits.
+  git checkout 46aa99e9843ac0148357bb36a9235cfd48903e73
 popd
 
 if [ ! -d $TP_DIR/parquet-cpp ]; then