Upgrade arrow to include the plasma TensorFlow op (#2412)

CMakeLists.txt

@@ -25,6 +25,9 @@ find_package(Plasma)
 # This ensures that things like gnu++11 get passed correctly
 set(CMAKE_CXX_STANDARD 11)
 
+# Use old C++ ABI to be compatible with TensorFlow
+set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -D_GLIBCXX_USE_CXX11_ABI=0")
+
 # We require a C++11 compliant compiler
 set(CMAKE_CXX_STANDARD_REQUIRED ON)
 

cmake/Modules/ThirdpartyToolchain.cmake

@@ -2,6 +2,10 @@ set(GFLAGS_VERSION "2.2.0")
 set(GTEST_VERSION "1.8.0")
 set(GBENCHMARK_VERSION "1.1.0")
 
+# Because we use the old C++ ABI to be compatible with TensorFlow,
+# we have to turn it on for dependencies too
+set(EP_CXX_FLAGS "${EP_CXX_FLAGS} -D_GLIBCXX_USE_CXX11_ABI=0")
+
 if(RAY_BUILD_TESTS OR RAY_BUILD_BENCHMARKS)
   add_custom_target(unittest ctest -L unittest)
 

src/ray/CMakeLists.txt

@@ -1,5 +1,5 @@
 cmake_minimum_required(VERSION 3.2)
-set(CMAKE_CXX_FLAGS "-g -Wall -Werror -std=c++11")
+set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -g -Wall -Werror -std=c++11")
 
 include_directories(${CMAKE_CURRENT_LIST_DIR}/../common/thirdparty/ae)
 

thirdparty/scripts/build_arrow.sh

@@ -40,10 +40,10 @@ else
   exit 1
 fi
 
-# The PR for this commit is https://github.com/apache/arrow/pull/2235. We
+# The PR for this commit is https://github.com/apache/arrow/pull/2104. We
 # include the link here to make it easier to find the right commit because
 # Arrow often rewrites git history and invalidates certain commits.
-TARGET_COMMIT_ID=fa08ddfbe8ada173b2c621172e176f159ce3b728
+TARGET_COMMIT_ID=cecbcf7de00d2bf255bdba97cee1d37130e5bb79
 build_arrow() {
   echo "building arrow"
 
@@ -98,7 +98,7 @@ build_arrow() {
       -DPYTHON_EXECUTABLE:FILEPATH=$PYTHON_EXECUTABLE \
       -DARROW_PYTHON=on \
       -DARROW_PLASMA=on \
-      -DPLASMA_PYTHON=on \
+      -DARROW_TENSORFLOW=on \
       -DARROW_JEMALLOC=off \
       -DARROW_WITH_BROTLI=off \
       -DARROW_WITH_LZ4=off \
@@ -124,11 +124,13 @@ build_arrow() {
   # find plasma.
   PKG_CONFIG_PATH=$ARROW_HOME/lib/pkgconfig \
   PYARROW_WITH_PLASMA=1 \
+  PYARROW_WITH_TENSORFLOW=1 \
   PYARROW_BUNDLE_ARROW_CPP=1 \
   $PYTHON_EXECUTABLE setup.py build
 
   PKG_CONFIG_PATH=$ARROW_HOME/lib/pkgconfig \
   PYARROW_WITH_PLASMA=1 \
+  PYARROW_WITH_TENSORFLOW=1 \
   PYARROW_BUNDLE_ARROW_CPP=1 \
   PARQUET_HOME=$TP_DIR/pkg/arrow/cpp/build/cpp-install \
   PYARROW_WITH_PARQUET=1 \