hiro/ray
1
0
Fork 0
mirror of https://github.com/vale981/ray synced 2025-03-06 02:21:39 -05:00
Code Issues Projects Releases Packages Wiki Activity Actions

Fix TensorFlow and PyTorch compatibility (#3574)

Browse source 
* remove tensorflow workaround
* update docker
* add boost threads
* add date_time, too
* change link order
* cosmetics
This commit is contained in:
Philipp Moritz 2018-12-22 13:25:48 -08:00 • committed by Alexey Tumanov
parent deb26b954e
commit e578a38116
7 changed files with 48 additions and 32 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

10
cmake/Modules/ArrowExternalProject.cmake
View file

@ -14,11 +14,15 @@
# - PLASMA_STATIC_LIB # - PLASMA_STATIC_LIB
# - PLASMA_SHARED_LIB # - PLASMA_SHARED_LIB
set(arrow_URL https://github.com/apache/arrow.git) set(arrow_URL https://github.com/ray-project/arrow.git)
# The PR for this commit is https://github.com/apache/arrow/pull/3154. We # This commit is based on https://github.com/apache/arrow/pull/3197. We
# include the link here to make it easier to find the right commit because # include the link here to make it easier to find the right commit because
# Arrow often rewrites git history and invalidates certain commits. # Arrow often rewrites git history and invalidates certain commits.
set(arrow_TAG 8c413036775796d9bcc52be56373bbb45de8c0ae) # It has been patched to fix an upstream symbol clash with TensorFlow,
# the patch is available at
# https://github.com/ray-project/arrow/commit/c347cd571e51723fc8512922f1b3a8e45e45b169
# See the discussion in https://github.com/apache/arrow/pull/3177
set(arrow_TAG c347cd571e51723fc8512922f1b3a8e45e45b169)
set(ARROW_INSTALL_PREFIX ${CMAKE_CURRENT_BINARY_DIR}/external/arrow-install) set(ARROW_INSTALL_PREFIX ${CMAKE_CURRENT_BINARY_DIR}/external/arrow-install)
set(ARROW_HOME ${ARROW_INSTALL_PREFIX}) set(ARROW_HOME ${ARROW_INSTALL_PREFIX})

9
cmake/Modules/BoostExternalProject.cmake
View file

@ -27,14 +27,15 @@ else()
set(BOOST_ROOT ${Boost_INSTALL_PREFIX}) set(BOOST_ROOT ${Boost_INSTALL_PREFIX})
set(Boost_LIBRARY_DIR ${Boost_INSTALL_PREFIX}/lib) set(Boost_LIBRARY_DIR ${Boost_INSTALL_PREFIX}/lib)
set(Boost_SYSTEM_LIBRARY ${Boost_LIBRARY_DIR}/${CMAKE_STATIC_LIBRARY_PREFIX}boost_system${CMAKE_STATIC_LIBRARY_SUFFIX}) set(Boost_SYSTEM_LIBRARY ${Boost_LIBRARY_DIR}/${CMAKE_STATIC_LIBRARY_PREFIX}boost_system${CMAKE_STATIC_LIBRARY_SUFFIX})
set(Boost_THREAD_LIBRARY ${Boost_LIBRARY_DIR}/${CMAKE_STATIC_LIBRARY_PREFIX}boost_thread${CMAKE_STATIC_LIBRARY_SUFFIX})
set(Boost_FILESYSTEM_LIBRARY ${Boost_LIBRARY_DIR}/${CMAKE_STATIC_LIBRARY_PREFIX}boost_filesystem${CMAKE_STATIC_LIBRARY_SUFFIX}) set(Boost_FILESYSTEM_LIBRARY ${Boost_LIBRARY_DIR}/${CMAKE_STATIC_LIBRARY_PREFIX}boost_filesystem${CMAKE_STATIC_LIBRARY_SUFFIX})
#set(boost_URL https://github.com/boostorg/boost.git) #set(boost_URL https://github.com/boostorg/boost.git)
#set(boost_TAG boost-1.65.1) #set(boost_TAG boost-1.68.0)
set(Boost_TAR_GZ_URL http://dl.bintray.com/boostorg/release/1.65.1/source/boost_1_65_1.tar.gz) set(Boost_TAR_GZ_URL http://dl.bintray.com/boostorg/release/1.68.0/source/boost_1_68_0.tar.gz)
set(Boost_BUILD_PRODUCTS ${Boost_SYSTEM_LIBRARY} ${Boost_FILESYSTEM_LIBRARY}) set(Boost_BUILD_PRODUCTS ${Boost_SYSTEM_LIBRARY} ${Boost_FILESYSTEM_LIBRARY})
set(Boost_URL_MD5 "ee64fd29a3fe42232c6ac3c419e523cf") set(Boost_URL_MD5 "5d8b4503582fffa9eefdb9045359c239")
set(Boost_USE_STATIC_LIBS ON) set(Boost_USE_STATIC_LIBS ON)
@ -48,6 +49,6 @@ else()
BUILD_IN_SOURCE 1 BUILD_IN_SOURCE 1
BUILD_BYPRODUCTS ${Boost_BUILD_PRODUCTS} BUILD_BYPRODUCTS ${Boost_BUILD_PRODUCTS}
CONFIGURE_COMMAND ./bootstrap.sh CONFIGURE_COMMAND ./bootstrap.sh
BUILD_COMMAND bash -c "./b2 cxxflags=-fPIC cflags=-fPIC variant=release link=static --with-filesystem --with-system --with-regex -j8 install --prefix=${Boost_INSTALL_PREFIX} > /dev/null" BUILD_COMMAND bash -c "./b2 cxxflags=-fPIC cflags=-fPIC variant=release link=static --with-filesystem --with-system --with-thread --with-atomic --with-chrono --with-date_time --with-regex -j8 install --prefix=${Boost_INSTALL_PREFIX} > /dev/null"
INSTALL_COMMAND "") INSTALL_COMMAND "")
endif () endif ()

6
cmake/Modules/ThirdpartyToolchain.cmake
View file

@ -54,11 +54,14 @@ ADD_THIRDPARTY_LIB(boost_system
STATIC_LIB ${Boost_SYSTEM_LIBRARY}) STATIC_LIB ${Boost_SYSTEM_LIBRARY})
ADD_THIRDPARTY_LIB(boost_filesystem ADD_THIRDPARTY_LIB(boost_filesystem
STATIC_LIB ${Boost_FILESYSTEM_LIBRARY}) STATIC_LIB ${Boost_FILESYSTEM_LIBRARY})
ADD_THIRDPARTY_LIB(boost_thread
STATIC_LIB ${Boost_THREAD_LIBRARY})
add_dependencies(boost_system boost_ep) add_dependencies(boost_system boost_ep)
add_dependencies(boost_filesystem boost_ep) add_dependencies(boost_filesystem boost_ep)
add_dependencies(boost_thread boost_ep)
add_custom_target(boost DEPENDS boost_system boost_filesystem) add_custom_target(boost DEPENDS boost_system boost_filesystem boost_thread)
# flatbuffers # flatbuffers
include(FlatBuffersExternalProject) include(FlatBuffersExternalProject)
@ -120,6 +123,7 @@ if ("${CMAKE_RAY_LANG_PYTHON}" STREQUAL "YES")
"PYARROW_WITH_TENSORFLOW=1" "PYARROW_WITH_TENSORFLOW=1"
"PYARROW_BUNDLE_ARROW_CPP=1" "PYARROW_BUNDLE_ARROW_CPP=1"
"PARQUET_HOME=${PARQUET_HOME}" "PARQUET_HOME=${PARQUET_HOME}"
"BOOST_ROOT=${BOOST_ROOT}"
"PYARROW_WITH_PARQUET=1" "PYARROW_WITH_PARQUET=1"
"PYARROW_PARALLEL=") "PYARROW_PARALLEL=")

21
src/ray/CMakeLists.txt
View file

@ -1,11 +1,6 @@
cmake_minimum_required(VERSION 3.2) cmake_minimum_required(VERSION 3.2)
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -g -Wall -Werror -std=c++11") set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -g -Wall -Werror -std=c++11")
add_subdirectory(util)
add_subdirectory(gcs)
add_subdirectory(object_manager)
add_subdirectory(raylet)
include_directories(thirdparty/ae) include_directories(thirdparty/ae)
set(HIREDIS_SRCS set(HIREDIS_SRCS
@ -56,8 +51,15 @@ set(RAY_SRCS
raylet/raylet.cc raylet/raylet.cc
) )
set(RAY_LIB_STATIC_LINK_LIBS ${PLASMA_STATIC_LIB} ${ARROW_STATIC_LIB}) set(RAY_SYSTEM_LIBS ${Boost_SYSTEM_LIBRARY} ${Boost_THREAD_LIBRARY} pthread)
if(UNIX AND NOT APPLE)
set(RAY_SYSTEM_LIBS ${RAY_SYSTEM_LIBS} -lrt)
endif()
set(RAY_LIB_STATIC_LINK_LIBS ${PLASMA_STATIC_LIB} ${ARROW_STATIC_LIB} ${RAY_SYSTEM_LIBS})
set(RAY_LIB_DEPENDENCIES set(RAY_LIB_DEPENDENCIES
boost_thread
arrow_ep arrow_ep
gen_gcs_fbs gen_gcs_fbs
gen_object_manager_fbs gen_object_manager_fbs
@ -89,6 +91,13 @@ ADD_RAY_LIB(ray
SHARED_LINK_LIBS "" SHARED_LINK_LIBS ""
STATIC_LINK_LIBS ${RAY_LIB_STATIC_LINK_LIBS}) STATIC_LINK_LIBS ${RAY_LIB_STATIC_LINK_LIBS})
set(RAY_TEST_LIBS ray_static ${PLASMA_STATIC_LIB} ${ARROW_STATIC_LIB} gtest gtest_main gmock_main ${RAY_SYSTEM_LIBS})
add_subdirectory(util)
add_subdirectory(gcs)
add_subdirectory(object_manager)
add_subdirectory(raylet)
add_custom_target(copy_redis ALL) add_custom_target(copy_redis ALL)
foreach(file "redis-cli" "redis-server") foreach(file "redis-cli" "redis-server")
add_custom_command(TARGET copy_redis POST_BUILD add_custom_command(TARGET copy_redis POST_BUILD

6
src/ray/object_manager/CMakeLists.txt
View file

@ -17,11 +17,11 @@ add_custom_command(
add_custom_target(gen_object_manager_fbs DEPENDS ${OBJECT_MANAGER_FBS_OUTPUT_FILES}) add_custom_target(gen_object_manager_fbs DEPENDS ${OBJECT_MANAGER_FBS_OUTPUT_FILES})
ADD_RAY_TEST(test/object_manager_test STATIC_LINK_LIBS ray_static ${PLASMA_STATIC_LIB} ${ARROW_STATIC_LIB} gtest gtest_main pthread ${Boost_SYSTEM_LIBRARY}) ADD_RAY_TEST(test/object_manager_test STATIC_LINK_LIBS ${RAY_TEST_LIBS})
ADD_RAY_TEST(test/object_manager_stress_test STATIC_LINK_LIBS ray_static ${PLASMA_STATIC_LIB} ${ARROW_STATIC_LIB} gtest gtest_main pthread ${Boost_SYSTEM_LIBRARY}) ADD_RAY_TEST(test/object_manager_stress_test STATIC_LINK_LIBS ${RAY_TEST_LIBS})
add_library(object_manager object_manager.cc object_manager.h ${OBJECT_MANAGER_FBS_OUTPUT_FILES}) add_library(object_manager object_manager.cc object_manager.h ${OBJECT_MANAGER_FBS_OUTPUT_FILES})
target_link_libraries(object_manager common ray_static ${PLASMA_STATIC_LIB} ${ARROW_STATIC_LIB} ${Boost_SYSTEM_LIBRARY}) target_link_libraries(object_manager common ray_static ${PLASMA_STATIC_LIB} ${ARROW_STATIC_LIB} ${Boost_SYSTEM_LIBRARY} ${Boost_THREAD_LIBRARY})
install(FILES install(FILES
object_manager object_manager

26
src/ray/raylet/CMakeLists.txt
View file

@ -30,19 +30,17 @@ add_custom_command(
add_dependencies(gen_node_manager_fbs flatbuffers_ep) add_dependencies(gen_node_manager_fbs flatbuffers_ep)
ADD_RAY_TEST(object_manager_integration_test STATIC_LINK_LIBS ray_static ${PLASMA_STATIC_LIB} ${ARROW_STATIC_LIB} gtest gtest_main pthread ${Boost_SYSTEM_LIBRARY}) ADD_RAY_TEST(object_manager_integration_test STATIC_LINK_LIBS ${RAY_TEST_LIBS})
ADD_RAY_TEST(worker_pool_test STATIC_LINK_LIBS ${RAY_TEST_LIBS})
ADD_RAY_TEST(worker_pool_test STATIC_LINK_LIBS ray_static ${PLASMA_STATIC_LIB} ${ARROW_STATIC_LIB} gtest gtest_main gmock_main pthread ${Boost_SYSTEM_LIBRARY}) ADD_RAY_TEST(client_connection_test STATIC_LINK_LIBS ${RAY_TEST_LIBS})
ADD_RAY_TEST(task_test STATIC_LINK_LIBS ${RAY_TEST_LIBS})
ADD_RAY_TEST(client_connection_test STATIC_LINK_LIBS ray_static gtest gtest_main gmock_main pthread ${Boost_SYSTEM_LIBRARY}) ADD_RAY_TEST(lineage_cache_test STATIC_LINK_LIBS ${RAY_TEST_LIBS})
ADD_RAY_TEST(task_test STATIC_LINK_LIBS ray_static gtest gtest_main gmock_main pthread ${Boost_SYSTEM_LIBRARY}) ADD_RAY_TEST(task_dependency_manager_test STATIC_LINK_LIBS ${RAY_TEST_LIBS})
ADD_RAY_TEST(lineage_cache_test STATIC_LINK_LIBS ray_static gtest gtest_main gmock_main pthread ${Boost_SYSTEM_LIBRARY}) ADD_RAY_TEST(reconstruction_policy_test STATIC_LINK_LIBS ${RAY_TEST_LIBS})
ADD_RAY_TEST(task_dependency_manager_test STATIC_LINK_LIBS ray_static gtest gtest_main gmock_main pthread ${Boost_SYSTEM_LIBRARY})
ADD_RAY_TEST(reconstruction_policy_test STATIC_LINK_LIBS ray_static gtest gtest_main gmock_main pthread ${Boost_SYSTEM_LIBRARY})
include_directories(${GCS_FBS_OUTPUT_DIRECTORY}) include_directories(${GCS_FBS_OUTPUT_DIRECTORY})
add_library(rayletlib raylet.cc ${NODE_MANAGER_FBS_OUTPUT_FILES}) add_library(rayletlib raylet.cc ${NODE_MANAGER_FBS_OUTPUT_FILES})
target_link_libraries(rayletlib ray_static ${Boost_SYSTEM_LIBRARY}) target_link_libraries(rayletlib ray_static ${Boost_SYSTEM_LIBRARY} ${Boost_THREAD_LIBRARY})
add_library(raylet_client STATIC raylet_client.cc) add_library(raylet_client STATIC raylet_client.cc)
@ -52,10 +50,10 @@ add_dependencies(rayletlib gen_gcs_fbs)
add_dependencies(raylet_client gen_gcs_fbs arrow_ep gen_node_manager_fbs) add_dependencies(raylet_client gen_gcs_fbs arrow_ep gen_node_manager_fbs)
add_executable(raylet main.cc) add_executable(raylet main.cc)
target_link_libraries(raylet rayletlib ${Boost_SYSTEM_LIBRARY} pthread) target_link_libraries(raylet rayletlib ${RAY_SYSTEM_LIBS})
add_executable(raylet_monitor monitor_main.cc) add_executable(raylet_monitor monitor_main.cc)
target_link_libraries(raylet_monitor rayletlib ${Boost_SYSTEM_LIBRARY} pthread) target_link_libraries(raylet_monitor rayletlib ${Boost_SYSTEM_LIBRARY} ${Boost_THREAD_LIBRARY} pthread)
install(FILES install(FILES
raylet raylet
@ -78,9 +76,9 @@ macro(set_raylet_library LANG)
if ("${LANG}" STREQUAL "python") if ("${LANG}" STREQUAL "python")
SET_TARGET_PROPERTIES(${RAYLET_LIBRARY_LANG} PROPERTIES SUFFIX .so) SET_TARGET_PROPERTIES(${RAYLET_LIBRARY_LANG} PROPERTIES SUFFIX .so)
endif() endif()
target_link_libraries(${RAYLET_LIBRARY_LANG} "-undefined dynamic_lookup" raylet_client ray_static ${PLASMA_STATIC_LIB} ${ARROW_STATIC_LIB} ${Boost_SYSTEM_LIBRARY}) target_link_libraries(${RAYLET_LIBRARY_LANG} "-undefined dynamic_lookup" raylet_client ray_static ${PLASMA_STATIC_LIB} ${ARROW_STATIC_LIB} ${Boost_SYSTEM_LIBRARY} ${Boost_THREAD_LIBRARY})
else(APPLE) else(APPLE)
target_link_libraries(${RAYLET_LIBRARY_LANG} raylet_client ray_static ${PLASMA_STATIC_LIB} ${ARROW_STATIC_LIB} ${Boost_SYSTEM_LIBRARY}) target_link_libraries(${RAYLET_LIBRARY_LANG} raylet_client ray_static ${PLASMA_STATIC_LIB} ${ARROW_STATIC_LIB} ${Boost_SYSTEM_LIBRARY} ${Boost_THREAD_LIBRARY})
endif(APPLE) endif(APPLE)
add_dependencies(${RAYLET_LIBRARY_LANG} gen_node_manager_fbs) add_dependencies(${RAYLET_LIBRARY_LANG} gen_node_manager_fbs)

2
thirdparty/scripts/build_boost.sh vendored
View file

@ -28,6 +28,6 @@ if [[ ! -d $TP_DIR/pkg/boost ]]; then
# Compile boost. # Compile boost.
pushd $TP_DIR/build/boost_$BOOST_VERSION_UNDERSCORE pushd $TP_DIR/build/boost_$BOOST_VERSION_UNDERSCORE
./bootstrap.sh ./bootstrap.sh
./bjam cxxflags=-fPIC cflags=-fPIC variant=release link=static --prefix=$TP_DIR/pkg/boost --with-filesystem --with-system --with-regex install > /dev/null ./bjam cxxflags=-fPIC cflags=-fPIC variant=release link=static --prefix=$TP_DIR/pkg/boost --with-filesystem --with-system --with-thread --with-regex install > /dev/null
popd popd
fi fi