Support building Java and Python version at the same time. (#2640)

* Support building Java and Python version at the same time.

* Remove duplicated definition.

* Refine the building process of local_scheduler

* Refine

* Add comment for languages

* Modify instruction and add python,jave building to CI.

* change according to comment

CMakeLists.txt

@@ -2,19 +2,6 @@ cmake_minimum_required(VERSION 3.4)
 
 project(ray)
 
-set(CMAKE_RAY_LANG_PYTHON "NO")
-set(CMAKE_RAY_LANG_JAVA "NO")
-if ("${CMAKE_RAY_LANGUAGE}" STREQUAL "python")
-  set(CMAKE_RAY_LANG_PYTHON "YES")
-elseif ("${CMAKE_RAY_LANGUAGE}" STREQUAL "java")
-  set(CMAKE_RAY_LANG_JAVA "YES")
-elseif ("${CMAKE_RAY_LANGUAGE}" STREQUAL "")
-  message(WARNING "Language is not set, choose Python as default.")
-  set(CMAKE_RAY_LANG_PYTHON "YES")
-else()
-  message(FATAL_ERROR "Unrecognized language, use -DCMAKE_RAY_LANGUAGE=java|python. Abort.")
-endif()
-
 set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${CMAKE_SOURCE_DIR}/cmake/Modules")
 
 include(${CMAKE_CURRENT_SOURCE_DIR}/thirdparty/scripts/thirdparty.cmake)
@@ -116,12 +103,12 @@ if ("${CMAKE_RAY_LANG_PYTHON}" STREQUAL "YES")
 
   # Make sure that the Python extensions are built before copying the files.
   if ("${CMAKE_RAY_LANG_PYTHON}" STREQUAL "YES")
-    get_local_scheduler_library("python" LOCAL_SCHEDULER_LIBRARY_LANG)
-    add_dependencies(copy_ray ${LOCAL_SCHEDULER_LIBRARY_LANG})
+    get_local_scheduler_library("python" LOCAL_SCHEDULER_LIBRARY_PYTHON)
+    add_dependencies(copy_ray ${LOCAL_SCHEDULER_LIBRARY_PYTHON})
   endif()
   if ("${CMAKE_RAY_LANG_JAVA}" STREQUAL "YES")
-    get_local_scheduler_library("java" LOCAL_SCHEDULER_LIBRARY_LANG)
-    add_dependencies(copy_ray ${LOCAL_SCHEDULER_LIBRARY_LANG})
+    get_local_scheduler_library("java" LOCAL_SCHEDULER_LIBRARY_JAVA)
+    add_dependencies(copy_ray ${LOCAL_SCHEDULER_LIBRARY_JAVA})
   endif()
 
   add_dependencies(copy_ray ray_redis_module)

build.sh

@@ -14,9 +14,10 @@ function usage()
   echo "Options:"
   echo "  -h|--help               print the help info"
   echo "  -d|--debug              CMAKE_BUILD_TYPE=Debug (default is RelWithDebInfo)"
-  echo "  -l|--language python(default) "
-  echo "                          build native library for python"
-  echo "                java      build native library for java"
+  echo "  -l|--language language1[,language2]"
+  echo "                          a list of languages to build native libraries."
+  echo "                          Supported languages include \"python\" and \"java\"."
+  echo "                          If not specified, only python library will be built."
   echo "  -p|--python             which python executable (default from which python)"
   echo
 }
@@ -32,7 +33,8 @@ else
   exit 1
 fi
 
-LANGUAGE="python"
+RAY_BUILD_PYTHON="YES"
+RAY_BUILD_JAVA="NO"
 PYTHON_EXECUTABLE=""
 BUILD_DIR=""
 if [ "$VALGRIND" = "1" ]; then
@@ -54,8 +56,16 @@ while [[ $# > 0 ]]; do
       ;;
     -l|--languags)
       LANGUAGE="$2"
-      if [ "$LANGUAGE" != "python" ] && [ "$LANGUAGE" != "java" ]; then
-        echo "Unrecognized language."
+      RAY_BUILD_PYTHON="NO"
+      RAY_BUILD_JAVA="NO"
+      if [[ "$LANGUAGE" == *"python"* ]]; then
+        RAY_BUILD_PYTHON="YES"
+      fi
+      if [[ "$LANGUAGE" == *"java"* ]]; then
+        RAY_BUILD_JAVA="YES"
+      fi
+      if [ "$RAY_BUILD_PYTHON" == "NO" ] && [ "$RAY_BUILD_JAVA" == "NO" ]; then
+        echo "Unrecognized language: $LANGUAGE"
         exit -1
       fi
       shift
@@ -79,7 +89,9 @@ if [[ -z  "$PYTHON_EXECUTABLE" ]]; then
 fi
 echo "Using Python executable $PYTHON_EXECUTABLE."
 
-bash $ROOT_DIR/setup_thirdparty.sh $PYTHON_EXECUTABLE $LANGUAGE
+RAY_BUILD_PYTHON=$RAY_BUILD_PYTHON \
+RAY_BUILD_JAVA=$RAY_BUILD_JAVA \
+bash $ROOT_DIR/setup_thirdparty.sh $PYTHON_EXECUTABLE
 
 # Now we build everything.
 BUILD_DIR="$ROOT_DIR/build/"
@@ -95,7 +107,8 @@ ARROW_HOME=$TP_PKG_DIR/arrow/cpp/build/cpp-install
 BOOST_ROOT=$TP_PKG_DIR/boost \
 PKG_CONFIG_PATH=$ARROW_HOME/lib/pkgconfig \
 cmake -DCMAKE_BUILD_TYPE=$CBUILD_TYPE \
-      -DCMAKE_RAY_LANGUAGE=$LANGUAGE \
+      -DCMAKE_RAY_LANG_JAVA=$RAY_BUILD_JAVA \
+      -DCMAKE_RAY_LANG_PYTHON=$RAY_BUILD_PYTHON \
       -DRAY_USE_NEW_GCS=$RAY_USE_NEW_GCS \
       -DPYTHON_EXECUTABLE:FILEPATH=$PYTHON_EXECUTABLE $ROOT_DIR
 
@@ -105,9 +118,9 @@ popd
 
 # Move stuff from Arrow to Ray.
 cp $ROOT_DIR/thirdparty/pkg/arrow/cpp/build/cpp-install/bin/plasma_store $BUILD_DIR/src/plasma/
-if [[ "$LANGUAGE" == "python" ]]; then
+if [[ "$RAY_BUILD_PYTHON" == "YES" ]]; then
   cp $ROOT_DIR/thirdparty/pkg/arrow/cpp/build/cpp-install/bin/plasma_store $BUILD_DIR/../python/ray/core/src/plasma/
 fi
-if [[ "$LANGUAGE" == "java" ]]; then
+if [[ "$RAY_BUILD_JAVA" == "YES" ]]; then
   cp $ROOT_DIR/thirdparty/build/arrow/cpp/build/release/libplasma_java.* $BUILD_DIR/src/plasma/
 fi

java/test.sh

@@ -7,7 +7,7 @@ set -e
 set -x
 
 ROOT_DIR=$(cd "$(dirname "${BASH_SOURCE:-$0}")"; pwd)
-$ROOT_DIR/../build.sh -l java
+$ROOT_DIR/../build.sh -l java,python
 
 pushd $ROOT_DIR/../thirdparty/build/arrow/java
 mvn clean install -pl plasma -am -Dmaven.test.skip

setup_thirdparty.sh

@@ -13,14 +13,11 @@ else
 fi
 echo "Using Python executable $PYTHON_EXECUTABLE."
 
-LANGUAGE="python"
-if [[ -n  "$2" ]]; then
-  LANGUAGE=$2
-fi
+RAY_BUILD_PYTHON=$RAY_BUILD_PYTHON \
+RAY_BUILD_JAVA=$RAY_BUILD_JAVA \
+$ROOT_DIR/thirdparty/scripts/setup.sh $PYTHON_EXECUTABLE
 
-$ROOT_DIR/thirdparty/scripts/setup.sh $PYTHON_EXECUTABLE $LANGUAGE
-
-if [[ "$LANGUAGE" == "java" ]]; then
+if [[ "$RAY_BUILD_JAVA" == "YES" ]]; then
     pushd $ROOT_DIR/thirdparty/build/arrow/java
     mvn clean install -pl plasma -am -Dmaven.test.skip
     popd

src/local_scheduler/CMakeLists.txt

@@ -17,9 +17,6 @@ endif()
 add_definitions(-fPIC)
 
 if ("${CMAKE_RAY_LANG_PYTHON}" STREQUAL "YES")
-  if(APPLE)
-    SET(CMAKE_SHARED_LIBRARY_SUFFIX ".so")
-  endif(APPLE)
   include_directories("${PYTHON_INCLUDE_DIRS}")
   include_directories("${NUMPY_INCLUDE_DIR}")
 endif()
@@ -76,7 +73,8 @@ macro(get_local_scheduler_library LANG VAR)
 endmacro()
 
 macro(set_local_scheduler_library LANG)
-  get_local_scheduler_library(${LANG} LOCAL_SCHEDULER_LIBRARY_LANG)
+  get_local_scheduler_library(${LANG} LOCAL_SCHEDULER_LIBRARY_${LANG})
+  set(LOCAL_SCHEDULER_LIBRARY_LANG ${LOCAL_SCHEDULER_LIBRARY_${LANG}})
   include_directories("${CMAKE_CURRENT_LIST_DIR}/../common/lib/${LANG}/")
 
   file(GLOB LOCAL_SCHEDULER_LIBRARY_${LANG}_SRC
@@ -86,6 +84,9 @@ macro(set_local_scheduler_library LANG)
     ${LOCAL_SCHEDULER_LIBRARY_${LANG}_SRC})
 
     if(APPLE)
+      if ("${LANG}" STREQUAL "python")
+        SET_TARGET_PROPERTIES(${LOCAL_SCHEDULER_LIBRARY_LANG} PROPERTIES SUFFIX .so)
+      endif()
       target_link_libraries(${LOCAL_SCHEDULER_LIBRARY_LANG} "-undefined dynamic_lookup" local_scheduler_client common ray_static ${PLASMA_STATIC_LIB} ${ARROW_STATIC_LIB} ${Boost_SYSTEM_LIBRARY})
     else(APPLE)
       target_link_libraries(${LOCAL_SCHEDULER_LIBRARY_LANG} local_scheduler_client common ray_static ${PLASMA_STATIC_LIB} ${ARROW_STATIC_LIB} ${Boost_SYSTEM_LIBRARY})

thirdparty/scripts/build_arrow.sh

@@ -15,12 +15,6 @@ else
 fi
 echo "Using Python executable $PYTHON_EXECUTABLE."
 
-LANGUAGE="python"
-if [[ -n  "$2" ]]; then
-  LANGUAGE=$2
-fi
-echo "Build language is $LANGUAGE."
-
 unamestr="$(uname)"
 
 if [[ "$unamestr" == "Linux" ]]; then
@@ -46,9 +40,8 @@ fi
 TARGET_COMMIT_ID=d48dce2cfebdbd044a8260d0a77f5fe3d89a4a2d
 build_arrow() {
   echo "building arrow"
-
   # Make sure arrow will be built again when building ray for java later than python
-  if [[ "$LANGUAGE" == "java" ]]; then
+  if [[ "$RAY_BUILD_JAVA" == "YES" ]]; then
     rm -rf $TP_DIR/build/arrow/cpp/build/CMakeCache.txt
   fi
 
@@ -78,7 +71,7 @@ build_arrow() {
   cd build
 
   BUILD_ARROW_PLASMA_JAVA_CLIENT=off
-  if [[ "$LANGUAGE" == "java" ]]; then
+  if [[ "$RAY_BUILD_JAVA" == "YES" ]]; then
     BUILD_ARROW_PLASMA_JAVA_CLIENT=on
   fi
 
@@ -147,8 +140,8 @@ build_arrow() {
   popd
 }
 # Download and compile arrow if it isn't already present or the commit-id mismatches.
-if [[ ! -d $TP_DIR/../python/ray/pyarrow_files/pyarrow ]] || \
-    [[ "$LANGUAGE" == "java" && ! -f $TP_DIR/build/arrow/cpp/build/release/libplasma_java.dylib ]]; then
+if [[ "$RAY_BUILD_PYTHON" == "YES" && ! -d $TP_DIR/../python/ray/pyarrow_files/pyarrow ]] || \
+    [[ "$RAY_BUILD_JAVA" == "YES" && ! -f $TP_DIR/build/arrow/cpp/build/release/libplasma_java.dylib ]]; then
   build_arrow
 else
   REBUILD=off

thirdparty/scripts/setup.sh

@@ -18,11 +18,12 @@ else
 fi
 echo "Using Python executable $PYTHON_EXECUTABLE."
 
-LANGUAGE="python"
-if [[ -n  "$2" ]]; then
-  LANGUAGE=$2
+if [[ "$RAY_BUILD_JAVA" == "YES" ]]; then
+echo "Java library will be built."
+fi
+if [[ "$RAY_BUILD_PYTHON" == "YES" ]]; then
+echo "Python library will be built."
 fi
-echo "Build language is $LANGUAGE."
 
 unamestr="$(uname)"
 
@@ -52,7 +53,9 @@ fi
 ##############################################
 # arrow
 ##############################################
-bash "$TP_SCRIPT_DIR/build_arrow.sh" $PYTHON_EXECUTABLE $LANGUAGE
+RAY_BUILD_PYTHON=$RAY_BUILD_PYTHON \
+RAY_BUILD_JAVA=$RAY_BUILD_JAVA \
+bash "$TP_SCRIPT_DIR/build_arrow.sh" $PYTHON_EXECUTABLE
 
 ##############################################
 # parquet (skipped as it is inlined in build_arrow.sh)