Package ray java jars into wheels (#6600)

.gitignore

@@ -3,6 +3,7 @@
 /python/ray/core
 /python/ray/pyarrow_files/
 /python/ray/pickle5_files/
+/python/ray/jars/
 /python/build
 /python/dist
 /python/python-driver-*

.travis.yml

@@ -98,7 +98,7 @@ matrix:
 
     # Build Linux wheels.
     - os: linux
-      env: LINUX_WHEELS=1 PYTHONWARNINGS=ignore
+      env: LINUX_WHEELS=1 PYTHONWARNINGS=ignore RAY_INSTALL_JAVA=1
       install:
         - eval `python $TRAVIS_BUILD_DIR/ci/travis/determine_tests_to_run.py`
         - if [ $RAY_CI_LINUX_WHEELS_AFFECTED != "1" ]; then exit; fi
@@ -124,7 +124,7 @@ matrix:
     # Build MacOS wheels.
     - os: osx
       osx_image: xcode7
-      env: MAC_WHEELS=1 PYTHONWARNINGS=ignore
+      env: MAC_WHEELS=1 PYTHONWARNINGS=ignore RAY_INSTALL_JAVA=1
       install:
         - eval `python $TRAVIS_BUILD_DIR/ci/travis/determine_tests_to_run.py`
         - if [ $RAY_CI_MACOS_WHEELS_AFFECTED != "1" ]; then exit; fi

build.sh

@@ -119,7 +119,7 @@ export PYTHON3_BIN_PATH="$PYTHON_EXECUTABLE"
 export PYTHON2_BIN_PATH="$PYTHON_EXECUTABLE"
 
 if [ "$RAY_BUILD_JAVA" == "YES" ]; then
-  "$BAZEL_EXECUTABLE" build //java:all --verbose_failures
+  "$BAZEL_EXECUTABLE" build //java:ray_java_pkg --verbose_failures
 fi
 
 if [ "$RAY_BUILD_PYTHON" == "YES" ]; then

java/BUILD.bazel

@@ -207,3 +207,30 @@ genrule(
     local = 1,
     tags = ["no-cache"],
 )
+
+java_binary(
+    name = "ray_dist",
+    # This rule is used to package all Ray Java code and the third-party dependencies into a
+    # fat jar file. It's not really an executable jar. So we set its `main_class` to empty.
+    main_class = "",
+    runtime_deps = [
+        "//java:org_ray_ray_api",
+        "//java:org_ray_ray_runtime",
+        "//streaming/java:org_ray_ray_streaming-api",
+        "//streaming/java:org_ray_ray_streaming-runtime",
+    ]
+)
+
+genrule(
+    name = "ray_java_pkg",
+    srcs = ["//java:ray_dist_deploy.jar"],
+    outs = ["ray_java_pkg.out"],
+    cmd = """
+        WORK_DIR=$$(pwd)
+        rm -rf $$WORK_DIR/python/ray/jars && mkdir -p $$WORK_DIR/python/ray/jars
+        cp -f $(location //java:ray_dist_deploy.jar) $$WORK_DIR/python/ray/jars/ray_dist.jar
+        echo $$(date) > $@
+    """,
+    local = 1,
+    tags = ["no-cache"],
+)

java/test.sh

@@ -21,6 +21,12 @@ echo "Linting Java code with checkstyle."
 # Thus, we add the `build_tests_only` option to avoid re-building everything.
 bazel test //java:all --test_tag_filters="checkstyle" --build_tests_only
 
+echo "Build java maven deps."
+bazel build //java:gen_maven_deps
+
+echo "Build test jar."
+bazel build //java:all_tests_deploy.jar
+
 echo "Running tests under cluster mode."
 # TODO(hchen): Ideally, we should use the following bazel command to run Java tests. However, if there're skipped tests,
 # TestNG will exit with code 2. And bazel treats it as test failure.

java/test/src/main/java/org/ray/api/test/BaseMultiLanguageTest.java

@@ -6,6 +6,8 @@ import java.util.List;
 import java.util.Map;
 import java.util.Map.Entry;
 import java.util.concurrent.TimeUnit;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
 
 import com.google.common.base.Strings;
 import com.google.common.collect.ImmutableList;
@@ -71,9 +73,15 @@ public abstract class BaseMultiLanguageTest {
 
     String nodeManagerPort = String.valueOf(NetworkUtil.getUnusedPort());
 
-    // Start ray cluster.
+    // jars in the `ray` wheel doesn't contains test classes, so we add test classes explicitly.
+    // Since mvn test classes contains `test` in path and bazel test classes is located at a jar
+    // with `test` included in the name, we can check classpath `test` to filter out test classes.
+    String classpath = Stream.of(System.getProperty("java.class.path").split(":"))
+        .filter(s -> !s.contains(" ") && s.contains("test"))
+        .collect(Collectors.joining(":"));
     String workerOptions =
-        " -classpath " + System.getProperty("java.class.path");
+        " -classpath " + classpath;
+    // Start ray cluster.
     List<String> startCommand = ImmutableList.of(
         "ray",
         "start",

python/ray/services.py

@@ -4,6 +4,7 @@ import logging
 import multiprocessing
 import os
 import random
+import re
 import resource
 import socket
 import subprocess
@@ -1248,6 +1249,18 @@ def start_raylet(redis_address,
     return process_info
 
 
+def get_ray_jars_dir():
+    """Return a directory where all ray-related jars and
+      their dependencies locate."""
+    current_dir = os.path.abspath(os.path.dirname(__file__))
+    jars_dir = os.path.abspath(os.path.join(current_dir, "jars"))
+    if not os.path.exists(jars_dir):
+        raise Exception("Ray jars is not packaged into ray. "
+                        "Please build ray with java enabled "
+                        "(set env var RAY_INSTALL_JAVA=1)")
+    return os.path.abspath(os.path.join(current_dir, "jars"))
+
+
 def build_java_worker_command(
         java_worker_options,
         redis_address,
@@ -1270,8 +1283,6 @@ def build_java_worker_command(
     Returns:
         The command string for starting Java worker.
     """
-    assert java_worker_options is not None
-
     command = "java "
 
     if redis_address is not None:
@@ -1294,10 +1305,29 @@ def build_java_worker_command(
     command += ("-Dray.raylet.config.num_workers_per_process_java=" +
                 "RAY_WORKER_NUM_WORKERS_PLACEHOLDER ")
 
-    if java_worker_options:
-        # Put `java_worker_options` in the last, so it can overwrite the
-        # above options.
-        command += java_worker_options + " "
+    # Add ray jars path to java classpath
+    ray_jars = os.path.join(get_ray_jars_dir(), "*")
+    cp_sep = ":"
+    import platform
+    if platform.system() == "Windows":
+        cp_sep = ";"
+    if java_worker_options is None:
+        java_worker_options = ""
+    options = re.split("\\s+", java_worker_options)
+    cp_index = -1
+    for i in range(len(options)):
+        option = options[i]
+        if option == "-cp" or option == "-classpath":
+            cp_index = i + 1
+            break
+    if cp_index != -1:
+        options[cp_index] = options[cp_index] + cp_sep + ray_jars
+    else:
+        options = ["-cp", ray_jars] + options
+    java_worker_options = " ".join(options)
+    # Put `java_worker_options` in the last, so it can overwrite the
+    # above options.
+    command += java_worker_options + " "
 
     command += "RAY_WORKER_DYNAMIC_OPTION_PLACEHOLDER_0 "
     command += "org.ray.runtime.runner.worker.DefaultWorker"

python/setup.py

@@ -28,6 +28,10 @@ ray_files = [
     "ray/streaming/_streaming.so",
 ]
 
+build_java = os.getenv("RAY_INSTALL_JAVA") == "1"
+if build_java:
+    ray_files.append("ray/jars/ray_dist.jar")
+
 # These are the directories where automatically generated Python protobuf
 # bindings are created.
 generated_python_directories = [
@@ -91,7 +95,7 @@ class build_ext(_build_ext.build_ext):
         # that certain flags will not be passed along such as --user or sudo.
         # TODO(rkn): Fix this.
         command = ["../build.sh", "-p", sys.executable]
-        if os.getenv("RAY_INSTALL_JAVA") == "1":
+        if build_java:
             # Also build binaries for Java if the above env variable exists.
             command += ["-l", "python,java"]
         subprocess.check_call(command)
@@ -141,7 +145,7 @@ class build_ext(_build_ext.build_ext):
             os.makedirs(parent_directory)
         if not os.path.exists(destination):
             print("Copying {} to {}.".format(source, destination))
-            shutil.copy(source, destination)
+            shutil.copy(source, destination, follow_symlinks=True)
 
 
 class BinaryDistribution(Distribution):