diff --git a/.gitignore b/.gitignore index 8e44e43d5..e9681ebb6 100644 --- a/.gitignore +++ b/.gitignore @@ -3,6 +3,7 @@ /python/ray/core /python/ray/pyarrow_files/ /python/ray/pickle5_files/ +/python/ray/jars/ /python/build /python/dist /python/python-driver-* diff --git a/.travis.yml b/.travis.yml index e52e264fb..8d928bb28 100644 --- a/.travis.yml +++ b/.travis.yml @@ -98,7 +98,7 @@ matrix: # Build Linux wheels. - os: linux - env: LINUX_WHEELS=1 PYTHONWARNINGS=ignore + env: LINUX_WHEELS=1 PYTHONWARNINGS=ignore RAY_INSTALL_JAVA=1 install: - eval `python $TRAVIS_BUILD_DIR/ci/travis/determine_tests_to_run.py` - if [ $RAY_CI_LINUX_WHEELS_AFFECTED != "1" ]; then exit; fi @@ -124,7 +124,7 @@ matrix: # Build MacOS wheels. - os: osx osx_image: xcode7 - env: MAC_WHEELS=1 PYTHONWARNINGS=ignore + env: MAC_WHEELS=1 PYTHONWARNINGS=ignore RAY_INSTALL_JAVA=1 install: - eval `python $TRAVIS_BUILD_DIR/ci/travis/determine_tests_to_run.py` - if [ $RAY_CI_MACOS_WHEELS_AFFECTED != "1" ]; then exit; fi diff --git a/build.sh b/build.sh index 2a061bb3d..97c16b22a 100755 --- a/build.sh +++ b/build.sh @@ -119,7 +119,7 @@ export PYTHON3_BIN_PATH="$PYTHON_EXECUTABLE" export PYTHON2_BIN_PATH="$PYTHON_EXECUTABLE" if [ "$RAY_BUILD_JAVA" == "YES" ]; then - "$BAZEL_EXECUTABLE" build //java:all --verbose_failures + "$BAZEL_EXECUTABLE" build //java:ray_java_pkg --verbose_failures fi if [ "$RAY_BUILD_PYTHON" == "YES" ]; then diff --git a/java/BUILD.bazel b/java/BUILD.bazel index 77865979c..c3224a961 100644 --- a/java/BUILD.bazel +++ b/java/BUILD.bazel @@ -207,3 +207,30 @@ genrule( local = 1, tags = ["no-cache"], ) + +java_binary( + name = "ray_dist", + # This rule is used to package all Ray Java code and the third-party dependencies into a + # fat jar file. It's not really an executable jar. So we set its `main_class` to empty. + main_class = "", + runtime_deps = [ + "//java:org_ray_ray_api", + "//java:org_ray_ray_runtime", + "//streaming/java:org_ray_ray_streaming-api", + "//streaming/java:org_ray_ray_streaming-runtime", + ] +) + +genrule( + name = "ray_java_pkg", + srcs = ["//java:ray_dist_deploy.jar"], + outs = ["ray_java_pkg.out"], + cmd = """ + WORK_DIR=$$(pwd) + rm -rf $$WORK_DIR/python/ray/jars && mkdir -p $$WORK_DIR/python/ray/jars + cp -f $(location //java:ray_dist_deploy.jar) $$WORK_DIR/python/ray/jars/ray_dist.jar + echo $$(date) > $@ + """, + local = 1, + tags = ["no-cache"], +) diff --git a/java/test.sh b/java/test.sh index 4b03cf1f5..59080fbb3 100755 --- a/java/test.sh +++ b/java/test.sh @@ -21,6 +21,12 @@ echo "Linting Java code with checkstyle." # Thus, we add the `build_tests_only` option to avoid re-building everything. bazel test //java:all --test_tag_filters="checkstyle" --build_tests_only +echo "Build java maven deps." +bazel build //java:gen_maven_deps + +echo "Build test jar." +bazel build //java:all_tests_deploy.jar + echo "Running tests under cluster mode." # TODO(hchen): Ideally, we should use the following bazel command to run Java tests. However, if there're skipped tests, # TestNG will exit with code 2. And bazel treats it as test failure. diff --git a/java/test/src/main/java/org/ray/api/test/BaseMultiLanguageTest.java b/java/test/src/main/java/org/ray/api/test/BaseMultiLanguageTest.java index d430adee7..9daaded4a 100644 --- a/java/test/src/main/java/org/ray/api/test/BaseMultiLanguageTest.java +++ b/java/test/src/main/java/org/ray/api/test/BaseMultiLanguageTest.java @@ -6,6 +6,8 @@ import java.util.List; import java.util.Map; import java.util.Map.Entry; import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; +import java.util.stream.Stream; import com.google.common.base.Strings; import com.google.common.collect.ImmutableList; @@ -71,9 +73,15 @@ public abstract class BaseMultiLanguageTest { String nodeManagerPort = String.valueOf(NetworkUtil.getUnusedPort()); - // Start ray cluster. + // jars in the `ray` wheel doesn't contains test classes, so we add test classes explicitly. + // Since mvn test classes contains `test` in path and bazel test classes is located at a jar + // with `test` included in the name, we can check classpath `test` to filter out test classes. + String classpath = Stream.of(System.getProperty("java.class.path").split(":")) + .filter(s -> !s.contains(" ") && s.contains("test")) + .collect(Collectors.joining(":")); String workerOptions = - " -classpath " + System.getProperty("java.class.path"); + " -classpath " + classpath; + // Start ray cluster. List startCommand = ImmutableList.of( "ray", "start", diff --git a/python/ray/services.py b/python/ray/services.py index be07a3cc9..1fccbc9b1 100644 --- a/python/ray/services.py +++ b/python/ray/services.py @@ -4,6 +4,7 @@ import logging import multiprocessing import os import random +import re import resource import socket import subprocess @@ -1248,6 +1249,18 @@ def start_raylet(redis_address, return process_info +def get_ray_jars_dir(): + """Return a directory where all ray-related jars and + their dependencies locate.""" + current_dir = os.path.abspath(os.path.dirname(__file__)) + jars_dir = os.path.abspath(os.path.join(current_dir, "jars")) + if not os.path.exists(jars_dir): + raise Exception("Ray jars is not packaged into ray. " + "Please build ray with java enabled " + "(set env var RAY_INSTALL_JAVA=1)") + return os.path.abspath(os.path.join(current_dir, "jars")) + + def build_java_worker_command( java_worker_options, redis_address, @@ -1270,8 +1283,6 @@ def build_java_worker_command( Returns: The command string for starting Java worker. """ - assert java_worker_options is not None - command = "java " if redis_address is not None: @@ -1294,10 +1305,29 @@ def build_java_worker_command( command += ("-Dray.raylet.config.num_workers_per_process_java=" + "RAY_WORKER_NUM_WORKERS_PLACEHOLDER ") - if java_worker_options: - # Put `java_worker_options` in the last, so it can overwrite the - # above options. - command += java_worker_options + " " + # Add ray jars path to java classpath + ray_jars = os.path.join(get_ray_jars_dir(), "*") + cp_sep = ":" + import platform + if platform.system() == "Windows": + cp_sep = ";" + if java_worker_options is None: + java_worker_options = "" + options = re.split("\\s+", java_worker_options) + cp_index = -1 + for i in range(len(options)): + option = options[i] + if option == "-cp" or option == "-classpath": + cp_index = i + 1 + break + if cp_index != -1: + options[cp_index] = options[cp_index] + cp_sep + ray_jars + else: + options = ["-cp", ray_jars] + options + java_worker_options = " ".join(options) + # Put `java_worker_options` in the last, so it can overwrite the + # above options. + command += java_worker_options + " " command += "RAY_WORKER_DYNAMIC_OPTION_PLACEHOLDER_0 " command += "org.ray.runtime.runner.worker.DefaultWorker" diff --git a/python/setup.py b/python/setup.py index f15ccf16d..8dea874f2 100644 --- a/python/setup.py +++ b/python/setup.py @@ -28,6 +28,10 @@ ray_files = [ "ray/streaming/_streaming.so", ] +build_java = os.getenv("RAY_INSTALL_JAVA") == "1" +if build_java: + ray_files.append("ray/jars/ray_dist.jar") + # These are the directories where automatically generated Python protobuf # bindings are created. generated_python_directories = [ @@ -91,7 +95,7 @@ class build_ext(_build_ext.build_ext): # that certain flags will not be passed along such as --user or sudo. # TODO(rkn): Fix this. command = ["../build.sh", "-p", sys.executable] - if os.getenv("RAY_INSTALL_JAVA") == "1": + if build_java: # Also build binaries for Java if the above env variable exists. command += ["-l", "python,java"] subprocess.check_call(command) @@ -141,7 +145,7 @@ class build_ext(_build_ext.build_ext): os.makedirs(parent_directory) if not os.path.exists(destination): print("Copying {} to {}.".format(source, destination)) - shutil.copy(source, destination) + shutil.copy(source, destination, follow_symlinks=True) class BinaryDistribution(Distribution):