hiro/ray
1
0
Fork 0
mirror of https://github.com/vale981/ray synced 2025-03-05 18:11:42 -05:00
Code Issues Projects Releases Packages Wiki Activity Actions

Package and upload ray cross-platform jar (#9540)

Browse source
This commit is contained in:
chaokunyang 2020-07-27 17:20:20 +08:00 • committed by GitHub
parent 2790818c53
commit 881032593d
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
11 changed files with 347 additions and 95 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
.gitignore vendored
View file

@ -10,6 +10,7 @@
/python/python-driver-*
/thirdparty/pkg/
/build/java
.jar
# Files generated by flatc should be ignored
/src/ray/gcs/format/*_generated.h

49
.travis.yml
View file

@ -99,33 +99,37 @@ matrix:
script:
- true # we still need this block to exist, otherwise it will fall back to the global one
# Build Linux wheels.
# Build Linux wheels and multi-platform jars.
- os: linux
env:
- LINUX_WHEELS=1
- LINUX_WHEELS=1 LINUX_JARS=1
- PYTHONWARNINGS=ignore
- RAY_INSTALL_JAVA=1
install:
- . ./ci/travis/ci.sh init RAY_CI_LINUX_WHEELS_AFFECTED
- . ./ci/travis/ci.sh init RAY_CI_LINUX_WHEELS_AFFECTED,RAY_CI_JAVA_AFFECTED,RAY_CI_STREAMING_JAVA_AFFECTED
before_script:
- . ./ci/travis/ci.sh build
script:
- . ./ci/travis/ci.sh test_wheels
- bash ./java/build-jar-multiplatform.sh linux
cache: false
# Build MacOS wheels.
# Build MacOS wheels and MacOS jars
- os: osx
osx_image: xcode7
env:
- MAC_WHEELS=1
- MAC_WHEELS=1 MULTIPLATFORM_JARS=1
- PYTHONWARNINGS=ignore
- RAY_INSTALL_JAVA=1
install:
- . ./ci/travis/ci.sh init RAY_CI_MACOS_WHEELS_AFFECTED
- . ./ci/travis/ci.sh init RAY_CI_MACOS_WHEELS_AFFECTED,RAY_CI_JAVA_AFFECTED,RAY_CI_STREAMING_JAVA_AFFECTED
before_script:
- . ./ci/travis/ci.sh build
script:
- . ./ci/travis/ci.sh test_wheels
- bash ./java/build-jar-multiplatform.sh darwin
- bash ./java/build-jar-multiplatform.sh multiplatform
#- bash ./java/build-jar-multiplatform.sh deploy
# RLlib: Learning tests (from rllib/tuned_examples/*.yaml).
- os: linux
@ -392,6 +396,39 @@ deploy:
repo: ray-project/ray
condition: $LINUX_WHEELS = 1 || $MAC_WHEELS = 1
# Upload jars so that we can debug locally for every commit
- provider: s3
edge: true # This supposedly opts in to deploy v2.
access_key_id: AKIAU6DMUCJUFL3EX3SM
secret_access_key:
secure: J1sX71fKFPQhgWzColllxfzcF877ScBZ1cIl71krZ6SO0LKnwsCScpQck5eZOyQo/Iverwye0iKtE87qNsiRi3+V2D9iulSr18T09j7+FjPKfxAmXmjfrNafoMXTDQroSJblCri5vl+DysISPqImJkWTNaYhGJ9QakoSd5djnAopLNWj6PCR3S50baS49+nB5nSIY3jMhtUzlaBdniFPFC81Cxyuafr4pv6McGRfR/dK+ZnPhdGtMnVeIJXB+ooZKQ26mDJKBPka4jm3u1Oa72b/Atu2RO3MwxTg79LTrMxXKh2OcCqhtD2Z3lz1OltvNSunCuwY8AejCJsfSLbM9mGDoz+xhNUWmYNy48YFf+61OY8PXi8S/9Q817yb3GpLbb2l/P+KMgq9eSEiELIOwuYsDxPX5TuAg6dx0wCNgDEBJoThSQjYl6MgJrLrs7p+JBxp3giedHiy0TLa5hCVKTj3euONAXDArYnnT+DvUIOkaeTk5DClRZbZ0sUXhLy//HuT5WJvjFBJJZ0u0f4RLVb5D7DI4uMZr7+yJPDR2AXCyW9YMaBEbmEYbPaKi283jlEyn7R33+AZlnXv0THHwZ4xvjKKG3/fBSXsOUmv5wmUveEqVGDj1mKPGj9NF8iA5qMm2AaZuJpEEBVBZtSlTZt6ZG7rzAJZGNL52t7xuMo=
bucket: ray-wheels
acl: public_read
region: us-west-2
local_dir: .jar
upload-dir: "jars/$TRAVIS_BRANCH/$TRAVIS_COMMIT"
skip_cleanup: true
on:
repo: ray-project/ray
all_branches: true
condition: $MULTIPLATFORM_JARS = 1 || $LINUX_JARS=1
- provider: s3
edge: true # This supposedly opts in to deploy v2.
access_key_id: AKIAU6DMUCJUFL3EX3SM
secret_access_key:
secure: J1sX71fKFPQhgWzColllxfzcF877ScBZ1cIl71krZ6SO0LKnwsCScpQck5eZOyQo/Iverwye0iKtE87qNsiRi3+V2D9iulSr18T09j7+FjPKfxAmXmjfrNafoMXTDQroSJblCri5vl+DysISPqImJkWTNaYhGJ9QakoSd5djnAopLNWj6PCR3S50baS49+nB5nSIY3jMhtUzlaBdniFPFC81Cxyuafr4pv6McGRfR/dK+ZnPhdGtMnVeIJXB+ooZKQ26mDJKBPka4jm3u1Oa72b/Atu2RO3MwxTg79LTrMxXKh2OcCqhtD2Z3lz1OltvNSunCuwY8AejCJsfSLbM9mGDoz+xhNUWmYNy48YFf+61OY8PXi8S/9Q817yb3GpLbb2l/P+KMgq9eSEiELIOwuYsDxPX5TuAg6dx0wCNgDEBJoThSQjYl6MgJrLrs7p+JBxp3giedHiy0TLa5hCVKTj3euONAXDArYnnT+DvUIOkaeTk5DClRZbZ0sUXhLy//HuT5WJvjFBJJZ0u0f4RLVb5D7DI4uMZr7+yJPDR2AXCyW9YMaBEbmEYbPaKi283jlEyn7R33+AZlnXv0THHwZ4xvjKKG3/fBSXsOUmv5wmUveEqVGDj1mKPGj9NF8iA5qMm2AaZuJpEEBVBZtSlTZt6ZG7rzAJZGNL52t7xuMo=
bucket: ray-wheels
acl: public_read
region: us-west-2
local_dir: .jar
upload-dir: "jars/latest"
skip_cleanup: true
on:
repo: ray-project/ray
branch: master
condition: $MULTIPLATFORM_JARS = 1 || $LINUX_JARS=1
- provider: script
edge: true # This supposedly opts in to deploy v2.
script: bash $TRAVIS_BUILD_DIR/ci/travis/build-autoscaler-images.sh || true

18
BUILD.bazel
View file

@ -1815,6 +1815,7 @@ cc_binary(
}),
linkshared = 1,
linkstatic = 1,
visibility = ["//java:__subpackages__"],
deps = [
"//:core_worker_lib",
"//:global_state_accessor_lib",
@ -1825,23 +1826,6 @@ cc_binary(
],
)
genrule(
name = "core_worker-jni-darwin-compat",
srcs = [":libcore_worker_library_java.so"],
outs = ["libcore_worker_library_java.dylib"],
cmd = "cp $< $@",
output_to_bindir = 1,
)
filegroup(
name = "core_worker_library_java",
srcs = select({
"@bazel_tools//src/conditions:darwin": [":libcore_worker_library_java.dylib"],
"//conditions:default": [":libcore_worker_library_java.so"],
}),
visibility = ["//java:__subpackages__"],
)
filegroup(
name = "python_sources",
srcs = glob([

57
bazel/ray.bzl
View file

@ -1,5 +1,6 @@
load("@com_github_google_flatbuffers//:build_defs.bzl", "flatbuffer_library_public")
load("@com_github_checkstyle_java//checkstyle:checkstyle.bzl", "checkstyle_test")
load("@bazel_skylib//rules:copy_file.bzl", "copy_file")
load("@bazel_common//tools/maven:pom_file.bzl", "pom_file")
COPTS = ["-DRAY_USE_GLOG"] + select({
@ -125,7 +126,7 @@ def copy_to_workspace(name, srcs, dstdir = ""):
dstdir = "." + ("/" + dstdir.replace("\\", "/")).rstrip("/") + "/",
),
# Keep this batch script equivalent to the Bash script above (or take out the batch script)
cmd_bat = r"""
cmd_bat = """
(
if not exist {dstdir} mkdir {dstdir}
) && (
@ -140,3 +141,57 @@ def copy_to_workspace(name, srcs, dstdir = ""):
),
local = 1,
)
def native_java_binary(module_name, name, native_binary_name):
"""Copy native binary file to different path based on operating systems"""
copy_file(
name = name + "_darwin",
src = native_binary_name,
out = module_name + "/src/main/resources/native/darwin/" + name,
)
copy_file(
name = name + "_linux",
src = native_binary_name,
out = module_name + "/src/main/resources/native/linux/" + name,
)
copy_file(
name = name + "_windows",
src = native_binary_name,
out = module_name + "/src/main/resources/native/windows/" + name,
)
native.filegroup(
name = name,
srcs = select({
"@bazel_tools//src/conditions:darwin": [name + "_darwin"],
"@bazel_tools//src/conditions:windows": [name + "_windows"],
"//conditions:default": [name + "_linux"],
}),
visibility = ["//visibility:public"],
)
def native_java_library(module_name, name, native_library_name):
"""Copy native library file to different path based on operating systems"""
copy_file(
name = name + "_darwin",
src = native_library_name,
out = module_name + "/src/main/resources/native/darwin/lib{}.dylib".format(name),
)
copy_file(
name = name + "_linux",
src = native_library_name,
out = module_name + "/src/main/resources/native/linux/lib{}.so".format(name),
)
native.filegroup(
name = name,
srcs = select({
"@bazel_tools//src/conditions:darwin": [name + "_darwin"],
"@bazel_tools//src/conditions:windows": [],
"//conditions:default": [name + "_linux"],
}),
visibility = ["//visibility:public"],
)

70
java/BUILD.bazel
View file

@ -1,5 +1,6 @@
load("//bazel:ray.bzl", "define_java_module")
load("@bazel_skylib//rules:copy_file.bzl", "copy_file")
load("//bazel:ray.bzl", "native_java_binary")
load("//bazel:ray.bzl", "native_java_library")
load("@rules_proto_grpc//java:defs.bzl", "java_proto_compile")
exports_files([
@ -143,43 +144,38 @@ filegroup(
],
)
# plasma_store_server binary should be located in jar root path
# and keep in sync with mvn resource
copy_file(
name = "cp_plasma_store_server",
src = "//:plasma_store_server",
out = "plasma_store_server",
)
native_java_binary("runtime", "raylet", "//:raylet")
# `//:redis-server`'s full path is `external/com_github_antirez_redis/redis-server`,
# This rule removes the prefix, and only keeps `redis-server`.
copy_file(
name = "redis-server-without-prefix",
src = "//:redis-server",
out = "redis-server",
)
native_java_binary("runtime", "plasma_store_server", "//:plasma_store_server")
native_java_binary("runtime", "redis-server", "//:redis-server")
native_java_binary("runtime", "gcs_server", "//:gcs_server")
native_java_binary("runtime", "libray_redis_module.so", "//:libray_redis_module.so")
native_java_library("runtime", "core_worker_library_java", "//:libcore_worker_library_java.so")
filegroup(
name = "java_native_deps",
srcs = [
":cp_plasma_store_server",
":redis-server-without-prefix",
"//:core_worker_library_java",
"//:gcs_server",
"//:libray_redis_module.so",
"//:raylet",
":core_worker_library_java",
":gcs_server",
":libray_redis_module.so",
":plasma_store_server",
":raylet",
":redis-server",
],
)
# Generates the dependencies needed by maven.
genrule(
name = "gen_maven_deps",
name = "cp_java_generated",
srcs = [
":all_java_proto",
":java_native_deps",
":copy_pom_file",
],
outs = ["gen_maven_deps.out"],
outs = ["cp_java_generated.out"],
cmd = """
WORK_DIR="$$(pwd)"
# Copy protobuf-generated files.
@ -187,8 +183,30 @@ genrule(
for f in $(locations //java:all_java_proto); do
unzip "$$f" -x META-INF/MANIFEST.MF -d "$$WORK_DIR/java/runtime/src/main/java"
done
# Copy native dependecies.
NATIVE_DEPS_DIR="$$WORK_DIR/java/runtime/native_dependencies/"
date > $@
""",
local = 1,
tags = ["no-cache"],
)
# Generates the dependencies needed by maven.
genrule(
name = "gen_maven_deps",
srcs = [
":cp_java_generated",
":java_native_deps",
],
outs = ["gen_maven_deps.out"],
cmd = """
WORK_DIR="$${PWD}"
# Copy native dependencies.
OS_NAME=""
case "$${OSTYPE}" in
linux*) OS_NAME="linux";;
darwin*) OS_NAME="darwin";;
*) echo "$${OSTYPE} is not supported currently"; exit 1;;
esac
NATIVE_DEPS_DIR="$$WORK_DIR/java/runtime/native_dependencies/native/$$OS_NAME"
rm -rf "$$NATIVE_DEPS_DIR"
mkdir -p "$$NATIVE_DEPS_DIR"
for f in $(locations //java:java_native_deps); do

166
java/build-jar-multiplatform.sh Executable file
View file

@ -0,0 +1,166 @@
#!/bin/bash
set -x
# Cause the script to exit if a single command fails.
set -e
ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE:-$0}")"; pwd)"
WORKSPACE_DIR="${ROOT_DIR}/.."
JAVA_DIRS_PATH=('java' 'streaming/java')
RAY_JAVA_MODULES=('api' 'runtime')
RAY_STREAMING_JAVA_MODULES=('streaming-api' 'streaming-runtime' 'streaming-state')
JAR_BASE_DIR="$WORKSPACE_DIR"/.jar
cd "$WORKSPACE_DIR/java"
# ray jar version, ex: 0.1-SNAPSHORT
version=$(python -c "import xml.etree.ElementTree as ET; r = ET.parse('pom.xml').getroot(); print(r.find(r.tag.replace('project', 'version')).text);" | tail -n 1)
cd -
build_jars() {
local platform="$1"
local bazel_build="${2:-true}"
echo "bazel_build $bazel_build"
echo "Start building jar for $platform"
local JAR_DIR="$JAR_BASE_DIR/$platform"
mkdir -p "$JAR_DIR"
for p in "${JAVA_DIRS_PATH[@]}"; do
cd "$WORKSPACE_DIR/$p"
if [[ $bazel_build == "true" ]]; then
echo "Starting building java native dependencies for $p"
bazel build gen_maven_deps
echo "Finished building java native dependencies for $p"
fi
echo "Start building jars for $p"
mvn -T16 clean package install -Dmaven.test.skip=true -Dcheckstyle.skip
mvn -T16 source:jar -Dmaven.test.skip=true -Dcheckstyle.skip
echo "Finished building jars for $p"
done
copy_jars "$JAR_DIR"
# ray runtime jar and streaming runtime jar are in a dir specifed by maven-jar-plugin
cp -f "$WORKSPACE_DIR"/build/java/ray*.jar "$JAR_DIR"
cp -f "$WORKSPACE_DIR"/streaming/build/java/streaming*.jar "$JAR_DIR"
echo "Finished building jar for $platform"
}
copy_jars() {
local JAR_DIR="$1"
echo "Copy to dir $JAR_DIR"
for module in "${RAY_JAVA_MODULES[@]}"; do
cp -f "$WORKSPACE_DIR"/java/"$module"/target/*jar "$JAR_DIR"
done
for module in "${RAY_STREAMING_JAVA_MODULES[@]}"; do
cp -f "$WORKSPACE_DIR"/streaming/java/"$module"/target/*jar "$JAR_DIR"
done
# ray runtime jar and streaming runtime jar are in a dir specifed by maven-jar-plugin
cp -f "$WORKSPACE_DIR"/build/java/ray*.jar "$JAR_DIR"
cp -f "$WORKSPACE_DIR"/streaming/build/java/streaming*.jar "$JAR_DIR"
}
# This function assuem all dependencies are installed already.
build_jars_linux() {
build_jars linux
}
# This function assuem all dependencies are installed already.
build_jars_darwin() {
build_jars darwin
}
build_jars_multiplatform() {
if [[ "${TRAVIS_REPO_SLUG-}" != "ray-project/ray" || "${TRAVIS_PULL_REQUEST-}" != "false" ]]; then
echo "Skip build multiplatform jars when this build is from a pull request or
not a build for commit in ray-project/ray."
return
fi
download_jars "ray-runtime-$version.jar" "streaming-runtime-$version.jar"
prepare_native
build_jars multiplatform false
}
# Download linux/windows ray-related jar from s3
# This function assumes darwin jars exist already.
download_jars() {
local wait_time=0
local sleep_time_units=60
for f in "$@"; do
for os in 'linux' 'windows'; do
if [[ "$os" == "windows" ]]; then
break
fi
local url="https://ray-wheels.s3-us-west-2.amazonaws.com/jars/$TRAVIS_BRANCH/$TRAVIS_COMMIT/$os/$f"
local dest_file="$JAR_BASE_DIR/$os/$f"
echo "Jar url: $url"
echo "Jar dest_file: $dest_file"
while true; do
if ! wget -q "$url" -O "$dest_file">/dev/null; then
echo "Waiting $url to be ready for $wait_time seconds..."
sleep $sleep_time_units
wait_time=$((wait_time + sleep_time_units))
if [[ wait_time == $((60 * 120)) ]]; then
echo "Download $url timeout"
exit 1
fi
else
echo "Download $url to $dest_file succeed"
break
fi
done
done
done
}
# prepare native binaries and libraries.
prepare_native() {
for os in 'darwin' 'linux'; do
cd "$JAR_BASE_DIR/$os"
jar xf "ray-runtime-$version.jar" "native/$os"
local native_dir="$WORKSPACE_DIR/java/runtime/native_dependencies/native/$os"
mkdir -p "$native_dir"
rm -rf "$native_dir"
mv "native/$os" "$native_dir"
jar xf "streaming-runtime-$version.jar" "native/$os"
local native_dir="$WORKSPACE_DIR/streaming/java/streaming-runtime/native_dependencies/native/$os"
mkdir -p "$native_dir"
rm -rf "$native_dir"
mv "native/$os" "$native_dir"
done
}
# This function assume all multiplatform binaries are prepared already.
deploy_jars() {
if [ "${TRAVIS-}" = true ]; then
if [[ "$TRAVIS_REPO_SLUG" != "ray-project/ray" ||
"$TRAVIS_PULL_REQUEST" != "false" || "$TRAVIS_BRANCH" != "master" ]]; then
echo "Skip deploying jars when this build is from a pull request or
not a build for commit of master branch in ray-project/ray"
return
fi
fi
echo "Start deploying jars"
cd "$WORKSPACE_DIR"/java
mvn -T16 deploy -Dmaven.test.skip=true -Dcheckstyle.skip
cd "$WORKSPACE_DIR"/streaming/java
mvn -T16 deploy -Dmaven.test.skip=true -Dcheckstyle.skip
echo "Finished deploying jars"
}
case "$1" in
linux) # build jars that only contains Linux binaries.
build_jars_linux
;;
darwin) # build jars that only contains macos binaries.
build_jars_darwin
;;
multiplatform) # downloading jars of multiple platforms and packaging them into one jar.
build_jars_multiplatform
;;
deploy) # Deploy jars to maven repository.
deploy_jars
;;
*)
echo "Execute command $*"
"$@"
;;
esac

3
java/runtime/src/main/java/io/ray/runtime/RayNativeRuntime.java
View file

@ -16,6 +16,7 @@ import io.ray.runtime.runner.RunManager;
import io.ray.runtime.task.NativeTaskExecutor;
import io.ray.runtime.task.NativeTaskSubmitter;
import io.ray.runtime.task.TaskExecutor;
import io.ray.runtime.util.BinaryFileUtil;
import io.ray.runtime.util.JniUtils;
import java.io.File;
import java.io.IOException;
@ -49,7 +50,7 @@ public final class RayNativeRuntime extends AbstractRayRuntime {
rayConfig.setSessionDir(sessionDir);
}
JniUtils.loadLibrary("core_worker_library_java", true);
JniUtils.loadLibrary(BinaryFileUtil.CORE_WORKER_JAVA_LIBRARY, true);
LOGGER.debug("Native libraries loaded.");
// Reset library path at runtime.
resetLibraryPath(rayConfig);

10
java/runtime/src/main/java/io/ray/runtime/runner/RunManager.java
View file

@ -237,7 +237,7 @@ public class RunManager {
}
// See `src/ray/gcs/gcs_server/gcs_server_main.cc` for the meaning of each parameter.
final File gcsServerFile = BinaryFileUtil.getFile(
final File gcsServerFile = BinaryFileUtil.getNativeFile(
rayConfig.sessionDir, BinaryFileUtil.GCS_SERVER_BINARY_NAME);
Preconditions.checkState(gcsServerFile.setExecutable(true));
List<String> command = ImmutableList.of(
@ -254,7 +254,7 @@ public class RunManager {
}
private String startRedisInstance(String ip, int port, String password, Integer shard) {
final File redisServerFile = BinaryFileUtil.getFile(
final File redisServerFile = BinaryFileUtil.getNativeFile(
rayConfig.sessionDir, BinaryFileUtil.REDIS_SERVER_BINARY_NAME);
Preconditions.checkState(redisServerFile.setExecutable(true));
List<String> command = Lists.newArrayList(
@ -268,7 +268,7 @@ public class RunManager {
"warning",
"--loadmodule",
// The redis module file.
BinaryFileUtil.getFile(
BinaryFileUtil.getNativeFile(
rayConfig.sessionDir, BinaryFileUtil.REDIS_MODULE_LIBRARY_NAME).getAbsolutePath()
);
@ -305,7 +305,7 @@ public class RunManager {
}
// See `src/ray/raylet/main.cc` for the meaning of each parameter.
final File rayletFile = BinaryFileUtil.getFile(
final File rayletFile = BinaryFileUtil.getNativeFile(
rayConfig.sessionDir, BinaryFileUtil.RAYLET_BINARY_NAME);
Preconditions.checkState(rayletFile.setExecutable(true));
List<String> command = ImmutableList.of(
@ -373,7 +373,7 @@ public class RunManager {
}
private void startObjectStore() {
final File objectStoreFile = BinaryFileUtil.getFile(
final File objectStoreFile = BinaryFileUtil.getNativeFile(
rayConfig.sessionDir, BinaryFileUtil.PLASMA_STORE_SERVER_BINARY_NAME);
Preconditions.checkState(objectStoreFile.setExecutable(true));
List<String> command = ImmutableList.of(

23
java/runtime/src/main/java/io/ray/runtime/util/BinaryFileUtil.java
View file

@ -9,6 +9,7 @@ import java.nio.channels.FileLock;
import java.nio.file.Files;
import java.nio.file.Paths;
import org.apache.commons.io.FileUtils;
import org.apache.commons.lang3.SystemUtils;
public class BinaryFileUtil {
public static final String REDIS_SERVER_BINARY_NAME = "redis-server";
@ -21,11 +22,10 @@ public class BinaryFileUtil {
public static final String REDIS_MODULE_LIBRARY_NAME = "libray_redis_module.so";
public static final String CORE_WORKER_JAVA_LIBRARY =
System.mapLibraryName("core_worker_library_java");
public static final String CORE_WORKER_JAVA_LIBRARY = "core_worker_library_java";
/**
* Extract a resource file to <code>destDir</code>.
* Extract a platform-native resource file to <code>destDir</code>.
* Note that this a process-safe operation. If multi processes extract the file to same
* directory concurrently, this operation will be protected by a file lock.
*
@ -33,7 +33,7 @@ public class BinaryFileUtil {
* @param fileName resource file name
* @return extracted resource file
*/
public static File getFile(String destDir, String fileName) {
public static File getNativeFile(String destDir, String fileName) {
final File dir = new File(destDir);
if (!dir.exists()) {
try {
@ -45,17 +45,26 @@ public class BinaryFileUtil {
String lockFilePath = destDir + File.separator + "file_lock";
try (FileLock ignored = new RandomAccessFile(lockFilePath, "rw")
.getChannel().lock()) {
String resourceDir;
if (SystemUtils.IS_OS_MAC) {
resourceDir = "native/darwin/";
} else if (SystemUtils.IS_OS_LINUX) {
resourceDir = "native/linux/";
} else {
throw new UnsupportedOperationException("Unsupported os " + SystemUtils.OS_NAME);
}
String resourcePath = resourceDir + fileName;
File file = new File(String.format("%s/%s", destDir, fileName));
if (file.exists()) {
return file;
}
// File does not exist.
try (InputStream is = BinaryFileUtil.class.getResourceAsStream("/" + fileName)) {
Preconditions.checkNotNull(is, "{} doesn't exist.", fileName);
try (InputStream is = BinaryFileUtil.class.getResourceAsStream("/" + resourcePath)) {
Preconditions.checkNotNull(is, "{} doesn't exist.", resourcePath);
Files.copy(is, Paths.get(file.getCanonicalPath()));
} catch (IOException e) {
throw new RuntimeException("Couldn't get temp file from resource " + fileName, e);
throw new RuntimeException("Couldn't get temp file from resource " + resourcePath, e);
}
return file;
} catch (IOException e) {

2
java/runtime/src/main/java/io/ray/runtime/util/JniUtils.java
View file

@ -39,7 +39,7 @@ public class JniUtils {
// Load native library.
String fileName = System.mapLibraryName(libraryName);
final String sessionDir = RayConfig.getInstance().sessionDir;
final File file = BinaryFileUtil.getFile(sessionDir, fileName);
final File file = BinaryFileUtil.getNativeFile(sessionDir, fileName);
if (exportSymbols) {
// Expose library symbols using RTLD_GLOBAL which may be depended by other shared

43
streaming/java/BUILD.bazel
View file

@ -1,4 +1,6 @@
load("//bazel:ray.bzl", "define_java_module")
load("//bazel:ray.bzl", "native_java_binary")
load("//bazel:ray.bzl", "native_java_library")
load("@rules_proto_grpc//java:defs.bzl", "java_proto_compile")
exports_files([
@ -82,40 +84,13 @@ define_java_module(
],
)
# `//streaming:streaming_java` will be located in jar `streaming` directory,
# but we need it located in jar root path.
# resource_strip_prefix = "streaming" will make other resources file located in wrong path.
# So we copy libs explicitly to remove `streaming` path.
native_java_library("streaming-runtime", "streaming_java", "//streaming:libstreaming_java.so")
filegroup(
name = "java_native_deps",
srcs = [":streaming_java"],
)
filegroup(
name = "streaming_java",
srcs = select({
"@bazel_tools//src/conditions:darwin": [":streaming_java_darwin"],
"//conditions:default": [":streaming_java_linux"],
}),
visibility = ["//visibility:public"],
)
genrule(
name = "streaming_java_darwin",
srcs = ["//streaming:libstreaming_java.so"],
outs = ["libstreaming_java.dylib"],
cmd = "cp $< $@",
output_to_bindir = 1,
)
genrule(
name = "streaming_java_linux",
srcs = ["//streaming:libstreaming_java.so"],
outs = ["libstreaming_java.so"],
cmd = "cp $< $@",
output_to_bindir = 1,
)
define_java_module(
name = "streaming-runtime",
additional_resources = [
@ -251,9 +226,15 @@ genrule(
],
outs = ["gen_maven_deps.out"],
cmd = """
WORK_DIR="$$(pwd)"
WORK_DIR="$${PWD}"
# Copy native dependencies.
NATIVE_DEPS_DIR="$$WORK_DIR/streaming/java/streaming-runtime/native_dependencies/"
OS_NAME=""
case "$${OSTYPE}" in
linux*) OS_NAME="linux";;
darwin*) OS_NAME="darwin";;
*) echo "$${OSTYPE} is not supported currently"; exit 1;;
esac
NATIVE_DEPS_DIR="$$WORK_DIR/streaming/java/streaming-runtime/native_dependencies/native/$$OS_NAME"
rm -rf "$$NATIVE_DEPS_DIR"
mkdir -p "$$NATIVE_DEPS_DIR"
for f in $(locations //streaming/java:java_native_deps); do