[Java] Package native dependencies into jar (#4367)

.gitignore

@@ -148,6 +148,7 @@ java/**/lib
 java/**/.settings
 java/**/.classpath
 java/**/.project
+java/runtime/native_dependencies/
 
 # python virtual env
 venv

CMakeLists.txt

@@ -152,8 +152,23 @@ if ("${CMAKE_RAY_LANG_JAVA}" STREQUAL "YES")
   get_raylet_library("java" RAYLET_LIBRARY_JAVA)
   add_dependencies(copy_ray ${RAYLET_LIBRARY_JAVA})
 
-  # copy libplasma_java files
+  # Copy java native dependencies.
   add_custom_command(TARGET copy_ray POST_BUILD
-    COMMAND bash -c "mkdir -p ${CMAKE_CURRENT_BINARY_DIR}/src/plasma"
-    COMMAND bash -c "cp ${ARROW_LIBRARY_DIR}/libplasma_java.* ${CMAKE_CURRENT_BINARY_DIR}/src/plasma/")
+    COMMAND mkdir -p ${CMAKE_SOURCE_DIR}/java/runtime/native_dependencies)
+  set(java_native_dependencies
+    "src/ray/thirdparty/redis/src/redis-server"
+    "src/ray/gcs/redis_module/libray_redis_module.so"
+    "src/ray/raylet/raylet"
+    "src/ray/raylet/libraylet_library_java.*")
+  foreach(file ${java_native_dependencies})
+  add_custom_command(TARGET copy_ray POST_BUILD
+    COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_BINARY_DIR}/${file}
+      ${CMAKE_SOURCE_DIR}/java/runtime/native_dependencies)
+  endforeach()
+add_custom_command(TARGET copy_ray POST_BUILD
+  COMMAND ${CMAKE_COMMAND} -E copy ${ARROW_HOME}/bin/plasma_store_server
+    ${CMAKE_SOURCE_DIR}/java/runtime/native_dependencies)
+  add_custom_command(TARGET copy_ray POST_BUILD
+    COMMAND $(CMAKE_COMMAND) -E copy ${ARROW_LIBRARY_DIR}/libplasma_java.*
+      ${CMAKE_SOURCE_DIR}/java/runtime/native_dependencies)
 endif()

java/README.rst

@@ -5,7 +5,7 @@ Configuration
 -------------
 Ray will read your configurations in the following order:
 
-* Java system properties: e.g., ``-Dray.home=/path/to/ray``.
+* Java system properties: e.g., ``-Dray.run-mode=SINGLE_PROCESS``.
 * A ``ray.conf`` file in the classpath: `example <https://github.com/ray-project/ray/blob/master/java/example.conf>`_.
 * Customise your own ``ray.conf`` path using system property ``-Dray.config=/path/to/ray.conf``
 

java/example.conf

@@ -6,11 +6,6 @@
 # For config file format, see 'https://github.com/lightbend/config/blob/master/HOCON.md'.
 
 ray {
-  // This is the path to the directory where Ray is installed, e.g.,
-  // something like /home/ubmutu/ray. This can be an absolute path or
-  // a relative path from the current working directory.
-  home = "/path/to/your/ray/home"
-
   // Run mode, available options are:
   //
   // `SINGLE_PROCESS`: Ray is running in one single Java process, without Raylet backend,

java/runtime/pom.xml

@@ -79,6 +79,14 @@
   </dependencies>
 
   <build>
+    <resources>
+      <resource>
+        <directory>src/main/resources</directory>
+      </resource>
+      <resource>
+        <directory>native_dependencies</directory>
+      </resource>
+    </resources>
     <plugins>
       <plugin>
         <groupId>org.apache.maven.plugins</groupId>

java/runtime/src/main/java/org/ray/runtime/RayNativeRuntime.java

@@ -2,8 +2,13 @@ package org.ray.runtime;
 
 import com.google.common.base.Preconditions;
 import com.google.common.base.Strings;
+import java.io.File;
+import java.io.InputStream;
 import java.lang.reflect.Field;
 import java.nio.ByteBuffer;
+import java.nio.file.Files;
+import java.nio.file.Paths;
+import java.nio.file.StandardCopyOption;
 import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.List;
@@ -72,11 +77,22 @@ public final class RayNativeRuntime extends AbstractRayRuntime {
 
   @Override
   public void start() throws Exception {
-    // Load native libraries.
     try {
+      // Reset library path at runtime.
       resetLibraryPath();
-      System.loadLibrary("raylet_library_java");
-      System.loadLibrary("plasma_java");
+
+      // Load native libraries.
+      String[] libraries = new String[]{"raylet_library_java", "plasma_java"};
+      for (String library : libraries) {
+        String fileName = System.mapLibraryName(library);
+        // Copy the file from resources to a temp dir, and load the native library.
+        File file = File.createTempFile(fileName, "");
+        file.deleteOnExit();
+        InputStream in = RayNativeRuntime.class.getResourceAsStream("/" + fileName);
+        Preconditions.checkNotNull(in, "{} doesn't exist.", fileName);
+        Files.copy(in, Paths.get(file.getAbsolutePath()), StandardCopyOption.REPLACE_EXISTING);
+        System.load(file.getAbsolutePath());
+      }
     } catch (Exception e) {
       LOGGER.error("Failed to load native libraries.", e);
       throw e;

java/runtime/src/main/java/org/ray/runtime/config/RayConfig.java

@@ -30,7 +30,6 @@ public class RayConfig {
   public static final String DEFAULT_CONFIG_FILE = "ray.default.conf";
   public static final String CUSTOM_CONFIG_FILE = "ray.conf";
 
-  public final String rayHome;
   public final String nodeIp;
   public final WorkerMode workerMode;
   public final RunMode runMode;
@@ -56,10 +55,6 @@ public class RayConfig {
   public final String rayletSocketName;
   public final List<String> rayletConfigParameters;
 
-  public final String redisServerExecutablePath;
-  public final String redisModulePath;
-  public final String plasmaStoreExecutablePath;
-  public final String rayletExecutablePath;
   public final String driverResourcePath;
   public final String pythonWorkerCommand;
 
@@ -72,9 +67,6 @@ public class RayConfig {
     if (workerMode == WorkerMode.WORKER) {
       Preconditions.checkArgument(redisAddress != null,
           "Redis address must be set in worker mode.");
-    } else {
-      Preconditions.checkArgument(!rayHome.isEmpty(),
-          "'ray.home' must be set in driver mode");
     }
   }
 
@@ -87,32 +79,24 @@ public class RayConfig {
   }
 
   public RayConfig(Config config) {
-    // worker mode
+    // Worker mode.
     WorkerMode localWorkerMode;
     try {
       localWorkerMode = config.getEnum(WorkerMode.class, "ray.worker.mode");
     } catch (ConfigException.Missing e) {
       localWorkerMode = WorkerMode.DRIVER;
     }
-
     workerMode = localWorkerMode;
     boolean isDriver = workerMode == WorkerMode.DRIVER;
-    // run mode
+    // Run mode.
     runMode = config.getEnum(RunMode.class, "ray.run-mode");
-    // ray home
-    String localRayHome = config.getString("ray.home");
-    if (!localRayHome.startsWith("/")) {
-      // If ray.home isn't an absolute path, prepend it with current work dir.
-      localRayHome = System.getProperty("user.dir") + "/" + localRayHome;
-    }
-    rayHome = removeTrailingSlash(localRayHome);
-    // node ip
+    // Node ip.
     String nodeIp = config.getString("ray.node-ip");
     if (nodeIp.isEmpty()) {
       nodeIp = NetworkUtil.getIpAddress(null);
     }
     this.nodeIp = nodeIp;
-    // resources
+    // Resources.
     resources = ResourceUtil.getResourcesMapFromString(
         config.getString("ray.resources"));
     if (isDriver) {
@@ -127,22 +111,22 @@ public class RayConfig {
         resources.put("GPU", 0.0);
       }
     }
-    // driver id
+    // Driver id.
     String driverId = config.getString("ray.driver.id");
     if (!driverId.isEmpty()) {
       this.driverId = UniqueId.fromHexString(driverId);
     } else {
       this.driverId = UniqueId.randomId();
     }
-    // log dir
+    // Log dir.
     logDir = removeTrailingSlash(config.getString("ray.log-dir"));
-    // redirect output
+    // Redirect output.
     redirectOutput = config.getBoolean("ray.redirect-output");
-    // custom library path
-    List<String> customLibraryPath = config.getStringList("ray.library.path");
-    // custom classpath
+    // Library path.
+    libraryPath = config.getStringList("ray.library.path");
+    // Custom classpath.
     classpath = config.getStringList("ray.classpath");
-    // custom worker jvm parameters
+    // Custom worker jvm parameters.
     if (config.hasPath("ray.worker.jvm-parameters")) {
       jvmParameters = config.getStringList("ray.worker.jvm-parameters");
     } else {
@@ -155,7 +139,7 @@ public class RayConfig {
       pythonWorkerCommand = null;
     }
 
-    // redis configurations
+    // Redis configurations.
     String redisAddress = config.getString("ray.redis.address");
     if (!redisAddress.isEmpty()) {
       setRedisAddress(redisAddress);
@@ -167,34 +151,22 @@ public class RayConfig {
     headRedisPassword = config.getString("ray.redis.head-password");
     redisPassword = config.getString("ray.redis.password");
 
-    // object store configurations
+    // Object store configurations.
     objectStoreSocketName = config.getString("ray.object-store.socket-name");
     objectStoreSize = config.getBytes("ray.object-store.size");
 
-    // raylet socket name
+    // Raylet socket name.
     rayletSocketName = config.getString("ray.raylet.socket-name");
 
-    // raylet parameters
-    rayletConfigParameters = new ArrayList<String>();
+    // Raylet parameters.
+    rayletConfigParameters = new ArrayList<>();
     Config rayletConfig = config.getConfig("ray.raylet.config");
     for (Map.Entry<String,ConfigValue> entry : rayletConfig.entrySet()) {
-      String parameter = entry.getKey() + "," + String.valueOf(entry.getValue().unwrapped());
+      String parameter = entry.getKey() + "," + entry.getValue().unwrapped();
       rayletConfigParameters.add(parameter);
     }
 
-    // library path
-    this.libraryPath = new ImmutableList.Builder<String>().add(
-        rayHome + "/build/src/plasma",
-        rayHome + "/build/src/ray/raylet"
-    ).addAll(customLibraryPath).build();
-
-    redisServerExecutablePath = rayHome +
-        "/build/src/ray/thirdparty/redis/src/redis-server";
-    redisModulePath = rayHome + "/build/src/ray/gcs/redis_module/libray_redis_module.so";
-    plasmaStoreExecutablePath = rayHome + "/build/src/plasma/plasma_store_server";
-    rayletExecutablePath = rayHome + "/build/src/ray/raylet/raylet";
-
-    // driver resource path
+    // Driver resource path.
     if (config.hasPath("ray.driver.resource-path")) {
       driverResourcePath = config.getString("ray.driver.resource-path");
     } else {
@@ -204,7 +176,7 @@ public class RayConfig {
     // Number of threads that execute tasks.
     numberExecThreadsForDevRuntime = config.getInt("ray.dev-runtime.execution-parallelism");
 
-    // validate config
+    // Validate config.
     validate();
     LOGGER.debug("Created config: {}", this);
   }
@@ -235,7 +207,6 @@ public class RayConfig {
   @Override
   public String toString() {
     return "RayConfig{"
-        + "rayHome='" + rayHome + '\''
         + ", nodeIp='" + nodeIp + '\''
         + ", workerMode=" + workerMode
         + ", runMode=" + runMode
@@ -255,10 +226,6 @@ public class RayConfig {
         + ", objectStoreSize=" + objectStoreSize
         + ", rayletSocketName='" + rayletSocketName + '\''
         + ", rayletConfigParameters=" + rayletConfigParameters
-        + ", redisServerExecutablePath='" + redisServerExecutablePath + '\''
-        + ", redisModulePath='" + redisModulePath + '\''
-        + ", plasmaStoreExecutablePath='" + plasmaStoreExecutablePath + '\''
-        + ", rayletExecutablePath='" + rayletExecutablePath + '\''
         + ", driverResourcePath='" + driverResourcePath + '\''
         + ", pythonWorkerCommand='" + pythonWorkerCommand + '\''
         + '}';

java/runtime/src/main/java/org/ray/runtime/runner/RunManager.java

@@ -5,9 +5,14 @@ import com.google.common.collect.ImmutableList;
 import com.google.common.collect.Lists;
 import java.io.File;
 import java.io.IOException;
+import java.io.InputStream;
+import java.nio.file.Files;
+import java.nio.file.Paths;
+import java.nio.file.StandardCopyOption;
 import java.time.LocalDateTime;
 import java.time.format.DateTimeFormatter;
 import java.util.ArrayList;
+import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.Random;
@@ -42,10 +47,13 @@ public class RunManager {
 
   private static final int KILL_PROCESS_WAIT_TIMEOUT_SECONDS = 1;
 
+  private final Map<String, File> tempFiles;
+
   public RunManager(RayConfig rayConfig) {
     this.rayConfig = rayConfig;
     processes = new ArrayList<>();
     random = new Random();
+    tempFiles = new HashMap<>();
   }
 
   public void cleanup() {
@@ -61,7 +69,7 @@ public class RunManager {
         p.waitFor(KILL_PROCESS_WAIT_TIMEOUT_SECONDS, TimeUnit.SECONDS);
       } catch (InterruptedException e) {
         LOGGER.warn("Got InterruptedException while waiting for process {}" +
-            " to be terminated.",  processes.get(i));
+            " to be terminated.", processes.get(i));
       }
 
       if (p.isAlive()) {
@@ -76,8 +84,30 @@ public class RunManager {
     FileUtil.mkDir(new File(rayConfig.objectStoreSocketName).getParentFile());
   }
 
+  /**
+   * Copy a file from resources to a temp dir, and return the file object.
+   */
+  private File getTempFile(String fileName) {
+    File file = tempFiles.get(fileName);
+    if (file == null) {
+      try {
+        file = File.createTempFile(fileName, "");
+        file.deleteOnExit();
+        try (InputStream in = RunManager.class.getResourceAsStream(fileName)) {
+          Files.copy(in, Paths.get(file.getCanonicalPath()), StandardCopyOption.REPLACE_EXISTING);
+        }
+        file.setExecutable(true);
+      } catch (IOException e) {
+        throw new RuntimeException("Couldn't get temp file " + fileName, e);
+      }
+      tempFiles.put(fileName, file);
+    }
+    return file;
+  }
+
   /**
    * Start a process.
+   *
    * @param command The command to start the process with.
    * @param env Environment variables.
    * @param name Process name.
@@ -126,6 +156,7 @@ public class RunManager {
 
   /**
    * Start all Ray processes on this node.
+   *
    * @param isHead Whether this node is the head node. If true, redis server will be started.
    */
   public void startRayProcesses(boolean isHead) {
@@ -171,7 +202,8 @@ public class RunManager {
 
   private String startRedisInstance(String ip, int port, String password, Integer shard) {
     List<String> command = Lists.newArrayList(
-        rayConfig.redisServerExecutablePath,
+        // The redis-server executable file.
+        getTempFile("/redis-server").getAbsolutePath(),
         "--protected-mode",
         "no",
         "--port",
@@ -179,7 +211,8 @@ public class RunManager {
         "--loglevel",
         "warning",
         "--loadmodule",
-        rayConfig.redisModulePath
+        // The redis module file.
+        getTempFile("/libray_redis_module.so").getAbsolutePath()
     );
 
     if (!StringUtil.isNullOrEmpty(password)) {
@@ -216,7 +249,8 @@ public class RunManager {
 
     // See `src/ray/raylet/main.cc` for the meaning of each parameter.
     List<String> command = ImmutableList.of(
-        rayConfig.rayletExecutablePath,
+        // The raylet executable file.
+        getTempFile("/raylet").getAbsolutePath(),
         rayConfig.rayletSocketName,
         rayConfig.objectStoreSocketName,
         "0",  // The object manager port.
@@ -291,7 +325,8 @@ public class RunManager {
 
   private void startObjectStore() {
     List<String> command = ImmutableList.of(
-        rayConfig.plasmaStoreExecutablePath,
+        // The plasma store executable file.
+        getTempFile("/plasma_store_server").getAbsolutePath(),
         "-s",
         rayConfig.objectStoreSocketName,
         "-m",

java/runtime/src/main/resources/ray.default.conf

@@ -7,11 +7,6 @@ ray {
   // Basic configurations
   // ----------------------
 
-  // This is the path to the directory where Ray is installed, e.g.,
-  // something like /home/ubmutu/ray. This can be an absolute path or
-  // a relative path from the current working directory.
-  home: ""
-
   // IP of this node. if not provided, IP will be automatically detected.
   node-ip: ""
 

java/test/src/main/java/org/ray/api/test/BaseTest.java

@@ -1,6 +1,9 @@
 package org.ray.api.test;
 
+import com.google.common.collect.ImmutableList;
+import java.io.File;
 import java.lang.reflect.Method;
+import java.util.List;
 import org.ray.api.Ray;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -11,27 +14,32 @@ public class BaseTest {
 
   private static final Logger LOGGER = LoggerFactory.getLogger(BaseTest.class);
 
+  private List<File> filesToDelete;
+
   @BeforeMethod
   public void setUpBase(Method method) {
     LOGGER.info("===== Running test: "
         + method.getDeclaringClass().getName() + "." + method.getName());
-    System.setProperty("ray.home", "../..");
     System.setProperty("ray.resources", "CPU:4,RES-A:4");
     Ray.init();
+    // These files need to be deleted after each test case.
+    filesToDelete = ImmutableList.of(
+        new File(Ray.getRuntimeContext().getRayletSocketName()),
+        new File(Ray.getRuntimeContext().getObjectStoreSocketName())
+    );
+    // Make sure the files will be deleted even if the test doesn't exit gracefully.
+    filesToDelete.forEach(File::deleteOnExit);
   }
 
   @AfterMethod
   public void tearDownBase() {
-    // TODO(qwang): This is double check to check that the socket file is removed actually.
-    // We could not enable this until `systemInfo` enabled.
-    //File rayletSocketFIle = new File(Ray.systemInfo().rayletSocketName());
     Ray.shutdown();
 
-    //remove raylet socket file
-    //rayletSocketFIle.delete();
+    for (File file : filesToDelete) {
+      file.delete();
+    }
 
-    // unset system properties
-    System.clearProperty("ray.home");
+    // Unset system properties.
     System.clearProperty("ray.resources");
   }
 

java/test/src/main/java/org/ray/api/test/MultiLanguageClusterTest.java

@@ -66,8 +66,7 @@ public class MultiLanguageClusterTest {
 
     // Start ray cluster.
     String testDir = System.getProperty("user.dir");
-    String workerOptions = String.format("-Dray.home=%s/../../", testDir);
-    workerOptions +=
+    String workerOptions =
         " -classpath " + String.format("%s/../../build/java/*:%s/target/*", testDir, testDir);
     final List<String> startCommand = ImmutableList.of(
         "ray",
@@ -85,7 +84,6 @@ public class MultiLanguageClusterTest {
     }
 
     // Connect to the cluster.
-    System.setProperty("ray.home", "../..");
     System.setProperty("ray.redis.address", "127.0.0.1:6379");
     System.setProperty("ray.object-store.socket-name", PLASMA_STORE_SOCKET_NAME);
     System.setProperty("ray.raylet.socket-name", RAYLET_SOCKET_NAME);
@@ -96,7 +94,6 @@ public class MultiLanguageClusterTest {
   public void tearDown() {
     // Disconnect to the cluster.
     Ray.shutdown();
-    System.clearProperty("ray.home");
     System.clearProperty("ray.redis.address");
     System.clearProperty("ray.object-store.socket-name");
     System.clearProperty("ray.raylet.socket-name");

java/test/src/main/java/org/ray/api/test/RayConfigTest.java

@@ -10,24 +10,12 @@ public class RayConfigTest {
   @Test
   public void testCreateRayConfig() {
     try {
-      System.setProperty("ray.home", "/path/to/ray");
       System.setProperty("ray.driver.resource-path", "path/to/ray/driver/resource/path");
       RayConfig rayConfig = RayConfig.create();
-
-      Assert.assertEquals("/path/to/ray", rayConfig.rayHome);
       Assert.assertEquals(WorkerMode.DRIVER, rayConfig.workerMode);
-
-      System.setProperty("ray.home", "");
-      rayConfig = RayConfig.create();
-
-      Assert.assertEquals(System.getProperty("user.dir"), rayConfig.rayHome);
-      Assert.assertEquals(System.getProperty("user.dir") +
-          "/build/src/ray/thirdparty/redis/src/redis-server", rayConfig.redisServerExecutablePath);
-
       Assert.assertEquals("path/to/ray/driver/resource/path", rayConfig.driverResourcePath);
     } finally {
-      //unset the system property
-      System.clearProperty("ray.home");
+      // Unset system properties.
       System.clearProperty("ray.driver.resource-path");
     }
 

java/tutorial/README.rst

@@ -12,7 +12,7 @@ To run them, execute the following command under ``ray/java`` folder.
 
 .. code-block:: shell
 
-    java -Dray.home=.. -classpath "tutorial/target/ray-tutorial-1.0.jar:tutorial/lib/*" org.ray.exercise.Exercise01
+    java -classpath "tutorial/target/ray-tutorial-1.0.jar:tutorial/lib/*" org.ray.exercise.Exercise01
 
 `Exercise 1 <https://github.com/ray-project/ray/tree/master/java/tutorial/src/main/java/org/ray/exercise/Exercise01.java>`_: Define a remote function, and execute multiple remote functions in parallel.
 

java/tutorial/src/main/resources/ray.conf

@@ -1,5 +1,4 @@
-ray{
-  home: ".."
+ray {
   run-mode: CLUSTER
   redirect-output: false
 }