[xray] Add Travis build for testing xray on Linux. (#2047)

* Run xray tests in travis.

* Comment out TaskTests.testSubmittingManyTasks.

* Comment out failing tests.

* Comment out hanging test.

* Linting

* Comment out failing test.

* Comment out failing test.

* Ignore test_dataframe.py for now.

* Comment out testDriverExitingQuickly.

.travis.yml

@@ -97,6 +97,58 @@ matrix:
         - PYTHON=3.5
         - RAY_USE_NEW_GCS=on
 
+    - os: linux
+      dist: trusty
+      env: PYTHON=3.5 RAY_USE_XRAY=1
+      install:
+        - ./.travis/install-dependencies.sh
+        - export PATH="$HOME/miniconda/bin:$PATH"
+        - ./.travis/install-ray.sh
+        - ./.travis/install-cython-examples.sh
+      script:
+        - export PATH="$HOME/miniconda/bin:$PATH"
+
+        - python python/ray/common/test/test.py
+        - python python/ray/common/redis_module/runtest.py
+        - python python/ray/plasma/test/test.py
+        # - python python/ray/local_scheduler/test/test.py
+        # - python python/ray/global_scheduler/test/test.py
+
+        - python -m pytest test/xray_test.py
+
+        - python test/runtest.py
+        - python test/array_test.py
+        - python test/actor_test.py
+        - python test/autoscaler_test.py
+        - python test/tensorflow_test.py
+        - python test/failure_test.py
+        - python test/microbenchmarks.py
+        - python test/stress_tests.py
+        # - python test/component_failures_test.py
+        - python test/multi_node_test.py
+        - python test/recursion_test.py
+        # - python test/monitor_test.py
+        - python test/cython_test.py
+        - python test/credis_test.py
+
+        # ray dataframe tests
+        # - python -m pytest python/ray/dataframe/test/test_dataframe.py
+        - python -m pytest python/ray/dataframe/test/test_concat.py
+        - python -m pytest python/ray/dataframe/test/test_io.py
+
+        # ray tune tests
+        # - python python/ray/tune/test/dependency_test.py
+        # - python -m pytest python/ray/tune/test/trial_runner_test.py
+        - python -m pytest python/ray/tune/test/trial_scheduler_test.py
+        # - python -m pytest python/ray/tune/test/tune_server_test.py
+
+        # ray rllib tests
+        - python -m pytest python/ray/rllib/test/test_catalog.py
+        - python -m pytest python/ray/rllib/test/test_filters.py
+        - python -m pytest python/ray/rllib/test/test_optimizers.py
+        - python -m pytest python/ray/rllib/test/test_evaluators.py
+
+
 install:
   - ./.travis/install-dependencies.sh
   - export PATH="$HOME/miniconda/bin:$PATH"

python/ray/scripts/scripts.py

@@ -4,6 +4,7 @@ from __future__ import print_function
 
 import click
 import json
+import os
 import subprocess
 
 import ray.services as services
@@ -144,7 +145,7 @@ def cli():
 @click.option(
     "--use-raylet",
     is_flag=True,
-    default=False,
+    default=None,
     help="use the raylet code path, this is not supported yet")
 def start(node_ip_address, redis_address, redis_port, num_redis_shards,
           redis_max_clients, redis_shard_ports, object_manager_port,
@@ -157,6 +158,11 @@ def start(node_ip_address, redis_address, redis_port, num_redis_shards,
     if redis_address is not None:
         redis_address = services.address_to_ip(redis_address)
 
+    if use_raylet is None and os.environ.get("RAY_USE_XRAY") == "1":
+        # This environment variable is used in our testing setup.
+        print("Detected environment variable 'RAY_USE_XRAY'.")
+        use_raylet = True
+
     try:
         resources = json.loads(resources)
     except Exception as e:

python/ray/worker.py

@@ -1440,6 +1440,11 @@ def _init(address_info=None,
         raise Exception("Driver_mode must be in [ray.SCRIPT_MODE, "
                         "ray.PYTHON_MODE, ray.SILENT_MODE].")
 
+    if use_raylet is None and os.environ.get("RAY_USE_XRAY") == "1":
+        # This environment variable is used in our testing setup.
+        print("Detected environment variable 'RAY_USE_XRAY'.")
+        use_raylet = True
+
     # Get addresses of existing services.
     if address_info is None:
         address_info = {}
@@ -1580,7 +1585,7 @@ def init(redis_address=None,
          huge_pages=False,
          include_webui=True,
          object_store_memory=None,
-         use_raylet=False):
+         use_raylet=None):
     """Connect to an existing Ray cluster or start one and connect to it.
 
     This method handles two cases. Either a Ray cluster already exists and we
@@ -1635,6 +1640,11 @@ def init(redis_address=None,
         Exception: An exception is raised if an inappropriate combination of
             arguments is passed in.
     """
+    if use_raylet is None and os.environ.get("RAY_USE_XRAY") == "1":
+        # This environment variable is used in our testing setup.
+        print("Detected environment variable 'RAY_USE_XRAY'.")
+        use_raylet = True
+
     # Convert hostnames to numerical IP address.
     if node_ip_address is not None:
         node_ip_address = services.address_to_ip(node_ip_address)

test/actor_test.py

@@ -18,6 +18,9 @@ class ActorAPI(unittest.TestCase):
     def tearDown(self):
         ray.worker.cleanup()
 
+    @unittest.skipIf(
+        os.environ.get("RAY_USE_XRAY") == "1",
+        "This test does not work with xray yet.")
     def testKeywordArgs(self):
         ray.init(num_workers=0, driver_mode=ray.SILENT_MODE)
 
@@ -68,6 +71,9 @@ class ActorAPI(unittest.TestCase):
         with self.assertRaises(Exception):
             ray.get(actor.get_values.remote())
 
+    @unittest.skipIf(
+        os.environ.get("RAY_USE_XRAY") == "1",
+        "This test does not work with xray yet.")
     def testVariableNumberOfArgs(self):
         ray.init(num_workers=0)
 
@@ -234,6 +240,9 @@ class ActorAPI(unittest.TestCase):
             def __init__(self):
                 pass
 
+    @unittest.skipIf(
+        os.environ.get("RAY_USE_XRAY") == "1",
+        "This test does not work with xray yet.")
     def testRandomIDGeneration(self):
         ray.init(num_workers=0)
 
@@ -327,6 +336,9 @@ class ActorMethods(unittest.TestCase):
         with self.assertRaises(Exception):
             t.f(1)
 
+    @unittest.skipIf(
+        os.environ.get("RAY_USE_XRAY") == "1",
+        "This test does not work with xray yet.")
     def testActorDeletion(self):
         ray.init(num_workers=0)
 
@@ -359,6 +371,9 @@ class ActorMethods(unittest.TestCase):
         # called.
         self.assertEqual(ray.get(Actor.remote().method.remote()), 1)
 
+    @unittest.skipIf(
+        os.environ.get("RAY_USE_XRAY") == "1",
+        "This test does not work with xray yet.")
     def testActorDeletionWithGPUs(self):
         ray.init(num_workers=0, num_gpus=1)
 
@@ -549,6 +564,9 @@ class ActorNesting(unittest.TestCase):
         actor2 = Actor2.remote(3, 4)
         self.assertEqual(ray.get(actor2.get_values.remote(5)), (3, 4))
 
+    @unittest.skipIf(
+        os.environ.get("RAY_USE_XRAY") == "1",
+        "This test does not work with xray yet.")
     def testDefineActorWithinRemoteFunction(self):
         # Make sure we can define and actors within remote funtions.
         ray.init(num_cpus=10)
@@ -684,6 +702,9 @@ class ActorsOnMultipleNodes(unittest.TestCase):
     def tearDown(self):
         ray.worker.cleanup()
 
+    @unittest.skipIf(
+        os.environ.get("RAY_USE_XRAY") == "1",
+        "This test does not work with xray yet.")
     def testActorsOnNodesWithNoCPUs(self):
         ray.init(num_cpus=0)
 
@@ -1098,6 +1119,9 @@ class ActorsWithGPUs(unittest.TestCase):
         ready_ids, remaining_ids = ray.wait(results, timeout=1000)
         self.assertEqual(len(ready_ids), 0)
 
+    @unittest.skipIf(
+        os.environ.get("RAY_USE_XRAY") == "1",
+        "This test does not work with xray yet.")
     def testActorsAndTasksWithGPUsVersionTwo(self):
         # Create tasks and actors that both use GPUs and make sure that they
         # are given different GPUs
@@ -1170,6 +1194,9 @@ class ActorsWithGPUs(unittest.TestCase):
             self.assertLess(interval1[1], interval2[0])
             self.assertLess(interval2[0], interval2[1])
 
+    @unittest.skipIf(
+        os.environ.get("RAY_USE_XRAY") == "1",
+        "This test does not work with xray yet.")
     def testBlockingActorTask(self):
         ray.init(num_cpus=1, num_gpus=1)
 
@@ -1763,6 +1790,9 @@ class DistributedActorHandles(unittest.TestCase):
 
         return Queue.remote()
 
+    @unittest.skipIf(
+        os.environ.get("RAY_USE_XRAY") == "1",
+        "This test does not work with xray yet.")
     def testFork(self):
         queue = self.setup_queue_actor()
 
@@ -1778,6 +1808,9 @@ class DistributedActorHandles(unittest.TestCase):
             filtered_items = [item[1] for item in items if item[0] == i]
             self.assertEqual(filtered_items, list(range(1)))
 
+    @unittest.skipIf(
+        os.environ.get("RAY_USE_XRAY") == "1",
+        "This test does not work with xray yet.")
     def testForkConsistency(self):
         queue = self.setup_queue_actor()
 
@@ -1871,6 +1904,9 @@ class ActorPlacementAndResources(unittest.TestCase):
     def tearDown(self):
         ray.worker.cleanup()
 
+    @unittest.skipIf(
+        os.environ.get("RAY_USE_XRAY") == "1",
+        "This test does not work with xray yet.")
     def testLifetimeAndTransientResources(self):
         ray.init(num_cpus=1)
 
@@ -1928,6 +1964,9 @@ class ActorPlacementAndResources(unittest.TestCase):
         for location in locations2:
             self.assertNotEqual(location, local_plasma)
 
+    @unittest.skipIf(
+        os.environ.get("RAY_USE_XRAY") == "1",
+        "This test does not work with xray yet.")
     def testCreatingMoreActorsThanResources(self):
         ray.init(
             num_workers=0,

test/failure_test.py

@@ -257,6 +257,9 @@ class WorkerDeath(unittest.TestCase):
     def tearDown(self):
         ray.worker.cleanup()
 
+    @unittest.skipIf(
+        os.environ.get("RAY_USE_XRAY") == "1",
+        "This test does not work with xray yet.")
     def testWorkerRaisingException(self):
         ray.init(num_workers=1, driver_mode=ray.SILENT_MODE)
 
@@ -272,6 +275,9 @@ class WorkerDeath(unittest.TestCase):
         wait_for_errors(b"worker_died", 1)
         self.assertEqual(len(ray.error_info()), 2)
 
+    @unittest.skipIf(
+        os.environ.get("RAY_USE_XRAY") == "1",
+        "This test does not work with xray yet.")
     def testWorkerDying(self):
         ray.init(num_workers=0, driver_mode=ray.SILENT_MODE)
 
@@ -288,6 +294,9 @@ class WorkerDeath(unittest.TestCase):
         self.assertIn("died or was killed while executing the task",
                       ray.error_info()[0][b"message"].decode("ascii"))
 
+    @unittest.skipIf(
+        os.environ.get("RAY_USE_XRAY") == "1",
+        "This test does not work with xray yet.")
     def testActorWorkerDying(self):
         ray.init(num_workers=0, driver_mode=ray.SILENT_MODE)
 
@@ -306,6 +315,9 @@ class WorkerDeath(unittest.TestCase):
         self.assertRaises(Exception, lambda: ray.get(consume.remote(obj)))
         wait_for_errors(b"worker_died", 1)
 
+    @unittest.skipIf(
+        os.environ.get("RAY_USE_XRAY") == "1",
+        "This test does not work with xray yet.")
     def testActorWorkerDyingFutureTasks(self):
         ray.init(num_workers=0, driver_mode=ray.SILENT_MODE)
 
@@ -328,6 +340,9 @@ class WorkerDeath(unittest.TestCase):
 
         wait_for_errors(b"worker_died", 1)
 
+    @unittest.skipIf(
+        os.environ.get("RAY_USE_XRAY") == "1",
+        "This test does not work with xray yet.")
     def testActorWorkerDyingNothingInProgress(self):
         ray.init(num_workers=0, driver_mode=ray.SILENT_MODE)
 

test/multi_node_test.py

@@ -2,12 +2,13 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
-import unittest
+import os
 import ray
 import subprocess
 import sys
 import tempfile
 import time
+import unittest
 
 from ray.test.test_utils import run_and_get_output
 
@@ -153,6 +154,9 @@ print("success")
         # Make sure the other driver succeeded.
         self.assertIn("success", out)
 
+    @unittest.skipIf(
+        os.environ.get("RAY_USE_XRAY") == "1",
+        "This test does not work with xray yet.")
     def testDriverExitingQuickly(self):
         # This test will create some drivers that submit some tasks and then
         # exit without waiting for the tasks to complete.

test/runtest.py

@@ -689,6 +689,9 @@ class APITest(unittest.TestCase):
         self.assertEqual(ray.get(k2.remote(1)), 2)
         self.assertEqual(ray.get(m.remote(1)), 2)
 
+    @unittest.skipIf(
+        os.environ.get("RAY_USE_XRAY") == "1",
+        "This test does not work with xray yet.")
     def testSubmitAPI(self):
         self.init_ray(num_gpus=1, resources={"Custom": 1}, num_workers=1)
 
@@ -720,6 +723,9 @@ class APITest(unittest.TestCase):
         results = ray.get([object_ids[i] for i in indices])
         self.assertEqual(results, indices)
 
+    @unittest.skipIf(
+        os.environ.get("RAY_USE_XRAY") == "1",
+        "This test does not work with xray yet.")
     def testWait(self):
         self.init_ray(num_cpus=1)
 
@@ -785,6 +791,9 @@ class APITest(unittest.TestCase):
         with self.assertRaises(TypeError):
             ray.wait([1])
 
+    @unittest.skipIf(
+        os.environ.get("RAY_USE_XRAY") == "1",
+        "This test does not work with xray yet.")
     def testMultipleWaitsAndGets(self):
         # It is important to use three workers here, so that the three tasks
         # launched in this experiment can run at the same time.
@@ -887,6 +896,9 @@ class APITest(unittest.TestCase):
 
         self.assertTrue("fake_directory" not in ray.get(get_path2.remote()))
 
+    @unittest.skipIf(
+        os.environ.get("RAY_USE_XRAY") == "1",
+        "This test does not work with xray yet.")
     def testLoggingAPI(self):
         self.init_ray(driver_mode=ray.SILENT_MODE)
 
@@ -1033,6 +1045,9 @@ class PythonModeTest(unittest.TestCase):
     def tearDown(self):
         ray.worker.cleanup()
 
+    @unittest.skipIf(
+        os.environ.get("RAY_USE_XRAY") == "1",
+        "This test does not work with xray yet.")
     def testPythonMode(self):
         reload(test_functions)
         ray.init(driver_mode=ray.PYTHON_MODE)
@@ -1229,6 +1244,9 @@ class ResourcesTest(unittest.TestCase):
         self.assertLess(duration, 1 + time_buffer)
         self.assertGreater(duration, 1)
 
+    @unittest.skipIf(
+        os.environ.get("RAY_USE_XRAY") == "1",
+        "This test does not work with xray yet.")
     def testGPUIDs(self):
         num_gpus = 10
         ray.init(num_cpus=10, num_gpus=num_gpus)
@@ -1659,6 +1677,9 @@ class CudaVisibleDevicesTest(unittest.TestCase):
         else:
             del os.environ["CUDA_VISIBLE_DEVICES"]
 
+    @unittest.skipIf(
+        os.environ.get("RAY_USE_XRAY") == "1",
+        "This test does not work with xray yet.")
     def testSpecificGPUs(self):
         allowed_gpu_ids = [4, 5, 6]
         os.environ["CUDA_VISIBLE_DEVICES"] = ",".join(
@@ -1699,6 +1720,9 @@ class WorkerPoolTests(unittest.TestCase):
 
         ray.get([f.remote() for _ in range(100)])
 
+    @unittest.skipIf(
+        os.environ.get("RAY_USE_XRAY") == "1",
+        "This test does not work with xray yet.")
     def testBlockingTasks(self):
         ray.init(num_workers=1)
 
@@ -1728,6 +1752,9 @@ class WorkerPoolTests(unittest.TestCase):
 
         ray.get(sleep.remote())
 
+    @unittest.skipIf(
+        os.environ.get("RAY_USE_XRAY") == "1",
+        "This test does not work with xray yet.")
     def testMaxCallTasks(self):
         ray.init(num_cpus=1)
 
@@ -1838,6 +1865,9 @@ class GlobalStateAPI(unittest.TestCase):
     def tearDown(self):
         ray.worker.cleanup()
 
+    @unittest.skipIf(
+        os.environ.get("RAY_USE_XRAY") == "1",
+        "This test does not work with xray yet.")
     def testGlobalStateAPI(self):
         with self.assertRaises(Exception):
             ray.global_state.object_table()
@@ -1995,6 +2025,9 @@ class GlobalStateAPI(unittest.TestCase):
 
         self.assertEqual(found_message, True)
 
+    @unittest.skipIf(
+        os.environ.get("RAY_USE_XRAY") == "1",
+        "This test does not work with xray yet.")
     def testTaskProfileAPI(self):
         ray.init(redirect_output=True)
 
@@ -2053,6 +2086,9 @@ class GlobalStateAPI(unittest.TestCase):
             self.assertIn("stderr_file", info)
             self.assertIn("stdout_file", info)
 
+    @unittest.skipIf(
+        os.environ.get("RAY_USE_XRAY") == "1",
+        "This test does not work with xray yet.")
     def testDumpTraceFile(self):
         ray.init(redirect_output=True)
 
@@ -2091,6 +2127,9 @@ class GlobalStateAPI(unittest.TestCase):
         # the visualization actually renders (e.g., the context of the dumped
         # trace could be malformed).
 
+    @unittest.skipIf(
+        os.environ.get("RAY_USE_XRAY") == "1",
+        "This test does not work with xray yet.")
     def testFlushAPI(self):
         ray.init(num_cpus=1)
 

test/stress_tests.py

@@ -71,6 +71,9 @@ class TaskTests(unittest.TestCase):
                 self.assertTrue(ray.services.all_processes_alive())
                 ray.worker.cleanup()
 
+    @unittest.skipIf(
+        os.environ.get("RAY_USE_XRAY") == "1",
+        "This test does not work with xray yet.")
     def testSubmittingManyTasks(self):
         ray.init()
 
@@ -118,6 +121,9 @@ class TaskTests(unittest.TestCase):
         self.assertTrue(ray.services.all_processes_alive())
         ray.worker.cleanup()
 
+    @unittest.skipIf(
+        os.environ.get("RAY_USE_XRAY") == "1",
+        "This test does not work with xray yet.")
     def testWait(self):
         for num_local_schedulers in [1, 4]:
             for num_workers_per_scheduler in [4]: