Rename redis-port to port and add default (#8406)

doc/source/using-ray-on-a-cluster.rst

@@ -25,11 +25,12 @@ Starting Ray on each machine
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
 On the head node (just choose some node to be the head node), run the following.
-If the ``--redis-port`` argument is omitted, Ray will choose a port at random.
+If the ``--port`` argument is omitted, Ray will choose port 6379, falling back to a
+random port.
 
 .. code-block:: bash
 
-  ray start --head --redis-port=6379
+  ray start --head --port=6379
 
 The command will print out the address of the Redis server that was started
 (and some other address information).

python/ray/actor.py

@@ -4,7 +4,6 @@ import weakref
 
 from abc import ABCMeta, abstractmethod
 from collections import namedtuple
-
 import ray.ray_constants as ray_constants
 import ray._raylet
 import ray.signature as signature

python/ray/autoscaler/aws/development-example.yaml

@@ -111,7 +111,7 @@ worker_setup_commands: []
 # Command to start ray on the head node. You don't need to change this.
 head_start_ray_commands:
     - ray stop
-    - ulimit -n 65536; ray start --head --num-redis-shards=10 --redis-port=6379 --autoscaling-config=~/ray_bootstrap_config.yaml
+    - ulimit -n 65536; ray start --head --num-redis-shards=10 --port=6379 --autoscaling-config=~/ray_bootstrap_config.yaml
 
 # Command to start ray on worker nodes. You don't need to change this.
 worker_start_ray_commands:

python/ray/autoscaler/aws/example-full.yaml

@@ -134,7 +134,7 @@ worker_setup_commands: []
 # Command to start ray on the head node. You don't need to change this.
 head_start_ray_commands:
     - ray stop
-    - ulimit -n 65536; ray start --head --redis-port=6379 --object-manager-port=8076 --autoscaling-config=~/ray_bootstrap_config.yaml
+    - ulimit -n 65536; ray start --head --port=6379 --object-manager-port=8076 --autoscaling-config=~/ray_bootstrap_config.yaml
 
 # Command to start ray on worker nodes. You don't need to change this.
 worker_start_ray_commands:

python/ray/autoscaler/aws/example-gpu-docker.yaml

@@ -119,7 +119,7 @@ worker_setup_commands: []
 # Command to start ray on the head node. You don't need to change this.
 head_start_ray_commands:
     - ray stop
-    - ulimit -n 65536; ray start --head --redis-port=6379 --object-manager-port=8076 --autoscaling-config=~/ray_bootstrap_config.yaml
+    - ulimit -n 65536; ray start --head --port=6379 --object-manager-port=8076 --autoscaling-config=~/ray_bootstrap_config.yaml
 
 # Command to start ray on worker nodes. You don't need to change this.
 worker_start_ray_commands:

python/ray/autoscaler/azure/example-full.yaml

@@ -122,7 +122,7 @@ worker_setup_commands: []
 # Command to start ray on the head node. You don't need to change this.
 head_start_ray_commands:
     - ray stop
-    - ulimit -n 65536; ray start --head --redis-port=6379 --object-manager-port=8076 --autoscaling-config=~/ray_bootstrap_config.yaml
+    - ulimit -n 65536; ray start --head --port=6379 --object-manager-port=8076 --autoscaling-config=~/ray_bootstrap_config.yaml
 
 # Command to start ray on worker nodes. You don't need to change this.
 worker_start_ray_commands:

python/ray/autoscaler/azure/example-gpu-docker.yaml

@@ -92,7 +92,7 @@ worker_setup_commands: []
 # Command to start ray on the head node. You don't need to change this.
 head_start_ray_commands:
     - ray stop
-    - ulimit -n 65536; ray start --head --redis-port=6379 --object-manager-port=8076 --autoscaling-config=~/ray_bootstrap_config.yaml
+    - ulimit -n 65536; ray start --head --port=6379 --object-manager-port=8076 --autoscaling-config=~/ray_bootstrap_config.yaml
 
 # Command to start ray on worker nodes. You don't need to change this.
 worker_start_ray_commands:

python/ray/autoscaler/azure/example-gpu.yaml

@@ -124,7 +124,7 @@ worker_setup_commands: []
 # Command to start ray on the head node. You don't need to change this.
 head_start_ray_commands:
     - ray stop
-    - ulimit -n 65536; ray start --head --redis-port=6379 --object-manager-port=8076 --autoscaling-config=~/ray_bootstrap_config.yaml
+    - ulimit -n 65536; ray start --head --port=6379 --object-manager-port=8076 --autoscaling-config=~/ray_bootstrap_config.yaml
 
 # Command to start ray on worker nodes. You don't need to change this.
 worker_start_ray_commands:

python/ray/autoscaler/gcp/example-full.yaml

@@ -149,7 +149,7 @@ head_start_ray_commands:
       ulimit -n 65536;
       ray start
       --head
-      --redis-port=6379
+      --port=6379
       --object-manager-port=8076
       --autoscaling-config=~/ray_bootstrap_config.yaml
 

python/ray/autoscaler/gcp/example-gpu-docker.yaml

@@ -158,7 +158,7 @@ head_start_ray_commands:
       ulimit -n 65536;
       ray start
       --head
-      --redis-port=6379
+      --port=6379
       --object-manager-port=8076
       --autoscaling-config=~/ray_bootstrap_config.yaml
 

python/ray/autoscaler/kubernetes/example-full.yaml

@@ -238,7 +238,7 @@ worker_setup_commands: []
 # Note webui-host is set to 0.0.0.0 so that kubernetes can port forward.
 head_start_ray_commands:
     - ray stop
-    - ulimit -n 65536; ray start --head --num-cpus=$MY_CPU_REQUEST --redis-port=6379 --object-manager-port=8076 --autoscaling-config=~/ray_bootstrap_config.yaml --webui-host 0.0.0.0
+    - ulimit -n 65536; ray start --head --num-cpus=$MY_CPU_REQUEST --port=6379 --object-manager-port=8076 --autoscaling-config=~/ray_bootstrap_config.yaml --webui-host 0.0.0.0
 
 # Command to start ray on worker nodes. You don't need to change this.
 worker_start_ray_commands:

python/ray/autoscaler/local/development-example.yaml

@@ -26,7 +26,7 @@ setup_commands:
 #    - source activate ray && cd ray/python && pip install -e .
 head_start_ray_commands:
     - source activate ray && ray stop
-    - source activate ray && ulimit -c unlimited && ray start --head --redis-port=6379 --autoscaling-config=~/ray_bootstrap_config.yaml
+    - source activate ray && ulimit -c unlimited && ray start --head --port=6379 --autoscaling-config=~/ray_bootstrap_config.yaml
 worker_start_ray_commands:
     - source activate ray && ray stop
     - source activate ray && ray start --address=$RAY_HEAD_IP:6379

python/ray/autoscaler/local/example-full.yaml

@@ -75,7 +75,7 @@ worker_setup_commands: []
 # Command to start ray on the head node. You don't need to change this.
 head_start_ray_commands:
     - ray stop
-    - ulimit -c unlimited && ray start --head --redis-port=6379 --autoscaling-config=~/ray_bootstrap_config.yaml
+    - ulimit -c unlimited && ray start --head --port=6379 --autoscaling-config=~/ray_bootstrap_config.yaml
 
 # Command to start ray on worker nodes. You don't need to change this.
 worker_start_ray_commands:

python/ray/ray_constants.py

@@ -32,6 +32,9 @@ OBJECT_STORE_MINIMUM_MEMORY_BYTES = 75 * 1024 * 1024
 DEFAULT_REDIS_MAX_MEMORY_BYTES = 10**10
 # The smallest cap on the memory used by Redis that we allow.
 REDIS_MINIMUM_MEMORY_BYTES = 10**7
+# If a user does not specify a port for the primary Ray service,
+# we attempt to start the service running at this port.
+DEFAULT_PORT = 6379
 
 # Default resource requirements for actors when no resource requirements are
 # specified.

python/ray/scripts/scripts.py

@@ -127,7 +127,15 @@ def dashboard(cluster_config_file, cluster_name, port):
     "--redis-port",
     required=False,
     type=str,
-    help="the port to use for starting Redis")
+    help="(DEPRECATED) the port to use for starting redis. "
+    "Please use --port instead now.")
+@click.option(
+    "--port",
+    required=False,
+    type=str,
+    help="the port of the head ray process. If not provided, tries to use "
+    "{0}, falling back to a random port if {0} is "
+    "not available".format(ray_constants.DEFAULT_PORT))
 @click.option(
     "--num-redis-shards",
     required=False,
@@ -279,7 +287,7 @@ def dashboard(cluster_config_file, cluster_name, port):
     is_flag=True,
     default=False,
     help="Specify whether load code from local file or GCS serialization.")
-def start(node_ip_address, redis_address, address, redis_port,
+def start(node_ip_address, redis_address, address, redis_port, port,
           num_redis_shards, redis_max_clients, redis_password,
           redis_shard_ports, object_manager_port, node_manager_port, memory,
           object_store_memory, redis_max_memory, num_cpus, num_gpus, resources,
@@ -291,6 +299,12 @@ def start(node_ip_address, redis_address, address, redis_port,
     if redis_address is not None:
         raise DeprecationWarning("The --redis-address argument is "
                                  "deprecated. Please use --address instead.")
+    if redis_port is not None:
+        logger.warn("The --redis-port argument will be deprecated soon. "
+                    "Please use --port instead.")
+        if port is not None and port != redis_port:
+            raise ValueError("Cannot specify both --port and --redis-port "
+                             "as port is a rename of deprecated redis-port")
 
     # Convert hostnames to numerical IP address.
     if node_ip_address is not None:
@@ -361,7 +375,7 @@ def start(node_ip_address, redis_address, address, redis_port,
         logger.info("Using IP address {} for this node.".format(
             ray_params.node_ip_address))
         ray_params.update_if_absent(
-            redis_port=redis_port,
+            redis_port=port or redis_port,
             redis_shard_ports=redis_shard_ports,
             redis_max_memory=redis_max_memory,
             num_redis_shards=num_redis_shards,
@@ -392,8 +406,9 @@ def start(node_ip_address, redis_address, address, redis_port,
                 if redis_password else ""))
     else:
         # Start Ray on a non-head node.
-        if redis_port is not None:
+        if not (redis_port is None and port is None):
-            raise Exception("If --head is not passed in, --redis-port is not "
+            raise Exception(
+                "If --head is not passed in, --port and --redis-port are not "
                 "allowed.")
         if redis_shard_ports is not None:
             raise Exception("If --head is not passed in, --redis-shard-ports "

python/ray/services.py

@@ -873,8 +873,9 @@ def _start_redis_instance(executable,
 
     Notes:
         If "port" is not None, then we will only use this port and try
-        only once. Otherwise, random ports will be used and the maximum
+        only once. Otherwise, we will first try the default redis port,
-        retries count is "num_retries".
+        and if it is unavailable, we will try random ports with
+        maximum retries of "num_retries".
 
     Args:
         executable (str): Full path of the redis-server executable.
@@ -912,7 +913,7 @@ def _start_redis_instance(executable,
         # This ensures that we will use the given port.
         num_retries = 1
     else:
-        port = new_port()
+        port = ray_constants.DEFAULT_PORT
 
     load_module_args = []
     for module in modules:

python/ray/tests/additional_property.yaml

@@ -27,4 +27,4 @@ setup_commands:
 # # Command to start ray on the head node. You don't need to change this.
 head_start_ray_commands:
     - ray stop
-    - ray start --head --redis-port=6379 --object-manager-port=8076 --autoscaling-config=~/ray_bootstrap_config.yaml
+    - ray start --head --port=6379 --object-manager-port=8076 --autoscaling-config=~/ray_bootstrap_config.yaml

python/ray/tests/test_multi_node.py

@@ -360,7 +360,7 @@ def test_calling_start_ray_head(call_ray_stop_only):
     subprocess.check_output(["ray", "stop"])
 
     # Test starting Ray with a redis port specified.
-    subprocess.check_output(["ray", "start", "--head", "--redis-port", "6379"])
+    subprocess.check_output(["ray", "start", "--head"])
     subprocess.check_output(["ray", "stop"])
 
     # Test starting Ray with a node IP address specified.
@@ -398,10 +398,9 @@ def test_calling_start_ray_head(call_ray_stop_only):
 
         # Test starting Ray with all arguments specified.
         subprocess.check_output([
-            "ray", "start", "--head", "--redis-port", "6379",
+            "ray", "start", "--head", "--redis-shard-ports", "6380,6381,6382",
-            "--redis-shard-ports", "6380,6381,6382", "--object-manager-port",
+            "--object-manager-port", "12345", "--num-cpus", "2", "--num-gpus",
-            "12345", "--num-cpus", "2", "--num-gpus", "0",
+            "0", "--redis-max-clients", "100", "--resources", "{\"Custom\": 1}"
-            "--redis-max-clients", "100", "--resources", "{\"Custom\": 1}"
         ])
         subprocess.check_output(["ray", "stop"])
 
@@ -438,10 +437,8 @@ def test_calling_start_ray_head(call_ray_stop_only):
 
 
 @pytest.mark.parametrize(
-    "call_ray_start", [
+    "call_ray_start",
-        "ray start --head --num-cpus=1 " +
+    ["ray start --head --num-cpus=1 " + "--node-ip-address=localhost"],
-        "--node-ip-address=localhost --redis-port=6379"
-    ],
     indirect=True)
 def test_using_hostnames(call_ray_start):
     ray.init(node_ip_address="localhost", address="localhost:6379")

python/ray/util/sgd/torch/examples/benchmarks/horovod-benchmark.yaml

@@ -75,7 +75,7 @@ worker_setup_commands:
 # # Command to start ray on the head node. You don't need to change this.
 head_start_ray_commands:
     - ray stop
-    - ray start --head --redis-port=6379 --object-manager-port=8076 --autoscaling-config=~/ray_bootstrap_config.yaml --object-store-memory=1000000000
+    - ray start --head --object-manager-port=8076 --autoscaling-config=~/ray_bootstrap_config.yaml --object-store-memory=1000000000
 
 # Command to start ray on worker nodes. You don't need to change this.
 worker_start_ray_commands:

python/ray/util/sgd/torch/examples/image_models/cluster.yaml

@@ -93,7 +93,7 @@ worker_setup_commands: []
 # # Command to start ray on the head node. You don't need to change this.
 head_start_ray_commands:
     - ray stop
-    - ray start --head --redis-port=6379 --object-manager-port=8076 --autoscaling-config=~/ray_bootstrap_config.yaml --object-store-memory=1000000000
+    - ray start --head --object-manager-port=8076 --autoscaling-config=~/ray_bootstrap_config.yaml --object-store-memory=1000000000
 
 # Command to start ray on worker nodes. You don't need to change this.
 worker_start_ray_commands:

python/ray/util/sgd/torch/examples/sgd-development.yaml

@@ -89,7 +89,7 @@ worker_setup_commands: []
 # # Command to start ray on the head node. You don't need to change this.
 head_start_ray_commands:
     - ray stop
-    - ray start --head --redis-port=6379 --object-manager-port=8076 --autoscaling-config=~/ray_bootstrap_config.yaml --object-store-memory=1000000000
+    - ray start --head --object-manager-port=8076 --autoscaling-config=~/ray_bootstrap_config.yaml --object-store-memory=1000000000
 
 # Command to start ray on worker nodes. You don't need to change this.
 worker_start_ray_commands: