[Doc] [Core] [runtime env] Add runtime env doc (#16290)

.buildkite/pipeline.yml

@@ -34,7 +34,7 @@
       --test_env=CONDA_DEFAULT_ENV
       --test_env=CI
       --test_env=RAY_CI_POST_WHEEL_TESTS=True
-      python/ray/tests/... python/ray/serve/... python/ray/tune/... rllib/...
+      python/ray/tests/... python/ray/serve/... python/ray/tune/... rllib/... doc/...
 
 - label: ":docker: Build Images: py36"
   conditions: ["RAY_CI_LINUX_WHEELS_AFFECTED"]

doc/BUILD

@@ -57,6 +57,14 @@ py_test(
     tags = ["exclusive"]
 )
 
+py_test(
+    name = "doc_code_runtime_env_example",
+    size = "small",
+    main = "examples/doc_code/runtime_env_example.py",
+    srcs = ["examples/doc_code/runtime_env_example.py"],
+    tags = ["exclusive", "post_wheel_build"]
+)
+
 # --------------------------------------------------------------------
 # Tests from the doc/source/tune/_tutorials directory.
 # Please keep these sorted alphabetically.

doc/examples/doc_code/runtime_env_example.py

@@ -0,0 +1,57 @@
+# flake8: noqa
+"""
+This file holds code for the runtime envs documentation.
+
+It ignores yapf because yapf doesn't allow comments right after code blocks,
+but we put comments right after code blocks to prevent large white spaces
+in the documentation.
+"""
+import ray
+
+# yapf: disable
+
+# __runtime_env_conda_def_start__
+runtime_env = {
+    "conda": {
+        "dependencies":
+        ["toolz", "dill", "pip", {
+            "pip": ["pendulum", "ray[serve]"]
+        }]
+    },
+    "env_vars": {"TF_WARNINGS": "none"}
+}
+# __runtime_env_conda_def_end__
+
+# __ray_init_start__
+ray.init(job_config=ray.job_config.JobConfig(runtime_env=runtime_env))
+# __ray_init_end__
+
+@ray.remote
+def f_job():
+    pass
+
+@ray.remote
+class Actor_job:
+    def g(self):
+        pass
+
+ray.get(f_job.remote())
+a = Actor_job.remote()
+ray.get(a.g.remote())
+
+ray.shutdown()
+
+ray.init()
+
+@ray.remote
+def f():
+    pass
+
+@ray.remote
+class Actor:
+    pass
+
+# __per_task_per_actor_start__
+f.options(runtime_env=runtime_env).remote()
+Actor.options(runtime_env=runtime_env).remote()
+# __per_task_per_actor_end__

doc/source/advanced.rst

@@ -423,33 +423,89 @@ To get information about the current available resource capacity of your cluster
 .. autofunction:: ray.available_resources
     :noindex:
 
-.. _conda-environments-for-tasks-and-actors:
+.. _runtime-environments:
 
-Conda Environments for Tasks and Actors
------------------------------------------
+Runtime Environments (Experimental)
+-----------------------------------
+On Mac OS and Linux, Ray 1.4+ supports dynamically setting the runtime environment of tasks, actors, and jobs so that they can depend on different Python libraries (e.g., conda environments, pip dependencies) while all running on the same Ray cluster.
 
-Starting with Ray 1.3.0, Ray supports starting tasks and actors in `conda environments <https://docs.conda.io/en/latest/>`_.
-This allows you to use tasks and actors with different (possibly conflicting) package dependencies within a single Ray runtime.
-You will need to have the desired conda environments installed beforehand on all nodes in your Ray cluster, and they
-must all use the same Python minor version (e.g., Python 3.8).  Note that this feature is currently unavailable on Windows.
+The ``runtime_env`` is a (JSON-serializable) dictionary that can be passed as an option to tasks and actors, and can also be passed to ``ray.init()`` and ``ray.client().connect()``.
+The runtime environment defines the dependencies required for your workload.
 
-To start a specific task or an actor in an existing conda environment, pass in the environment name to your task or 
-actor via the ``runtime_env`` parameter as follows:
+You can specify a runtime environment for your whole job using ``ray.init()`` or Ray Client...
+
+.. literalinclude:: ../examples/doc_code/runtime_env_example.py
+   :language: python
+   :start-after: __ray_init_start__
+   :end-before: __ray_init_end__
+
+..
+  TODO(architkulkarni): run Ray Client doc example in CI
 
 .. code-block:: python
 
-    result = ray.get(my_task.options(runtime_env={"conda_env": "my_env"}).remote())
+    ray.client("localhost:10001").env(runtime_env).connect()
+
+...or specify per-actor or per-task using ``.options()``:
+
+.. literalinclude:: ../examples/doc_code/runtime_env_example.py
+   :language: python
+   :start-after: __per_task_per_actor_start__
+   :end-before: __per_task_per_actor_end__
+
+The ``runtime_env`` is a Python dictionary including one or more of the following arguments:
+
+- ``working_dir`` (Path): Specifies the working directory for your job. This must be an existing local directory.
+  It will be cached on the cluster, so the next time you connect with Ray Client you will be able to skip uploading the directory contents.
+  Furthermore, if you locally make a small change to your directory, the next time you connect only the updated part will be uploaded.
+
+  - Examples
+
+    - ``"."  # cwd``
+
+    - ``"/code/my_project"``
+
+  Note: Setting this option per-task or per-actor is currently unsupported.
+
+- ``pip`` (List[str] | str): Either a list of pip packages, or a string containing the path to a pip 
+  `“requirements.txt” <https://pip.pypa.io/en/stable/user_guide/#requirements-files>`_ file.
+  This will be dynamically installed in the ``runtime_env``.
+  To use a library like Ray Serve or Ray Tune, you will need to include ``"ray[serve]"`` or ``"ray[tune]"`` here.
+
+  - Example: ``["requests==1.0.0", "aiohttp"]``
+
+  - Example: ``"./requirements.txt"``
+
+- ``conda`` (dict | str): Either a dict representing the conda environment YAML, a string containing the path to a 
+  `conda “environment.yml” <https://conda.io/projects/conda/en/latest/user-guide/tasks/manage-environments.html#create-env-file-manually>`_ file,
+  or the name of a local conda env already installed on each node in your cluster (e.g., ``"pytorch_p36"``).
+  In the first two cases, the Ray and Python dependencies will be automatically injected into the environment to ensure compatibility, so there is no need to manually include them.  
+  Note that the ``conda`` and ``pip`` keys of ``runtime_env`` cannot both be specified at the same time---to use them together, please use ``conda`` and add your pip dependencies in the ``"pip"`` field in your conda ``environment.yaml``.
+
+  - Example: ``{"conda": {"dependencies": ["pytorch", “torchvision”, "pip", {"pip": ["pendulum"]}]}}``
+
+  - Example: ``"./environment.yml"``
+
+  - Example: ``"pytorch_p36"``
+
+- ``env_vars`` (Dict[str, str]): Environment variables to set.
+
+  - Example: ``{"OMP_NUM_THREADS": "32", "TF_WARNINGS": "none"}``
+
+The runtime env is inheritable, so it will apply to all tasks/actors within a job and all child tasks/actors of a task or actor, once set.
+
+If a child actor or task specifies a new ``runtime_env``, it will be merged with the parent’s ``runtime_env`` via a simple dict update.
+
+Here are some examples of runtime envs combining multiple options:
+
+..
+  TODO(architkulkarni): run working_dir doc example in CI
 
 .. code-block:: python
+    
+    runtime_env = {"working_dir": "/code/my_project", "pip": ["pendulum=2.1.2"]}
 
-    my_actor = MyActor.options(runtime_env={"conda_env": "my_env"}).remote()
-
-Nested tasks and actors will inherit the conda environment of their parent by default.
-
-To have Ray start all tasks and actors in a specific conda environment by default, you may
-pass in the desired conda environment name into ``ray.init()``:
-
-.. code-block:: python
-
-    from ray.job_config import JobConfig
-    ray.init(job_config=JobConfig(runtime_env={"conda_env": "my_env"}))
+.. literalinclude:: ../examples/doc_code/runtime_env_example.py
+   :language: python
+   :start-after: __runtime_env_conda_def_start__
+   :end-before: __runtime_env_conda_def_end__

doc/source/serve/core-apis.rst

@@ -245,7 +245,7 @@ your deployment.
 You must have a conda environment set up for each set of
 dependencies you want to isolate.  If using a multi-node cluster, the
 desired conda environment must be present on all nodes.  
-See :ref:`conda-environments-for-tasks-and-actors` for details.
+See :ref:`runtime-environments` for details.
 
 Here's an example script.  For it to work, first create a conda
 environment named ``ray-tf1`` with Ray Serve and Tensorflow 1 installed,

python/ray/_private/runtime_env.py

@@ -43,14 +43,10 @@ class RuntimeEnvDict:
             Examples:
                 ["/path/to/other_module", "/other_path/local_project.zip"]
         pip (List[str] | str): Either a list of pip packages, or a string
-            containing the path to a pip requirements.txt file.  If a relative
-            path is specified and working_dir is specified, the path is
-            interpreted relative to working_dir.
+            containing the path to a pip requirements.txt file.
         conda (dict | str): Either the conda YAML config, the name of a
             local conda env (e.g., "pytorch_p36"), or the path to a conda
-            environment.yaml file. If a relative path is specified and
-            working_dir is specified, the path is interpreted relative to
-            working_dir.
+            environment.yaml file.
             The Ray dependency will be automatically injected into the conda
             env to ensure compatibility with the cluster Ray. The conda name
             may be mangled automatically to avoid conflicts between runtime

python/ray/client_builder.py

@@ -69,7 +69,7 @@ class ClientBuilder:
         Set an environment for the session.
         Args:
             env (Dict[st, Any]): A runtime environment to use for this
-            connection. See ``runtime_env.py`` for what values are
+            connection. See :ref:`runtime-environments` for what values are
             accepted in this dict.
         """
         self._job_config.set_runtime_env(env)

python/ray/worker.py

@@ -1907,9 +1907,12 @@ def remote(*args, **kwargs):
             the default is 4 (default), and a value of -1 indicates
             infinite retries.
         runtime_env (Dict[str, Any]): Specifies the runtime environment for
-            this actor or task and its children. See``runtime_env.py`` for
-            detailed documentation.  Note: can only be set via `.options()`.
-        override_environment_variables (Dict[str, str]): This specifies
+            this actor or task and its children. See
+            :ref:`runtime-environments` for detailed documentation.
+            Note: can only be set via `.options()`.
+        override_environment_variables (Dict[str, str]): (Deprecated in Ray
+            1.4.0, will be removed in Ray 1.5--please use the ``env_vars``
+            field of :ref:`runtime-environments` instead.) This specifies
             environment variables to override for the actor or task.  The
             overrides are propagated to all child actors and tasks.  This
             is a dictionary mapping variable names to their values.  Existing