Project fixes and cleanups (#5632)

doc/source/projects.rst

@@ -16,19 +16,23 @@ Quick start (CLI)
     # .rayproject subdirectory of the current directory.
     $ ray project create <project-name>
 
-    # Create a new session from the given project.
-    # Launch a cluster and run the appropriate command.
-    $ ray session start <command> [arguments]
+    # Create a new session from the given project.  Launch a cluster and run
+    # the command, which must be specified in the project.yaml file. If no
+    # command is specified, the "default" command in .rayproject/project.yaml
+    # will be used. Alternatively, use --shell to run a raw shell command.
+    $ ray session start <command-name> [arguments] [--shell]
 
     # Open a console for the given session.
     $ ray session attach
 
-    # Stop the given session and all of its worker nodes. The nodes/clusters
-    # are not actually terminated.
+    # Stop the given session and terminate all of its worker nodes.
     $ ray session stop
 
 Examples
 --------
+See `the readme <https://github.com/ray-project/ray/blob/master/python/ray/projects/examples/README.md>`__
+for instructions on how to run these examples:
+
 - `Open Tacotron <https://github.com/ray-project/ray/blob/master/python/ray/projects/examples/open-tacotron/.rayproject/project.yaml>`__:
   A TensorFlow implementation of Google's Tacotron speech synthesis with pre-trained model (unofficial)
 - `PyTorch Transformers <https://github.com/ray-project/ray/blob/master/python/ray/projects/examples/pytorch-transformers/.rayproject/project.yaml>`__:

python/ray/projects/examples/README.md

@@ -0,0 +1,41 @@
+Ray Projects
+============
+
+To run these example projects, we first have to make sure the full
+repository is checked out into the project directory.
+
+Open Tacotron
+-------------
+
+```shell
+cd open-tacotron
+# Check out the original repository
+git init
+git remote add origin https://github.com/keithito/tacotron.git
+git fetch
+git checkout -t origin/master
+
+# Serve the model
+ray session start serve
+
+# Terminate the session
+ray session stop
+```
+
+PyTorch Transformers
+--------------------
+
+```shell
+cd python-transformers
+# Check out the original repository
+git init
+git remote add origin https://github.com/huggingface/pytorch-transformers.git
+git fetch
+git checkout -t origin/master
+
+# Now we can start the training
+ray session start train --dataset SST-2
+
+# Terminate the session
+ray session stop
+```

python/ray/projects/examples/open-tacotron/.rayproject/project.yaml

@@ -7,7 +7,7 @@ repo: https://github.com/keithito/tacotron
 cluster: .rayproject/cluster.yaml
 
 environment:
-  requirements: requirements.txt
+  requirements: .rayproject/requirements.txt
 
   shell:
     - curl http://data.keithito.com/data/speech/tacotron-20180906.tar.gz | tar xzC /tmp

python/ray/projects/examples/pytorch-transformers/.rayproject/project.yaml

@@ -7,7 +7,7 @@ repo: https://github.com/huggingface/pytorch-transformers
 cluster: .rayproject/cluster.yaml
 
 environment:
-  requirements: requirements.txt
+  requirements: .rayproject/requirements.txt
 
 commands:
   - name: train

python/ray/projects/scripts.py

@@ -3,13 +3,14 @@ from __future__ import division
 from __future__ import print_function
 
 import argparse
-import logging
-import os
-import sys
-from shutil import copyfile
-import time
 import click
 import jsonschema
+import logging
+import os
+from shutil import copyfile
+import subprocess
+import sys
+import time
 
 import ray
 from ray.autoscaler.commands import (
@@ -20,7 +21,7 @@ from ray.autoscaler.commands import (
     teardown_cluster,
 )
 
-logging.basicConfig(format=ray.ray_constants.LOGGER_FORMAT)
+logging.basicConfig(format=ray.ray_constants.LOGGER_FORMAT, level=logging.INFO)
 logger = logging.getLogger(__file__)
 
 # File layout for generated project files
@@ -100,6 +101,14 @@ def create(project_name, cluster_yaml, requirements):
 
         requirements = REQUIREMENTS_TXT
 
+    repo = None
+    try:
+        repo = subprocess.check_output(
+            "git remote get-url origin".split(" ")).strip()
+        logger.info("Setting repo URL to %s", repo)
+    except subprocess.CalledProcessError:
+        pass
+
     with open(PROJECT_TEMPLATE) as f:
         project_template = f.read()
         # NOTE(simon):
@@ -109,7 +118,12 @@ def create(project_name, cluster_yaml, requirements):
                                                     cluster_yaml)
         project_template = project_template.replace(r"{{requirements}}",
                                                     requirements)
-
+        if repo is None:
+            project_template = project_template.replace(
+                r"{{repo_string}}", "# repo: {}".format("..."))
+        else:
+            project_template = project_template.replace(
+                r"{{repo_string}}", "repo: {}".format(repo))
     with open(PROJECT_YAML, "w") as f:
         f.write(project_template)
 
@@ -159,10 +173,18 @@ def stop():
     help="Start a session based on current project config")
 @click.argument("command", required=False)
 @click.argument("args", nargs=-1, type=click.UNPROCESSED)
-def start(command, args):
+@click.option(
+    "--shell",
+    help=(
+        "If set, run the command as a raw shell command instead of looking up "
+        "the command in the project config"),
+    is_flag=True)
+def start(command, args, shell):
     project_definition = load_project_or_throw()
 
-    if command:
+    if shell:
+        command_to_run = command
+    elif command:
         command_to_run = _get_command_to_run(command, project_definition, args)
     else:
         command_to_run = _get_command_to_run("default", project_definition,
@@ -192,22 +214,19 @@ def start(command, args):
         override_cluster_name=None,
     )
 
-    logger.info("[2/4] Syncing the repo")
-    if "repo" in project_definition:
-        # HACK: Skip git clone if exists so the this command can be idempotent
-        # More advanced repo update behavior can be found at
-        # https://github.com/jupyterhub/nbgitpuller/blob/master/nbgitpuller/pull.py
-        session_exec_cluster(
-            cluster_yaml,
-            "git clone {repo} {directory} || true".format(
-                repo=project_definition["repo"],
-                directory=project_definition["name"]),
-        )
-    else:
-        session_exec_cluster(
-            cluster_yaml,
-            "mkdir {directory} || true".format(
-                directory=project_definition["name"]))
+    logger.info("[2/4] Syncing the project")
+    project_root = ray.projects.find_root(os.getcwd())
+    # This is so that rsync syncs directly to the target directory, instead of
+    # nesting inside the target directory.
+    if not project_root.endswith("/"):
+        project_root += "/"
+    rsync(
+        cluster_yaml,
+        source=project_root,
+        target="~/{}/".format(working_directory),
+        override_cluster_name=None,
+        down=False,
+    )
 
     logger.info("[3/4] Setting up environment")
     _setup_environment(

python/ray/projects/templates/project_template.yaml

@@ -3,7 +3,8 @@
 name: {{name}}
 
 # description: A short description of the project.
-# repo: The URL of the repo this project is part of.
+# The URL of the repo this project is part of.
+{{repo_string}}
 
 cluster: {{cluster}}
 
@@ -16,5 +17,5 @@ environment:
     - echo "Setting up the environment"
 
 commands:
-  - name: first-command
+  - name: default
     command: echo "Starting ray job"

python/ray/tests/test_projects.py

@@ -118,7 +118,9 @@ def test_session_start_default_project():
 
     # Part 2/3: Rsync Calls
     rsync_call = mock_calls["rsync"]
-    assert rsync_call.call_count == 1
+    # 1 for rsyncing the project directory, 1 for rsyncing the
+    # requirements.txt.
+    assert rsync_call.call_count == 2
     _, kwargs = rsync_call.call_args
     assert kwargs["source"] == loaded_project["environment"]["requirements"]
 
@@ -141,16 +143,6 @@ def test_session_start_default_project():
             cmd for cmd in commands_executed if "pip install -r" not in cmd
         ]
 
-    # if we don't have a repo, we will be creating a directory
-    if "repo" not in loaded_project:
-        mkdir_command = "mkdir {project_name}".format(
-            project_name=loaded_project["name"])
-        assert any(mkdir_command in cmd for cmd in commands_executed)
-        # pop the `pip install` off commands executed
-        commands_executed = [
-            cmd for cmd in commands_executed if mkdir_command not in cmd
-        ]
-
     assert expected_commands == commands_executed
 
 
@@ -163,25 +155,6 @@ def test_session_start_docker_fail():
             "not implemented") in result.output
 
 
-def test_session_git_repo_cloned():
-    result, mock_calls, test_dir = run_test_project(
-        "session-tests/git-repo-pass", start, [])
-
-    loaded_project = ray.projects.load_project(test_dir)
-    assert result.exit_code == 0
-
-    exec_cluster_call = mock_calls["exec_cluster"]
-    commands_executed = []
-    for _, kwargs in exec_cluster_call.call_args_list:
-        command_executed = kwargs["cmd"]
-        # Filter out the cd call that was appended to each command
-        cd_project_dir_call = "cd {}; ".format(loaded_project["name"])
-        command_executed = command_executed.replace(cd_project_dir_call, "")
-        commands_executed.append(command_executed)
-
-    assert any("git clone" in cmd for cmd in commands_executed)
-
-
 def test_session_invalid_config_errored():
     result, _, _ = run_test_project("session-tests/invalid-config-fail", start,
                                     [])