From 6fdd2e6d91c49d068db474f9d84650df1165c781 Mon Sep 17 00:00:00 2001 From: Max Pumperla Date: Fri, 13 May 2022 16:27:31 +0200 Subject: [PATCH] fixes Signed-off-by: Max Pumperla --- doc/source/_static/css/custom.css | 12 ++ doc/source/_toc.yml | 112 ++++++++---------- doc/source/ray-air/package-ref.rst | 4 +- doc/source/ray-contribute/docs.ipynb | 4 +- .../ray-contribute/getting-involved.rst | 2 +- doc/source/ray-contribute/index.md | 33 ++++++ doc/source/ray-contribute/whitepaper.rst | 8 -- .../ray-core/objects/fault-tolerance.rst | 2 +- doc/source/ray-more-libs/index.rst | 12 +- doc/source/ray-observability/index.rst | 4 +- doc/source/ray-overview/index.md | 63 +++++++++- doc/source/ray-overview/learn-more.md | 54 --------- 12 files changed, 172 insertions(+), 138 deletions(-) create mode 100644 doc/source/ray-contribute/index.md delete mode 100644 doc/source/ray-contribute/whitepaper.rst delete mode 100644 doc/source/ray-overview/learn-more.md diff --git a/doc/source/_static/css/custom.css b/doc/source/_static/css/custom.css index ceec74ac3..6456f89e4 100644 --- a/doc/source/_static/css/custom.css +++ b/doc/source/_static/css/custom.css @@ -10,6 +10,18 @@ margin: auto; } +li.toctree-l1 { + font-weight: 600; +} + +li.toctree-l2 { + font-weight: 520; +} + +li.toctree-l3 { + font-weight: normal; +} + /* Hide confusing "<-" back arrow in navigation for larger displays */ @media (min-width: 768px) { #navbar-toggler { diff --git a/doc/source/_toc.yml b/doc/source/_toc.yml index 5ab6318b7..151a655db 100644 --- a/doc/source/_toc.yml +++ b/doc/source/_toc.yml @@ -1,14 +1,11 @@ format: jb-book root: index parts: - - caption: Overview + - caption: "" chapters: - file: ray-overview/index - file: ray-overview/installation - file: ray-overview/ray-libraries - - - caption: Ray ML - chapters: - file: data/dataset title: Ray Data sections: @@ -18,7 +15,6 @@ parts: - file: data/examples/big_data_ingestion - file: data/package-ref - file: data/integrations - - file: train/train title: Ray Train sections: @@ -27,7 +23,6 @@ parts: - file: train/faq - file: train/architecture - file: train/api - - file: tune/index title: Ray Tune sections: @@ -57,8 +52,7 @@ parts: title: "Scalability Benchmarks" - file: tune/examples/index - file: tune/faq - - file: tune/api_docs/overview.rst - + - file: tune/api_docs/overview - file: serve/index title: Ray Serve sections: @@ -79,7 +73,6 @@ parts: - file: serve/tutorials/index - file: serve/faq - file: serve/package-ref - - file: rllib/index title: Ray RLlib sections: @@ -96,20 +89,35 @@ parts: - file: rllib/rllib-dev - file: rllib/rllib-examples - file: rllib/package_ref/index - - - file: workflows/concepts - title: Ray Workflows + - file: ray-core/walkthrough + title: Ray Core sections: - - file: workflows/basics - - file: workflows/management - - file: workflows/actors - - file: workflows/metadata - - file: workflows/events - - file: workflows/comparison - - file: workflows/advanced - - file: workflows/package-ref + - file: ray-core/key-concepts + title: "Key Concepts" + - file: ray-core/user-guide + title: "User Guides" + - file: ray-core/examples/overview + title: "Examples" + sections: + - file: ray-core/examples/plot_example-a3c + - file: ray-core/examples/plot_example-lm + - file: ray-core/examples/plot_hyperparameter + - file: ray-core/examples/plot_lbfgs + - file: ray-core/examples/plot_parameter_server + - file: ray-core/examples/plot_pong_example + - file: ray-core/package-ref + + - file: cluster/quickstart + title: Ray Clusters + sections: + - file: cluster/user-guide + - file: cluster/cloud + - file: cluster/deploy + - file: cluster/api + - file: cluster/usage-stats + - file: ray-more-libs/index - title: More Ray ML Libraries + title: More Ray Libraries sections: - file: ray-air/getting-started sections: @@ -117,6 +125,17 @@ parts: - file: ray-air/deployment - file: ray-air/examples/index - file: ray-air/package-ref + - file: workflows/concepts + title: Ray Workflows + sections: + - file: workflows/basics + - file: workflows/management + - file: workflows/actors + - file: workflows/metadata + - file: workflows/events + - file: workflows/comparison + - file: workflows/advanced + - file: workflows/package-ref - file: ray-more-libs/joblib - file: ray-more-libs/lightgbm-ray - file: ray-more-libs/multiprocessing @@ -127,48 +146,19 @@ parts: - file: ray-core/examples/dask_xgboost/dask_xgboost - file: ray-core/examples/modin_xgboost/modin_xgboost - - caption: Ray Core - chapters: - - file: ray-core/walkthrough - title: Getting Started - - file: ray-core/key-concepts - title: "Key Concepts" - - file: ray-core/user-guide - title: "User Guides" - - file: ray-core/examples/overview - title: "Examples" - sections: - - file: ray-core/examples/plot_example-a3c - - file: ray-core/examples/plot_example-lm - - file: ray-core/examples/plot_hyperparameter - - file: ray-core/examples/plot_lbfgs - - file: ray-core/examples/plot_parameter_server - - file: ray-core/examples/plot_pong_example - - file: ray-core/package-ref - - - caption: Ray Clusters - chapters: - - file: cluster/quickstart - - file: cluster/user-guide - - file: cluster/cloud - - file: cluster/deploy - - file: cluster/api - - file: cluster/usage-stats - - - caption: References - chapters: - file: ray-references/api + title: API References - - caption: Developer Guides - chapters: - - file: ray-contribute/getting-involved + - file: ray-contribute/index + title: Developer Guides sections: - - file: ray-contribute/development - - file: ray-contribute/docs - - file: ray-contribute/fake-autoscaler - - file: ray-core/examples/testing-tips - - file: ray-core/configure - - file: ray-observability/index - - file: ray-contribute/whitepaper + - file: ray-contribute/getting-involved + sections: + - file: ray-contribute/development + - file: ray-contribute/docs + - file: ray-contribute/fake-autoscaler + - file: ray-core/examples/testing-tips + - file: ray-core/configure + - file: ray-observability/index # TODO: Add examples section diff --git a/doc/source/ray-air/package-ref.rst b/doc/source/ray-air/package-ref.rst index 5aeb0e507..e1051795a 100644 --- a/doc/source/ray-air/package-ref.rst +++ b/doc/source/ray-air/package-ref.rst @@ -1,7 +1,7 @@ .. _air-api-ref: -AIR API -======= +Ray AIR API +=========== .. contents:: :local: diff --git a/doc/source/ray-contribute/docs.ipynb b/doc/source/ray-contribute/docs.ipynb index e842fb542..daa7ef167 100644 --- a/doc/source/ray-contribute/docs.ipynb +++ b/doc/source/ray-contribute/docs.ipynb @@ -419,7 +419,7 @@ "## Where to go from here?\n", "\n", "There are many other ways to contribute to Ray other than documentation.\n", - "See {ref}`our contributor guide ` for more information." + "See {ref}`our contributor guide ` for more information." ] } ], @@ -432,4 +432,4 @@ }, "nbformat": 4, "nbformat_minor": 5 -} +} \ No newline at end of file diff --git a/doc/source/ray-contribute/getting-involved.rst b/doc/source/ray-contribute/getting-involved.rst index 83fe8ab54..4473a2789 100644 --- a/doc/source/ray-contribute/getting-involved.rst +++ b/doc/source/ray-contribute/getting-involved.rst @@ -1,4 +1,4 @@ -.. _getting-involved: +.. _getting-involved-ref: Getting Involved / Contributing =============================== diff --git a/doc/source/ray-contribute/index.md b/doc/source/ray-contribute/index.md new file mode 100644 index 000000000..4df954c98 --- /dev/null +++ b/doc/source/ray-contribute/index.md @@ -0,0 +1,33 @@ +# Developer Guides + +Learn more about how to contribute to Ray, get involved with our developer community, +how to configure Ray, learn about how to debug and profile Ray, and more. + + +```{eval-rst} +.. panels:: + :container: container pb-4 + :column: col-md-4 px-2 py-2 + :img-top-cls: pt-5 w-50 d-block mx-auto + + --- + :img-top: /images/ray_logo.png + .. link-button:: getting-involved-ref + :type: ref + :text: How to get involved and contribute to Ray? + :classes: btn-link btn-block stretched-link + + --- + :img-top: /images/ray_logo.png + .. link-button:: configuring-ray + :type: ref + :text: How to use and configure Ray? + :classes: btn-link btn-block stretched-link + + --- + :img-top: /images/ray_logo.png + .. link-button:: ray-observability + :type: ref + :text: How to use Ray's advanced observability features? + :classes: btn-link btn-block stretched-link +``` diff --git a/doc/source/ray-contribute/whitepaper.rst b/doc/source/ray-contribute/whitepaper.rst deleted file mode 100644 index e2d5fa50a..000000000 --- a/doc/source/ray-contribute/whitepaper.rst +++ /dev/null @@ -1,8 +0,0 @@ -.. _whitepaper: - -Architecture Whitepaper -======================= - -For an in-depth overview of Ray internals, check out the `Ray 1.0 Architecture whitepaper `__. - -For more about the scalability and performance of the Ray dataplane, see the `Exoshuffle paper `__. diff --git a/doc/source/ray-core/objects/fault-tolerance.rst b/doc/source/ray-core/objects/fault-tolerance.rst index 4c0cec654..5c9ff705d 100644 --- a/doc/source/ray-core/objects/fault-tolerance.rst +++ b/doc/source/ray-core/objects/fault-tolerance.rst @@ -10,4 +10,4 @@ Ray delegates the metadata tracking of an object to its *owner process*. Typical The owner of the object tracks the location and reference count for an object. If the owner process is unexpectedly killed, then the object cannot be recovered, even via lineage reconstruction. -For more information about how object ownership works, see the :ref:`Ray Architecture Whitepaper `. +For more information about how object ownership works, see the :ref:`Ray Architecture Whitepaper `. diff --git a/doc/source/ray-more-libs/index.rst b/doc/source/ray-more-libs/index.rst index 697e678c7..e01283f4c 100644 --- a/doc/source/ray-more-libs/index.rst +++ b/doc/source/ray-more-libs/index.rst @@ -1,12 +1,14 @@ -More Ray ML Libraries -===================== +More Ray Libraries +================== -.. TODO: we added the three Ray Core examples below, since they don't really belong there. - Going forward, make sure that all "Ray Lightning" and XGBoost topics are in one document or group, - and not next to each other. +.. TODO: we added the three Ray Core examples below, since they don't really belong + there. Going forward, make sure that all "Ray Lightning" and XGBoost topics are + in one document or group, and not next to each other. Ray has a variety of different extra integrations with ecosystem libraries. +- :ref:`air` +- :ref:`workflows` - :ref:`ray-joblib` - :ref:`lightgbm-ray` - :ref:`ray-multiprocessing` diff --git a/doc/source/ray-observability/index.rst b/doc/source/ray-observability/index.rst index 7c7081a17..a262d0519 100644 --- a/doc/source/ray-observability/index.rst +++ b/doc/source/ray-observability/index.rst @@ -1,5 +1,7 @@ +.. _ray-observability: + Observability -=============== +============= .. toctree:: :maxdepth: 1 diff --git a/doc/source/ray-overview/index.md b/doc/source/ray-overview/index.md index f04a1d8bb..a7ddb69e4 100644 --- a/doc/source/ray-overview/index.md +++ b/doc/source/ray-overview/index.md @@ -3,7 +3,7 @@ (gentle-intro)= -# Getting Started Guide +# Getting Started This tutorial will give you a quick tour of Ray's features. To get started, we'll start by installing Ray. @@ -516,9 +516,66 @@ ray submit cluster.yaml example.py --start ````` +(learn_more)= + +## Learn More + +Here are some talks, papers, and press coverage involving Ray and its libraries. +Please raise an issue if any of the below links are broken, or if you'd like to add your own talk! + + +### Blog and Press + +- [Modern Parallel and Distributed Python: A Quick Tutorial on Ray](https://towardsdatascience.com/modern-parallel-and-distributed-python-a-quick-tutorial-on-ray-99f8d70369b8) +- [Why Every Python Developer Will Love Ray](https://www.datanami.com/2019/11/05/why-every-python-developer-will-love-ray/) +- [Ray: A Distributed System for AI (BAIR)](http://bair.berkeley.edu/blog/2018/01/09/ray/) +- [10x Faster Parallel Python Without Python Multiprocessing](https://towardsdatascience.com/10x-faster-parallel-python-without-python-multiprocessing-e5017c93cce1) +- [Implementing A Parameter Server in 15 Lines of Python with Ray](https://ray-project.github.io/2018/07/15/parameter-server-in-fifteen-lines.html) +- [Ray Distributed AI Framework Curriculum](https://rise.cs.berkeley.edu/blog/ray-intel-curriculum/) +- [RayOnSpark: Running Emerging AI Applications on Big Data Clusters with Ray and Analytics Zoo](https://medium.com/riselab/rayonspark-running-emerging-ai-applications-on-big-data-clusters-with-ray-and-analytics-zoo-923e0136ed6a) +- [First user tips for Ray](https://rise.cs.berkeley.edu/blog/ray-tips-for-first-time-users/) +- [Tune: a Python library for fast hyperparameter tuning at any scale](https://towardsdatascience.com/fast-hyperparameter-tuning-at-scale-d428223b081c) +- [Cutting edge hyperparameter tuning with Ray Tune](https://medium.com/riselab/cutting-edge-hyperparameter-tuning-with-ray-tune-be6c0447afdf) +- [New Library Targets High Speed Reinforcement Learning](https://www.datanami.com/2018/02/01/rays-new-library-targets-high-speed-reinforcement-learning/) +- [Scaling Multi Agent Reinforcement Learning](http://bair.berkeley.edu/blog/2018/12/12/rllib/) +- [Functional RL with Keras and Tensorflow Eager](https://bair.berkeley.edu/blog/2019/10/14/functional-rl/) +- [How to Speed up Pandas by 4x with one line of code](https://www.kdnuggets.com/2019/11/speed-up-pandas-4x.html) +- [Quick Tip -- Speed up Pandas using Modin](https://pythondata.com/quick-tip-speed-up-pandas-using-modin/) +- [Ray Blog](https://medium.com/distributed-computing-with-ray) + + +### Talks (Videos) + +- [Unifying Large Scale Data Preprocessing and Machine Learning Pipelines with Ray Datasets \| PyData 2021](https://zoom.us/rec/share/0cjbk_YdCTbiTm7gNhzSeNxxTCCEy1pCDUkkjfBjtvOsKGA8XmDOx82jflHdQCUP.fsjQkj5PWSYplOTz?startTime=1635456658000) [(slides)](https://docs.google.com/presentation/d/19F_wxkpo1JAROPxULmJHYZd3sKryapkbMd0ib3ndMiU/edit?usp=sharing) +- [Programming at any Scale with Ray \| SF Python Meetup Sept 2019](https://www.youtube.com/watch?v=LfpHyIXBhlE) +- [Ray for Reinforcement Learning \| Data Council 2019](https://www.youtube.com/watch?v=Ayc0ca150HI) +- [Scaling Interactive Pandas Workflows with Modin](https://www.youtube.com/watch?v=-HjLd_3ahCw) +- [Ray: A Distributed Execution Framework for AI \| SciPy 2018](https://www.youtube.com/watch?v=D_oz7E4v-U0) +- [Ray: A Cluster Computing Engine for Reinforcement Learning Applications \| Spark Summit](https://www.youtube.com/watch?v=xadZRRB_TeI) +- [RLlib: Ray Reinforcement Learning Library \| RISECamp 2018](https://www.youtube.com/watch?v=eeRGORQthaQ) +- [Enabling Composition in Distributed Reinforcement Learning \| Spark Summit 2018](https://www.youtube.com/watch?v=jAEPqjkjth4) +- [Tune: Distributed Hyperparameter Search \| RISECamp 2018](https://www.youtube.com/watch?v=38Yd_dXW51Q) + + +### Slides + +- [Talk given at UC Berkeley DS100](https://docs.google.com/presentation/d/1sF5T_ePR9R6fAi2R6uxehHzXuieme63O2n_5i9m7mVE/edit?usp=sharing) +- [Talk given in October 2019](https://docs.google.com/presentation/d/13K0JsogYQX3gUCGhmQ1PQ8HILwEDFysnq0cI2b88XbU/edit?usp=sharing) +- [Talk given at RISECamp 2019](https://docs.google.com/presentation/d/1v3IldXWrFNMK-vuONlSdEuM82fuGTrNUDuwtfx4axsQ/edit?usp=sharing) + + +(papers)= + +### Papers + +- [Ray 1.0 Architecture whitepaper](https://docs.google.com/document/d/1lAy0Owi-vPz2jEqBSaHNQcy2IBSDEHyXNOQZlGuj93c/preview) +- [Exoshuffle: large-scale data shuffle in Ray](https://arxiv.org/abs/2203.05072) +- [RLlib paper](https://arxiv.org/abs/1712.09381) +- [RLlib flow paper](https://arxiv.org/abs/2011.12719) +- [Tune paper](https://arxiv.org/abs/1807.05118) +- [Ray paper (old)](https://arxiv.org/abs/1712.05889) +- [Ray HotOS paper (old)](https://arxiv.org/abs/1703.03924) -```{include} learn-more.md -``` ```{include} /_includes/overview/announcement_bottom.md ``` \ No newline at end of file diff --git a/doc/source/ray-overview/learn-more.md b/doc/source/ray-overview/learn-more.md deleted file mode 100644 index 0e6c74cf7..000000000 --- a/doc/source/ray-overview/learn-more.md +++ /dev/null @@ -1,54 +0,0 @@ -# Learn More - -Here are some talks, papers, and press coverage involving Ray and its libraries. -Please raise an issue if any of the below links are broken, or if you'd like to add your own talk! - -## Blog and Press - -- [Modern Parallel and Distributed Python: A Quick Tutorial on Ray](https://towardsdatascience.com/modern-parallel-and-distributed-python-a-quick-tutorial-on-ray-99f8d70369b8) -- [Why Every Python Developer Will Love Ray](https://www.datanami.com/2019/11/05/why-every-python-developer-will-love-ray/) -- [Ray: A Distributed System for AI (BAIR)](http://bair.berkeley.edu/blog/2018/01/09/ray/) -- [10x Faster Parallel Python Without Python Multiprocessing](https://towardsdatascience.com/10x-faster-parallel-python-without-python-multiprocessing-e5017c93cce1) -- [Implementing A Parameter Server in 15 Lines of Python with Ray](https://ray-project.github.io/2018/07/15/parameter-server-in-fifteen-lines.html) -- [Ray Distributed AI Framework Curriculum](https://rise.cs.berkeley.edu/blog/ray-intel-curriculum/) -- [RayOnSpark: Running Emerging AI Applications on Big Data Clusters with Ray and Analytics Zoo](https://medium.com/riselab/rayonspark-running-emerging-ai-applications-on-big-data-clusters-with-ray-and-analytics-zoo-923e0136ed6a) -- [First user tips for Ray](https://rise.cs.berkeley.edu/blog/ray-tips-for-first-time-users/) -- [Tune: a Python library for fast hyperparameter tuning at any scale](https://towardsdatascience.com/fast-hyperparameter-tuning-at-scale-d428223b081c) -- [Cutting edge hyperparameter tuning with Ray Tune](https://medium.com/riselab/cutting-edge-hyperparameter-tuning-with-ray-tune-be6c0447afdf) -- [New Library Targets High Speed Reinforcement Learning](https://www.datanami.com/2018/02/01/rays-new-library-targets-high-speed-reinforcement-learning/) -- [Scaling Multi Agent Reinforcement Learning](http://bair.berkeley.edu/blog/2018/12/12/rllib/) -- [Functional RL with Keras and Tensorflow Eager](https://bair.berkeley.edu/blog/2019/10/14/functional-rl/) -- [How to Speed up Pandas by 4x with one line of code](https://www.kdnuggets.com/2019/11/speed-up-pandas-4x.html) -- [Quick Tip -- Speed up Pandas using Modin](https://pythondata.com/quick-tip-speed-up-pandas-using-modin/) -- [Ray Blog](https://medium.com/distributed-computing-with-ray) - - -## Talks (Videos) - -- [Unifying Large Scale Data Preprocessing and Machine Learning Pipelines with Ray Datasets \| PyData 2021](https://zoom.us/rec/share/0cjbk_YdCTbiTm7gNhzSeNxxTCCEy1pCDUkkjfBjtvOsKGA8XmDOx82jflHdQCUP.fsjQkj5PWSYplOTz?startTime=1635456658000) [(slides)](https://docs.google.com/presentation/d/19F_wxkpo1JAROPxULmJHYZd3sKryapkbMd0ib3ndMiU/edit?usp=sharing) -- [Programming at any Scale with Ray \| SF Python Meetup Sept 2019](https://www.youtube.com/watch?v=LfpHyIXBhlE) -- [Ray for Reinforcement Learning \| Data Council 2019](https://www.youtube.com/watch?v=Ayc0ca150HI) -- [Scaling Interactive Pandas Workflows with Modin](https://www.youtube.com/watch?v=-HjLd_3ahCw) -- [Ray: A Distributed Execution Framework for AI \| SciPy 2018](https://www.youtube.com/watch?v=D_oz7E4v-U0) -- [Ray: A Cluster Computing Engine for Reinforcement Learning Applications \| Spark Summit](https://www.youtube.com/watch?v=xadZRRB_TeI) -- [RLlib: Ray Reinforcement Learning Library \| RISECamp 2018](https://www.youtube.com/watch?v=eeRGORQthaQ) -- [Enabling Composition in Distributed Reinforcement Learning \| Spark Summit 2018](https://www.youtube.com/watch?v=jAEPqjkjth4) -- [Tune: Distributed Hyperparameter Search \| RISECamp 2018](https://www.youtube.com/watch?v=38Yd_dXW51Q) - - -## Slides - -- [Talk given at UC Berkeley DS100](https://docs.google.com/presentation/d/1sF5T_ePR9R6fAi2R6uxehHzXuieme63O2n_5i9m7mVE/edit?usp=sharing) -- [Talk given in October 2019](https://docs.google.com/presentation/d/13K0JsogYQX3gUCGhmQ1PQ8HILwEDFysnq0cI2b88XbU/edit?usp=sharing) -- [Talk given at RISECamp 2019](https://docs.google.com/presentation/d/1v3IldXWrFNMK-vuONlSdEuM82fuGTrNUDuwtfx4axsQ/edit?usp=sharing) - - -## Papers - -- [Ray 1.0 Architecture whitepaper](https://docs.google.com/document/d/1lAy0Owi-vPz2jEqBSaHNQcy2IBSDEHyXNOQZlGuj93c/preview) -- [Exoshuffle: large-scale data shuffle in Ray](https://arxiv.org/abs/2203.05072) -- [RLlib paper](https://arxiv.org/abs/1712.09381) -- [RLlib flow paper](https://arxiv.org/abs/2011.12719) -- [Tune paper](https://arxiv.org/abs/1807.05118) -- [Ray paper (old)](https://arxiv.org/abs/1712.05889) -- [Ray HotOS paper (old)](https://arxiv.org/abs/1703.03924)