From 749c8f83b7b83ac5b25c52c335abe4cafaa15670 Mon Sep 17 00:00:00 2001 From: Jiajun Yao Date: Mon, 6 Feb 2023 10:53:37 -0800 Subject: [PATCH 01/55] Restructure core API docs Signed-off-by: Jiajun Yao --- doc/source/ray-core/api.rst | 6 +++--- doc/source/ray-core/api/core.rst | 9 +++++++++ 2 files changed, 12 insertions(+), 3 deletions(-) create mode 100644 doc/source/ray-core/api/core.rst diff --git a/doc/source/ray-core/api.rst b/doc/source/ray-core/api.rst index 0d86189932f9..67a4f9631dc1 100644 --- a/doc/source/ray-core/api.rst +++ b/doc/source/ray-core/api.rst @@ -1,10 +1,10 @@ -API References -============== +Ray Core API References +======================= .. toctree:: :maxdepth: 2 - :caption: Ray Core API References + api/core.rst package-ref.rst cli.rst ../ray-observability/state/cli.rst diff --git a/doc/source/ray-core/api/core.rst b/doc/source/ray-core/api/core.rst new file mode 100644 index 000000000000..b83577fb980e --- /dev/null +++ b/doc/source/ray-core/api/core.rst @@ -0,0 +1,9 @@ +Core API +======== + +.. autosummary:: + :toctree: doc/ + + ray.init + ray.remote + ray.shutdown \ No newline at end of file From 9cdfc3e436238f1848f2fa02e0488158b2f81986 Mon Sep 17 00:00:00 2001 From: Jiajun Yao Date: Mon, 6 Feb 2023 12:50:45 -0800 Subject: [PATCH 02/55] up Signed-off-by: Jiajun Yao --- doc/source/ray-core/api.rst | 4 ++-- doc/source/ray-core/examples/monte_carlo_pi.rst | 2 +- doc/source/ray-core/package-ref.rst | 7 ------- doc/source/ray-core/scheduling/resources.rst | 2 +- 4 files changed, 4 insertions(+), 11 deletions(-) diff --git a/doc/source/ray-core/api.rst b/doc/source/ray-core/api.rst index 67a4f9631dc1..f55164430049 100644 --- a/doc/source/ray-core/api.rst +++ b/doc/source/ray-core/api.rst @@ -1,5 +1,5 @@ -Ray Core API References -======================= +API References +============== .. toctree:: :maxdepth: 2 diff --git a/doc/source/ray-core/examples/monte_carlo_pi.rst b/doc/source/ray-core/examples/monte_carlo_pi.rst index faf1caa75139..11b84d8e39d7 100644 --- a/doc/source/ray-core/examples/monte_carlo_pi.rst +++ b/doc/source/ray-core/examples/monte_carlo_pi.rst @@ -19,7 +19,7 @@ To get started, install Ray via ``pip install -U ray``. See :ref:`Installing Ray Starting Ray ------------ -First, let's include all modules needed for this tutorial and start a local Ray cluster with :ref:`ray.init() `: +First, let's include all modules needed for this tutorial and start a local Ray cluster with :meth:`~ray.init`: .. literalinclude:: ../doc_code/monte_carlo_pi.py :language: python diff --git a/doc/source/ray-core/package-ref.rst b/doc/source/ray-core/package-ref.rst index 190d158b318d..6d7c53230773 100644 --- a/doc/source/ray-core/package-ref.rst +++ b/doc/source/ray-core/package-ref.rst @@ -4,13 +4,6 @@ Ray Core API Python API ---------- -.. _ray-init-ref: - -ray.init -~~~~~~~~ - -.. autofunction:: ray.init - .. _ray-is_initialized-ref: ray.is_initialized diff --git a/doc/source/ray-core/scheduling/resources.rst b/doc/source/ray-core/scheduling/resources.rst index 04e7b811be74..bbc3e80d7476 100644 --- a/doc/source/ray-core/scheduling/resources.rst +++ b/doc/source/ray-core/scheduling/resources.rst @@ -78,7 +78,7 @@ There are several ways to do that depending on how you start the Ray cluster: .. tabbed:: ray.init() - If you are using :ref:`ray.init() ` to start a single node Ray cluster, you can do the following to manually specify node resources: + If you are using :meth:`~ray.init` to start a single node Ray cluster, you can do the following to manually specify node resources: .. literalinclude:: ../doc_code/resources.py :language: python From 8e3191107c73d017711fe074914f2e9031be2a5d Mon Sep 17 00:00:00 2001 From: Jiajun Yao Date: Mon, 6 Feb 2023 13:15:06 -0800 Subject: [PATCH 03/55] up Signed-off-by: Jiajun Yao --- doc/source/ray-core/api/core.rst | 7 +++++-- .../ray-core/examples/monte_carlo_pi.rst | 2 +- doc/source/ray-core/package-ref.rst | 18 ------------------ doc/source/ray-core/scheduling/resources.rst | 2 +- 4 files changed, 7 insertions(+), 22 deletions(-) diff --git a/doc/source/ray-core/api/core.rst b/doc/source/ray-core/api/core.rst index b83577fb980e..2180194cc142 100644 --- a/doc/source/ray-core/api/core.rst +++ b/doc/source/ray-core/api/core.rst @@ -4,6 +4,9 @@ Core API .. autosummary:: :toctree: doc/ - ray.init ray.remote - ray.shutdown \ No newline at end of file + .. _ray-options-ref: + ray.remote_function.RemoteFunction.options + ray.actor.ActorClass.options + ray.init + ray.shutdown diff --git a/doc/source/ray-core/examples/monte_carlo_pi.rst b/doc/source/ray-core/examples/monte_carlo_pi.rst index 11b84d8e39d7..b76ee9817839 100644 --- a/doc/source/ray-core/examples/monte_carlo_pi.rst +++ b/doc/source/ray-core/examples/monte_carlo_pi.rst @@ -19,7 +19,7 @@ To get started, install Ray via ``pip install -U ray``. See :ref:`Installing Ray Starting Ray ------------ -First, let's include all modules needed for this tutorial and start a local Ray cluster with :meth:`~ray.init`: +First, let's include all modules needed for this tutorial and start a local Ray cluster with :meth:`ray.init() `: .. literalinclude:: ../doc_code/monte_carlo_pi.py :language: python diff --git a/doc/source/ray-core/package-ref.rst b/doc/source/ray-core/package-ref.rst index 6d7c53230773..bad9eb020e2f 100644 --- a/doc/source/ray-core/package-ref.rst +++ b/doc/source/ray-core/package-ref.rst @@ -13,17 +13,6 @@ ray.is_initialized .. _ray-remote-ref: -ray.remote -~~~~~~~~~~ - -.. autofunction:: ray.remote - -.. _ray-options-ref: - -.. autofunction:: ray.remote_function.RemoteFunction.options - -.. autofunction:: ray.actor.ActorClass.options - .. _scheduling-strategy-ref: .. autofunction:: ray.util.scheduling_strategies.PlacementGroupSchedulingStrategy @@ -80,13 +69,6 @@ ray.get_gpu_ids .. autofunction:: ray.get_gpu_ids -.. _ray-shutdown-ref: - -ray.shutdown -~~~~~~~~~~~~ - -.. autofunction:: ray.shutdown - .. _ray-method-ref: ray.method diff --git a/doc/source/ray-core/scheduling/resources.rst b/doc/source/ray-core/scheduling/resources.rst index bbc3e80d7476..8ca4a5f8adc0 100644 --- a/doc/source/ray-core/scheduling/resources.rst +++ b/doc/source/ray-core/scheduling/resources.rst @@ -78,7 +78,7 @@ There are several ways to do that depending on how you start the Ray cluster: .. tabbed:: ray.init() - If you are using :meth:`~ray.init` to start a single node Ray cluster, you can do the following to manually specify node resources: + If you are using :meth:`ray.init() ` to start a single node Ray cluster, you can do the following to manually specify node resources: .. literalinclude:: ../doc_code/resources.py :language: python From 9ad9adad9efea449d73fe36fe13b088136fc3d71 Mon Sep 17 00:00:00 2001 From: Jiajun Yao Date: Mon, 6 Feb 2023 14:01:14 -0800 Subject: [PATCH 04/55] up Signed-off-by: Jiajun Yao --- doc/source/ray-core/actors.rst | 2 +- doc/source/ray-core/api/core.rst | 7 ++- doc/source/ray-core/configure.rst | 10 ++--- .../ray-core/examples/monte_carlo_pi.rst | 8 ++-- .../ray-core/fault_tolerance/actors.rst | 2 +- doc/source/ray-core/fault_tolerance/tasks.rst | 2 +- doc/source/ray-core/objects.rst | 4 +- doc/source/ray-core/package-ref.rst | 44 ------------------- .../closure-capture-large-objects.rst | 4 +- .../ray-core/patterns/limit-pending-tasks.rst | 2 +- doc/source/ray-core/patterns/nested-tasks.rst | 2 +- .../patterns/pass-large-arg-by-value.rst | 2 +- doc/source/ray-core/patterns/pipelining.rst | 2 +- doc/source/ray-core/patterns/ray-get-loop.rst | 2 +- .../patterns/ray-get-submission-order.rst | 4 +- .../patterns/ray-get-too-many-objects.rst | 2 +- .../patterns/redefine-task-actor-loop.rst | 2 +- .../ray-core/patterns/return-ray-put.rst | 2 +- .../ray-core/patterns/unnecessary-ray-get.rst | 2 +- doc/source/ray-core/scheduling/index.rst | 2 +- doc/source/ray-core/scheduling/resources.rst | 3 +- doc/source/ray-core/tasks.rst | 6 +-- .../ray-core/tasks/using-ray-with-gpus.rst | 2 +- doc/source/ray-more-libs/joblib.rst | 4 +- 24 files changed, 42 insertions(+), 80 deletions(-) diff --git a/doc/source/ray-core/actors.rst b/doc/source/ray-core/actors.rst index 0b6d4c1c5f0c..83c87c7cc9e2 100644 --- a/doc/source/ray-core/actors.rst +++ b/doc/source/ray-core/actors.rst @@ -337,7 +337,7 @@ By default, Ray actors won't be :ref:`restarted ` and actor tasks won't be retried when actors crash unexpectedly. You can change this behavior by setting ``max_restarts`` and ``max_task_retries`` options -in :ref:`ray.remote() ` and :ref:`.options() `. +in :meth:`ray.remote() ` and :meth:`.options() `. See :ref:`Ray fault tolerance ` for more details. FAQ: Actors, Workers and Resources diff --git a/doc/source/ray-core/api/core.rst b/doc/source/ray-core/api/core.rst index 2180194cc142..b053684ab0ae 100644 --- a/doc/source/ray-core/api/core.rst +++ b/doc/source/ray-core/api/core.rst @@ -5,8 +5,13 @@ Core API :toctree: doc/ ray.remote - .. _ray-options-ref: ray.remote_function.RemoteFunction.options ray.actor.ActorClass.options + ray.method + ray.get + ray.wait + ray.put + ray.kill + ray.cancel ray.init ray.shutdown diff --git a/doc/source/ray-core/configure.rst b/doc/source/ray-core/configure.rst index 23eafaf5c88d..3287ca3a2903 100644 --- a/doc/source/ray-core/configure.rst +++ b/doc/source/ray-core/configure.rst @@ -57,12 +57,12 @@ If using the command line, connect to the Ray cluster as follow: .. _omp-num-thread-note: .. note:: - Ray sets the environment variable ``OMP_NUM_THREADS=`` if ``num_cpus`` is set on - the task/actor via :ref:`ray.remote() ` and :ref:`.options() `. - Ray sets ``OMP_NUM_THREADS=1`` if ``num_cpus`` is not specified; this + Ray sets the environment variable ``OMP_NUM_THREADS=`` if ``num_cpus`` is set on + the task/actor via :meth:`ray.remote() ` and :meth:`task.options() `/:meth:`actor.options() `. + Ray sets ``OMP_NUM_THREADS=1`` if ``num_cpus`` is not specified; this is done to avoid performance degradation with many workers (issue #6998). You can - also override this by explicitly setting ``OMP_NUM_THREADS`` to override anything Ray sets by default. - ``OMP_NUM_THREADS`` is commonly used in numpy, PyTorch, and Tensorflow to perform multi-threaded + also override this by explicitly setting ``OMP_NUM_THREADS`` to override anything Ray sets by default. + ``OMP_NUM_THREADS`` is commonly used in numpy, PyTorch, and Tensorflow to perform multi-threaded linear algebra. In multi-worker setting, we want one thread per worker instead of many threads per worker to avoid contention. Some other libraries may have their own way to configure parallelism. For example, if you're using OpenCV, you should manually set the number of diff --git a/doc/source/ray-core/examples/monte_carlo_pi.rst b/doc/source/ray-core/examples/monte_carlo_pi.rst index b76ee9817839..1aff1234e343 100644 --- a/doc/source/ray-core/examples/monte_carlo_pi.rst +++ b/doc/source/ray-core/examples/monte_carlo_pi.rst @@ -41,7 +41,7 @@ Ray actors are essentially stateful services that anyone with an instance (a han :start-after: __defining_actor_start__ :end-before: __defining_actor_end__ -We define a Ray actor by decorating a normal Python class with :ref:`ray.remote `. +We define a Ray actor by decorating a normal Python class with :meth:`ray.remote `. The progress actor has ``report_progress()`` method that will be called by sampling tasks to update their progress individually and ``get_progress()`` method to get the overall progress. @@ -55,7 +55,7 @@ Ray tasks are stateless functions. They execute asynchronously, and run in paral :start-after: __defining_task_start__ :end-before: __defining_task_end__ -To convert a normal Python function as a Ray task, we decorate the function with :ref:`ray.remote `. +To convert a normal Python function as a Ray task, we decorate the function with :meth:`ray.remote `. The sampling task takes a progress actor handle as an input and reports progress to it. The above code shows an example of calling actor methods from tasks. @@ -97,7 +97,7 @@ While sampling tasks are running, we can periodically query the progress by call To call an actor method, use ``actor_handle.method.remote()``. This invocation immediately returns an ``ObjectRef`` as a future and then executes the method asynchronously on the remote actor process. -To fetch the actual returned value of ``ObjectRef``, we use the blocking :ref:`ray.get() `. +To fetch the actual returned value of ``ObjectRef``, we use the blocking :meth:`ray.get() `. Calculating π ------------- @@ -108,7 +108,7 @@ Finally, we get number of samples inside the circle from the remote sampling tas :start-after: __calculating_pi_start__ :end-before: __calculating_pi_end__ -As we can see from the above code, besides a single ``ObjectRef``, :ref:`ray.get() ` can also take a list of ``ObjectRef`` and return a list of results. +As we can see from the above code, besides a single ``ObjectRef``, :meth:`ray.get() ` can also take a list of ``ObjectRef`` and return a list of results. If you run this tutorial, you will see output like: diff --git a/doc/source/ray-core/fault_tolerance/actors.rst b/doc/source/ray-core/fault_tolerance/actors.rst index d1ea3d67e524..4184f2399e34 100644 --- a/doc/source/ray-core/fault_tolerance/actors.rst +++ b/doc/source/ray-core/fault_tolerance/actors.rst @@ -24,7 +24,7 @@ After the specified number of restarts, subsequent actor methods will raise a ``RayActorError``. By default, actor tasks execute with at-most-once semantics -(``max_task_retries=0`` in the ``@ray.remote`` :ref:`decorator `). This means that if an +(``max_task_retries=0`` in the ``@ray.remote`` :meth:`decorator `). This means that if an actor task is submitted to an actor that is unreachable, Ray will report the error with ``RayActorError``, a Python-level exception that is thrown when ``ray.get`` is called on the future returned by the task. Note that this diff --git a/doc/source/ray-core/fault_tolerance/tasks.rst b/doc/source/ray-core/fault_tolerance/tasks.rst index 69652b1852e8..a7e67aacb59d 100644 --- a/doc/source/ray-core/fault_tolerance/tasks.rst +++ b/doc/source/ray-core/fault_tolerance/tasks.rst @@ -71,7 +71,7 @@ If a task is hanging, you may want to cancel the task to continue to make progress. You can do this by calling ``ray.cancel`` on an ``ObjectRef`` returned by the task. By default, this will send a KeyboardInterrupt to the task's worker if it is mid-execution. Passing ``force=True`` to ``ray.cancel`` -will force-exit the worker. See :ref:`the API reference ` for +will force-exit the worker. See :meth:`the API reference ` for ``ray.cancel`` for more details. Note that currently, Ray will not automatically retry tasks that have been diff --git a/doc/source/ray-core/objects.rst b/doc/source/ray-core/objects.rst index 7c7f0b69eb56..e90d0d9d4e99 100644 --- a/doc/source/ray-core/objects.rst +++ b/doc/source/ray-core/objects.rst @@ -12,7 +12,7 @@ similar. Object refs can be created in two ways. 1. They are returned by remote function calls. - 2. They are returned by ``put`` (:ref:`docstring `). + 2. They are returned by :meth:`ray.put() `. .. tabbed:: Python @@ -48,7 +48,7 @@ Object refs can be created in two ways. Fetching Object Data -------------------- -You can use the ``get`` method (:ref:`docstring `) to fetch the result of a remote object from an object ref. +You can use the :meth:`ray.get() ` method to fetch the result of a remote object from an object ref. If the current node's object store does not contain the object, the object is downloaded. .. tabbed:: Python diff --git a/doc/source/ray-core/package-ref.rst b/doc/source/ray-core/package-ref.rst index bad9eb020e2f..5438765c6dd7 100644 --- a/doc/source/ray-core/package-ref.rst +++ b/doc/source/ray-core/package-ref.rst @@ -11,49 +11,12 @@ ray.is_initialized .. autofunction:: ray.is_initialized -.. _ray-remote-ref: - .. _scheduling-strategy-ref: .. autofunction:: ray.util.scheduling_strategies.PlacementGroupSchedulingStrategy .. autofunction:: ray.util.scheduling_strategies.NodeAffinitySchedulingStrategy -.. _ray-get-ref: - -ray.get -~~~~~~~ - -.. autofunction:: ray.get - -.. _ray-wait-ref: - -ray.wait -~~~~~~~~ - -.. autofunction:: ray.wait - -.. _ray-put-ref: - -ray.put -~~~~~~~ - -.. autofunction:: ray.put - -.. _ray-kill-ref: - -ray.kill -~~~~~~~~ - -.. autofunction:: ray.kill - -.. _ray-cancel-ref: - -ray.cancel -~~~~~~~~~~ - -.. autofunction:: ray.cancel - .. _ray-get_actor-ref: @@ -69,13 +32,6 @@ ray.get_gpu_ids .. autofunction:: ray.get_gpu_ids -.. _ray-method-ref: - -ray.method -~~~~~~~~~~ - -.. autofunction:: ray.method - .. _ray-actor-pool-ref: ray.util.ActorPool diff --git a/doc/source/ray-core/patterns/closure-capture-large-objects.rst b/doc/source/ray-core/patterns/closure-capture-large-objects.rst index b5cace64173c..fceea4382451 100644 --- a/doc/source/ray-core/patterns/closure-capture-large-objects.rst +++ b/doc/source/ray-core/patterns/closure-capture-large-objects.rst @@ -3,13 +3,13 @@ Anti-pattern: Closure capturing large objects harms performance **TLDR:** Avoid closure capturing large objects in remote functions or classes, use object store instead. -When you define a :ref:`ray.remote ` function or class, +When you define a :meth:`ray.remote ` function or class, it is easy to accidentally capture large (more than a few MB) objects implicitly in the definition. This can lead to slow performance or even OOM since Ray is not designed to handle serialized functions or classes that are very large. For such large objects, there are two options to resolve this problem: -- Use :ref:`ray.put() ` to put the large objects in the Ray object store, and then pass object references as arguments to the remote functions or classes (*"better approach #1"* below) +- Use :meth:`ray.put() ` to put the large objects in the Ray object store, and then pass object references as arguments to the remote functions or classes (*"better approach #1"* below) - Create the large objects inside the remote functions or classes by passing a lambda method (*"better approach #2"*). This is also the only option for using unserializable objects. diff --git a/doc/source/ray-core/patterns/limit-pending-tasks.rst b/doc/source/ray-core/patterns/limit-pending-tasks.rst index fd5ccddbc42d..39e4a4450a2e 100644 --- a/doc/source/ray-core/patterns/limit-pending-tasks.rst +++ b/doc/source/ray-core/patterns/limit-pending-tasks.rst @@ -3,7 +3,7 @@ Pattern: Using ray.wait to limit the number of pending tasks ============================================================ -In this pattern, we use :ref:`ray.wait() ` to limit the number of pending tasks. +In this pattern, we use :meth:`ray.wait() ` to limit the number of pending tasks. If we continuously submit tasks faster than their process time, we will accumulate tasks in the pending task queue, which can eventually cause OOM. With ``ray.wait()``, we can apply backpressure and limit the number of pending tasks so that the pending task queue won't grow indefinitely and cause OOM. diff --git a/doc/source/ray-core/patterns/nested-tasks.rst b/doc/source/ray-core/patterns/nested-tasks.rst index 170b8f63525d..3b0db17703ec 100644 --- a/doc/source/ray-core/patterns/nested-tasks.rst +++ b/doc/source/ray-core/patterns/nested-tasks.rst @@ -28,7 +28,7 @@ Code example :start-after: __pattern_start__ :end-before: __pattern_end__ -We call :ref:`ray.get() ` after both ``quick_sort_distributed`` function invocations take place. +We call :meth:`ray.get() ` after both ``quick_sort_distributed`` function invocations take place. This allows you to maximize parallelism in the workload. See :doc:`ray-get-loop` for more details. Notice in the execution times above that with smaller tasks, the non-distributed version is faster. However, as the task execution diff --git a/doc/source/ray-core/patterns/pass-large-arg-by-value.rst b/doc/source/ray-core/patterns/pass-large-arg-by-value.rst index 37a6816f2245..58cfe5e3178c 100644 --- a/doc/source/ray-core/patterns/pass-large-arg-by-value.rst +++ b/doc/source/ray-core/patterns/pass-large-arg-by-value.rst @@ -3,7 +3,7 @@ Anti-pattern: Passing the same large argument by value repeatedly harms performance =================================================================================== -**TLDR:** Avoid passing the same large argument by value to multiple tasks, use :ref:`ray.put() ` and pass by reference instead. +**TLDR:** Avoid passing the same large argument by value to multiple tasks, use :meth:`ray.put() ` and pass by reference instead. When passing a large argument (>100KB) by value to a task, Ray will implicitly store the argument in the object store and the worker process will fetch the argument to the local object store from the caller's object store before running the task. diff --git a/doc/source/ray-core/patterns/pipelining.rst b/doc/source/ray-core/patterns/pipelining.rst index 42ad5e5320c2..f4d8393abcaa 100644 --- a/doc/source/ray-core/patterns/pipelining.rst +++ b/doc/source/ray-core/patterns/pipelining.rst @@ -23,5 +23,5 @@ Code example .. literalinclude:: ../doc_code/pattern_pipelining.py In the example above, a worker actor pulls work off of a queue and then does some computation on it. -Without pipelining, we call :ref:`ray.get() ` immediately after requesting a work item, so we block while that RPC is in flight, causing idle CPU time. +Without pipelining, we call :meth:`ray.get() ` immediately after requesting a work item, so we block while that RPC is in flight, causing idle CPU time. With pipelining, we instead preemptively request the next work item before processing the current one, so we can use the CPU while the RPC is in flight which increases the CPU utilization. diff --git a/doc/source/ray-core/patterns/ray-get-loop.rst b/doc/source/ray-core/patterns/ray-get-loop.rst index 5e75191a5e1f..54c6b5c9de6a 100644 --- a/doc/source/ray-core/patterns/ray-get-loop.rst +++ b/doc/source/ray-core/patterns/ray-get-loop.rst @@ -3,7 +3,7 @@ Anti-pattern: Calling ray.get in a loop harms parallelism ========================================================= -**TLDR:** Avoid calling :ref:`ray.get() ` in a loop since it's a blocking call; use ``ray.get()`` only for the final result. +**TLDR:** Avoid calling :meth:`ray.get() ` in a loop since it's a blocking call; use ``ray.get()`` only for the final result. A call to ``ray.get()`` fetches the results of remotely executed functions. However, it is a blocking call, which means that it always waits until the requested result is available. If you call ``ray.get()`` in a loop, the loop will not continue to run until the call to ``ray.get()`` is resolved. diff --git a/doc/source/ray-core/patterns/ray-get-submission-order.rst b/doc/source/ray-core/patterns/ray-get-submission-order.rst index 9a4d57b32537..56ab0f1bc3dc 100644 --- a/doc/source/ray-core/patterns/ray-get-submission-order.rst +++ b/doc/source/ray-core/patterns/ray-get-submission-order.rst @@ -1,12 +1,12 @@ Anti-pattern: Processing results in submission order using ray.get increases runtime ==================================================================================== -**TLDR:** Avoid processing independent results in submission order using :ref:`ray.get() ` since results may be ready in a different order than the submission order. +**TLDR:** Avoid processing independent results in submission order using :meth:`ray.get() ` since results may be ready in a different order than the submission order. A batch of tasks is submitted, and we need to process their results individually once they’re done. If each task takes a different amount of time to finish and we process results in submission order, we may waste time waiting for all of the slower (straggler) tasks that were submitted earlier to finish while later faster tasks have already finished. -Instead, we want to process the tasks in the order that they finish using :ref:`ray.wait() ` to speed up total time to completion. +Instead, we want to process the tasks in the order that they finish using :meth:`ray.wait() ` to speed up total time to completion. .. figure:: ../images/ray-get-submission-order.svg diff --git a/doc/source/ray-core/patterns/ray-get-too-many-objects.rst b/doc/source/ray-core/patterns/ray-get-too-many-objects.rst index bd03de0962a3..b0b8504e044e 100644 --- a/doc/source/ray-core/patterns/ray-get-too-many-objects.rst +++ b/doc/source/ray-core/patterns/ray-get-too-many-objects.rst @@ -1,7 +1,7 @@ Anti-pattern: Fetching too many objects at once with ray.get causes failure =========================================================================== -**TLDR:** Avoid calling :ref:`ray.get() ` on too many objects since this will lead to heap out-of-memory or object store out-of-space. Instead fetch and process one batch at a time. +**TLDR:** Avoid calling :meth:`ray.get() ` on too many objects since this will lead to heap out-of-memory or object store out-of-space. Instead fetch and process one batch at a time. If you have a large number of tasks that you want to run in parallel, trying to do ``ray.get()`` on all of them at once could lead to failure with heap out-of-memory or object store out-of-space since Ray needs to fetch all the objects to the caller at the same time. Instead you should get and process the results one batch at a time. Once a batch is processed, Ray will evict objects in that batch to make space for future batches. diff --git a/doc/source/ray-core/patterns/redefine-task-actor-loop.rst b/doc/source/ray-core/patterns/redefine-task-actor-loop.rst index 939b16dc585f..3903b20c06b4 100644 --- a/doc/source/ray-core/patterns/redefine-task-actor-loop.rst +++ b/doc/source/ray-core/patterns/redefine-task-actor-loop.rst @@ -3,7 +3,7 @@ Anti-pattern: Redefining the same remote function or class harms performance **TLDR:** Avoid redefining the same remote function or class. -Decorating the same function or class multiple times using the :ref:`ray.remote ` decorator leads to slow performance in Ray. +Decorating the same function or class multiple times using the :meth:`ray.remote ` decorator leads to slow performance in Ray. For each Ray remote function or class, Ray will pickle it and upload to GCS. Later on, the worker that runs the task or actor will download and unpickle it. Each decoration of the same function or class generates a new remote function or class from Ray's perspective. diff --git a/doc/source/ray-core/patterns/return-ray-put.rst b/doc/source/ray-core/patterns/return-ray-put.rst index 79dfd4035c36..1df727e583f4 100644 --- a/doc/source/ray-core/patterns/return-ray-put.rst +++ b/doc/source/ray-core/patterns/return-ray-put.rst @@ -1,7 +1,7 @@ Anti-pattern: Returning ray.put() ObjectRefs from a task harms performance and fault tolerance ============================================================================================== -**TLDR:** Avoid calling :ref:`ray.put() ` on task return values and returning the resulting ObjectRefs. +**TLDR:** Avoid calling :meth:`ray.put() ` on task return values and returning the resulting ObjectRefs. Instead, return these values directly if possible. Returning ray.put() ObjectRefs are considered anti-patterns for the following reasons: diff --git a/doc/source/ray-core/patterns/unnecessary-ray-get.rst b/doc/source/ray-core/patterns/unnecessary-ray-get.rst index 121cecabd297..334f69e10648 100644 --- a/doc/source/ray-core/patterns/unnecessary-ray-get.rst +++ b/doc/source/ray-core/patterns/unnecessary-ray-get.rst @@ -3,7 +3,7 @@ Anti-pattern: Calling ray.get unnecessarily harms performance ============================================================= -**TLDR:** Avoid calling :ref:`ray.get() ` unnecessarily for intermediate steps. Work with object references directly, and only call ``ray.get()`` at the end to get the final result. +**TLDR:** Avoid calling :meth:`ray.get() ` unnecessarily for intermediate steps. Work with object references directly, and only call ``ray.get()`` at the end to get the final result. When ``ray.get()`` is called, objects must be transferred to the worker/node that calls ``ray.get()``. If you don't need to manipulate the object, you probably don't need to call ``ray.get()`` on it! diff --git a/doc/source/ray-core/scheduling/index.rst b/doc/source/ray-core/scheduling/index.rst index 560a2aa2cc05..6da36cdd15de 100644 --- a/doc/source/ray-core/scheduling/index.rst +++ b/doc/source/ray-core/scheduling/index.rst @@ -31,7 +31,7 @@ If all nodes are infeasible, the task or actor cannot be scheduled until feasibl Scheduling Strategies --------------------- -Tasks or actors support a :ref:`scheduling_strategy ` option to specify the strategy used to decide the best node among feasible nodes. +Tasks or actors support a :meth:`scheduling_strategy ` option to specify the strategy used to decide the best node among feasible nodes. Currently the supported strategies are the followings. "DEFAULT" diff --git a/doc/source/ray-core/scheduling/resources.rst b/doc/source/ray-core/scheduling/resources.rst index 8ca4a5f8adc0..42f239706b23 100644 --- a/doc/source/ray-core/scheduling/resources.rst +++ b/doc/source/ray-core/scheduling/resources.rst @@ -136,7 +136,8 @@ The default resource requirements for actors was chosen for historical reasons. It's recommended to always explicitly set ``num_cpus`` for actors to avoid any surprises. If resources are specified explicitly, they are required for both scheduling and running.) -You can also explicitly specify a task's or actor's resource requirements (for example, one task may require a GPU) instead of using default ones via :ref:`ray.remote() ` and :ref:`.options() `. +You can also explicitly specify a task's or actor's resource requirements (for example, one task may require a GPU) instead of using default ones via :meth:`ray.remote() ` +and :meth:`task.options() `/:meth:`actor.options() `. .. tabbed:: Python diff --git a/doc/source/ray-core/tasks.rst b/doc/source/ray-core/tasks.rst index f3e65113ca0c..292e270b2c8d 100644 --- a/doc/source/ray-core/tasks.rst +++ b/doc/source/ray-core/tasks.rst @@ -164,7 +164,7 @@ Waiting for Partial Results --------------------------- Calling **ray.get** on Ray task results will block until the task finished execution. After launching a number of tasks, you may want to know which ones have -finished executing without blocking on all of them. This could be achieved by (:ref:`ray-wait-ref`). The function +finished executing without blocking on all of them. This could be achieved by :meth:`ray.wait() `. The function works as follows. .. tabbed:: Python @@ -215,7 +215,7 @@ For tasks that return multiple objects, Ray also supports remote generators that Cancelling tasks ---------------- -Ray tasks can be canceled by calling ``ray.cancel`` (:ref:`docstring `) on the returned Object ref. +Ray tasks can be canceled by calling :meth:`ray.cancel() ` on the returned Object ref. .. tabbed:: Python @@ -242,7 +242,7 @@ By default, Ray will :ref:`retry ` failed tasks due to system failures and specified application-level failures. You can change this behavior by setting ``max_retries`` and ``retry_exceptions`` options -in :ref:`ray.remote() ` and :ref:`.options() `. +in :meth:`ray.remote() ` and :meth:`.options() `. See :ref:`Ray fault tolerance ` for more details. diff --git a/doc/source/ray-core/tasks/using-ray-with-gpus.rst b/doc/source/ray-core/tasks/using-ray-with-gpus.rst index d6c74be1e113..0ba3fe2fba4c 100644 --- a/doc/source/ray-core/tasks/using-ray-with-gpus.rst +++ b/doc/source/ray-core/tasks/using-ray-with-gpus.rst @@ -99,7 +99,7 @@ task tries to use the same GPU. To address the problem, Ray disables the worker process reuse between GPU tasks by default, where the GPU resources is released after the task process exits. Since this adds overhead to GPU task scheduling, you can re-enable worker reuse by setting ``max_calls=0`` -in the :ref:`ray.remote ` decorator. +in the :meth:`ray.remote ` decorator. .. literalinclude:: ../doc_code/gpus.py :language: python diff --git a/doc/source/ray-more-libs/joblib.rst b/doc/source/ray-more-libs/joblib.rst index d55efbf21f8d..5eeff08fda62 100644 --- a/doc/source/ray-more-libs/joblib.rst +++ b/doc/source/ray-more-libs/joblib.rst @@ -51,8 +51,8 @@ a multi-node Ray cluster instead. with joblib.parallel_backend('ray'): search.fit(digits.data, digits.target) -You can also set the ``ray_remote_args`` argument in ``parallel_backend`` to :ref:`configure -the Ray Actors ` making up the Pool. This can be used to eg. :ref:`assign resources +You can also set the ``ray_remote_args`` argument in ``parallel_backend`` to :meth:`configure +the Ray Actors ` making up the Pool. This can be used to eg. :ref:`assign resources to Actors, such as GPUs `. .. code-block:: python From b81b9aa955e39996ed1da69c11c68a42c7534af9 Mon Sep 17 00:00:00 2001 From: Jiajun Yao Date: Mon, 6 Feb 2023 14:22:25 -0800 Subject: [PATCH 05/55] up Signed-off-by: Jiajun Yao --- doc/source/ray-core/api/exceptions.rst | 9 +++ doc/source/ray-core/api/runtime-env.rst | 8 +++ doc/source/ray-core/api/scheduling.rst | 25 ++++++++ doc/source/ray-core/package-ref.rst | 59 ------------------- .../ray-core/scheduling/placement-group.rst | 14 ++--- doc/source/ray-observability/overview.rst | 12 ++-- 6 files changed, 55 insertions(+), 72 deletions(-) create mode 100644 doc/source/ray-core/api/exceptions.rst create mode 100644 doc/source/ray-core/api/runtime-env.rst create mode 100644 doc/source/ray-core/api/scheduling.rst diff --git a/doc/source/ray-core/api/exceptions.rst b/doc/source/ray-core/api/exceptions.rst new file mode 100644 index 000000000000..37a53a1d34dc --- /dev/null +++ b/doc/source/ray-core/api/exceptions.rst @@ -0,0 +1,9 @@ +Exceptions +========== + +.. autosummary:: + :toctree: doc/ + + ray.exceptions.RayError + ray.exceptions.RayTaskError + ray.exceptions.RayActorError diff --git a/doc/source/ray-core/api/runtime-env.rst b/doc/source/ray-core/api/runtime-env.rst new file mode 100644 index 000000000000..607766a9535a --- /dev/null +++ b/doc/source/ray-core/api/runtime-env.rst @@ -0,0 +1,8 @@ +Runtime Env API +=============== + +.. autosummary:: + :toctree: doc/ + + ray.runtime_env.RuntimeEnvConfig + ray.runtime_env.RuntimeEnv diff --git a/doc/source/ray-core/api/scheduling.rst b/doc/source/ray-core/api/scheduling.rst new file mode 100644 index 000000000000..2971aae8e3c8 --- /dev/null +++ b/doc/source/ray-core/api/scheduling.rst @@ -0,0 +1,25 @@ +Scheduling API +============== + +Scheduling Strategy +------------------- + +.. autosummary:: + :toctree: doc/ + + ray.util.scheduling_strategies.PlacementGroupSchedulingStrategy + ray.util.scheduling_strategies.NodeAffinitySchedulingStrategy + +.. _ray-placement-group-ref: + +Placement Group +--------------- + +.. autosummary:: + :toctree: doc/ + + ray.util.placement_group.placement_group + ray.util.placement_group.PlacementGroup + ray.util.placement_group.placement_group_table + ray.util.placement_group.remove_placement_group + ray.util.placement_group.get_current_placement_group diff --git a/doc/source/ray-core/package-ref.rst b/doc/source/ray-core/package-ref.rst index 5438765c6dd7..01a35fcdbfdf 100644 --- a/doc/source/ray-core/package-ref.rst +++ b/doc/source/ray-core/package-ref.rst @@ -11,12 +11,6 @@ ray.is_initialized .. autofunction:: ray.is_initialized -.. _scheduling-strategy-ref: - -.. autofunction:: ray.util.scheduling_strategies.PlacementGroupSchedulingStrategy - -.. autofunction:: ray.util.scheduling_strategies.NodeAffinitySchedulingStrategy - .. _ray-get_actor-ref: @@ -83,39 +77,6 @@ ray.cross_language .. autofunction:: ray.cross_language.java_actor_class -.. _ray-placement-group-ref: - -Placement Group APIs --------------------- - -placement_group -~~~~~~~~~~~~~~~ - -.. autofunction:: ray.util.placement_group.placement_group - - -PlacementGroup (class) -~~~~~~~~~~~~~~~~~~~~~~ - -.. autoclass:: ray.util.placement_group.PlacementGroup - :members: - -placement_group_table -~~~~~~~~~~~~~~~~~~~~~ - -.. autofunction:: ray.util.placement_group.placement_group_table - - -remove_placement_group -~~~~~~~~~~~~~~~~~~~~~~ - -.. autofunction:: ray.util.placement_group.remove_placement_group - -get_current_placement_group -~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -.. autofunction:: ray.util.placement_group.get_current_placement_group - .. _custom-metric-api-ref: Custom Metrics APIs @@ -149,17 +110,6 @@ Runtime Context APIs .. autoclass:: ray.runtime_context.RuntimeContext :members: -.. _runtime-env-apis: - -Runtime Env APIs ----------------- - -.. autoclass:: ray.runtime_env.RuntimeEnvConfig - :members: - -.. autoclass:: ray.runtime_env.RuntimeEnv - :members: - .. _package-ref-debugging-apis: Debugging APIs @@ -174,19 +124,10 @@ Debugging APIs Exceptions ---------- -.. autoclass:: ray.exceptions.RayError - :members: - -.. _ray-core-exceptions-ray-task-error: -.. autoclass:: ray.exceptions.RayTaskError - :members: .. autoclass:: ray.exceptions.TaskCancelledError :members: .. autoclass:: ray.exceptions.GetTimeoutError :members: -.. _ray-core-exceptions-ray-actor-error: -.. autoclass:: ray.exceptions.RayActorError - :members: .. _ray-core-exceptions-runtime-env-setup-error: .. autoclass:: ray.exceptions.RuntimeEnvSetupError :members: diff --git a/doc/source/ray-core/scheduling/placement-group.rst b/doc/source/ray-core/scheduling/placement-group.rst index a5c613874d13..d6d01f05b899 100644 --- a/doc/source/ray-core/scheduling/placement-group.rst +++ b/doc/source/ray-core/scheduling/placement-group.rst @@ -360,7 +360,7 @@ Let's create a placement group. Recall that each bundle is a collection of resou Now let's define an actor that uses GPU. We'll also define a task that use ``extra_resources``. You can schedule actors/tasks on the placement group using -:ref:`options(scheduling_strategy=PlacementGroupSchedulingStrategy(...)) `. +:class:`options(scheduling_strategy=PlacementGroupSchedulingStrategy(...)) `. .. tabbed:: Python @@ -470,7 +470,7 @@ You can schedule actors/tasks on the placement group using }; RAY_REMOTE(&Counter::Ping, &Counter::GetValue, CreateCounter); - + // Create GPU actors on a gpu bundle. for (int index = 0; index < 2; index++) { ray::Actor(CreateCounter) @@ -660,9 +660,9 @@ Placement Group Lifetimes .. tabbed:: Python By default, the lifetimes of placement groups are not detached and will be destroyed - when the driver is terminated (but, if it is created from a detached actor, it is - killed when the detached actor is killed). If you'd like to keep the placement group - alive regardless of its job or detached actor, you should specify + when the driver is terminated (but, if it is created from a detached actor, it is + killed when the detached actor is killed). If you'd like to keep the placement group + alive regardless of its job or detached actor, you should specify `lifetime="detached"`. For example: .. code-block:: python @@ -671,8 +671,8 @@ Placement Group Lifetimes pg = placement_group([{"CPU": 2}, {"CPU": 2}], strategy="STRICT_SPREAD", lifetime="detached") ray.get(pg.ready()) - The placement group's lifetime will be independent of the driver now. This means it - is possible to retrieve the placement group from other drivers regardless of when + The placement group's lifetime will be independent of the driver now. This means it + is possible to retrieve the placement group from other drivers regardless of when the current driver exits. Let's see an example: .. code-block:: python diff --git a/doc/source/ray-observability/overview.rst b/doc/source/ray-observability/overview.rst index 3da5515c01c7..6b9376397e91 100644 --- a/doc/source/ray-observability/overview.rst +++ b/doc/source/ray-observability/overview.rst @@ -26,8 +26,8 @@ Exceptions Creating a new task or submitting an actor task generates an object reference. When ``ray.get`` is called on the object reference, the API raises an exception if anything goes wrong with a related task, actor or object. For example, -- :ref:`RayTaskError ` is raised when there's an error from user code that throws an exception. -- :ref:`RayActorError ` is raised when an actor is dead (by a system failure such as node failure or user-level failure such as an exception from ``__init__`` method). +- :class:`RayTaskError ` is raised when there's an error from user code that throws an exception. +- :class:`RayActorError ` is raised when an actor is dead (by a system failure such as node failure or user-level failure such as an exception from ``__init__`` method). - :ref:`RuntimeEnvSetupError ` is raised when the actor or task couldn't be started because :ref:`a runtime environment ` failed to be created. See :ref:`Exceptions Reference ` for more details. @@ -134,16 +134,16 @@ Here's an example output. Metrics ------- -Ray collects and exposes the physical stats (e.g., CPU, memory, GRAM, disk, and network usage of each node), -internal stats (e.g., number of actors in the cluster, number of worker failures of the cluster), +Ray collects and exposes the physical stats (e.g., CPU, memory, GRAM, disk, and network usage of each node), +internal stats (e.g., number of actors in the cluster, number of worker failures of the cluster), and custom metrics (e.g., metrics defined by users). All stats can be exported as time series data (to Prometheus by default) and used -to monitor the cluster over time. +to monitor the cluster over time. See :ref:`Ray Metrics ` for more details. Profiling --------- -Ray is compatible with Python profiling tools such as ``CProfile``. It also supports its built-in profiling tool such as :ref:```ray timeline`` `. +Ray is compatible with Python profiling tools such as ``CProfile``. It also supports its built-in profiling tool such as :ref:```ray timeline`` `. See :ref:`Profiling ` for more details. From 55ac49e15e5d0a21ebb2a8ddefe54d35b3f230ff Mon Sep 17 00:00:00 2001 From: Jiajun Yao Date: Mon, 6 Feb 2023 14:33:59 -0800 Subject: [PATCH 06/55] up Signed-off-by: Jiajun Yao --- doc/source/ray-core/api/core.rst | 2 + doc/source/ray-core/api/exceptions.rst | 23 ++++++++ doc/source/ray-core/package-ref.rst | 64 ----------------------- doc/source/ray-observability/overview.rst | 2 +- 4 files changed, 26 insertions(+), 65 deletions(-) diff --git a/doc/source/ray-core/api/core.rst b/doc/source/ray-core/api/core.rst index b053684ab0ae..d83287091198 100644 --- a/doc/source/ray-core/api/core.rst +++ b/doc/source/ray-core/api/core.rst @@ -8,6 +8,7 @@ Core API ray.remote_function.RemoteFunction.options ray.actor.ActorClass.options ray.method + ray.get_actor ray.get ray.wait ray.put @@ -15,3 +16,4 @@ Core API ray.cancel ray.init ray.shutdown + ray.is_initialized diff --git a/doc/source/ray-core/api/exceptions.rst b/doc/source/ray-core/api/exceptions.rst index 37a53a1d34dc..d041612573d4 100644 --- a/doc/source/ray-core/api/exceptions.rst +++ b/doc/source/ray-core/api/exceptions.rst @@ -1,3 +1,5 @@ +.. _ray-core-exceptions: + Exceptions ========== @@ -7,3 +9,24 @@ Exceptions ray.exceptions.RayError ray.exceptions.RayTaskError ray.exceptions.RayActorError + ray.exceptions.TaskCancelledError + ray.exceptions.TaskUnschedulableError + ray.exceptions.ActorUnschedulableError + ray.exceptions.AsyncioActorExit + ray.exceptions.LocalRayletDiedError + ray.exceptions.WorkerCrashedError + ray.exceptions.TaskPlacementGroupRemoved + ray.exceptions.ActorPlacementGroupRemoved + ray.exceptions.ObjectStoreFullError + ray.exceptions.OutOfDiskError + ray.exceptions.ObjectLostError + ray.exceptions.ObjectFetchTimedOutError + ray.exceptions.GetTimeoutError + ray.exceptions.OwnerDiedError + ray.exceptions.PlasmaObjectNotAvailable + ray.exceptions.ObjectReconstructionFailedError + ray.exceptions.ObjectReconstructionFailedMaxAttemptsExceededError + ray.exceptions.ObjectReconstructionFailedLineageEvictedError + ray.exceptions.RuntimeEnvSetupError + ray.exceptions.CrossLanguageError + ray.exceptions.RaySystemError diff --git a/doc/source/ray-core/package-ref.rst b/doc/source/ray-core/package-ref.rst index 01a35fcdbfdf..61194ac45f2b 100644 --- a/doc/source/ray-core/package-ref.rst +++ b/doc/source/ray-core/package-ref.rst @@ -4,21 +4,6 @@ Ray Core API Python API ---------- -.. _ray-is_initialized-ref: - -ray.is_initialized -~~~~~~~~~~~~~~~~~~ - -.. autofunction:: ray.is_initialized - -.. _ray-get_actor-ref: - - -ray.get_actor -~~~~~~~~~~~~~~~ - -.. autofunction:: ray.get_actor - .. _ray-get_gpu_ids-ref: ray.get_gpu_ids @@ -119,52 +104,3 @@ Debugging APIs .. autofunction:: ray.util.inspect_serializability -.. _ray-core-exceptions: - -Exceptions ----------- - -.. autoclass:: ray.exceptions.TaskCancelledError - :members: -.. autoclass:: ray.exceptions.GetTimeoutError - :members: -.. _ray-core-exceptions-runtime-env-setup-error: -.. autoclass:: ray.exceptions.RuntimeEnvSetupError - :members: -.. autoclass:: ray.exceptions.TaskUnschedulableError - :members: -.. autoclass:: ray.exceptions.ActorUnschedulableError - :members: -.. autoclass:: ray.exceptions.TaskPlacementGroupRemoved - :members: -.. autoclass:: ray.exceptions.ActorPlacementGroupRemoved - :members: -.. autoclass:: ray.exceptions.LocalRayletDiedError - :members: -.. autoclass:: ray.exceptions.WorkerCrashedError - :members: -.. autoclass:: ray.exceptions.RaySystemError - :members: -.. autoclass:: ray.exceptions.ObjectStoreFullError - :members: -.. autoclass:: ray.exceptions.OutOfDiskError - :members: -.. _ray-core-exceptions-object-lost-error: -.. autoclass:: ray.exceptions.ObjectLostError - :members: -.. autoclass:: ray.exceptions.ObjectFetchTimedOutError - :members: -.. autoclass:: ray.exceptions.OwnerDiedError - :members: -.. autoclass:: ray.exceptions.ObjectReconstructionFailedError - :members: -.. autoclass:: ray.exceptions.ObjectReconstructionFailedMaxAttemptsExceededError - :members: -.. autoclass:: ray.exceptions.ObjectReconstructionFailedLineageEvictedError - :members: -.. autoclass:: ray.exceptions.PlasmaObjectNotAvailable - :members: -.. autoclass:: ray.exceptions.AsyncioActorExit - :members: -.. autoclass:: ray.exceptions.CrossLanguageError - :members: diff --git a/doc/source/ray-observability/overview.rst b/doc/source/ray-observability/overview.rst index 6b9376397e91..f45bbfbf69a5 100644 --- a/doc/source/ray-observability/overview.rst +++ b/doc/source/ray-observability/overview.rst @@ -28,7 +28,7 @@ the API raises an exception if anything goes wrong with a related task, actor or - :class:`RayTaskError ` is raised when there's an error from user code that throws an exception. - :class:`RayActorError ` is raised when an actor is dead (by a system failure such as node failure or user-level failure such as an exception from ``__init__`` method). -- :ref:`RuntimeEnvSetupError ` is raised when the actor or task couldn't be started because :ref:`a runtime environment ` failed to be created. +- :class:`RuntimeEnvSetupError ` is raised when the actor or task couldn't be started because :ref:`a runtime environment ` failed to be created. See :ref:`Exceptions Reference ` for more details. From 175a824b2ef3b7e9dde17f92a89cfc580e2cdb7d Mon Sep 17 00:00:00 2001 From: Jiajun Yao Date: Mon, 6 Feb 2023 15:12:42 -0800 Subject: [PATCH 07/55] up Signed-off-by: Jiajun Yao --- doc/source/ray-core/actors/actor-utils.rst | 4 +- doc/source/ray-core/api.rst | 3 ++ doc/source/ray-core/api/core.rst | 43 +++++++++++++-- doc/source/ray-core/api/utility.rst | 11 ++++ doc/source/ray-core/package-ref.rst | 54 ------------------- .../ray-core/tasks/using-ray-with-gpus.rst | 2 +- 6 files changed, 55 insertions(+), 62 deletions(-) create mode 100644 doc/source/ray-core/api/utility.rst diff --git a/doc/source/ray-core/actors/actor-utils.rst b/doc/source/ray-core/actors/actor-utils.rst index 01e35107d9dc..6e5b3da20d4d 100644 --- a/doc/source/ray-core/actors/actor-utils.rst +++ b/doc/source/ray-core/actors/actor-utils.rst @@ -11,7 +11,7 @@ Actor Pool .. literalinclude:: ../doc_code/actor-pool.py - See the :ref:`package reference ` for more information. + See the :class:`package reference ` for more information. .. tabbed:: Java @@ -25,7 +25,7 @@ Message passing using Ray Queue ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Sometimes just using one signal to synchronize is not enough. If you need to send data among many tasks or -actors, you can use :ref:`ray.util.queue.Queue `. +actors, you can use :class:`ray.util.queue.Queue `. .. literalinclude:: ../doc_code/actor-queue.py diff --git a/doc/source/ray-core/api.rst b/doc/source/ray-core/api.rst index f55164430049..3a28da66926d 100644 --- a/doc/source/ray-core/api.rst +++ b/doc/source/ray-core/api.rst @@ -5,6 +5,9 @@ API References :maxdepth: 2 api/core.rst + api/scheduling.rst + api/runtime-env.rst + api/utility.rst package-ref.rst cli.rst ../ray-observability/state/cli.rst diff --git a/doc/source/ray-core/api/core.rst b/doc/source/ray-core/api/core.rst index d83287091198..5db9fe741abe 100644 --- a/doc/source/ray-core/api/core.rst +++ b/doc/source/ray-core/api/core.rst @@ -1,19 +1,52 @@ Core API ======== +.. autosummary:: + :toctree: doc/ + + ray.init + ray.shutdown + ray.is_initialized + +Task +---- + .. autosummary:: :toctree: doc/ ray.remote ray.remote_function.RemoteFunction.options + ray.cancel + +Actor +----- + +.. autosummary:: + :toctree: doc/ + + ray.remote ray.actor.ActorClass.options ray.method ray.get_actor + ray.kill + +Object +------ + +.. autosummary:: + :toctree: doc/ + ray.get ray.wait ray.put - ray.kill - ray.cancel - ray.init - ray.shutdown - ray.is_initialized + +.. _runtime-context-apis: + +Runtime Context +--------------- +.. autosummary:: + :toctree: doc/ + + ray.runtime_context.get_runtime_context + ray.runtime_context.RuntimeContext + ray.get_gpu_ids diff --git a/doc/source/ray-core/api/utility.rst b/doc/source/ray-core/api/utility.rst new file mode 100644 index 000000000000..f3a65f91faa5 --- /dev/null +++ b/doc/source/ray-core/api/utility.rst @@ -0,0 +1,11 @@ +Utility +======= + +.. autosummary:: + :toctree: doc/ + + ray.util.ActorPool + ray.util.queue.Queue + ray.nodes + ray.cluster_resources + ray.available_resources diff --git a/doc/source/ray-core/package-ref.rst b/doc/source/ray-core/package-ref.rst index 61194ac45f2b..e637749f3706 100644 --- a/doc/source/ray-core/package-ref.rst +++ b/doc/source/ray-core/package-ref.rst @@ -4,36 +4,6 @@ Ray Core API Python API ---------- -.. _ray-get_gpu_ids-ref: - -ray.get_gpu_ids -~~~~~~~~~~~~~~~ - -.. autofunction:: ray.get_gpu_ids - -.. _ray-actor-pool-ref: - -ray.util.ActorPool -~~~~~~~~~~~~~~~~~~ - -.. autoclass:: ray.util.ActorPool - :members: - -ray.util.queue.Queue -~~~~~~~~~~~~~~~~~~~~ - -.. _ray-queue-ref: - -.. autoclass:: ray.util.queue.Queue - :members: - -.. _ray-nodes-ref: - -ray.nodes -~~~~~~~~~ - -.. autofunction:: ray.nodes - .. _ray-timeline-ref: ray.timeline @@ -41,20 +11,6 @@ ray.timeline .. autofunction:: ray.timeline -.. _ray-cluster_resources-ref: - -ray.cluster_resources -~~~~~~~~~~~~~~~~~~~~~ - -.. autofunction:: ray.cluster_resources - -.. _ray-available_resources-ref: - -ray.available_resources -~~~~~~~~~~~~~~~~~~~~~~~ - -.. autofunction:: ray.available_resources - ray.cross_language ~~~~~~~~~~~~~~~~~~ @@ -85,16 +41,6 @@ Histogram .. autoclass:: ray.util.metrics.Histogram :members: -.. _runtime-context-apis: - -Runtime Context APIs --------------------- - -.. autofunction:: ray.runtime_context.get_runtime_context - -.. autoclass:: ray.runtime_context.RuntimeContext - :members: - .. _package-ref-debugging-apis: Debugging APIs diff --git a/doc/source/ray-core/tasks/using-ray-with-gpus.rst b/doc/source/ray-core/tasks/using-ray-with-gpus.rst index 0ba3fe2fba4c..5200d5a7b31a 100644 --- a/doc/source/ray-core/tasks/using-ray-with-gpus.rst +++ b/doc/source/ray-core/tasks/using-ray-with-gpus.rst @@ -40,7 +40,7 @@ and assign GPUs to the task or actor by setting the ``CUDA_VISIBLE_DEVICES`` env :start-after: __get_gpu_ids_start__ :end-before: __get_gpu_ids_end__ -Inside a task or actor, :ref:`ray.get_gpu_ids() ` will return a +Inside a task or actor, :meth:`ray.get_gpu_ids() ` will return a list of GPU IDs that are available to the task or actor. Typically, it is not necessary to call ``ray.get_gpu_ids()`` because Ray will automatically set the ``CUDA_VISIBLE_DEVICES`` environment variable, From b34a733459c273e6098b8a262531cd8e700e6abd Mon Sep 17 00:00:00 2001 From: Jiajun Yao Date: Mon, 6 Feb 2023 21:47:57 -0800 Subject: [PATCH 08/55] up Signed-off-by: Jiajun Yao --- doc/source/ray-core/api.rst | 2 +- doc/source/ray-core/api/core.rst | 8 +++++ doc/source/ray-core/api/utility.rst | 24 +++++++++++++ doc/source/ray-core/package-ref.rst | 52 ----------------------------- 4 files changed, 33 insertions(+), 53 deletions(-) delete mode 100644 doc/source/ray-core/package-ref.rst diff --git a/doc/source/ray-core/api.rst b/doc/source/ray-core/api.rst index 3a28da66926d..7162e7c4d829 100644 --- a/doc/source/ray-core/api.rst +++ b/doc/source/ray-core/api.rst @@ -8,7 +8,7 @@ API References api/scheduling.rst api/runtime-env.rst api/utility.rst - package-ref.rst + api/exceptions.rst cli.rst ../ray-observability/state/cli.rst ../ray-observability/state/ray-state-api-reference.rst diff --git a/doc/source/ray-core/api/core.rst b/doc/source/ray-core/api/core.rst index 5db9fe741abe..00dde503c5b8 100644 --- a/doc/source/ray-core/api/core.rst +++ b/doc/source/ray-core/api/core.rst @@ -50,3 +50,11 @@ Runtime Context ray.runtime_context.get_runtime_context ray.runtime_context.RuntimeContext ray.get_gpu_ids + +Cross Language +-------------- +.. autosummary:: + :toctree: doc/ + + ray.cross_language.java_function + ray.cross_language.java_actor_class diff --git a/doc/source/ray-core/api/utility.rst b/doc/source/ray-core/api/utility.rst index f3a65f91faa5..36da67415d4f 100644 --- a/doc/source/ray-core/api/utility.rst +++ b/doc/source/ray-core/api/utility.rst @@ -9,3 +9,27 @@ Utility ray.nodes ray.cluster_resources ray.available_resources + +.. _custom-metric-api-ref: + +Custom Metrics +-------------- + +.. autosummary:: + :toctree: doc/ + + ray.util.metrics.Counter + ray.util.metrics.Gauge + ray.util.metrics.Histogram + +.. _package-ref-debugging-apis: + +Debugging +--------- + +.. autosummary:: + :toctree: doc/ + + ray.util.pdb.set_trace + ray.util.inspect_serializability + ray.timeline diff --git a/doc/source/ray-core/package-ref.rst b/doc/source/ray-core/package-ref.rst deleted file mode 100644 index e637749f3706..000000000000 --- a/doc/source/ray-core/package-ref.rst +++ /dev/null @@ -1,52 +0,0 @@ -Ray Core API -============ - -Python API ----------- - -.. _ray-timeline-ref: - -ray.timeline -~~~~~~~~~~~~ - -.. autofunction:: ray.timeline - -ray.cross_language -~~~~~~~~~~~~~~~~~~ - -.. autofunction:: ray.cross_language.java_function - -.. autofunction:: ray.cross_language.java_actor_class - -.. _custom-metric-api-ref: - -Custom Metrics APIs -------------------- - -Counter -~~~~~~~ - -.. autoclass:: ray.util.metrics.Counter - :members: - -Gauge -~~~~~ - -.. autoclass:: ray.util.metrics.Gauge - :members: - -Histogram -~~~~~~~~~ - -.. autoclass:: ray.util.metrics.Histogram - :members: - -.. _package-ref-debugging-apis: - -Debugging APIs --------------- - -.. autofunction:: ray.util.pdb.set_trace - -.. autofunction:: ray.util.inspect_serializability - From 9286053822d0dcabbf37024a641ad47822eb82dc Mon Sep 17 00:00:00 2001 From: Jiajun Yao Date: Mon, 6 Feb 2023 23:07:32 -0800 Subject: [PATCH 09/55] up Signed-off-by: Jiajun Yao --- .../ray-core/{api.rst => reference.rst} | 12 +- doc/source/ray-core/{ => reference}/cli.rst | 0 .../ray-core/{api => reference}/core.rst | 12 +- .../{api => reference}/exceptions.rst | 2 +- .../{api => reference}/runtime-env.rst | 2 +- .../{api => reference}/scheduling.rst | 4 +- .../ray-core/{api => reference}/utility.rst | 6 +- .../state/ray-state-api-reference.rst | 177 ++++++------------ 8 files changed, 76 insertions(+), 139 deletions(-) rename doc/source/ray-core/{api.rst => reference.rst} (50%) rename doc/source/ray-core/{ => reference}/cli.rst (100%) rename doc/source/ray-core/{api => reference}/core.rst (86%) rename doc/source/ray-core/{api => reference}/exceptions.rst (98%) rename doc/source/ray-core/{api => reference}/runtime-env.rst (86%) rename doc/source/ray-core/{api => reference}/scheduling.rst (93%) rename doc/source/ray-core/{api => reference}/utility.rst (89%) diff --git a/doc/source/ray-core/api.rst b/doc/source/ray-core/reference.rst similarity index 50% rename from doc/source/ray-core/api.rst rename to doc/source/ray-core/reference.rst index 7162e7c4d829..5f28d4696cb5 100644 --- a/doc/source/ray-core/api.rst +++ b/doc/source/ray-core/reference.rst @@ -4,11 +4,11 @@ API References .. toctree:: :maxdepth: 2 - api/core.rst - api/scheduling.rst - api/runtime-env.rst - api/utility.rst - api/exceptions.rst - cli.rst + reference/core.rst + reference/scheduling.rst + reference/runtime-env.rst + reference/utility.rst + reference/exceptions.rst + reference/cli.rst ../ray-observability/state/cli.rst ../ray-observability/state/ray-state-api-reference.rst diff --git a/doc/source/ray-core/cli.rst b/doc/source/ray-core/reference/cli.rst similarity index 100% rename from doc/source/ray-core/cli.rst rename to doc/source/ray-core/reference/cli.rst diff --git a/doc/source/ray-core/api/core.rst b/doc/source/ray-core/reference/core.rst similarity index 86% rename from doc/source/ray-core/api/core.rst rename to doc/source/ray-core/reference/core.rst index 00dde503c5b8..a0148c18bc90 100644 --- a/doc/source/ray-core/api/core.rst +++ b/doc/source/ray-core/reference/core.rst @@ -2,7 +2,7 @@ Core API ======== .. autosummary:: - :toctree: doc/ + :toctree: api/ ray.init ray.shutdown @@ -12,7 +12,7 @@ Task ---- .. autosummary:: - :toctree: doc/ + :toctree: api/ ray.remote ray.remote_function.RemoteFunction.options @@ -22,7 +22,7 @@ Actor ----- .. autosummary:: - :toctree: doc/ + :toctree: api/ ray.remote ray.actor.ActorClass.options @@ -34,7 +34,7 @@ Object ------ .. autosummary:: - :toctree: doc/ + :toctree: api/ ray.get ray.wait @@ -45,7 +45,7 @@ Object Runtime Context --------------- .. autosummary:: - :toctree: doc/ + :toctree: api/ ray.runtime_context.get_runtime_context ray.runtime_context.RuntimeContext @@ -54,7 +54,7 @@ Runtime Context Cross Language -------------- .. autosummary:: - :toctree: doc/ + :toctree: api/ ray.cross_language.java_function ray.cross_language.java_actor_class diff --git a/doc/source/ray-core/api/exceptions.rst b/doc/source/ray-core/reference/exceptions.rst similarity index 98% rename from doc/source/ray-core/api/exceptions.rst rename to doc/source/ray-core/reference/exceptions.rst index d041612573d4..54ff8e971e82 100644 --- a/doc/source/ray-core/api/exceptions.rst +++ b/doc/source/ray-core/reference/exceptions.rst @@ -4,7 +4,7 @@ Exceptions ========== .. autosummary:: - :toctree: doc/ + :toctree: api/ ray.exceptions.RayError ray.exceptions.RayTaskError diff --git a/doc/source/ray-core/api/runtime-env.rst b/doc/source/ray-core/reference/runtime-env.rst similarity index 86% rename from doc/source/ray-core/api/runtime-env.rst rename to doc/source/ray-core/reference/runtime-env.rst index 607766a9535a..7de5e56371ab 100644 --- a/doc/source/ray-core/api/runtime-env.rst +++ b/doc/source/ray-core/reference/runtime-env.rst @@ -2,7 +2,7 @@ Runtime Env API =============== .. autosummary:: - :toctree: doc/ + :toctree: api/ ray.runtime_env.RuntimeEnvConfig ray.runtime_env.RuntimeEnv diff --git a/doc/source/ray-core/api/scheduling.rst b/doc/source/ray-core/reference/scheduling.rst similarity index 93% rename from doc/source/ray-core/api/scheduling.rst rename to doc/source/ray-core/reference/scheduling.rst index 2971aae8e3c8..f3d990bb08ed 100644 --- a/doc/source/ray-core/api/scheduling.rst +++ b/doc/source/ray-core/reference/scheduling.rst @@ -5,7 +5,7 @@ Scheduling Strategy ------------------- .. autosummary:: - :toctree: doc/ + :toctree: api/ ray.util.scheduling_strategies.PlacementGroupSchedulingStrategy ray.util.scheduling_strategies.NodeAffinitySchedulingStrategy @@ -16,7 +16,7 @@ Placement Group --------------- .. autosummary:: - :toctree: doc/ + :toctree: api/ ray.util.placement_group.placement_group ray.util.placement_group.PlacementGroup diff --git a/doc/source/ray-core/api/utility.rst b/doc/source/ray-core/reference/utility.rst similarity index 89% rename from doc/source/ray-core/api/utility.rst rename to doc/source/ray-core/reference/utility.rst index 36da67415d4f..d2f18b011d03 100644 --- a/doc/source/ray-core/api/utility.rst +++ b/doc/source/ray-core/reference/utility.rst @@ -2,7 +2,7 @@ Utility ======= .. autosummary:: - :toctree: doc/ + :toctree: api/ ray.util.ActorPool ray.util.queue.Queue @@ -16,7 +16,7 @@ Custom Metrics -------------- .. autosummary:: - :toctree: doc/ + :toctree: api/ ray.util.metrics.Counter ray.util.metrics.Gauge @@ -28,7 +28,7 @@ Debugging --------- .. autosummary:: - :toctree: doc/ + :toctree: api/ ray.util.pdb.set_trace ray.util.inspect_serializability diff --git a/doc/source/ray-observability/state/ray-state-api-reference.rst b/doc/source/ray-observability/state/ray-state-api-reference.rst index b5ae074368fb..63b56321103b 100644 --- a/doc/source/ray-observability/state/ray-state-api-reference.rst +++ b/doc/source/ray-observability/state/ray-state-api-reference.rst @@ -3,7 +3,7 @@ Ray State API .. _state-api-ref: -.. note:: +.. note:: APIs are :ref:`alpha `. This feature requires a full installation of Ray using ``pip install "ray[default]"``. @@ -14,40 +14,54 @@ For the CLI reference see :ref:`Ray State CLI Reference ` or State Python SDK ----------------- -State APIs are also exported as functions. +State APIs are also exported as functions. Summary APIs ~~~~~~~~~~~~ -.. autofunction:: ray.experimental.state.api.summarize_actors -.. autofunction:: ray.experimental.state.api.summarize_objects -.. autofunction:: ray.experimental.state.api.summarize_tasks + +.. autosummary:: + :toctree: api/ + + ray.experimental.state.api.summarize_actors + ray.experimental.state.api.summarize_objects + ray.experimental.state.api.summarize_tasks List APIs ~~~~~~~~~~ -.. autofunction:: ray.experimental.state.api.list_actors -.. autofunction:: ray.experimental.state.api.list_placement_groups -.. autofunction:: ray.experimental.state.api.list_nodes -.. autofunction:: ray.experimental.state.api.list_jobs -.. autofunction:: ray.experimental.state.api.list_workers -.. autofunction:: ray.experimental.state.api.list_tasks -.. autofunction:: ray.experimental.state.api.list_objects -.. autofunction:: ray.experimental.state.api.list_runtime_envs +.. autosummary:: + :toctree: api/ + + ray.experimental.state.api.list_actors + ray.experimental.state.api.list_placement_groups + ray.experimental.state.api.list_nodes + ray.experimental.state.api.list_jobs + ray.experimental.state.api.list_workers + ray.experimental.state.api.list_tasks + ray.experimental.state.api.list_objects + ray.experimental.state.api.list_runtime_envs Get APIs ~~~~~~~~~ -.. autofunction:: ray.experimental.state.api.get_actor -.. autofunction:: ray.experimental.state.api.get_placement_group -.. autofunction:: ray.experimental.state.api.get_node -.. autofunction:: ray.experimental.state.api.get_worker -.. autofunction:: ray.experimental.state.api.get_task -.. autofunction:: ray.experimental.state.api.get_objects +.. autosummary:: + :toctree: api/ + + ray.experimental.state.api.get_actor + ray.experimental.state.api.get_placement_group + ray.experimental.state.api.get_node + ray.experimental.state.api.get_worker + ray.experimental.state.api.get_task + ray.experimental.state.api.get_objects Log APIs ~~~~~~~~ -.. autofunction:: ray.experimental.state.api.list_logs -.. autofunction:: ray.experimental.state.api.get_log + +.. autosummary:: + :toctree: api/ + + ray.experimental.state.api.list_logs + ray.experimental.state.api.get_log .. _state-api-schema: @@ -55,126 +69,49 @@ State APIs Schema ----------------- .. _state-api-schema-actor: - -ActorState -~~~~~~~~~~ - -.. autoclass:: ray.experimental.state.common.ActorState - :members: - .. _state-api-schema-task: - -TaskState -~~~~~~~~~ - -.. autoclass:: ray.experimental.state.common.TaskState - :members: - .. _state-api-schema-node: - -NodeState -~~~~~~~~~ - -.. autoclass:: ray.experimental.state.common.NodeState - :members: - .. _state-api-schema-pg: - -PlacementGroupState -~~~~~~~~~~~~~~~~~~~ - -.. autoclass:: ray.experimental.state.common.PlacementGroupState - :members: - .. _state-api-schema-worker: - -WorkerState -~~~~~~~~~~~ - -.. autoclass:: ray.experimental.state.common.WorkerState - :members: - .. _state-api-schema-obj: - -ObjectState -~~~~~~~~~~~ - -.. autoclass:: ray.experimental.state.common.ObjectState - :members: - .. _state-api-schema-runtime-env: - -RuntimeEnvState -~~~~~~~~~~~~~~~ - -.. autoclass:: ray.experimental.state.common.RuntimeEnvState - :members: - .. _state-api-schema-job: - -JobState -~~~~~~~~ - -.. autoclass:: ray.experimental.state.common.JobState - :members: - .. _state-api-schema-summary: - -StateSummary -~~~~~~~~~~~~ - -.. autoclass:: ray.experimental.state.common.StateSummary - :members: - .. _state-api-schema-task-summary: - -TaskSummary -~~~~~~~~~~~ - .. _state-api-schema-task-summaries: - -.. autoclass:: ray.experimental.state.common.TaskSummaries - :members: - .. _state-api-schema-task-summary-per-key: - -.. autoclass:: ray.experimental.state.common.TaskSummaryPerFuncOrClassName - :members: - .. _state-api-schema-actor-summary: - -ActorSummary -~~~~~~~~~~~~ - .. _state-api-schema-actor-summaries: - -.. autoclass:: ray.experimental.state.common.ActorSummaries - :members: - .. _state-api-schema-actor-summary-per-key: - -.. autoclass:: ray.experimental.state.common.ActorSummaryPerClass - :members: - .. _state-api-schema-object-summary: - -ObjectSummary -~~~~~~~~~~~~~ - .. _state-api-schema-object-summaries: - -.. autoclass:: ray.experimental.state.common.ObjectSummaries - :members: - .. _state-api-schema-object-summary-per-key: -.. autoclass:: ray.experimental.state.common.ObjectSummaryPerKey - :members: +.. autosummary:: + :toctree: api/ + + ray.experimental.state.common.ActorState + ray.experimental.state.common.TaskState + ray.experimental.state.common.NodeState + ray.experimental.state.common.PlacementGroupState + ray.experimental.state.common.WorkerState + ray.experimental.state.common.ObjectState + ray.experimental.state.common.RuntimeEnvState + ray.experimental.state.common.JobState + ray.experimental.state.common.StateSummary + ray.experimental.state.common.TaskSummaries + ray.experimental.state.common.TaskSummaryPerFuncOrClassName + ray.experimental.state.common.ActorSummaries + ray.experimental.state.common.ActorSummaryPerClass + ray.experimental.state.common.ObjectSummaries + ray.experimental.state.common.ObjectSummaryPerKey State APIs Exceptions --------------------- .. _state-api-exceptions: -.. autoclass:: ray.experimental.state.exception.RayStateApiException - :members: \ No newline at end of file +.. autosummary:: + :toctree: api/ + + ray.experimental.state.exception.RayStateApiException From f84e755f5520d14757504eb543e74293a4956c61 Mon Sep 17 00:00:00 2001 From: Jiajun Yao Date: Mon, 6 Feb 2023 23:27:18 -0800 Subject: [PATCH 10/55] up Signed-off-by: Jiajun Yao --- doc/source/_templates/autosummary/class.rst | 33 +++++++++++++++++++++ 1 file changed, 33 insertions(+) create mode 100644 doc/source/_templates/autosummary/class.rst diff --git a/doc/source/_templates/autosummary/class.rst b/doc/source/_templates/autosummary/class.rst new file mode 100644 index 000000000000..a9c9bd2b6507 --- /dev/null +++ b/doc/source/_templates/autosummary/class.rst @@ -0,0 +1,33 @@ +{% extends "!autosummary/class.rst" %} + +{% block methods %} +{% if methods %} + +.. + HACK -- the point here is that we don't want this to appear in the output, but the autosummary should still generate the pages. + .. autosummary:: + :toctree: + {% for item in all_methods %} + {%- if not item.startswith('_') or item in ['__call__'] %} + {{ name }}.{{ item }} + {%- endif -%} + {%- endfor %} + +{% endif %} +{% endblock %} + +{% block attributes %} +{% if attributes %} + +.. + HACK -- the point here is that we don't want this to appear in the output, but the autosummary should still generate the pages. + .. autosummary:: + :toctree: + {% for item in all_attributes %} + {%- if not item.startswith('_') %} + {{ name }}.{{ item }} + {%- endif -%} + {%- endfor %} + +{% endif %} +{% endblock %} From d97a2d3e53fe426fc948c81a77a1b800542fc5e0 Mon Sep 17 00:00:00 2001 From: Jiajun Yao Date: Tue, 7 Feb 2023 09:16:26 -0800 Subject: [PATCH 11/55] up Signed-off-by: Jiajun Yao --- doc/source/_toc.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/source/_toc.yml b/doc/source/_toc.yml index 223b1f9276ba..ffe4f32dbe6a 100644 --- a/doc/source/_toc.yml +++ b/doc/source/_toc.yml @@ -33,7 +33,7 @@ parts: - file: ray-core/examples/batch_training - file: ray-core/examples/automl_for_time_series - file: ray-core/examples/web-crawler - - file: ray-core/api + - file: ray-core/reference - file: cluster/getting-started title: "Ray Clusters" From fe9d1db19753c146852b8cc9cecdec0cd9e3a06c Mon Sep 17 00:00:00 2001 From: Jiajun Yao Date: Tue, 7 Feb 2023 09:31:28 -0800 Subject: [PATCH 12/55] up Signed-off-by: Jiajun Yao --- doc/source/_toc.yml | 2 +- doc/source/ray-core/{reference.rst => reference/index.rst} | 4 ++-- .../ray-state-api-reference.rst => reference/state/api.rst} | 0 doc/source/ray-observability/{ => reference}/state/cli.rst | 0 4 files changed, 3 insertions(+), 3 deletions(-) rename doc/source/ray-core/{reference.rst => reference/index.rst} (69%) rename doc/source/ray-observability/{state/ray-state-api-reference.rst => reference/state/api.rst} (100%) rename doc/source/ray-observability/{ => reference}/state/cli.rst (100%) diff --git a/doc/source/_toc.yml b/doc/source/_toc.yml index ffe4f32dbe6a..150cb8119526 100644 --- a/doc/source/_toc.yml +++ b/doc/source/_toc.yml @@ -33,7 +33,7 @@ parts: - file: ray-core/examples/batch_training - file: ray-core/examples/automl_for_time_series - file: ray-core/examples/web-crawler - - file: ray-core/reference + - file: ray-core/reference/index - file: cluster/getting-started title: "Ray Clusters" diff --git a/doc/source/ray-core/reference.rst b/doc/source/ray-core/reference/index.rst similarity index 69% rename from doc/source/ray-core/reference.rst rename to doc/source/ray-core/reference/index.rst index 5f28d4696cb5..48e0d1fee1df 100644 --- a/doc/source/ray-core/reference.rst +++ b/doc/source/ray-core/reference/index.rst @@ -10,5 +10,5 @@ API References reference/utility.rst reference/exceptions.rst reference/cli.rst - ../ray-observability/state/cli.rst - ../ray-observability/state/ray-state-api-reference.rst + ../ray-observability/reference/state/cli.rst + ../ray-observability/reference/state/api.rst diff --git a/doc/source/ray-observability/state/ray-state-api-reference.rst b/doc/source/ray-observability/reference/state/api.rst similarity index 100% rename from doc/source/ray-observability/state/ray-state-api-reference.rst rename to doc/source/ray-observability/reference/state/api.rst diff --git a/doc/source/ray-observability/state/cli.rst b/doc/source/ray-observability/reference/state/cli.rst similarity index 100% rename from doc/source/ray-observability/state/cli.rst rename to doc/source/ray-observability/reference/state/cli.rst From 31903e38e46cdc0aa0c37e6eda4871654fbcf625 Mon Sep 17 00:00:00 2001 From: Jiajun Yao Date: Tue, 7 Feb 2023 09:55:13 -0800 Subject: [PATCH 13/55] up Signed-off-by: Jiajun Yao --- .../ray-observability/reference/state/api.rst | 19 ---- .../ray-observability/state/state-api.rst | 98 +++++++++---------- python/ray/experimental/state/api.py | 50 +++++----- python/ray/experimental/state/state_cli.py | 14 +-- 4 files changed, 81 insertions(+), 100 deletions(-) diff --git a/doc/source/ray-observability/reference/state/api.rst b/doc/source/ray-observability/reference/state/api.rst index 63b56321103b..5426473e6f7e 100644 --- a/doc/source/ray-observability/reference/state/api.rst +++ b/doc/source/ray-observability/reference/state/api.rst @@ -68,25 +68,6 @@ Log APIs State APIs Schema ----------------- -.. _state-api-schema-actor: -.. _state-api-schema-task: -.. _state-api-schema-node: -.. _state-api-schema-pg: -.. _state-api-schema-worker: -.. _state-api-schema-obj: -.. _state-api-schema-runtime-env: -.. _state-api-schema-job: -.. _state-api-schema-summary: -.. _state-api-schema-task-summary: -.. _state-api-schema-task-summaries: -.. _state-api-schema-task-summary-per-key: -.. _state-api-schema-actor-summary: -.. _state-api-schema-actor-summaries: -.. _state-api-schema-actor-summary-per-key: -.. _state-api-schema-object-summary: -.. _state-api-schema-object-summaries: -.. _state-api-schema-object-summary-per-key: - .. autosummary:: :toctree: api/ diff --git a/doc/source/ray-observability/state/state-api.rst b/doc/source/ray-observability/state/state-api.rst index 04bd682e6d07..78778ef3b753 100644 --- a/doc/source/ray-observability/state/state-api.rst +++ b/doc/source/ray-observability/state/state-api.rst @@ -7,7 +7,7 @@ Monitoring Ray States Ray state APIs allow users to conveniently access the current state (snapshot) of Ray through CLI or Python SDK. -.. note:: +.. note:: APIs are :ref:`alpha `. This feature requires a full installation of Ray using ``pip install "ray[default]"``. This feature also requires the dashboard component to be available. The dashboard component needs to be included when starting the ray cluster, which is the default behavior for ``ray start`` and ``ray.init()``. For more in-depth debugging, you could check the dashboard log at ``/dashboard.log``, which is usually ``/tmp/ray/session_latest/logs/dashboard.log``. @@ -27,12 +27,12 @@ Run any workload. In this example, you will use the following script that runs 2 def task_running_300_seconds(): print("Start!") time.sleep(300) - + @ray.remote class Actor: def __init__(self): print("Actor created") - + # Create 2 tasks tasks = [task_running_300_seconds.remote() for _ in range(2)] @@ -84,7 +84,7 @@ Let's list all actors. .. code-block:: python - from ray.experimental.state.api import list_actors + from ray.experimental.state.api import list_actors print(list_actors()) .. code-block:: text @@ -100,15 +100,15 @@ Let's list all actors. 0 31405554844820381c2f0f8501000000 Actor 96956 ALIVE 1 f36758a9f8871a9ca993b1d201000000 Actor 96955 ALIVE -You can get the state of a single task using the get API. +You can get the state of a single task using the get API. .. tabbed:: CLI .. code-block:: bash # In this case, 31405554844820381c2f0f8501000000 - ray get actors - + ray get actors + .. tabbed:: Python SDK .. code-block:: python @@ -139,7 +139,7 @@ You can also access logs through ``ray logs`` API. ray list actors # In this case, ACTOR_ID is 31405554844820381c2f0f8501000000 - ray logs actor --id + ray logs actor --id .. tabbed:: Python SDK @@ -164,13 +164,13 @@ Key Concepts Ray state APIs allow you to access **states** of **resources** through **summary**, **list**, and **get** APIs. It also supports **logs** API to access logs. - **states**: The state of the cluster of corresponding resources. States consist of immutable metadata (e.g., actor's name) and mutable states (e.g., actor's scheduling state or pid). -- **resources**: Resources created by Ray. E.g., actors, tasks, objects, placement groups, and etc. +- **resources**: Resources created by Ray. E.g., actors, tasks, objects, placement groups, and etc. - **summary**: API to return the summarized view of resources. - **list**: API to return every individual entity of resources. - **get**: API to return a single entity of resources in detail. - **logs**: API to access the log of actors, tasks, workers, or system log files. -Summary +Summary ------- Return the summarized information of the given Ray resource (objects, actors, tasks). It is recommended to start monitoring states through summary APIs first. When you find anomalies @@ -193,7 +193,7 @@ E.g., Summarize all actors from ray.experimental.state.api import summarize_actors print(summarize_actors()) -E.g., Summarize all tasks +E.g., Summarize all tasks ~~~~~~~~~~~~~~~~~~~~~~~~~ .. tabbed:: CLI @@ -209,7 +209,7 @@ E.g., Summarize all tasks from ray.experimental.state.api import summarize_tasks print(summarize_tasks()) -E.g., Summarize all objects +E.g., Summarize all objects ~~~~~~~~~~~~~~~~~~~~~~~~~~~~ .. note:: @@ -223,7 +223,7 @@ E.g., Summarize all objects .. code-block:: bash - ray summary objects + ray summary objects .. tabbed:: Python SDK @@ -235,34 +235,34 @@ E.g., Summarize all objects List ---- -Get a list of resources, possible resources include: +Get a list of resources, possible resources include: -- :ref:`Actors `, e.g., actor id, state, pid, death_cause. (:ref:`output schema `) -- :ref:`Tasks `, e.g., name, scheduling state, type, runtime env info (:ref:`output schema `) -- :ref:`Objects `, e.g., object id, callsites, reference types. (:ref:`output schema `) -- :ref:`Jobs `, e.g., start/end time, entrypoint, status. (:ref:`output schema `) -- :ref:`Placement Groups `, e.g., name, bundles, stats. (:ref:`output schema `) -- Nodes (Ray worker nodes), e.g., node id, node ip, node state. (:ref:`output schema `) -- Workers (Ray worker processes), e.g., worker id, type, exit type and details. (:ref:`output schema `) -- :ref:`Runtime environments `, e.g., runtime envs, creation time, nodes (:ref:`output schema `) +- :ref:`Actors `, e.g., actor id, state, pid, death_cause. (:class:`output schema `) +- :ref:`Tasks `, e.g., name, scheduling state, type, runtime env info (:class:`output schema `) +- :ref:`Objects `, e.g., object id, callsites, reference types. (:class:`output schema `) +- :ref:`Jobs `, e.g., start/end time, entrypoint, status. (:class:`output schema `) +- :ref:`Placement Groups `, e.g., name, bundles, stats. (:class:`output schema `) +- Nodes (Ray worker nodes), e.g., node id, node ip, node state. (:class:`output schema `) +- Workers (Ray worker processes), e.g., worker id, type, exit type and details. (:class:`output schema `) +- :ref:`Runtime environments `, e.g., runtime envs, creation time, nodes (:class:`output schema `) -E.g., List all nodes +E.g., List all nodes ~~~~~~~~~~~~~~~~~~~~~ .. tabbed:: CLI .. code-block:: bash - ray list nodes + ray list nodes .. tabbed:: Python SDK .. code-block:: python - from ray.experimental.state.api import list_nodes() + from ray.experimental.state.api import list_nodes() list_nodes() -E.g., List all placement groups +E.g., List all placement groups ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ .. tabbed:: CLI @@ -275,10 +275,10 @@ E.g., List all placement groups .. code-block:: python - from ray.experimental.state.api import list_placement_groups + from ray.experimental.state.api import list_placement_groups list_placement_groups() - + E.g., List local referenced objects created by a process ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -294,7 +294,7 @@ E.g., List local referenced objects created by a process .. code-block:: python - from ray.experimental.state.api import list_objects + from ray.experimental.state.api import list_objects list_objects(filters=[("pid", "=", ), ("reference_type", "=", "LOCAL_REFERENCE")]) E.g., List alive actors @@ -310,7 +310,7 @@ E.g., List alive actors .. code-block:: python - from ray.experimental.state.api import list_actors + from ray.experimental.state.api import list_actors list_actors(filters=[("state", "=", "ALIVE")]) E.g., List running tasks @@ -326,7 +326,7 @@ E.g., List running tasks .. code-block:: python - from ray.experimental.state.api import list_tasks + from ray.experimental.state.api import list_tasks list_tasks(filters=[("state", "=", "RUNNING")]) E.g., List non-running tasks @@ -342,7 +342,7 @@ E.g., List non-running tasks .. code-block:: python - from ray.experimental.state.api import list_tasks + from ray.experimental.state.api import list_tasks list_tasks(filters=[("state", "!=", "RUNNING")]) E.g., List running tasks that have a name func @@ -358,7 +358,7 @@ E.g., List running tasks that have a name func .. code-block:: python - from ray.experimental.state.api import list_tasks + from ray.experimental.state.api import list_tasks list_tasks(filters=[("state", "=", "RUNNING"), ("name", "=", "task_running_300_seconds()")]) E.g., List tasks with more details @@ -376,7 +376,7 @@ E.g., List tasks with more details .. code-block:: python - from ray.experimental.state.api import list_tasks + from ray.experimental.state.api import list_tasks list_tasks(detail=True) Get @@ -389,13 +389,13 @@ E.g., Get a task info .. code-block:: bash - ray get tasks + ray get tasks .. tabbed:: Python SDK .. code-block:: python - from ray.experimental.state.api import get_task + from ray.experimental.state.api import get_task get_task(id=) E.g., Get a node info @@ -405,13 +405,13 @@ E.g., Get a node info .. code-block:: bash - ray get nodes + ray get nodes .. tabbed:: Python SDK .. code-block:: python - from ray.experimental.state.api import get_node + from ray.experimental.state.api import get_node get_node(id=) Logs @@ -435,11 +435,11 @@ E.g., Get all retrievable log file names from a head node in a cluster .. code-block:: python - # You could get the node id / node ip from `ray list nodes` - from ray.experimental.state.api import list_logs - # `ray logs` by default print logs from a head node. - # So in order to list the same logs, you should provide the head node id. - # You could get the node id / node ip from `ray list nodes` + # You could get the node id / node ip from `ray list nodes` + from ray.experimental.state.api import list_logs + # `ray logs` by default print logs from a head node. + # So in order to list the same logs, you should provide the head node id. + # You could get the node id / node ip from `ray list nodes` list_logs(node_id=) E.g., Get a particular log file from a node @@ -452,13 +452,13 @@ E.g., Get a particular log file from a node # You could get the node id / node ip from `ray list nodes` ray logs cluster gcs_server.out --node-id # `ray logs cluster` is alias to `ray logs` when querying with globs. - ray logs gcs_server.out --node-id + ray logs gcs_server.out --node-id .. tabbed:: Python SDK .. code-block:: python - from ray.experimental.state.api import get_log + from ray.experimental.state.api import get_log # Node IP could be retrieved from list_nodes() or ray.nodes() for line in get_log(filename="gcs_server.out", node_id=): @@ -471,7 +471,7 @@ E.g., Stream a log file from a node .. code-block:: bash - # You could get the node id / node ip from `ray list nodes` + # You could get the node id / node ip from `ray list nodes` ray logs raylet.out --node-ip --follow # Or, ray logs cluster raylet.out --node-ip --follow @@ -481,7 +481,7 @@ E.g., Stream a log file from a node .. code-block:: python - from ray.experimental.state.api import get_log + from ray.experimental.state.api import get_log # Node IP could be retrieved from list_nodes() or ray.nodes() # The loop will block with `follow=True` @@ -508,7 +508,7 @@ E.g., Stream log from an actor with actor id for line in get_log(actor_id=, follow=True): print(line) -E.g., Stream log from a pid +E.g., Stream log from a pid ~~~~~~~~~~~~~~~~~~~~~~~~~~~ .. tabbed:: CLI @@ -521,7 +521,7 @@ E.g., Stream log from a pid .. code-block:: python - from ray.experimental.state.api import get_log + from ray.experimental.state.api import get_log # Node IP could be retrieved from list_nodes() or ray.nodes() # You could get the pid of the worker running the actor easily when output diff --git a/python/ray/experimental/state/api.py b/python/ray/experimental/state/api.py index daefed8b423b..73e194afea00 100644 --- a/python/ray/experimental/state/api.py +++ b/python/ray/experimental/state/api.py @@ -550,7 +550,7 @@ def get_actor( Returns: None if actor not found, or dictionarified - :ref:`ActorState `. + :class:`ActorState `. Raises: Exceptions: :ref:`RayStateApiException ` if the CLI @@ -589,7 +589,7 @@ def get_placement_group( Returns: None if actor not found, or dictionarified - :ref:`PlacementGroupState `. + :class:`PlacementGroupState `. Raises: Exceptions: :ref:`RayStateApiException ` if the CLI @@ -621,7 +621,7 @@ def get_node( Returns: None if actor not found, or dictionarified - :ref:`NodeState `. + :class:`NodeState `. Raises: Exceptions: :ref:`RayStateApiException ` @@ -653,7 +653,7 @@ def get_worker( Returns: None if actor not found, or dictionarified - :ref:`WorkerState `. + :class:`WorkerState `. Raises: Exceptions: :ref:`RayStateApiException ` if the CLI @@ -685,7 +685,7 @@ def get_task( Returns: None if task not found, or a list of dictionarified - :ref:`TaskState ` from the task attempts. + :class:`TaskState ` from the task attempts. Raises: Exceptions: :ref:`RayStateApiException ` if the CLI @@ -719,7 +719,7 @@ def get_objects( failed query information. Returns: - List of dictionarified :ref:`ObjectState `. + List of dictionarified :class:`ObjectState `. Raises: Exceptions: :ref:`RayStateApiException ` if the CLI @@ -753,7 +753,7 @@ def list_actors( timeout: Max timeout value for the state APIs requests made. detail: When True, more details info (specified in `ActorState`) will be queried and returned. See - :ref:`ActorState `. + :class:`ActorState `. raise_on_missing_output: When True, exceptions will be raised if there is missing data due to truncation/data source unavailable. _explain: Print the API information such as API latency or @@ -761,7 +761,7 @@ def list_actors( Returns: List of dictionarified - :ref:`ActorState `. + :class:`ActorState `. Raises: Exceptions: :ref:`RayStateApiException ` if the CLI @@ -800,7 +800,7 @@ def list_placement_groups( timeout: Max timeout value for the state APIs requests made. detail: When True, more details info (specified in `PlacementGroupState`) will be queried and returned. See - :ref:`PlacementGroupState `. + :class:`PlacementGroupState `. raise_on_missing_output: When True, exceptions will be raised if there is missing data due to truncation/data source unavailable. _explain: Print the API information such as API latency or @@ -808,7 +808,7 @@ def list_placement_groups( Returns: List of dictionarified - :ref:`PlacementGroupState `. + :class:`PlacementGroupState `. Raises: Exceptions: :ref:`RayStateApiException ` if the CLI @@ -844,7 +844,7 @@ def list_nodes( timeout: Max timeout value for the state APIs requests made. detail: When True, more details info (specified in `NodeState`) will be queried and returned. See - :ref:`NodeState `. + :class:`NodeState `. raise_on_missing_output: When True, exceptions will be raised if there is missing data due to truncation/data source unavailable. _explain: Print the API information such as API latency or @@ -852,7 +852,7 @@ def list_nodes( Returns: List of dictionarified - :ref:`NodeState `. + :class:`NodeState `. Raises: Exceptions: :ref:`RayStateApiException ` @@ -888,7 +888,7 @@ def list_jobs( timeout: Max timeout value for the state APIs requests made. detail: When True, more details info (specified in `JobState`) will be queried and returned. See - :ref:`JobState `. + :class:`JobState `. raise_on_missing_output: When True, exceptions will be raised if there is missing data due to truncation/data source unavailable. _explain: Print the API information such as API latency or @@ -896,7 +896,7 @@ def list_jobs( Returns: List of dictionarified - :ref:`JobState `. + :class:`JobState `. Raises: Exceptions: :ref:`RayStateApiException ` if the CLI @@ -932,7 +932,7 @@ def list_workers( timeout: Max timeout value for the state APIs requests made. detail: When True, more details info (specified in `WorkerState`) will be queried and returned. See - :ref:`WorkerState `. + :class:`WorkerState `. raise_on_missing_output: When True, exceptions will be raised if there is missing data due to truncation/data source unavailable. _explain: Print the API information such as API latency or @@ -940,7 +940,7 @@ def list_workers( Returns: List of dictionarified - :ref:`WorkerState `. + :class:`WorkerState `. Raises: Exceptions: :ref:`RayStateApiException ` if the CLI @@ -976,7 +976,7 @@ def list_tasks( timeout: Max timeout value for the state APIs requests made. detail: When True, more details info (specified in `WorkerState`) will be queried and returned. See - :ref:`WorkerState `. + :class:`WorkerState `. raise_on_missing_output: When True, exceptions will be raised if there is missing data due to truncation/data source unavailable. _explain: Print the API information such as API latency or @@ -984,7 +984,7 @@ def list_tasks( Returns: List of dictionarified - :ref:`WorkerState `. + :class:`WorkerState `. Raises: Exceptions: :ref:`RayStateApiException ` if the CLI @@ -1020,7 +1020,7 @@ def list_objects( timeout: Max timeout value for the state APIs requests made. detail: When True, more details info (specified in `ObjectState`) will be queried and returned. See - :ref:`ObjectState `. + :class:`ObjectState `. raise_on_missing_output: When True, exceptions will be raised if there is missing data due to truncation/data source unavailable. _explain: Print the API information such as API latency or @@ -1028,7 +1028,7 @@ def list_objects( Returns: List of dictionarified - :ref:`ObjectState `. + :class:`ObjectState `. Raises: Exceptions: :ref:`RayStateApiException ` if the CLI @@ -1064,7 +1064,7 @@ def list_runtime_envs( timeout: Max timeout value for the state APIs requests made. detail: When True, more details info (specified in `RuntimeEnvState`) will be queried and returned. See - :ref:`RuntimeEnvState `. + :class:`RuntimeEnvState `. raise_on_missing_output: When True, exceptions will be raised if there is missing data due to truncation/data source unavailable. _explain: Print the API information such as API latency or @@ -1072,7 +1072,7 @@ def list_runtime_envs( Returns: List of dictionarified - :ref:`RuntimeEnvState `. + :class:`RuntimeEnvState `. Raises: Exceptions: :ref:`RayStateApiException ` if the CLI @@ -1291,7 +1291,7 @@ def summarize_tasks( failed query information. Return: - Dictionarified :ref:`TaskSummaries ` + Dictionarified :class:`TaskSummaries ` Raises: Exceptions: :ref:`RayStateApiException ` @@ -1323,7 +1323,7 @@ def summarize_actors( failed query information. Return: - Dictionarified :ref:`ActorSummaries ` + Dictionarified :class:`ActorSummaries ` Raises: Exceptions: :ref:`RayStateApiException ` if the CLI @@ -1355,7 +1355,7 @@ def summarize_objects( failed query information. Return: - Dictionarified :ref:`ObjectSummaries ` + Dictionarified :class:`ObjectSummaries ` Raises: Exceptions: :ref:`RayStateApiException ` if the CLI diff --git a/python/ray/experimental/state/state_cli.py b/python/ray/experimental/state/state_cli.py index 1f297c9eaba0..11c4caf6ca7f 100644 --- a/python/ray/experimental/state/state_cli.py +++ b/python/ray/experimental/state/state_cli.py @@ -353,7 +353,7 @@ def ray_get( The output schema is defined at :ref:`State API Schema section. ` For example, the output schema of `ray get tasks ` is - :ref:`ray.experimental.state.common.TaskState `. + :class:`ray.experimental.state.common.TaskState `. Usage: @@ -466,7 +466,7 @@ def ray_list( The output schema is defined at :ref:`State API Schema section. ` For example, the output schema of `ray list tasks` is - :ref:`ray.experimental.state.common.TaskState `. + :class:`ray.experimental.state.common.TaskState `. Usage: @@ -581,7 +581,7 @@ def task_summary(ctx, timeout: float, address: str): task function names. The output schema is - :ref:`ray.experimental.state.common.TaskSummaries `. + :class:`ray.experimental.state.common.TaskSummaries `. Raises: :ref:`RayStateApiException ` @@ -612,8 +612,8 @@ def actor_summary(ctx, timeout: float, address: str): actor class names. The output schema is - :ref:`ray.experimental.state.common.ActorSummaries - `. + :class:`ray.experimental.state.common.ActorSummaries + `. Raises: :ref:`RayStateApiException ` @@ -663,8 +663,8 @@ def object_summary(ctx, timeout: float, address: str): ``` The output schema is - :ref:`ray.experimental.state.common.ObjectSummaries - `. + :class:`ray.experimental.state.common.ObjectSummaries + `. Raises: :ref:`RayStateApiException ` From bf86626bb19b8eec60aca6ef78e2689a4cca453f Mon Sep 17 00:00:00 2001 From: Jiajun Yao Date: Tue, 7 Feb 2023 11:01:18 -0800 Subject: [PATCH 14/55] up Signed-off-by: Jiajun Yao --- doc/source/_templates/autosummary/class.rst | 59 ++++++++++++--------- doc/source/ray-core/reference/index.rst | 4 +- 2 files changed, 35 insertions(+), 28 deletions(-) diff --git a/doc/source/_templates/autosummary/class.rst b/doc/source/_templates/autosummary/class.rst index a9c9bd2b6507..601820b87686 100644 --- a/doc/source/_templates/autosummary/class.rst +++ b/doc/source/_templates/autosummary/class.rst @@ -1,33 +1,40 @@ -{% extends "!autosummary/class.rst" %} +{{ fullname }} +{{ underline }} -{% block methods %} -{% if methods %} +.. currentmodule:: {{ module }} -.. - HACK -- the point here is that we don't want this to appear in the output, but the autosummary should still generate the pages. - .. autosummary:: - :toctree: +.. autoclass:: {{ objname }} + :no-members: + :no-inherited-members: + :no-special-members: + + {% block methods %} + {% if methods %} + .. HACK -- the point here is that we don't want this to appear in the output, but the autosummary should still generate the pages. + .. autosummary:: + :toctree: {% for item in all_methods %} - {%- if not item.startswith('_') or item in ['__call__'] %} - {{ name }}.{{ item }} - {%- endif -%} + {%- if not item.startswith('_') or item in ['__call__', '__mul__', '__getitem__', '__len__'] %} + {{ name }}.{{ item }} + {%- endif -%} {%- endfor %} + {% for item in inherited_members %} + {%- if item in ['__call__', '__mul__', '__getitem__', '__len__'] %} + {{ name }}.{{ item }} + {%- endif -%} + {%- endfor %} + {% endif %} + {% endblock %} -{% endif %} -{% endblock %} - -{% block attributes %} -{% if attributes %} - -.. - HACK -- the point here is that we don't want this to appear in the output, but the autosummary should still generate the pages. - .. autosummary:: - :toctree: + {% block attributes %} + {% if attributes %} + .. HACK -- the point here is that we don't want this to appear in the output, but the autosummary should still generate the pages. + .. autosummary:: + :toctree: {% for item in all_attributes %} - {%- if not item.startswith('_') %} - {{ name }}.{{ item }} - {%- endif -%} + {%- if not item.startswith('_') %} + {{ name }}.{{ item }} + {%- endif -%} {%- endfor %} - -{% endif %} -{% endblock %} + {% endif %} + {% endblock %} diff --git a/doc/source/ray-core/reference/index.rst b/doc/source/ray-core/reference/index.rst index 48e0d1fee1df..d26f8bba0254 100644 --- a/doc/source/ray-core/reference/index.rst +++ b/doc/source/ray-core/reference/index.rst @@ -10,5 +10,5 @@ API References reference/utility.rst reference/exceptions.rst reference/cli.rst - ../ray-observability/reference/state/cli.rst - ../ray-observability/reference/state/api.rst + ../../ray-observability/reference/state/cli.rst + ../../ray-observability/reference/state/api.rst From 298de259fcb9f1b30644e0d7d18a4cc689fcb5cb Mon Sep 17 00:00:00 2001 From: Jiajun Yao Date: Tue, 7 Feb 2023 11:13:39 -0800 Subject: [PATCH 15/55] up Signed-off-by: Jiajun Yao --- doc/source/_templates/autosummary/class.rst | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/doc/source/_templates/autosummary/class.rst b/doc/source/_templates/autosummary/class.rst index 601820b87686..4a36427018c2 100644 --- a/doc/source/_templates/autosummary/class.rst +++ b/doc/source/_templates/autosummary/class.rst @@ -1,5 +1,4 @@ -{{ fullname }} -{{ underline }} +{{ fullname | escape | underline }} .. currentmodule:: {{ module }} From aaa5f002f94607470138f9659db388cc967248c8 Mon Sep 17 00:00:00 2001 From: Jiajun Yao Date: Tue, 7 Feb 2023 11:49:05 -0800 Subject: [PATCH 16/55] up Signed-off-by: Jiajun Yao --- doc/source/_templates/autosummary/class.rst | 64 ++++++++++----------- 1 file changed, 29 insertions(+), 35 deletions(-) diff --git a/doc/source/_templates/autosummary/class.rst b/doc/source/_templates/autosummary/class.rst index 4a36427018c2..e3ff806df7c0 100644 --- a/doc/source/_templates/autosummary/class.rst +++ b/doc/source/_templates/autosummary/class.rst @@ -1,39 +1,33 @@ -{{ fullname | escape | underline }} +{{ fullname | escape | underline}} .. currentmodule:: {{ module }} .. autoclass:: {{ objname }} - :no-members: - :no-inherited-members: - :no-special-members: - - {% block methods %} - {% if methods %} - .. HACK -- the point here is that we don't want this to appear in the output, but the autosummary should still generate the pages. - .. autosummary:: - :toctree: - {% for item in all_methods %} - {%- if not item.startswith('_') or item in ['__call__', '__mul__', '__getitem__', '__len__'] %} - {{ name }}.{{ item }} - {%- endif -%} - {%- endfor %} - {% for item in inherited_members %} - {%- if item in ['__call__', '__mul__', '__getitem__', '__len__'] %} - {{ name }}.{{ item }} - {%- endif -%} - {%- endfor %} - {% endif %} - {% endblock %} - - {% block attributes %} - {% if attributes %} - .. HACK -- the point here is that we don't want this to appear in the output, but the autosummary should still generate the pages. - .. autosummary:: - :toctree: - {% for item in all_attributes %} - {%- if not item.startswith('_') %} - {{ name }}.{{ item }} - {%- endif -%} - {%- endfor %} - {% endif %} - {% endblock %} + + {% block methods %} + {% if methods %} + .. rubric:: {{ _('Methods') }} + + .. autosummary:: + :toctree: api/ + + {% for item in methods %} + {{ name }}.{{ item }} + {%- endfor %} + + {% endif %} + {% endblock %} + + {% block attributes %} + {% if attributes %} + .. rubric:: {{ _('Attributes') }} + + .. autosummary:: + :toctree: api/ + + {% for item in attributes %} + {{ name }}.{{ item }} + {%- endfor %} + + {% endif %} + {% endblock %} From 0771fcf77efc7fa9735b2055268e937b3b247b57 Mon Sep 17 00:00:00 2001 From: Jiajun Yao Date: Tue, 7 Feb 2023 13:18:49 -0800 Subject: [PATCH 17/55] up Signed-off-by: Jiajun Yao --- doc/source/ray-references/api.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/source/ray-references/api.rst b/doc/source/ray-references/api.rst index 6aef18962dbc..b2ea8110a7ae 100644 --- a/doc/source/ray-references/api.rst +++ b/doc/source/ray-references/api.rst @@ -13,4 +13,4 @@ API References ../rllib/package_ref/index.rst ../workflows/package-ref.rst ../cluster/package-overview.rst - ../ray-core/package-ref.rst + ../ray-core/reference/index.rst From f8cbdc8dd8d052f042084fa2a1924684fa433bd0 Mon Sep 17 00:00:00 2001 From: Jiajun Yao Date: Tue, 7 Feb 2023 14:35:17 -0800 Subject: [PATCH 18/55] up Signed-off-by: Jiajun Yao --- doc/source/ray-core/reference/index.rst | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/doc/source/ray-core/reference/index.rst b/doc/source/ray-core/reference/index.rst index d26f8bba0254..deffe5ec1b77 100644 --- a/doc/source/ray-core/reference/index.rst +++ b/doc/source/ray-core/reference/index.rst @@ -4,11 +4,11 @@ API References .. toctree:: :maxdepth: 2 - reference/core.rst - reference/scheduling.rst - reference/runtime-env.rst - reference/utility.rst - reference/exceptions.rst - reference/cli.rst + core.rst + scheduling.rst + runtime-env.rst + utility.rst + exceptions.rst + cli.rst ../../ray-observability/reference/state/cli.rst ../../ray-observability/reference/state/api.rst From 7aa8d6f13e41b45b74a3e9e995886488a422f548 Mon Sep 17 00:00:00 2001 From: Jiajun Yao Date: Tue, 7 Feb 2023 20:48:59 -0800 Subject: [PATCH 19/55] up Signed-off-by: Jiajun Yao --- doc/source/_templates/autosummary/class.rst | 29 +-------------------- 1 file changed, 1 insertion(+), 28 deletions(-) diff --git a/doc/source/_templates/autosummary/class.rst b/doc/source/_templates/autosummary/class.rst index e3ff806df7c0..f3e9c2c18cbd 100644 --- a/doc/source/_templates/autosummary/class.rst +++ b/doc/source/_templates/autosummary/class.rst @@ -3,31 +3,4 @@ .. currentmodule:: {{ module }} .. autoclass:: {{ objname }} - - {% block methods %} - {% if methods %} - .. rubric:: {{ _('Methods') }} - - .. autosummary:: - :toctree: api/ - - {% for item in methods %} - {{ name }}.{{ item }} - {%- endfor %} - - {% endif %} - {% endblock %} - - {% block attributes %} - {% if attributes %} - .. rubric:: {{ _('Attributes') }} - - .. autosummary:: - :toctree: api/ - - {% for item in attributes %} - {{ name }}.{{ item }} - {%- endfor %} - - {% endif %} - {% endblock %} + :members: From 9239df9ba1e2cfac1411e1dbe2fff3818be96f91 Mon Sep 17 00:00:00 2001 From: Jiajun Yao Date: Tue, 7 Feb 2023 22:28:11 -0800 Subject: [PATCH 20/55] up Signed-off-by: Jiajun Yao --- doc/source/ray-core/reference/index.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/source/ray-core/reference/index.rst b/doc/source/ray-core/reference/index.rst index deffe5ec1b77..e03e42a081b3 100644 --- a/doc/source/ray-core/reference/index.rst +++ b/doc/source/ray-core/reference/index.rst @@ -1,5 +1,5 @@ -API References -============== +Ray Core API +============ .. toctree:: :maxdepth: 2 From 96ae8cfe4944acb14559c38ac7286fd3cdada0f4 Mon Sep 17 00:00:00 2001 From: Jiajun Yao Date: Tue, 7 Feb 2023 22:28:41 -0800 Subject: [PATCH 21/55] Revert "up" This reverts commit 7aa8d6f13e41b45b74a3e9e995886488a422f548. --- doc/source/_templates/autosummary/class.rst | 29 ++++++++++++++++++++- 1 file changed, 28 insertions(+), 1 deletion(-) diff --git a/doc/source/_templates/autosummary/class.rst b/doc/source/_templates/autosummary/class.rst index f3e9c2c18cbd..e3ff806df7c0 100644 --- a/doc/source/_templates/autosummary/class.rst +++ b/doc/source/_templates/autosummary/class.rst @@ -3,4 +3,31 @@ .. currentmodule:: {{ module }} .. autoclass:: {{ objname }} - :members: + + {% block methods %} + {% if methods %} + .. rubric:: {{ _('Methods') }} + + .. autosummary:: + :toctree: api/ + + {% for item in methods %} + {{ name }}.{{ item }} + {%- endfor %} + + {% endif %} + {% endblock %} + + {% block attributes %} + {% if attributes %} + .. rubric:: {{ _('Attributes') }} + + .. autosummary:: + :toctree: api/ + + {% for item in attributes %} + {{ name }}.{{ item }} + {%- endfor %} + + {% endif %} + {% endblock %} From d0d5424552259d8cb3d6e0768e2757e381f72c45 Mon Sep 17 00:00:00 2001 From: Jiajun Yao Date: Tue, 7 Feb 2023 22:35:06 -0800 Subject: [PATCH 22/55] test Signed-off-by: Jiajun Yao --- doc/source/data/api/grouped_dataset.rst | 2 -- 1 file changed, 2 deletions(-) diff --git a/doc/source/data/api/grouped_dataset.rst b/doc/source/data/api/grouped_dataset.rst index b426a91aa74d..a8ec7b4db461 100644 --- a/doc/source/data/api/grouped_dataset.rst +++ b/doc/source/data/api/grouped_dataset.rst @@ -19,7 +19,6 @@ Computations / Descriptive Stats -------------------------------- .. autosummary:: - :toctree: doc/ grouped_dataset.GroupedDataset.count grouped_dataset.GroupedDataset.sum @@ -32,7 +31,6 @@ Function Application -------------------- .. autosummary:: - :toctree: doc/ grouped_dataset.GroupedDataset.aggregate grouped_dataset.GroupedDataset.map_groups From d0a95628fd18b9eddf30e185eb127efec2132a9a Mon Sep 17 00:00:00 2001 From: Jiajun Yao Date: Tue, 7 Feb 2023 23:46:51 -0800 Subject: [PATCH 23/55] up Signed-off-by: Jiajun Yao --- doc/source/_templates/autosummary/class.rst | 4 ++-- doc/source/data/api/grouped_dataset.rst | 2 ++ doc/source/ray-core/reference/cli.rst | 4 ++-- doc/source/ray-observability/reference/state/api.rst | 4 ++-- doc/source/ray-observability/reference/state/cli.rst | 10 +++++----- 5 files changed, 13 insertions(+), 11 deletions(-) diff --git a/doc/source/_templates/autosummary/class.rst b/doc/source/_templates/autosummary/class.rst index e3ff806df7c0..47cadf758a42 100644 --- a/doc/source/_templates/autosummary/class.rst +++ b/doc/source/_templates/autosummary/class.rst @@ -9,7 +9,7 @@ .. rubric:: {{ _('Methods') }} .. autosummary:: - :toctree: api/ + :toctree: {% for item in methods %} {{ name }}.{{ item }} @@ -23,7 +23,7 @@ .. rubric:: {{ _('Attributes') }} .. autosummary:: - :toctree: api/ + :toctree: {% for item in attributes %} {{ name }}.{{ item }} diff --git a/doc/source/data/api/grouped_dataset.rst b/doc/source/data/api/grouped_dataset.rst index a8ec7b4db461..b426a91aa74d 100644 --- a/doc/source/data/api/grouped_dataset.rst +++ b/doc/source/data/api/grouped_dataset.rst @@ -19,6 +19,7 @@ Computations / Descriptive Stats -------------------------------- .. autosummary:: + :toctree: doc/ grouped_dataset.GroupedDataset.count grouped_dataset.GroupedDataset.sum @@ -31,6 +32,7 @@ Function Application -------------------- .. autosummary:: + :toctree: doc/ grouped_dataset.GroupedDataset.aggregate grouped_dataset.GroupedDataset.map_groups diff --git a/doc/source/ray-core/reference/cli.rst b/doc/source/ray-core/reference/cli.rst index 7f5ee4c3ef57..c9041ad3e18c 100644 --- a/doc/source/ray-core/reference/cli.rst +++ b/doc/source/ray-core/reference/cli.rst @@ -1,5 +1,5 @@ -Ray Core CLI -============ +Core CLI +======== .. _ray-cli: diff --git a/doc/source/ray-observability/reference/state/api.rst b/doc/source/ray-observability/reference/state/api.rst index 5426473e6f7e..af7b4fefc500 100644 --- a/doc/source/ray-observability/reference/state/api.rst +++ b/doc/source/ray-observability/reference/state/api.rst @@ -1,5 +1,5 @@ -Ray State API -============= +State API +========= .. _state-api-ref: diff --git a/doc/source/ray-observability/reference/state/cli.rst b/doc/source/ray-observability/reference/state/cli.rst index da765ac79c15..f85874ca83f4 100644 --- a/doc/source/ray-observability/reference/state/cli.rst +++ b/doc/source/ray-observability/reference/state/cli.rst @@ -1,5 +1,5 @@ -Ray State CLI -============= +State CLI +========= .. _state-api-cli-ref: @@ -7,7 +7,7 @@ State ----- This section contains commands to access the :ref:`live state of Ray resources (actor, task, object, etc.) `. -.. note:: +.. note:: APIs are :ref:`alpha `. This feature requires a full installation of Ray using ``pip install "ray[default]"``. This feature also requires the dashboard component to be available. The dashboard component needs to be included when starting the ray cluster, which is the default behavior for ``ray start`` and ``ray.init()``. For more in-depth debugging, you could check the dashboard log at ``/dashboard.log``, which is usually ``/tmp/ray/session_latest/logs/dashboard.log``. @@ -34,11 +34,11 @@ Log --- This section contains commands to :ref:`access logs ` from Ray clusters. -.. note:: +.. note:: APIs are :ref:`alpha `. This feature requires a full installation of Ray using ``pip install "ray[default]"``. -Log CLI allows users to access the log from the cluster. +Log CLI allows users to access the log from the cluster. Note that only the logs from alive nodes are available through this API. .. click:: ray.experimental.state.state_cli:logs_state_cli_group From 741332c70b76711d10ba310d28bf8259fea7e946 Mon Sep 17 00:00:00 2001 From: Jiajun Yao Date: Wed, 8 Feb 2023 00:45:48 -0800 Subject: [PATCH 24/55] lint Signed-off-by: Jiajun Yao --- python/ray/experimental/state/api.py | 20 ++++++++++++-------- python/ray/experimental/state/state_cli.py | 6 +++--- 2 files changed, 15 insertions(+), 11 deletions(-) diff --git a/python/ray/experimental/state/api.py b/python/ray/experimental/state/api.py index 73e194afea00..510d8ffa0feb 100644 --- a/python/ray/experimental/state/api.py +++ b/python/ray/experimental/state/api.py @@ -589,7 +589,7 @@ def get_placement_group( Returns: None if actor not found, or dictionarified - :class:`PlacementGroupState `. + :class:`~ray.experimental.state.common.PlacementGroupState`. Raises: Exceptions: :ref:`RayStateApiException ` if the CLI @@ -685,7 +685,8 @@ def get_task( Returns: None if task not found, or a list of dictionarified - :class:`TaskState ` from the task attempts. + :class:`~ray.experimental.state.common.TaskState` + from the task attempts. Raises: Exceptions: :ref:`RayStateApiException ` if the CLI @@ -719,7 +720,8 @@ def get_objects( failed query information. Returns: - List of dictionarified :class:`ObjectState `. + List of dictionarified + :class:`~ray.experimental.state.common.ObjectState`. Raises: Exceptions: :ref:`RayStateApiException ` if the CLI @@ -800,7 +802,7 @@ def list_placement_groups( timeout: Max timeout value for the state APIs requests made. detail: When True, more details info (specified in `PlacementGroupState`) will be queried and returned. See - :class:`PlacementGroupState `. + :class:`~ray.experimental.state.common.PlacementGroupState`. raise_on_missing_output: When True, exceptions will be raised if there is missing data due to truncation/data source unavailable. _explain: Print the API information such as API latency or @@ -808,7 +810,7 @@ def list_placement_groups( Returns: List of dictionarified - :class:`PlacementGroupState `. + :class:`~ray.experimental.state.common.PlacementGroupState`. Raises: Exceptions: :ref:`RayStateApiException ` if the CLI @@ -1291,7 +1293,8 @@ def summarize_tasks( failed query information. Return: - Dictionarified :class:`TaskSummaries ` + Dictionarified + :class:`~ray.experimental.state.common.TaskSummaries` Raises: Exceptions: :ref:`RayStateApiException ` @@ -1323,7 +1326,8 @@ def summarize_actors( failed query information. Return: - Dictionarified :class:`ActorSummaries ` + Dictionarified + :class:`~ray.experimental.state.common.ActorSummaries` Raises: Exceptions: :ref:`RayStateApiException ` if the CLI @@ -1355,7 +1359,7 @@ def summarize_objects( failed query information. Return: - Dictionarified :class:`ObjectSummaries ` + Dictionarified :class:`~ray.experimental.state.common.ObjectSummaries` Raises: Exceptions: :ref:`RayStateApiException ` if the CLI diff --git a/python/ray/experimental/state/state_cli.py b/python/ray/experimental/state/state_cli.py index 11c4caf6ca7f..1b4e5242ad12 100644 --- a/python/ray/experimental/state/state_cli.py +++ b/python/ray/experimental/state/state_cli.py @@ -353,7 +353,7 @@ def ray_get( The output schema is defined at :ref:`State API Schema section. ` For example, the output schema of `ray get tasks ` is - :class:`ray.experimental.state.common.TaskState `. + :class:`~ray.experimental.state.common.TaskState`. Usage: @@ -466,7 +466,7 @@ def ray_list( The output schema is defined at :ref:`State API Schema section. ` For example, the output schema of `ray list tasks` is - :class:`ray.experimental.state.common.TaskState `. + :class:`~ray.experimental.state.common.TaskState`. Usage: @@ -581,7 +581,7 @@ def task_summary(ctx, timeout: float, address: str): task function names. The output schema is - :class:`ray.experimental.state.common.TaskSummaries `. + :class:`~ray.experimental.state.common.TaskSummaries`. Raises: :ref:`RayStateApiException ` From 6908a2cc8deaf1818cdcb679df730e5400652cd7 Mon Sep 17 00:00:00 2001 From: Jiajun Yao Date: Wed, 8 Feb 2023 09:42:30 -0800 Subject: [PATCH 25/55] fix Signed-off-by: Jiajun Yao --- python/ray/data/block.py | 14 +++++------- python/ray/data/datasource/partitioning.py | 26 ++++++++++------------ rllib/policy/sample_batch.py | 2 +- 3 files changed, 19 insertions(+), 23 deletions(-) diff --git a/python/ray/data/block.py b/python/ray/data/block.py index 3de5992d8511..2e0ba3d23a1b 100644 --- a/python/ray/data/block.py +++ b/python/ray/data/block.py @@ -210,20 +210,18 @@ def build(self) -> "BlockExecStats": @dataclass class BlockMetadata: """Metadata about the block. - - Attributes: - num_rows: The number of rows contained in this block, or None. - size_bytes: The approximate size in bytes of this block, or None. - schema: The pyarrow schema or types of the block elements, or None. - input_files: The list of file paths used to generate this block, or - the empty list if indeterminate. - exec_stats: Execution stats for this block. """ + #: The number of rows contained in this block, or None. num_rows: Optional[int] + #: The approximate size in bytes of this block, or None. size_bytes: Optional[int] + #: The pyarrow schema or types of the block elements, or None. schema: Optional[Union[type, "pyarrow.lib.Schema"]] + #: The list of file paths used to generate this block, or + #: the empty list if indeterminate. input_files: Optional[List[str]] + #: Execution stats for this block. exec_stats: Optional[BlockExecStats] def __post_init__(self): diff --git a/python/ray/data/datasource/partitioning.py b/python/ray/data/datasource/partitioning.py index b7255d0a7721..30462514c36a 100644 --- a/python/ray/data/datasource/partitioning.py +++ b/python/ray/data/datasource/partitioning.py @@ -42,25 +42,23 @@ class Partitioning: Path-based partition formats embed all partition keys and values directly in their dataset file paths. - - Attributes: - style: The partition style - may be either HIVE or DIRECTORY. - base_dir: "/"-delimited base directory that all partitioned paths should - exist under (exclusive). File paths either outside of, or at the first - level of, this directory will be considered unpartitioned. Specify - `None` or an empty string to search for partitions in all file path - directories. - field_names: The partition key field names (i.e. column names for tabular - datasets). When non-empty, the order and length of partition key - field names must match the order and length of partition values. - Required when parsing DIRECTORY partitioned paths or generating - HIVE partitioned paths. - filesystem: Filesystem that will be used for partition path file I/O. """ + #: The partition style - may be either HIVE or DIRECTORY. style: PartitionStyle + #: "/"-delimited base directory that all partitioned paths should + #: exist under (exclusive). File paths either outside of, or at the first + #: level of, this directory will be considered unpartitioned. Specify + #: `None` or an empty string to search for partitions in all file path + #: directories. base_dir: Optional[str] = None + #: The partition key field names (i.e. column names for tabular + #: datasets). When non-empty, the order and length of partition key + #: field names must match the order and length of partition values. + #: Required when parsing DIRECTORY partitioned paths or generating + #: HIVE partitioned paths. field_names: Optional[List[str]] = None + #: Filesystem that will be used for partition path file I/O. filesystem: Optional["pyarrow.fs.FileSystem"] = None def __post_init__(self): diff --git a/rllib/policy/sample_batch.py b/rllib/policy/sample_batch.py index 98fe4213b1dd..0f83b70fb0f9 100644 --- a/rllib/policy/sample_batch.py +++ b/rllib/policy/sample_batch.py @@ -804,7 +804,7 @@ def to_device(self, device, framework="torch"): def size_bytes(self) -> int: """Returns sum over number of bytes of all data buffers. - For numpy arrays, we use `.nbytes`. For all other value types, we use + For numpy arrays, we use ``.nbytes``. For all other value types, we use sys.getsizeof(...). Returns: From 7e6a48051ea079268e61d7ceaed0bb86cbda97a9 Mon Sep 17 00:00:00 2001 From: Jiajun Yao Date: Wed, 8 Feb 2023 15:59:55 -0800 Subject: [PATCH 26/55] lint Signed-off-by: Jiajun Yao --- python/ray/data/block.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/python/ray/data/block.py b/python/ray/data/block.py index 2e0ba3d23a1b..d3629ab6fd35 100644 --- a/python/ray/data/block.py +++ b/python/ray/data/block.py @@ -209,8 +209,7 @@ def build(self) -> "BlockExecStats": @DeveloperAPI @dataclass class BlockMetadata: - """Metadata about the block. - """ + """Metadata about the block.""" #: The number of rows contained in this block, or None. num_rows: Optional[int] From 2f910ec4376b937e2524744f90571cece83426c2 Mon Sep 17 00:00:00 2001 From: Jiajun Yao Date: Wed, 8 Feb 2023 16:07:30 -0800 Subject: [PATCH 27/55] up Signed-off-by: Jiajun Yao --- doc/source/_toc.yml | 2 +- doc/source/ray-core/{reference => api}/cli.rst | 0 doc/source/ray-core/{reference => api}/core.rst | 12 ++++++------ .../ray-core/{reference => api}/exceptions.rst | 2 +- doc/source/ray-core/{reference => api}/index.rst | 4 ++-- .../ray-core/{reference => api}/runtime-env.rst | 2 +- .../ray-core/{reference => api}/scheduling.rst | 4 ++-- doc/source/ray-core/{reference => api}/utility.rst | 6 +++--- .../{reference => api}/state/api.rst | 12 ++++++------ .../{reference => api}/state/cli.rst | 0 doc/source/ray-references/api.rst | 2 +- 11 files changed, 23 insertions(+), 23 deletions(-) rename doc/source/ray-core/{reference => api}/cli.rst (100%) rename doc/source/ray-core/{reference => api}/core.rst (86%) rename doc/source/ray-core/{reference => api}/exceptions.rst (98%) rename doc/source/ray-core/{reference => api}/index.rst (60%) rename doc/source/ray-core/{reference => api}/runtime-env.rst (86%) rename doc/source/ray-core/{reference => api}/scheduling.rst (93%) rename doc/source/ray-core/{reference => api}/utility.rst (89%) rename doc/source/ray-observability/{reference => api}/state/api.rst (95%) rename doc/source/ray-observability/{reference => api}/state/cli.rst (100%) diff --git a/doc/source/_toc.yml b/doc/source/_toc.yml index 150cb8119526..754d4429065a 100644 --- a/doc/source/_toc.yml +++ b/doc/source/_toc.yml @@ -33,7 +33,7 @@ parts: - file: ray-core/examples/batch_training - file: ray-core/examples/automl_for_time_series - file: ray-core/examples/web-crawler - - file: ray-core/reference/index + - file: ray-core/api/index - file: cluster/getting-started title: "Ray Clusters" diff --git a/doc/source/ray-core/reference/cli.rst b/doc/source/ray-core/api/cli.rst similarity index 100% rename from doc/source/ray-core/reference/cli.rst rename to doc/source/ray-core/api/cli.rst diff --git a/doc/source/ray-core/reference/core.rst b/doc/source/ray-core/api/core.rst similarity index 86% rename from doc/source/ray-core/reference/core.rst rename to doc/source/ray-core/api/core.rst index a0148c18bc90..00dde503c5b8 100644 --- a/doc/source/ray-core/reference/core.rst +++ b/doc/source/ray-core/api/core.rst @@ -2,7 +2,7 @@ Core API ======== .. autosummary:: - :toctree: api/ + :toctree: doc/ ray.init ray.shutdown @@ -12,7 +12,7 @@ Task ---- .. autosummary:: - :toctree: api/ + :toctree: doc/ ray.remote ray.remote_function.RemoteFunction.options @@ -22,7 +22,7 @@ Actor ----- .. autosummary:: - :toctree: api/ + :toctree: doc/ ray.remote ray.actor.ActorClass.options @@ -34,7 +34,7 @@ Object ------ .. autosummary:: - :toctree: api/ + :toctree: doc/ ray.get ray.wait @@ -45,7 +45,7 @@ Object Runtime Context --------------- .. autosummary:: - :toctree: api/ + :toctree: doc/ ray.runtime_context.get_runtime_context ray.runtime_context.RuntimeContext @@ -54,7 +54,7 @@ Runtime Context Cross Language -------------- .. autosummary:: - :toctree: api/ + :toctree: doc/ ray.cross_language.java_function ray.cross_language.java_actor_class diff --git a/doc/source/ray-core/reference/exceptions.rst b/doc/source/ray-core/api/exceptions.rst similarity index 98% rename from doc/source/ray-core/reference/exceptions.rst rename to doc/source/ray-core/api/exceptions.rst index 54ff8e971e82..d041612573d4 100644 --- a/doc/source/ray-core/reference/exceptions.rst +++ b/doc/source/ray-core/api/exceptions.rst @@ -4,7 +4,7 @@ Exceptions ========== .. autosummary:: - :toctree: api/ + :toctree: doc/ ray.exceptions.RayError ray.exceptions.RayTaskError diff --git a/doc/source/ray-core/reference/index.rst b/doc/source/ray-core/api/index.rst similarity index 60% rename from doc/source/ray-core/reference/index.rst rename to doc/source/ray-core/api/index.rst index e03e42a081b3..eb5cdd9d0ef5 100644 --- a/doc/source/ray-core/reference/index.rst +++ b/doc/source/ray-core/api/index.rst @@ -10,5 +10,5 @@ Ray Core API utility.rst exceptions.rst cli.rst - ../../ray-observability/reference/state/cli.rst - ../../ray-observability/reference/state/api.rst + ../../ray-observability/api/state/cli.rst + ../../ray-observability/api/state/api.rst diff --git a/doc/source/ray-core/reference/runtime-env.rst b/doc/source/ray-core/api/runtime-env.rst similarity index 86% rename from doc/source/ray-core/reference/runtime-env.rst rename to doc/source/ray-core/api/runtime-env.rst index 7de5e56371ab..607766a9535a 100644 --- a/doc/source/ray-core/reference/runtime-env.rst +++ b/doc/source/ray-core/api/runtime-env.rst @@ -2,7 +2,7 @@ Runtime Env API =============== .. autosummary:: - :toctree: api/ + :toctree: doc/ ray.runtime_env.RuntimeEnvConfig ray.runtime_env.RuntimeEnv diff --git a/doc/source/ray-core/reference/scheduling.rst b/doc/source/ray-core/api/scheduling.rst similarity index 93% rename from doc/source/ray-core/reference/scheduling.rst rename to doc/source/ray-core/api/scheduling.rst index f3d990bb08ed..2971aae8e3c8 100644 --- a/doc/source/ray-core/reference/scheduling.rst +++ b/doc/source/ray-core/api/scheduling.rst @@ -5,7 +5,7 @@ Scheduling Strategy ------------------- .. autosummary:: - :toctree: api/ + :toctree: doc/ ray.util.scheduling_strategies.PlacementGroupSchedulingStrategy ray.util.scheduling_strategies.NodeAffinitySchedulingStrategy @@ -16,7 +16,7 @@ Placement Group --------------- .. autosummary:: - :toctree: api/ + :toctree: doc/ ray.util.placement_group.placement_group ray.util.placement_group.PlacementGroup diff --git a/doc/source/ray-core/reference/utility.rst b/doc/source/ray-core/api/utility.rst similarity index 89% rename from doc/source/ray-core/reference/utility.rst rename to doc/source/ray-core/api/utility.rst index d2f18b011d03..36da67415d4f 100644 --- a/doc/source/ray-core/reference/utility.rst +++ b/doc/source/ray-core/api/utility.rst @@ -2,7 +2,7 @@ Utility ======= .. autosummary:: - :toctree: api/ + :toctree: doc/ ray.util.ActorPool ray.util.queue.Queue @@ -16,7 +16,7 @@ Custom Metrics -------------- .. autosummary:: - :toctree: api/ + :toctree: doc/ ray.util.metrics.Counter ray.util.metrics.Gauge @@ -28,7 +28,7 @@ Debugging --------- .. autosummary:: - :toctree: api/ + :toctree: doc/ ray.util.pdb.set_trace ray.util.inspect_serializability diff --git a/doc/source/ray-observability/reference/state/api.rst b/doc/source/ray-observability/api/state/api.rst similarity index 95% rename from doc/source/ray-observability/reference/state/api.rst rename to doc/source/ray-observability/api/state/api.rst index af7b4fefc500..46c98a2280b6 100644 --- a/doc/source/ray-observability/reference/state/api.rst +++ b/doc/source/ray-observability/api/state/api.rst @@ -20,7 +20,7 @@ Summary APIs ~~~~~~~~~~~~ .. autosummary:: - :toctree: api/ + :toctree: doc/ ray.experimental.state.api.summarize_actors ray.experimental.state.api.summarize_objects @@ -30,7 +30,7 @@ List APIs ~~~~~~~~~~ .. autosummary:: - :toctree: api/ + :toctree: doc/ ray.experimental.state.api.list_actors ray.experimental.state.api.list_placement_groups @@ -45,7 +45,7 @@ Get APIs ~~~~~~~~~ .. autosummary:: - :toctree: api/ + :toctree: doc/ ray.experimental.state.api.get_actor ray.experimental.state.api.get_placement_group @@ -58,7 +58,7 @@ Log APIs ~~~~~~~~ .. autosummary:: - :toctree: api/ + :toctree: doc/ ray.experimental.state.api.list_logs ray.experimental.state.api.get_log @@ -69,7 +69,7 @@ State APIs Schema ----------------- .. autosummary:: - :toctree: api/ + :toctree: doc/ ray.experimental.state.common.ActorState ray.experimental.state.common.TaskState @@ -93,6 +93,6 @@ State APIs Exceptions .. _state-api-exceptions: .. autosummary:: - :toctree: api/ + :toctree: doc/ ray.experimental.state.exception.RayStateApiException diff --git a/doc/source/ray-observability/reference/state/cli.rst b/doc/source/ray-observability/api/state/cli.rst similarity index 100% rename from doc/source/ray-observability/reference/state/cli.rst rename to doc/source/ray-observability/api/state/cli.rst diff --git a/doc/source/ray-references/api.rst b/doc/source/ray-references/api.rst index b2ea8110a7ae..07089a160a18 100644 --- a/doc/source/ray-references/api.rst +++ b/doc/source/ray-references/api.rst @@ -13,4 +13,4 @@ API References ../rllib/package_ref/index.rst ../workflows/package-ref.rst ../cluster/package-overview.rst - ../ray-core/reference/index.rst + ../ray-core/api/index.rst From f306a66ecb5de5830de176b6ede480f4449501b0 Mon Sep 17 00:00:00 2001 From: Jiajun Yao Date: Thu, 9 Feb 2023 10:27:17 -0800 Subject: [PATCH 28/55] test Signed-off-by: Jiajun Yao --- doc/source/_templates/autosummary/class.rst | 29 +-------------------- 1 file changed, 1 insertion(+), 28 deletions(-) diff --git a/doc/source/_templates/autosummary/class.rst b/doc/source/_templates/autosummary/class.rst index 47cadf758a42..f3e9c2c18cbd 100644 --- a/doc/source/_templates/autosummary/class.rst +++ b/doc/source/_templates/autosummary/class.rst @@ -3,31 +3,4 @@ .. currentmodule:: {{ module }} .. autoclass:: {{ objname }} - - {% block methods %} - {% if methods %} - .. rubric:: {{ _('Methods') }} - - .. autosummary:: - :toctree: - - {% for item in methods %} - {{ name }}.{{ item }} - {%- endfor %} - - {% endif %} - {% endblock %} - - {% block attributes %} - {% if attributes %} - .. rubric:: {{ _('Attributes') }} - - .. autosummary:: - :toctree: - - {% for item in attributes %} - {{ name }}.{{ item }} - {%- endfor %} - - {% endif %} - {% endblock %} + :members: From 19e8ad74e6524c9172e5b257357a98ae226041ba Mon Sep 17 00:00:00 2001 From: Jiajun Yao Date: Thu, 9 Feb 2023 11:49:05 -0800 Subject: [PATCH 29/55] Revert "test" This reverts commit f306a66ecb5de5830de176b6ede480f4449501b0. --- doc/source/_templates/autosummary/class.rst | 29 ++++++++++++++++++++- 1 file changed, 28 insertions(+), 1 deletion(-) diff --git a/doc/source/_templates/autosummary/class.rst b/doc/source/_templates/autosummary/class.rst index f3e9c2c18cbd..47cadf758a42 100644 --- a/doc/source/_templates/autosummary/class.rst +++ b/doc/source/_templates/autosummary/class.rst @@ -3,4 +3,31 @@ .. currentmodule:: {{ module }} .. autoclass:: {{ objname }} - :members: + + {% block methods %} + {% if methods %} + .. rubric:: {{ _('Methods') }} + + .. autosummary:: + :toctree: + + {% for item in methods %} + {{ name }}.{{ item }} + {%- endfor %} + + {% endif %} + {% endblock %} + + {% block attributes %} + {% if attributes %} + .. rubric:: {{ _('Attributes') }} + + .. autosummary:: + :toctree: + + {% for item in attributes %} + {{ name }}.{{ item }} + {%- endfor %} + + {% endif %} + {% endblock %} From 2979edb493110f5e29908b585ad7285a6beaf489 Mon Sep 17 00:00:00 2001 From: Jiajun Yao Date: Thu, 9 Feb 2023 15:32:20 -0800 Subject: [PATCH 30/55] fix Signed-off-by: Jiajun Yao --- .../ray-observability/api/state/api.rst | 2 - python/ray/experimental/state/api.py | 78 +++++++++---------- python/ray/experimental/state/state_cli.py | 32 ++++---- 3 files changed, 55 insertions(+), 57 deletions(-) diff --git a/doc/source/ray-observability/api/state/api.rst b/doc/source/ray-observability/api/state/api.rst index 46c98a2280b6..bfd37f82c391 100644 --- a/doc/source/ray-observability/api/state/api.rst +++ b/doc/source/ray-observability/api/state/api.rst @@ -90,8 +90,6 @@ State APIs Schema State APIs Exceptions --------------------- -.. _state-api-exceptions: - .. autosummary:: :toctree: doc/ diff --git a/python/ray/experimental/state/api.py b/python/ray/experimental/state/api.py index 510d8ffa0feb..883c43cacbe0 100644 --- a/python/ray/experimental/state/api.py +++ b/python/ray/experimental/state/api.py @@ -553,9 +553,9 @@ def get_actor( :class:`ActorState `. Raises: - Exceptions: :ref:`RayStateApiException ` if the CLI + Exceptions: :class:`RayStateApiException ` if the CLI failed to query the data. - """ + """ # noqa: E501 return StateApiClient(address=address).get( StateResource.ACTORS, id, GetApiOptions(timeout=timeout), _explain=_explain ) @@ -592,9 +592,9 @@ def get_placement_group( :class:`~ray.experimental.state.common.PlacementGroupState`. Raises: - Exceptions: :ref:`RayStateApiException ` if the CLI + Exceptions: :class:`RayStateApiException ` if the CLI failed to query the data. - """ + """ # noqa: E501 return StateApiClient(address=address).get( StateResource.PLACEMENT_GROUPS, id, @@ -624,9 +624,9 @@ def get_node( :class:`NodeState `. Raises: - Exceptions: :ref:`RayStateApiException ` + Exceptions: :class:`RayStateApiException ` if the CLI is failed to query the data. - """ + """ # noqa: E501 return StateApiClient(address=address).get( StateResource.NODES, id, @@ -656,9 +656,9 @@ def get_worker( :class:`WorkerState `. Raises: - Exceptions: :ref:`RayStateApiException ` if the CLI + Exceptions: :class:`RayStateApiException ` if the CLI failed to query the data. - """ + """ # noqa: E501 return StateApiClient(address=address).get( StateResource.WORKERS, id, @@ -689,9 +689,9 @@ def get_task( from the task attempts. Raises: - Exceptions: :ref:`RayStateApiException ` if the CLI + Exceptions: :class:`RayStateApiException ` if the CLI failed to query the data. - """ + """ # noqa: E501 return StateApiClient(address=address).get( StateResource.TASKS, id, @@ -724,9 +724,9 @@ def get_objects( :class:`~ray.experimental.state.common.ObjectState`. Raises: - Exceptions: :ref:`RayStateApiException ` if the CLI + Exceptions: :class:`RayStateApiException ` if the CLI failed to query the data. - """ + """ # noqa: E501 return StateApiClient(address=address).get( StateResource.OBJECTS, id, @@ -766,9 +766,9 @@ def list_actors( :class:`ActorState `. Raises: - Exceptions: :ref:`RayStateApiException ` if the CLI + Exceptions: :class:`RayStateApiException ` if the CLI failed to query the data. - """ + """ # noqa: E501 return StateApiClient(address=address).list( StateResource.ACTORS, options=ListApiOptions( @@ -813,9 +813,9 @@ def list_placement_groups( :class:`~ray.experimental.state.common.PlacementGroupState`. Raises: - Exceptions: :ref:`RayStateApiException ` if the CLI + Exceptions: :class:`RayStateApiException ` if the CLI failed to query the data. - """ + """ # noqa: E501 return StateApiClient(address=address).list( StateResource.PLACEMENT_GROUPS, options=ListApiOptions( @@ -857,9 +857,9 @@ def list_nodes( :class:`NodeState `. Raises: - Exceptions: :ref:`RayStateApiException ` + Exceptions: :class:`RayStateApiException ` if the CLI failed to query the data. - """ + """ # noqa: E501 return StateApiClient(address=address).list( StateResource.NODES, options=ListApiOptions( @@ -901,9 +901,9 @@ def list_jobs( :class:`JobState `. Raises: - Exceptions: :ref:`RayStateApiException ` if the CLI + Exceptions: :class:`RayStateApiException ` if the CLI failed to query the data. - """ + """ # noqa: E501 return StateApiClient(address=address).list( StateResource.JOBS, options=ListApiOptions( @@ -945,9 +945,9 @@ def list_workers( :class:`WorkerState `. Raises: - Exceptions: :ref:`RayStateApiException ` if the CLI + Exceptions: :class:`RayStateApiException ` if the CLI failed to query the data. - """ + """ # noqa: E501 return StateApiClient(address=address).list( StateResource.WORKERS, options=ListApiOptions( @@ -989,9 +989,9 @@ def list_tasks( :class:`WorkerState `. Raises: - Exceptions: :ref:`RayStateApiException ` if the CLI + Exceptions: :class:`RayStateApiException ` if the CLI failed to query the data. - """ + """ # noqa: E501 return StateApiClient(address=address).list( StateResource.TASKS, options=ListApiOptions( @@ -1033,9 +1033,9 @@ def list_objects( :class:`ObjectState `. Raises: - Exceptions: :ref:`RayStateApiException ` if the CLI + Exceptions: :class:`RayStateApiException ` if the CLI failed to query the data. - """ + """ # noqa: E501 return StateApiClient(address=address).list( StateResource.OBJECTS, options=ListApiOptions( @@ -1077,9 +1077,9 @@ def list_runtime_envs( :class:`RuntimeEnvState `. Raises: - Exceptions: :ref:`RayStateApiException ` if the CLI - failed to query the data. - """ + Exceptions: :class:`RayStateApiException ` + if the CLI failed to query the data. + """ # noqa: E501 return StateApiClient(address=address).list( StateResource.RUNTIME_ENVS, options=ListApiOptions( @@ -1163,9 +1163,9 @@ def get_log( A Generator of log line, None for SendType and ReturnType. Raises: - Exceptions: :ref:`RayStateApiException ` if the CLI + Exceptions: :class:`RayStateApiException ` if the CLI failed to query the data. - """ + """ # noqa: E501 api_server_url = ray_address_to_api_server_url(address) media_type = "stream" if follow else "file" @@ -1234,10 +1234,10 @@ def list_logs( values are list of log filenames. Raises: - Exceptions: :ref:`RayStateApiException ` if the CLI + Exceptions: :class:`RayStateApiException ` if the CLI failed to query the data, or ConnectionError if failed to resolve the ray address. - """ + """ # noqa: E501 assert ( node_ip is not None or node_id is not None ), "At least one of node ip and node id is required" @@ -1297,9 +1297,9 @@ def summarize_tasks( :class:`~ray.experimental.state.common.TaskSummaries` Raises: - Exceptions: :ref:`RayStateApiException ` + Exceptions: :class:`RayStateApiException ` if the CLI is failed to query the data. - """ + """ # noqa: E501 return StateApiClient(address=address).summary( SummaryResource.TASKS, options=SummaryApiOptions(timeout=timeout), @@ -1330,9 +1330,9 @@ def summarize_actors( :class:`~ray.experimental.state.common.ActorSummaries` Raises: - Exceptions: :ref:`RayStateApiException ` if the CLI + Exceptions: :class:`RayStateApiException ` if the CLI failed to query the data. - """ + """ # noqa: E501 return StateApiClient(address=address).summary( SummaryResource.ACTORS, options=SummaryApiOptions(timeout=timeout), @@ -1362,9 +1362,9 @@ def summarize_objects( Dictionarified :class:`~ray.experimental.state.common.ObjectSummaries` Raises: - Exceptions: :ref:`RayStateApiException ` if the CLI + Exceptions: :class:`RayStateApiException ` if the CLI failed to query the data. - """ + """ # noqa: E501 return StateApiClient(address=address).summary( SummaryResource.OBJECTS, options=SummaryApiOptions(timeout=timeout), diff --git a/python/ray/experimental/state/state_cli.py b/python/ray/experimental/state/state_cli.py index 1b4e5242ad12..55f94ad6bc31 100644 --- a/python/ray/experimental/state/state_cli.py +++ b/python/ray/experimental/state/state_cli.py @@ -378,9 +378,9 @@ def ray_get( id: The id of the resource. Raises: - :ref:`RayStateApiException ` + :class:`RayStateApiException ` if the CLI is failed to query the data. - """ + """ # noqa: E501 # All resource names use '_' rather than '-'. But users options have '-' resource = StateResource(resource.replace("-", "_")) @@ -517,9 +517,9 @@ def ray_list( resource: The type of the resource to query. Raises: - :ref:`RayStateApiException ` + :class:`RayStateApiException ` if the CLI is failed to query the data. - """ + """ # noqa: E501 # All resource names use '_' rather than '-'. But users options have '-' resource = StateResource(resource.replace("-", "_")) format = AvailableFormat(format) @@ -584,9 +584,9 @@ def task_summary(ctx, timeout: float, address: str): :class:`~ray.experimental.state.common.TaskSummaries`. Raises: - :ref:`RayStateApiException ` + :class:`RayStateApiException ` if the CLI is failed to query the data. - """ + """ # noqa: E501 print( format_summary_output( summarize_tasks( @@ -616,9 +616,9 @@ def actor_summary(ctx, timeout: float, address: str): `. Raises: - :ref:`RayStateApiException ` + :class:`RayStateApiException ` if the CLI is failed to query the data. - """ + """ # noqa: E501 print( format_summary_output( summarize_actors( @@ -667,9 +667,9 @@ def object_summary(ctx, timeout: float, address: str): `. Raises: - :ref:`RayStateApiException ` + :class:`RayStateApiException ` if the CLI is failed to query the data. - """ + """ # noqa: E501 print( format_object_summary_output( summarize_objects( @@ -920,9 +920,9 @@ def log_cluster( ``` Raises: - :ref:`RayStateApiException ` if the CLI + :class:`RayStateApiException ` if the CLI is failed to query the data. - """ + """ # noqa: E501 if node_id is None and node_ip is None: node_ip = _get_head_node_ip(address) @@ -1029,10 +1029,10 @@ def log_actor( ``` Raises: - :ref:`RayStateApiException ` + :class:`RayStateApiException ` if the CLI is failed to query the data. MissingParameter if inputs are missing. - """ + """ # noqa: E501 if pid is None and id is None: raise click.MissingParameter( @@ -1102,10 +1102,10 @@ def log_worker( ``` Raises: - :ref:`RayStateApiException ` + :class:`RayStateApiException ` if the CLI is failed to query the data. MissingParameter if inputs are missing. - """ + """ # noqa: E501 _print_log( address=address, From c9b66e4c6a1eb52de68061c6d4d38a4b8b474a17 Mon Sep 17 00:00:00 2001 From: Jiajun Yao Date: Thu, 9 Feb 2023 20:39:58 -0800 Subject: [PATCH 31/55] warn Signed-off-by: Jiajun Yao --- doc/source/conf.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/doc/source/conf.py b/doc/source/conf.py index 95750148649f..4a6ca87f9487 100644 --- a/doc/source/conf.py +++ b/doc/source/conf.py @@ -3,6 +3,7 @@ from pathlib import Path import os import sys +import warnings sys.path.insert(0, os.path.abspath(".")) from custom_directives import * @@ -104,6 +105,11 @@ # This is used to suppress warnings about explicit "toctree" directives. suppress_warnings = ["etoc.toctree"] +# It's a known bug (https://github.com/sphinx-doc/sphinx/issues/9884) +# that autosummary will generate warning for inherited instance attribute. +warnings.filterwarnings(action="ignore", message=".*autosummary: failed to import.*") +warnings.filterwarnings(action="ignore", message=".*autosummary: \[autosummary\] failed to import.*") + versionwarning_admonition_type = "note" versionwarning_banner_title = "Join the Ray Discuss Forums!" From 203241ff8581ad576e3594e481a32bd5b41aafbd Mon Sep 17 00:00:00 2001 From: Jiajun Yao Date: Thu, 9 Feb 2023 21:43:05 -0800 Subject: [PATCH 32/55] try Signed-off-by: Jiajun Yao --- doc/source/conf.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/doc/source/conf.py b/doc/source/conf.py index 4a6ca87f9487..b6639125f936 100644 --- a/doc/source/conf.py +++ b/doc/source/conf.py @@ -103,12 +103,13 @@ # autodoc_mock_imports = ["ray.experimental.tf_utils"] # This is used to suppress warnings about explicit "toctree" directives. -suppress_warnings = ["etoc.toctree"] +suppress_warnings = ["etoc.toctree", "autosummary"] # It's a known bug (https://github.com/sphinx-doc/sphinx/issues/9884) # that autosummary will generate warning for inherited instance attribute. -warnings.filterwarnings(action="ignore", message=".*autosummary: failed to import.*") -warnings.filterwarnings(action="ignore", message=".*autosummary: \[autosummary\] failed to import.*") +warnings.filterwarnings(action="ignore", message=".*autosummary.*") +warnings.filterwarnings(action="ignore", message="import") +warnings.filterwarnings(action="ignore", message=".*import.*") versionwarning_admonition_type = "note" versionwarning_banner_title = "Join the Ray Discuss Forums!" From db0261d5dae6c6953944e9be25dbb4775f2bcbea Mon Sep 17 00:00:00 2001 From: Jiajun Yao Date: Fri, 10 Feb 2023 08:57:06 -0800 Subject: [PATCH 33/55] comments Signed-off-by: Jiajun Yao --- doc/source/conf.py | 3 ++- doc/source/ray-core/api/cli.rst | 4 ++-- doc/source/ray-core/api/core.rst | 16 ++++++++-------- 3 files changed, 12 insertions(+), 11 deletions(-) diff --git a/doc/source/conf.py b/doc/source/conf.py index b6639125f936..92108c1d6389 100644 --- a/doc/source/conf.py +++ b/doc/source/conf.py @@ -103,10 +103,11 @@ # autodoc_mock_imports = ["ray.experimental.tf_utils"] # This is used to suppress warnings about explicit "toctree" directives. -suppress_warnings = ["etoc.toctree", "autosummary"] +suppress_warnings = ["etoc.toctree", "autosummary", "autodoc", "autodoc.import_object"] # It's a known bug (https://github.com/sphinx-doc/sphinx/issues/9884) # that autosummary will generate warning for inherited instance attribute. +autosummary_mock_imports = ["ray.experimental.state.common.JobState.entrypoint"] warnings.filterwarnings(action="ignore", message=".*autosummary.*") warnings.filterwarnings(action="ignore", message="import") warnings.filterwarnings(action="ignore", message=".*import.*") diff --git a/doc/source/ray-core/api/cli.rst b/doc/source/ray-core/api/cli.rst index c9041ad3e18c..7f5ee4c3ef57 100644 --- a/doc/source/ray-core/api/cli.rst +++ b/doc/source/ray-core/api/cli.rst @@ -1,5 +1,5 @@ -Core CLI -======== +Ray Core CLI +============ .. _ray-cli: diff --git a/doc/source/ray-core/api/core.rst b/doc/source/ray-core/api/core.rst index 00dde503c5b8..b1f4d3fa2bbb 100644 --- a/doc/source/ray-core/api/core.rst +++ b/doc/source/ray-core/api/core.rst @@ -1,5 +1,5 @@ -Core API -======== +Ray Core API +============ .. autosummary:: :toctree: doc/ @@ -8,8 +8,8 @@ Core API ray.shutdown ray.is_initialized -Task ----- +Tasks +----- .. autosummary:: :toctree: doc/ @@ -18,8 +18,8 @@ Task ray.remote_function.RemoteFunction.options ray.cancel -Actor ------ +Actors +------ .. autosummary:: :toctree: doc/ @@ -30,8 +30,8 @@ Actor ray.get_actor ray.kill -Object ------- +Objects +------- .. autosummary:: :toctree: doc/ From beb1bf23dfeb2a76b88ce9fa100a1ccc3e51de8e Mon Sep 17 00:00:00 2001 From: Jiajun Yao Date: Fri, 10 Feb 2023 09:54:44 -0800 Subject: [PATCH 34/55] up Signed-off-by: Jiajun Yao --- doc/source/conf.py | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/doc/source/conf.py b/doc/source/conf.py index 92108c1d6389..de0d23d30f3d 100644 --- a/doc/source/conf.py +++ b/doc/source/conf.py @@ -3,7 +3,6 @@ from pathlib import Path import os import sys -import warnings sys.path.insert(0, os.path.abspath(".")) from custom_directives import * @@ -103,14 +102,11 @@ # autodoc_mock_imports = ["ray.experimental.tf_utils"] # This is used to suppress warnings about explicit "toctree" directives. -suppress_warnings = ["etoc.toctree", "autosummary", "autodoc", "autodoc.import_object"] +suppress_warnings = ["etoc.toctree"] # It's a known bug (https://github.com/sphinx-doc/sphinx/issues/9884) # that autosummary will generate warning for inherited instance attribute. -autosummary_mock_imports = ["ray.experimental.state.common.JobState.entrypoint"] -warnings.filterwarnings(action="ignore", message=".*autosummary.*") -warnings.filterwarnings(action="ignore", message="import") -warnings.filterwarnings(action="ignore", message=".*import.*") +autosummary_mock_imports = ["ray.experimental.state.common"] versionwarning_admonition_type = "note" versionwarning_banner_title = "Join the Ray Discuss Forums!" From e2411831a02a6c724e68228b80b9071e333a70ae Mon Sep 17 00:00:00 2001 From: Jiajun Yao Date: Fri, 10 Feb 2023 10:50:53 -0800 Subject: [PATCH 35/55] up Signed-off-by: Jiajun Yao --- doc/source/_templates/autosummary/class.rst | 7 +++++++ doc/source/conf.py | 4 ---- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/doc/source/_templates/autosummary/class.rst b/doc/source/_templates/autosummary/class.rst index 47cadf758a42..76f94ff35b16 100644 --- a/doc/source/_templates/autosummary/class.rst +++ b/doc/source/_templates/autosummary/class.rst @@ -26,7 +26,14 @@ :toctree: {% for item in attributes %} + {# + It's a known bug (https://github.com/sphinx-doc/sphinx/issues/9884) + that autosummary will generate warning for inherited instance attribute. + We remove them for now. + #} + {%- if item not in inherited_members %} {{ name }}.{{ item }} + {%- endif -%} {%- endfor %} {% endif %} diff --git a/doc/source/conf.py b/doc/source/conf.py index de0d23d30f3d..95750148649f 100644 --- a/doc/source/conf.py +++ b/doc/source/conf.py @@ -104,10 +104,6 @@ # This is used to suppress warnings about explicit "toctree" directives. suppress_warnings = ["etoc.toctree"] -# It's a known bug (https://github.com/sphinx-doc/sphinx/issues/9884) -# that autosummary will generate warning for inherited instance attribute. -autosummary_mock_imports = ["ray.experimental.state.common"] - versionwarning_admonition_type = "note" versionwarning_banner_title = "Join the Ray Discuss Forums!" From 019fc859a7c19c6abd1b7c4275baaa4dcbd71840 Mon Sep 17 00:00:00 2001 From: Jiajun Yao Date: Fri, 10 Feb 2023 12:00:25 -0800 Subject: [PATCH 36/55] try Signed-off-by: Jiajun Yao --- doc/source/_templates/autosummary/class.rst | 21 --------------------- 1 file changed, 21 deletions(-) diff --git a/doc/source/_templates/autosummary/class.rst b/doc/source/_templates/autosummary/class.rst index 76f94ff35b16..554d2618222e 100644 --- a/doc/source/_templates/autosummary/class.rst +++ b/doc/source/_templates/autosummary/class.rst @@ -17,24 +17,3 @@ {% endif %} {% endblock %} - - {% block attributes %} - {% if attributes %} - .. rubric:: {{ _('Attributes') }} - - .. autosummary:: - :toctree: - - {% for item in attributes %} - {# - It's a known bug (https://github.com/sphinx-doc/sphinx/issues/9884) - that autosummary will generate warning for inherited instance attribute. - We remove them for now. - #} - {%- if item not in inherited_members %} - {{ name }}.{{ item }} - {%- endif -%} - {%- endfor %} - - {% endif %} - {% endblock %} From a24f3ba2865a70e8c30a3525a3ad7a9e0fc1f394 Mon Sep 17 00:00:00 2001 From: Jiajun Yao Date: Fri, 10 Feb 2023 15:13:28 -0800 Subject: [PATCH 37/55] up Signed-off-by: Jiajun Yao --- doc/source/ray-core/api/core.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/source/ray-core/api/core.rst b/doc/source/ray-core/api/core.rst index b1f4d3fa2bbb..c78e9afc418c 100644 --- a/doc/source/ray-core/api/core.rst +++ b/doc/source/ray-core/api/core.rst @@ -1,5 +1,5 @@ -Ray Core API -============ +Core API +======== .. autosummary:: :toctree: doc/ From 7f1b7ae6f38440ffd27327ece5c0f8135358f6b6 Mon Sep 17 00:00:00 2001 From: Jiajun Yao Date: Fri, 10 Feb 2023 15:13:48 -0800 Subject: [PATCH 38/55] Revert "try" This reverts commit 019fc859a7c19c6abd1b7c4275baaa4dcbd71840. --- doc/source/_templates/autosummary/class.rst | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/doc/source/_templates/autosummary/class.rst b/doc/source/_templates/autosummary/class.rst index 554d2618222e..76f94ff35b16 100644 --- a/doc/source/_templates/autosummary/class.rst +++ b/doc/source/_templates/autosummary/class.rst @@ -17,3 +17,24 @@ {% endif %} {% endblock %} + + {% block attributes %} + {% if attributes %} + .. rubric:: {{ _('Attributes') }} + + .. autosummary:: + :toctree: + + {% for item in attributes %} + {# + It's a known bug (https://github.com/sphinx-doc/sphinx/issues/9884) + that autosummary will generate warning for inherited instance attribute. + We remove them for now. + #} + {%- if item not in inherited_members %} + {{ name }}.{{ item }} + {%- endif -%} + {%- endfor %} + + {% endif %} + {% endblock %} From b7b6b5a121e4428e4c9e3aa73d6eb2c9d4e9f8aa Mon Sep 17 00:00:00 2001 From: Jiajun Yao Date: Fri, 10 Feb 2023 15:17:47 -0800 Subject: [PATCH 39/55] try Signed-off-by: Jiajun Yao --- doc/source/_templates/autosummary/class.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/source/_templates/autosummary/class.rst b/doc/source/_templates/autosummary/class.rst index 76f94ff35b16..38dd4c09a84f 100644 --- a/doc/source/_templates/autosummary/class.rst +++ b/doc/source/_templates/autosummary/class.rst @@ -31,7 +31,7 @@ that autosummary will generate warning for inherited instance attribute. We remove them for now. #} - {%- if item not in inherited_members %} + {%- if item in inherited_members %} {{ name }}.{{ item }} {%- endif -%} {%- endfor %} From d915e3b3e2dfbbd7784012b869d0c2e5307fcddc Mon Sep 17 00:00:00 2001 From: Jiajun Yao Date: Sat, 11 Feb 2023 21:07:14 -0800 Subject: [PATCH 40/55] try Signed-off-by: Jiajun Yao --- doc/source/_templates/autosummary/class.rst | 7 ------- .../autosummary/class_without_autosummary.rst | 12 ++++++++++++ doc/source/ray-observability/api/state/api.rst | 1 + 3 files changed, 13 insertions(+), 7 deletions(-) create mode 100644 doc/source/_templates/autosummary/class_without_autosummary.rst diff --git a/doc/source/_templates/autosummary/class.rst b/doc/source/_templates/autosummary/class.rst index 38dd4c09a84f..47cadf758a42 100644 --- a/doc/source/_templates/autosummary/class.rst +++ b/doc/source/_templates/autosummary/class.rst @@ -26,14 +26,7 @@ :toctree: {% for item in attributes %} - {# - It's a known bug (https://github.com/sphinx-doc/sphinx/issues/9884) - that autosummary will generate warning for inherited instance attribute. - We remove them for now. - #} - {%- if item in inherited_members %} {{ name }}.{{ item }} - {%- endif -%} {%- endfor %} {% endif %} diff --git a/doc/source/_templates/autosummary/class_without_autosummary.rst b/doc/source/_templates/autosummary/class_without_autosummary.rst new file mode 100644 index 000000000000..61a594cef1c7 --- /dev/null +++ b/doc/source/_templates/autosummary/class_without_autosummary.rst @@ -0,0 +1,12 @@ +{# + It's a known bug (https://github.com/sphinx-doc/sphinx/issues/9884) + that autosummary will generate warning for inherited instance attributes. + Those warnings will fail our build. + As a temporary workaround, we don't autosummary classes with inherited instance attributes. +#} +{{ fullname | escape | underline}} + +.. currentmodule:: {{ module }} + +.. autoclass:: {{ objname }} + :members: diff --git a/doc/source/ray-observability/api/state/api.rst b/doc/source/ray-observability/api/state/api.rst index bfd37f82c391..eda123718a72 100644 --- a/doc/source/ray-observability/api/state/api.rst +++ b/doc/source/ray-observability/api/state/api.rst @@ -70,6 +70,7 @@ State APIs Schema .. autosummary:: :toctree: doc/ + :template: autosummary/class_without_autosummary.rst ray.experimental.state.common.ActorState ray.experimental.state.common.TaskState From fe9f44de0921bdcc93b9c910a2215c5d62c318a4 Mon Sep 17 00:00:00 2001 From: Jiajun Yao Date: Sun, 12 Feb 2023 21:21:35 -0800 Subject: [PATCH 41/55] fix Signed-off-by: Jiajun Yao --- .../_templates/autosummary/class_without_autosummary.rst | 5 +++++ doc/source/tune/api/schedulers.rst | 5 +++++ doc/source/tune/api/suggestion.rst | 2 ++ 3 files changed, 12 insertions(+) diff --git a/doc/source/_templates/autosummary/class_without_autosummary.rst b/doc/source/_templates/autosummary/class_without_autosummary.rst index 61a594cef1c7..bf143b8bd333 100644 --- a/doc/source/_templates/autosummary/class_without_autosummary.rst +++ b/doc/source/_templates/autosummary/class_without_autosummary.rst @@ -4,6 +4,11 @@ Those warnings will fail our build. As a temporary workaround, we don't autosummary classes with inherited instance attributes. #} +{# + It also seems that autosummary doesn't work with type alias + so we don't autosummary those as well. + See ray.tune.schedulers.ASHAScheduler as an example. +#} {{ fullname | escape | underline}} .. currentmodule:: {{ module }} diff --git a/doc/source/tune/api/schedulers.rst b/doc/source/tune/api/schedulers.rst index a7ee7772092a..d09d89a7a303 100644 --- a/doc/source/tune/api/schedulers.rst +++ b/doc/source/tune/api/schedulers.rst @@ -75,6 +75,11 @@ This is the default used if no value is provided for the ``brackets`` argument. :toctree: doc/ AsyncHyperBandScheduler + +.. autosummary:: + :toctree: doc/ + :template: autosummary/class_without_autosummary.rst + ASHAScheduler .. _tune-original-hyperband: diff --git a/doc/source/tune/api/suggestion.rst b/doc/source/tune/api/suggestion.rst index bce657c9ecc5..fa00acaac2f7 100644 --- a/doc/source/tune/api/suggestion.rst +++ b/doc/source/tune/api/suggestion.rst @@ -177,6 +177,7 @@ See the `BlendSearch paper `_ and docu .. autosummary:: :toctree: doc/ + :template: autosummary/class_without_autosummary.rst flaml.BlendSearch @@ -200,6 +201,7 @@ FLAML `CFO documentation Date: Mon, 13 Feb 2023 07:40:21 -0800 Subject: [PATCH 42/55] fix Signed-off-by: Jiajun Yao --- python/ray/tune/execution/placement_groups.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/python/ray/tune/execution/placement_groups.py b/python/ray/tune/execution/placement_groups.py index 12f820e5df6b..df988be401ea 100644 --- a/python/ray/tune/execution/placement_groups.py +++ b/python/ray/tune/execution/placement_groups.py @@ -86,8 +86,8 @@ class PlacementGroupFactory(ResourceRequest): - "STRICT_PACK": Packs Bundles into one node. The group is not allowed to span multiple nodes. - "STRICT_SPREAD": Packs Bundles across distinct nodes. - *args: Passed to the call of ``placement_group()`` - **kwargs: Passed to the call of ``placement_group()`` + args: Passed to the call of ``placement_group()`` + kwargs: Passed to the call of ``placement_group()`` """ From a6d319b0a439462caee9c33545d2ecafbaea6266 Mon Sep 17 00:00:00 2001 From: Jiajun Yao Date: Mon, 13 Feb 2023 09:33:01 -0800 Subject: [PATCH 43/55] reduce time Signed-off-by: Jiajun Yao --- .../_templates/autosummary/class_without_autosummary.rst | 3 +++ doc/source/ray-core/api/exceptions.rst | 1 + doc/source/ray-core/api/scheduling.rst | 1 + doc/source/ray-core/api/utility.rst | 1 + 4 files changed, 6 insertions(+) diff --git a/doc/source/_templates/autosummary/class_without_autosummary.rst b/doc/source/_templates/autosummary/class_without_autosummary.rst index bf143b8bd333..aec544c3fc0c 100644 --- a/doc/source/_templates/autosummary/class_without_autosummary.rst +++ b/doc/source/_templates/autosummary/class_without_autosummary.rst @@ -9,6 +9,9 @@ so we don't autosummary those as well. See ray.tune.schedulers.ASHAScheduler as an example. #} +{# + We also don't autosummary some classes to reduce doc build time. +#} {{ fullname | escape | underline}} .. currentmodule:: {{ module }} diff --git a/doc/source/ray-core/api/exceptions.rst b/doc/source/ray-core/api/exceptions.rst index d041612573d4..8fd9ea33d452 100644 --- a/doc/source/ray-core/api/exceptions.rst +++ b/doc/source/ray-core/api/exceptions.rst @@ -5,6 +5,7 @@ Exceptions .. autosummary:: :toctree: doc/ + :template: autosummary/class_without_autosummary.rst ray.exceptions.RayError ray.exceptions.RayTaskError diff --git a/doc/source/ray-core/api/scheduling.rst b/doc/source/ray-core/api/scheduling.rst index 2971aae8e3c8..40d1a66ccecd 100644 --- a/doc/source/ray-core/api/scheduling.rst +++ b/doc/source/ray-core/api/scheduling.rst @@ -6,6 +6,7 @@ Scheduling Strategy .. autosummary:: :toctree: doc/ + :template: autosummary/class_without_autosummary.rst ray.util.scheduling_strategies.PlacementGroupSchedulingStrategy ray.util.scheduling_strategies.NodeAffinitySchedulingStrategy diff --git a/doc/source/ray-core/api/utility.rst b/doc/source/ray-core/api/utility.rst index 36da67415d4f..445e58de1456 100644 --- a/doc/source/ray-core/api/utility.rst +++ b/doc/source/ray-core/api/utility.rst @@ -17,6 +17,7 @@ Custom Metrics .. autosummary:: :toctree: doc/ + :template: autosummary/class_without_autosummary.rst ray.util.metrics.Counter ray.util.metrics.Gauge From 87d6710c824a23b9c6dca6b2f815430809527592 Mon Sep 17 00:00:00 2001 From: Jiajun Yao Date: Mon, 13 Feb 2023 10:48:28 -0800 Subject: [PATCH 44/55] fix Signed-off-by: Jiajun Yao --- python/ray/air/execution/resources/request.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/python/ray/air/execution/resources/request.py b/python/ray/air/execution/resources/request.py index 8a66fb5cfe00..0f1a53c0102f 100644 --- a/python/ray/air/execution/resources/request.py +++ b/python/ray/air/execution/resources/request.py @@ -52,8 +52,8 @@ class ResourceRequest: - "STRICT_PACK": Packs Bundles into one node. The group is not allowed to span multiple nodes. - "STRICT_SPREAD": Packs Bundles across distinct nodes. - *args: Passed to the call of ``placement_group()``, if applicable. - **kwargs: Passed to the call of ``placement_group()``, if applicable. + args: Passed to the call of ``placement_group()``, if applicable. + kwargs: Passed to the call of ``placement_group()``, if applicable. """ From 1272e726a12b92c83b485d57a9191f011ea31567 Mon Sep 17 00:00:00 2001 From: Jiajun Yao Date: Mon, 13 Feb 2023 11:59:45 -0800 Subject: [PATCH 45/55] fix Signed-off-by: Jiajun Yao --- python/ray/air/execution/resources/request.py | 5 +++-- python/ray/tune/execution/placement_groups.py | 4 ++-- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/python/ray/air/execution/resources/request.py b/python/ray/air/execution/resources/request.py index 0f1a53c0102f..87455a6b3ad8 100644 --- a/python/ray/air/execution/resources/request.py +++ b/python/ray/air/execution/resources/request.py @@ -52,8 +52,8 @@ class ResourceRequest: - "STRICT_PACK": Packs Bundles into one node. The group is not allowed to span multiple nodes. - "STRICT_SPREAD": Packs Bundles across distinct nodes. - args: Passed to the call of ``placement_group()``, if applicable. - kwargs: Passed to the call of ``placement_group()``, if applicable. + *args: Passed to the call of ``placement_group()``, if applicable. + **kwargs: Passed to the call of ``placement_group()``, if applicable. """ @@ -108,6 +108,7 @@ def head_bundle_is_empty(self): @property @DeveloperAPI def head_cpus(self) -> float: + """Returns the number of cpus in the head bundle.""" return 0.0 if self._head_bundle_is_empty else self._bundles[0].get("CPU", 0.0) @property diff --git a/python/ray/tune/execution/placement_groups.py b/python/ray/tune/execution/placement_groups.py index df988be401ea..12f820e5df6b 100644 --- a/python/ray/tune/execution/placement_groups.py +++ b/python/ray/tune/execution/placement_groups.py @@ -86,8 +86,8 @@ class PlacementGroupFactory(ResourceRequest): - "STRICT_PACK": Packs Bundles into one node. The group is not allowed to span multiple nodes. - "STRICT_SPREAD": Packs Bundles across distinct nodes. - args: Passed to the call of ``placement_group()`` - kwargs: Passed to the call of ``placement_group()`` + *args: Passed to the call of ``placement_group()`` + **kwargs: Passed to the call of ``placement_group()`` """ From 11333ba87634ca3199516e80e6e60151bec4ee32 Mon Sep 17 00:00:00 2001 From: Jiajun Yao Date: Mon, 13 Feb 2023 12:43:12 -0800 Subject: [PATCH 46/55] reduce doc build time Signed-off-by: Jiajun Yao --- doc/source/ray-core/api/runtime-env.rst | 1 + doc/source/tune/api/integration.rst | 6 ++++++ doc/source/tune/api/logging.rst | 3 +++ doc/source/tune/api/schedulers.rst | 14 +++++++++----- doc/source/tune/api/suggestion.rst | 14 ++++++++++++++ 5 files changed, 33 insertions(+), 5 deletions(-) diff --git a/doc/source/ray-core/api/runtime-env.rst b/doc/source/ray-core/api/runtime-env.rst index 607766a9535a..170d0c077031 100644 --- a/doc/source/ray-core/api/runtime-env.rst +++ b/doc/source/ray-core/api/runtime-env.rst @@ -3,6 +3,7 @@ Runtime Env API .. autosummary:: :toctree: doc/ + :template: autosummary/class_without_autosummary.rst ray.runtime_env.RuntimeEnvConfig ray.runtime_env.RuntimeEnv diff --git a/doc/source/tune/api/integration.rst b/doc/source/tune/api/integration.rst index 0ad8ff86d2a6..d5954dbfa76f 100644 --- a/doc/source/tune/api/integration.rst +++ b/doc/source/tune/api/integration.rst @@ -23,6 +23,7 @@ Comet (air.integrations.comet) .. autosummary:: :toctree: doc/ + :template: autosummary/class_without_autosummary.rst ~air.integrations.comet.CometLoggerCallback @@ -64,6 +65,7 @@ Keras (air.integrations.keras) .. autosummary:: :toctree: doc/ + :template: autosummary/class_without_autosummary.rst ~air.integrations.keras.ReportCheckpointCallback @@ -75,6 +77,7 @@ MXNet (tune.integration.mxnet) .. autosummary:: :toctree: doc/ + :template: autosummary/class_without_autosummary.rst ~tune.integration.mxnet.TuneReportCallback ~tune.integration.mxnet.TuneCheckpointCallback @@ -87,6 +90,7 @@ PyTorch Lightning (tune.integration.pytorch_lightning) .. autosummary:: :toctree: doc/ + :template: autosummary/class_without_autosummary.rst ~tune.integration.pytorch_lightning.TuneReportCallback ~tune.integration.pytorch_lightning.TuneReportCheckpointCallback @@ -98,6 +102,7 @@ XGBoost (tune.integration.xgboost) .. autosummary:: :toctree: doc/ + :template: autosummary/class_without_autosummary.rst ~tune.integration.xgboost.TuneReportCallback ~tune.integration.xgboost.TuneReportCheckpointCallback @@ -110,6 +115,7 @@ LightGBM (tune.integration.lightgbm) .. autosummary:: :toctree: doc/ + :template: autosummary/class_without_autosummary.rst ~tune.integration.lightgbm.TuneReportCallback ~tune.integration.lightgbm.TuneReportCheckpointCallback diff --git a/doc/source/tune/api/logging.rst b/doc/source/tune/api/logging.rst index e5eb8732b626..805704c8ed62 100644 --- a/doc/source/tune/api/logging.rst +++ b/doc/source/tune/api/logging.rst @@ -38,6 +38,7 @@ Tune Built-in Loggers .. autosummary:: :toctree: doc/ + :template: autosummary/class_without_autosummary.rst tune.logger.JsonLoggerCallback tune.logger.CSVLoggerCallback @@ -53,6 +54,7 @@ You can see the :doc:`tutorial here `. .. autosummary:: :toctree: doc/ + :template: autosummary/class_without_autosummary.rst air.integrations.mlflow.MLflowLoggerCallback @@ -65,6 +67,7 @@ You can see the :doc:`tutorial here `. .. autosummary:: :toctree: doc/ + :template: autosummary/class_without_autosummary.rst air.integrations.wandb.WandbLoggerCallback diff --git a/doc/source/tune/api/schedulers.rst b/doc/source/tune/api/schedulers.rst index d09d89a7a303..b1a10724b1c1 100644 --- a/doc/source/tune/api/schedulers.rst +++ b/doc/source/tune/api/schedulers.rst @@ -71,15 +71,11 @@ Even though the original paper mentions a bracket count of 3, discussions with t that the value should be left to 1 bracket. This is the default used if no value is provided for the ``brackets`` argument. -.. autosummary:: - :toctree: doc/ - - AsyncHyperBandScheduler - .. autosummary:: :toctree: doc/ :template: autosummary/class_without_autosummary.rst + AsyncHyperBandScheduler ASHAScheduler .. _tune-original-hyperband: @@ -92,6 +88,7 @@ Tune implements the `standard version of HyperBand `_ for more details. .. autosummary:: :toctree: doc/ + :template: autosummary/class_without_autosummary.rst bohb.TuneBOHB @@ -212,6 +216,7 @@ Dragonfly (tune.search.dragonfly.DragonflySearch) .. autosummary:: :toctree: doc/ + :template: autosummary/class_without_autosummary.rst dragonfly.DragonflySearch @@ -222,6 +227,7 @@ HEBO (tune.search.hebo.HEBOSearch) .. autosummary:: :toctree: doc/ + :template: autosummary/class_without_autosummary.rst hebo.HEBOSearch @@ -232,6 +238,7 @@ HyperOpt (tune.search.hyperopt.HyperOptSearch) .. autosummary:: :toctree: doc/ + :template: autosummary/class_without_autosummary.rst hyperopt.HyperOptSearch @@ -242,6 +249,7 @@ Nevergrad (tune.search.nevergrad.NevergradSearch) .. autosummary:: :toctree: doc/ + :template: autosummary/class_without_autosummary.rst nevergrad.NevergradSearch @@ -252,6 +260,7 @@ Optuna (tune.search.optuna.OptunaSearch) .. autosummary:: :toctree: doc/ + :template: autosummary/class_without_autosummary.rst optuna.OptunaSearch @@ -265,6 +274,7 @@ to specify your search space. .. autosummary:: :toctree: doc/ + :template: autosummary/class_without_autosummary.rst sigopt.SigOptSearch @@ -275,6 +285,7 @@ Scikit-Optimize (tune.search.skopt.SkOptSearch) .. autosummary:: :toctree: doc/ + :template: autosummary/class_without_autosummary.rst skopt.SkOptSearch @@ -285,6 +296,7 @@ ZOOpt (tune.search.zoopt.ZOOptSearch) .. autosummary:: :toctree: doc/ + :template: autosummary/class_without_autosummary.rst zoopt.ZOOptSearch @@ -308,6 +320,7 @@ will run ``repeat`` trials of the configuration. It will then average the .. autosummary:: :toctree: doc/ + :template: autosummary/class_without_autosummary.rst Repeater @@ -321,6 +334,7 @@ This is useful when a given optimization algorithm does not parallelize very wel .. autosummary:: :toctree: doc/ + :template: autosummary/class_without_autosummary.rst ConcurrencyLimiter From b80c383cf895b87ed64305a5bcc095653df735cb Mon Sep 17 00:00:00 2001 From: Jiajun Yao Date: Mon, 13 Feb 2023 14:44:12 -0800 Subject: [PATCH 47/55] try Signed-off-by: Jiajun Yao --- doc/source/_templates/autosummary/class.rst | 2 -- 1 file changed, 2 deletions(-) diff --git a/doc/source/_templates/autosummary/class.rst b/doc/source/_templates/autosummary/class.rst index 47cadf758a42..b121e7577a25 100644 --- a/doc/source/_templates/autosummary/class.rst +++ b/doc/source/_templates/autosummary/class.rst @@ -9,7 +9,6 @@ .. rubric:: {{ _('Methods') }} .. autosummary:: - :toctree: {% for item in methods %} {{ name }}.{{ item }} @@ -23,7 +22,6 @@ .. rubric:: {{ _('Attributes') }} .. autosummary:: - :toctree: {% for item in attributes %} {{ name }}.{{ item }} From 2b7844043fdd2c0a477071c2f9019af04055c2c9 Mon Sep 17 00:00:00 2001 From: Jiajun Yao Date: Mon, 13 Feb 2023 16:10:09 -0800 Subject: [PATCH 48/55] Revert "try" This reverts commit b80c383cf895b87ed64305a5bcc095653df735cb. --- doc/source/_templates/autosummary/class.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/doc/source/_templates/autosummary/class.rst b/doc/source/_templates/autosummary/class.rst index b121e7577a25..47cadf758a42 100644 --- a/doc/source/_templates/autosummary/class.rst +++ b/doc/source/_templates/autosummary/class.rst @@ -9,6 +9,7 @@ .. rubric:: {{ _('Methods') }} .. autosummary:: + :toctree: {% for item in methods %} {{ name }}.{{ item }} @@ -22,6 +23,7 @@ .. rubric:: {{ _('Attributes') }} .. autosummary:: + :toctree: {% for item in attributes %} {{ name }}.{{ item }} From ba4ca5ea4a1aaaa4ab30139dfcac258a148bafda Mon Sep 17 00:00:00 2001 From: Jiajun Yao Date: Mon, 13 Feb 2023 20:20:00 -0800 Subject: [PATCH 49/55] up Signed-off-by: Jiajun Yao --- doc/source/_templates/autosummary/class.rst | 29 +----------- .../autosummary/class_with_autosummary.rst | 45 +++++++++++++++++++ .../autosummary/class_without_autosummary.rst | 20 --------- doc/source/ray-core/api/exceptions.rst | 1 - doc/source/ray-core/api/runtime-env.rst | 1 - doc/source/ray-core/api/scheduling.rst | 1 - doc/source/ray-core/api/utility.rst | 1 - .../ray-observability/api/state/api.rst | 1 - doc/source/tune/api/integration.rst | 6 --- doc/source/tune/api/logging.rst | 3 -- doc/source/tune/api/schedulers.rst | 9 ---- doc/source/tune/api/suggestion.rst | 16 ------- 12 files changed, 46 insertions(+), 87 deletions(-) create mode 100644 doc/source/_templates/autosummary/class_with_autosummary.rst delete mode 100644 doc/source/_templates/autosummary/class_without_autosummary.rst diff --git a/doc/source/_templates/autosummary/class.rst b/doc/source/_templates/autosummary/class.rst index 47cadf758a42..8643ade9ab82 100644 --- a/doc/source/_templates/autosummary/class.rst +++ b/doc/source/_templates/autosummary/class.rst @@ -3,31 +3,4 @@ .. currentmodule:: {{ module }} .. autoclass:: {{ objname }} - - {% block methods %} - {% if methods %} - .. rubric:: {{ _('Methods') }} - - .. autosummary:: - :toctree: - - {% for item in methods %} - {{ name }}.{{ item }} - {%- endfor %} - - {% endif %} - {% endblock %} - - {% block attributes %} - {% if attributes %} - .. rubric:: {{ _('Attributes') }} - - .. autosummary:: - :toctree: - - {% for item in attributes %} - {{ name }}.{{ item }} - {%- endfor %} - - {% endif %} - {% endblock %} + :members: diff --git a/doc/source/_templates/autosummary/class_with_autosummary.rst b/doc/source/_templates/autosummary/class_with_autosummary.rst new file mode 100644 index 000000000000..b409424e83e9 --- /dev/null +++ b/doc/source/_templates/autosummary/class_with_autosummary.rst @@ -0,0 +1,45 @@ +{# + Generating pages for class methods and attributes + significantly increases the doc build time and causes timeouts. + For now, opt in explicitly via `:template: autosummary/class_with_autosummary.rst` +#} +{# + It's a known bug (https://github.com/sphinx-doc/sphinx/issues/9884) + that autosummary will generate warning for inherited instance attributes. + Those warnings will fail our build. + For now, we don't autosummary classes with inherited instance attributes. +#} + +{{ fullname | escape | underline}} + +.. currentmodule:: {{ module }} + +.. autoclass:: {{ objname }} + + {% block methods %} + {% if methods %} + .. rubric:: {{ _('Methods') }} + + .. autosummary:: + :toctree: + + {% for item in methods %} + {{ name }}.{{ item }} + {%- endfor %} + + {% endif %} + {% endblock %} + + {% block attributes %} + {% if attributes %} + .. rubric:: {{ _('Attributes') }} + + .. autosummary:: + :toctree: + + {% for item in attributes %} + {{ name }}.{{ item }} + {%- endfor %} + + {% endif %} + {% endblock %} diff --git a/doc/source/_templates/autosummary/class_without_autosummary.rst b/doc/source/_templates/autosummary/class_without_autosummary.rst deleted file mode 100644 index aec544c3fc0c..000000000000 --- a/doc/source/_templates/autosummary/class_without_autosummary.rst +++ /dev/null @@ -1,20 +0,0 @@ -{# - It's a known bug (https://github.com/sphinx-doc/sphinx/issues/9884) - that autosummary will generate warning for inherited instance attributes. - Those warnings will fail our build. - As a temporary workaround, we don't autosummary classes with inherited instance attributes. -#} -{# - It also seems that autosummary doesn't work with type alias - so we don't autosummary those as well. - See ray.tune.schedulers.ASHAScheduler as an example. -#} -{# - We also don't autosummary some classes to reduce doc build time. -#} -{{ fullname | escape | underline}} - -.. currentmodule:: {{ module }} - -.. autoclass:: {{ objname }} - :members: diff --git a/doc/source/ray-core/api/exceptions.rst b/doc/source/ray-core/api/exceptions.rst index 8fd9ea33d452..d041612573d4 100644 --- a/doc/source/ray-core/api/exceptions.rst +++ b/doc/source/ray-core/api/exceptions.rst @@ -5,7 +5,6 @@ Exceptions .. autosummary:: :toctree: doc/ - :template: autosummary/class_without_autosummary.rst ray.exceptions.RayError ray.exceptions.RayTaskError diff --git a/doc/source/ray-core/api/runtime-env.rst b/doc/source/ray-core/api/runtime-env.rst index 170d0c077031..607766a9535a 100644 --- a/doc/source/ray-core/api/runtime-env.rst +++ b/doc/source/ray-core/api/runtime-env.rst @@ -3,7 +3,6 @@ Runtime Env API .. autosummary:: :toctree: doc/ - :template: autosummary/class_without_autosummary.rst ray.runtime_env.RuntimeEnvConfig ray.runtime_env.RuntimeEnv diff --git a/doc/source/ray-core/api/scheduling.rst b/doc/source/ray-core/api/scheduling.rst index 40d1a66ccecd..2971aae8e3c8 100644 --- a/doc/source/ray-core/api/scheduling.rst +++ b/doc/source/ray-core/api/scheduling.rst @@ -6,7 +6,6 @@ Scheduling Strategy .. autosummary:: :toctree: doc/ - :template: autosummary/class_without_autosummary.rst ray.util.scheduling_strategies.PlacementGroupSchedulingStrategy ray.util.scheduling_strategies.NodeAffinitySchedulingStrategy diff --git a/doc/source/ray-core/api/utility.rst b/doc/source/ray-core/api/utility.rst index 445e58de1456..36da67415d4f 100644 --- a/doc/source/ray-core/api/utility.rst +++ b/doc/source/ray-core/api/utility.rst @@ -17,7 +17,6 @@ Custom Metrics .. autosummary:: :toctree: doc/ - :template: autosummary/class_without_autosummary.rst ray.util.metrics.Counter ray.util.metrics.Gauge diff --git a/doc/source/ray-observability/api/state/api.rst b/doc/source/ray-observability/api/state/api.rst index eda123718a72..bfd37f82c391 100644 --- a/doc/source/ray-observability/api/state/api.rst +++ b/doc/source/ray-observability/api/state/api.rst @@ -70,7 +70,6 @@ State APIs Schema .. autosummary:: :toctree: doc/ - :template: autosummary/class_without_autosummary.rst ray.experimental.state.common.ActorState ray.experimental.state.common.TaskState diff --git a/doc/source/tune/api/integration.rst b/doc/source/tune/api/integration.rst index d5954dbfa76f..0ad8ff86d2a6 100644 --- a/doc/source/tune/api/integration.rst +++ b/doc/source/tune/api/integration.rst @@ -23,7 +23,6 @@ Comet (air.integrations.comet) .. autosummary:: :toctree: doc/ - :template: autosummary/class_without_autosummary.rst ~air.integrations.comet.CometLoggerCallback @@ -65,7 +64,6 @@ Keras (air.integrations.keras) .. autosummary:: :toctree: doc/ - :template: autosummary/class_without_autosummary.rst ~air.integrations.keras.ReportCheckpointCallback @@ -77,7 +75,6 @@ MXNet (tune.integration.mxnet) .. autosummary:: :toctree: doc/ - :template: autosummary/class_without_autosummary.rst ~tune.integration.mxnet.TuneReportCallback ~tune.integration.mxnet.TuneCheckpointCallback @@ -90,7 +87,6 @@ PyTorch Lightning (tune.integration.pytorch_lightning) .. autosummary:: :toctree: doc/ - :template: autosummary/class_without_autosummary.rst ~tune.integration.pytorch_lightning.TuneReportCallback ~tune.integration.pytorch_lightning.TuneReportCheckpointCallback @@ -102,7 +98,6 @@ XGBoost (tune.integration.xgboost) .. autosummary:: :toctree: doc/ - :template: autosummary/class_without_autosummary.rst ~tune.integration.xgboost.TuneReportCallback ~tune.integration.xgboost.TuneReportCheckpointCallback @@ -115,7 +110,6 @@ LightGBM (tune.integration.lightgbm) .. autosummary:: :toctree: doc/ - :template: autosummary/class_without_autosummary.rst ~tune.integration.lightgbm.TuneReportCallback ~tune.integration.lightgbm.TuneReportCheckpointCallback diff --git a/doc/source/tune/api/logging.rst b/doc/source/tune/api/logging.rst index 805704c8ed62..e5eb8732b626 100644 --- a/doc/source/tune/api/logging.rst +++ b/doc/source/tune/api/logging.rst @@ -38,7 +38,6 @@ Tune Built-in Loggers .. autosummary:: :toctree: doc/ - :template: autosummary/class_without_autosummary.rst tune.logger.JsonLoggerCallback tune.logger.CSVLoggerCallback @@ -54,7 +53,6 @@ You can see the :doc:`tutorial here `. .. autosummary:: :toctree: doc/ - :template: autosummary/class_without_autosummary.rst air.integrations.mlflow.MLflowLoggerCallback @@ -67,7 +65,6 @@ You can see the :doc:`tutorial here `. .. autosummary:: :toctree: doc/ - :template: autosummary/class_without_autosummary.rst air.integrations.wandb.WandbLoggerCallback diff --git a/doc/source/tune/api/schedulers.rst b/doc/source/tune/api/schedulers.rst index b1a10724b1c1..a7ee7772092a 100644 --- a/doc/source/tune/api/schedulers.rst +++ b/doc/source/tune/api/schedulers.rst @@ -73,7 +73,6 @@ This is the default used if no value is provided for the ``brackets`` argument. .. autosummary:: :toctree: doc/ - :template: autosummary/class_without_autosummary.rst AsyncHyperBandScheduler ASHAScheduler @@ -88,7 +87,6 @@ Tune implements the `standard version of HyperBand `_ for more details. .. autosummary:: :toctree: doc/ - :template: autosummary/class_without_autosummary.rst bohb.TuneBOHB @@ -181,7 +177,6 @@ See the `BlendSearch paper `_ and docu .. autosummary:: :toctree: doc/ - :template: autosummary/class_without_autosummary.rst flaml.BlendSearch @@ -205,7 +200,6 @@ FLAML `CFO documentation Date: Mon, 13 Feb 2023 20:24:35 -0800 Subject: [PATCH 50/55] up Signed-off-by: Jiajun Yao --- doc/source/data/api/dataset.rst | 11 +++++++++++ doc/source/data/api/dataset_context.rst | 2 ++ doc/source/data/api/dataset_iterator.rst | 1 + doc/source/data/api/dataset_pipeline.rst | 8 ++++++++ doc/source/data/api/grouped_dataset.rst | 3 +++ 5 files changed, 25 insertions(+) diff --git a/doc/source/data/api/dataset.rst b/doc/source/data/api/dataset.rst index fe006b68ab4b..22d5aece5f8a 100644 --- a/doc/source/data/api/dataset.rst +++ b/doc/source/data/api/dataset.rst @@ -10,6 +10,7 @@ Constructor .. autosummary:: :toctree: doc/ + :template: autosummary/class_with_autosummary.rst Dataset @@ -18,6 +19,7 @@ Basic Transformations .. autosummary:: :toctree: doc/ + :template: autosummary/class_with_autosummary.rst Dataset.map Dataset.map_batches @@ -34,6 +36,7 @@ Sorting, Shuffling, Repartitioning .. autosummary:: :toctree: doc/ + :template: autosummary/class_with_autosummary.rst Dataset.sort Dataset.random_shuffle @@ -45,6 +48,7 @@ Splitting and Merging Datasets .. autosummary:: :toctree: doc/ + :template: autosummary/class_with_autosummary.rst Dataset.split Dataset.split_at_indices @@ -58,6 +62,7 @@ Grouped and Global Aggregations .. autosummary:: :toctree: doc/ + :template: autosummary/class_with_autosummary.rst Dataset.groupby Dataset.aggregate @@ -72,6 +77,7 @@ Converting to Pipeline .. autosummary:: :toctree: doc/ + :template: autosummary/class_with_autosummary.rst Dataset.repeat Dataset.window @@ -81,6 +87,7 @@ Consuming Datasets .. autosummary:: :toctree: doc/ + :template: autosummary/class_with_autosummary.rst Dataset.show Dataset.take @@ -96,6 +103,7 @@ I/O and Conversion .. autosummary:: :toctree: doc/ + :template: autosummary/class_with_autosummary.rst Dataset.write_parquet Dataset.write_json @@ -121,6 +129,7 @@ Inspecting Metadata .. autosummary:: :toctree: doc/ + :template: autosummary/class_with_autosummary.rst Dataset.count Dataset.schema @@ -136,6 +145,7 @@ Execution .. autosummary:: :toctree: doc/ + :template: autosummary/class_with_autosummary.rst Dataset.fully_executed Dataset.is_fully_executed @@ -146,6 +156,7 @@ Serialization .. autosummary:: :toctree: doc/ + :template: autosummary/class_with_autosummary.rst Dataset.has_serializable_lineage Dataset.serialize_lineage diff --git a/doc/source/data/api/dataset_context.rst b/doc/source/data/api/dataset_context.rst index 107f9567dc25..64683b9248ba 100644 --- a/doc/source/data/api/dataset_context.rst +++ b/doc/source/data/api/dataset_context.rst @@ -10,6 +10,7 @@ Constructor .. autosummary:: :toctree: doc/ + :template: autosummary/class_with_autosummary.rst context.DatasetContext @@ -18,5 +19,6 @@ Get DatasetContext .. autosummary:: :toctree: doc/ + :template: autosummary/class_with_autosummary.rst context.DatasetContext.get_current diff --git a/doc/source/data/api/dataset_iterator.rst b/doc/source/data/api/dataset_iterator.rst index d474b837fb62..5aec25e8765a 100644 --- a/doc/source/data/api/dataset_iterator.rst +++ b/doc/source/data/api/dataset_iterator.rst @@ -9,6 +9,7 @@ DatasetIterator API .. autosummary:: :toctree: doc/ + :template: autosummary/class_with_autosummary.rst DatasetIterator.iter_batches DatasetIterator.iter_torch_batches diff --git a/doc/source/data/api/dataset_pipeline.rst b/doc/source/data/api/dataset_pipeline.rst index 70919f562253..077ea9db2118 100644 --- a/doc/source/data/api/dataset_pipeline.rst +++ b/doc/source/data/api/dataset_pipeline.rst @@ -10,6 +10,7 @@ Constructor .. autosummary:: :toctree: doc/ + :template: autosummary/class_with_autosummary.rst DatasetPipeline @@ -18,6 +19,7 @@ Basic Transformations .. autosummary:: :toctree: doc/ + :template: autosummary/class_with_autosummary.rst DatasetPipeline.map DatasetPipeline.map_batches @@ -33,6 +35,7 @@ Sorting, Shuffling, Repartitioning .. autosummary:: :toctree: doc/ + :template: autosummary/class_with_autosummary.rst DatasetPipeline.sort_each_window DatasetPipeline.random_shuffle_each_window @@ -44,6 +47,7 @@ Splitting DatasetPipelines .. autosummary:: :toctree: doc/ + :template: autosummary/class_with_autosummary.rst DatasetPipeline.split DatasetPipeline.split_at_indices @@ -53,6 +57,7 @@ Creating DatasetPipelines .. autosummary:: :toctree: doc/ + :template: autosummary/class_with_autosummary.rst DatasetPipeline.repeat DatasetPipeline.rewindow @@ -63,6 +68,7 @@ Consuming DatasetPipelines .. autosummary:: :toctree: doc/ + :template: autosummary/class_with_autosummary.rst DatasetPipeline.show DatasetPipeline.show_windows @@ -79,6 +85,7 @@ I/O and Conversion .. autosummary:: :toctree: doc/ + :template: autosummary/class_with_autosummary.rst DatasetPipeline.write_json DatasetPipeline.write_csv @@ -92,6 +99,7 @@ Inspecting Metadata .. autosummary:: :toctree: doc/ + :template: autosummary/class_with_autosummary.rst DatasetPipeline.schema DatasetPipeline.count diff --git a/doc/source/data/api/grouped_dataset.rst b/doc/source/data/api/grouped_dataset.rst index b426a91aa74d..7f5a9f8c3202 100644 --- a/doc/source/data/api/grouped_dataset.rst +++ b/doc/source/data/api/grouped_dataset.rst @@ -12,6 +12,7 @@ Constructor .. autosummary:: :toctree: doc/ + :template: autosummary/class_with_autosummary.rst grouped_dataset.GroupedDataset @@ -20,6 +21,7 @@ Computations / Descriptive Stats .. autosummary:: :toctree: doc/ + :template: autosummary/class_with_autosummary.rst grouped_dataset.GroupedDataset.count grouped_dataset.GroupedDataset.sum @@ -33,6 +35,7 @@ Function Application .. autosummary:: :toctree: doc/ + :template: autosummary/class_with_autosummary.rst grouped_dataset.GroupedDataset.aggregate grouped_dataset.GroupedDataset.map_groups From 30cf81cadf102af613dc5b3622c98c053cecb288 Mon Sep 17 00:00:00 2001 From: Jiajun Yao Date: Mon, 13 Feb 2023 21:11:45 -0800 Subject: [PATCH 51/55] fix Signed-off-by: Jiajun Yao --- .../job-submission/jobs-package-ref.rst | 3 ++- doc/source/data/api/dataset.rst | 10 ---------- doc/source/data/api/dataset_context.rst | 1 - doc/source/data/api/dataset_iterator.rst | 1 - doc/source/data/api/dataset_pipeline.rst | 7 ------- doc/source/data/api/grouped_dataset.rst | 2 -- doc/source/data/api/random_access_dataset.rst | 1 + doc/source/serve/api/python_api.md | 1 + doc/source/tune/api/logging.rst | 1 + doc/source/tune/api/stoppers.rst | 1 + doc/source/tune/api/suggestion.rst | 1 + 11 files changed, 7 insertions(+), 22 deletions(-) diff --git a/doc/source/cluster/running-applications/job-submission/jobs-package-ref.rst b/doc/source/cluster/running-applications/job-submission/jobs-package-ref.rst index 3e6afaec76ec..beaf1127f449 100644 --- a/doc/source/cluster/running-applications/job-submission/jobs-package-ref.rst +++ b/doc/source/cluster/running-applications/job-submission/jobs-package-ref.rst @@ -8,7 +8,7 @@ Python SDK API Reference For an overview with examples see :ref:`Ray Jobs `. For the CLI reference see :ref:`Ray Job Submission CLI Reference `. - + .. _job-submission-client-ref: JobSubmissionClient @@ -16,6 +16,7 @@ JobSubmissionClient .. autosummary:: :toctree: doc/ + :template: autosummary/class_with_autosummary.rst JobSubmissionClient JobSubmissionClient.submit_job diff --git a/doc/source/data/api/dataset.rst b/doc/source/data/api/dataset.rst index 22d5aece5f8a..8bcd4107560c 100644 --- a/doc/source/data/api/dataset.rst +++ b/doc/source/data/api/dataset.rst @@ -19,7 +19,6 @@ Basic Transformations .. autosummary:: :toctree: doc/ - :template: autosummary/class_with_autosummary.rst Dataset.map Dataset.map_batches @@ -36,7 +35,6 @@ Sorting, Shuffling, Repartitioning .. autosummary:: :toctree: doc/ - :template: autosummary/class_with_autosummary.rst Dataset.sort Dataset.random_shuffle @@ -48,7 +46,6 @@ Splitting and Merging Datasets .. autosummary:: :toctree: doc/ - :template: autosummary/class_with_autosummary.rst Dataset.split Dataset.split_at_indices @@ -62,7 +59,6 @@ Grouped and Global Aggregations .. autosummary:: :toctree: doc/ - :template: autosummary/class_with_autosummary.rst Dataset.groupby Dataset.aggregate @@ -77,7 +73,6 @@ Converting to Pipeline .. autosummary:: :toctree: doc/ - :template: autosummary/class_with_autosummary.rst Dataset.repeat Dataset.window @@ -87,7 +82,6 @@ Consuming Datasets .. autosummary:: :toctree: doc/ - :template: autosummary/class_with_autosummary.rst Dataset.show Dataset.take @@ -103,7 +97,6 @@ I/O and Conversion .. autosummary:: :toctree: doc/ - :template: autosummary/class_with_autosummary.rst Dataset.write_parquet Dataset.write_json @@ -129,7 +122,6 @@ Inspecting Metadata .. autosummary:: :toctree: doc/ - :template: autosummary/class_with_autosummary.rst Dataset.count Dataset.schema @@ -145,7 +137,6 @@ Execution .. autosummary:: :toctree: doc/ - :template: autosummary/class_with_autosummary.rst Dataset.fully_executed Dataset.is_fully_executed @@ -156,7 +147,6 @@ Serialization .. autosummary:: :toctree: doc/ - :template: autosummary/class_with_autosummary.rst Dataset.has_serializable_lineage Dataset.serialize_lineage diff --git a/doc/source/data/api/dataset_context.rst b/doc/source/data/api/dataset_context.rst index 64683b9248ba..2288352d59f3 100644 --- a/doc/source/data/api/dataset_context.rst +++ b/doc/source/data/api/dataset_context.rst @@ -19,6 +19,5 @@ Get DatasetContext .. autosummary:: :toctree: doc/ - :template: autosummary/class_with_autosummary.rst context.DatasetContext.get_current diff --git a/doc/source/data/api/dataset_iterator.rst b/doc/source/data/api/dataset_iterator.rst index 5aec25e8765a..d474b837fb62 100644 --- a/doc/source/data/api/dataset_iterator.rst +++ b/doc/source/data/api/dataset_iterator.rst @@ -9,7 +9,6 @@ DatasetIterator API .. autosummary:: :toctree: doc/ - :template: autosummary/class_with_autosummary.rst DatasetIterator.iter_batches DatasetIterator.iter_torch_batches diff --git a/doc/source/data/api/dataset_pipeline.rst b/doc/source/data/api/dataset_pipeline.rst index 077ea9db2118..7ec82d999f81 100644 --- a/doc/source/data/api/dataset_pipeline.rst +++ b/doc/source/data/api/dataset_pipeline.rst @@ -19,7 +19,6 @@ Basic Transformations .. autosummary:: :toctree: doc/ - :template: autosummary/class_with_autosummary.rst DatasetPipeline.map DatasetPipeline.map_batches @@ -35,7 +34,6 @@ Sorting, Shuffling, Repartitioning .. autosummary:: :toctree: doc/ - :template: autosummary/class_with_autosummary.rst DatasetPipeline.sort_each_window DatasetPipeline.random_shuffle_each_window @@ -47,7 +45,6 @@ Splitting DatasetPipelines .. autosummary:: :toctree: doc/ - :template: autosummary/class_with_autosummary.rst DatasetPipeline.split DatasetPipeline.split_at_indices @@ -57,7 +54,6 @@ Creating DatasetPipelines .. autosummary:: :toctree: doc/ - :template: autosummary/class_with_autosummary.rst DatasetPipeline.repeat DatasetPipeline.rewindow @@ -68,7 +64,6 @@ Consuming DatasetPipelines .. autosummary:: :toctree: doc/ - :template: autosummary/class_with_autosummary.rst DatasetPipeline.show DatasetPipeline.show_windows @@ -85,7 +80,6 @@ I/O and Conversion .. autosummary:: :toctree: doc/ - :template: autosummary/class_with_autosummary.rst DatasetPipeline.write_json DatasetPipeline.write_csv @@ -99,7 +93,6 @@ Inspecting Metadata .. autosummary:: :toctree: doc/ - :template: autosummary/class_with_autosummary.rst DatasetPipeline.schema DatasetPipeline.count diff --git a/doc/source/data/api/grouped_dataset.rst b/doc/source/data/api/grouped_dataset.rst index 7f5a9f8c3202..8e496bd92fd3 100644 --- a/doc/source/data/api/grouped_dataset.rst +++ b/doc/source/data/api/grouped_dataset.rst @@ -21,7 +21,6 @@ Computations / Descriptive Stats .. autosummary:: :toctree: doc/ - :template: autosummary/class_with_autosummary.rst grouped_dataset.GroupedDataset.count grouped_dataset.GroupedDataset.sum @@ -35,7 +34,6 @@ Function Application .. autosummary:: :toctree: doc/ - :template: autosummary/class_with_autosummary.rst grouped_dataset.GroupedDataset.aggregate grouped_dataset.GroupedDataset.map_groups diff --git a/doc/source/data/api/random_access_dataset.rst b/doc/source/data/api/random_access_dataset.rst index e3a171af18d2..f1e1c231f6b2 100644 --- a/doc/source/data/api/random_access_dataset.rst +++ b/doc/source/data/api/random_access_dataset.rst @@ -12,6 +12,7 @@ Constructor .. autosummary:: :toctree: doc/ + :template: autosummary/class_with_autosummary.rst random_access_dataset.RandomAccessDataset diff --git a/doc/source/serve/api/python_api.md b/doc/source/serve/api/python_api.md index 9b288318842f..205f77b24dc3 100644 --- a/doc/source/serve/api/python_api.md +++ b/doc/source/serve/api/python_api.md @@ -24,6 +24,7 @@ ```{eval-rst} .. autosummary:: :toctree: doc/ + :template: autosummary/class_with_autosummary.rst serve.handle.RayServeHandle serve.handle.RayServeHandle.remote diff --git a/doc/source/tune/api/logging.rst b/doc/source/tune/api/logging.rst index e5eb8732b626..80e37d77884c 100644 --- a/doc/source/tune/api/logging.rst +++ b/doc/source/tune/api/logging.rst @@ -75,6 +75,7 @@ LoggerCallback Interface .. autosummary:: :toctree: doc/ + :template: autosummary/class_with_autosummary.rst tune.logger.LoggerCallback tune.logger.LoggerCallback.log_trial_start diff --git a/doc/source/tune/api/stoppers.rst b/doc/source/tune/api/stoppers.rst index 28dfa38d8044..c1355933cd56 100644 --- a/doc/source/tune/api/stoppers.rst +++ b/doc/source/tune/api/stoppers.rst @@ -23,6 +23,7 @@ Stopper Interface (tune.Stopper) .. autosummary:: :toctree: doc/ + :template: autosummary/class_with_autosummary.rst Stopper Stopper.__call__ diff --git a/doc/source/tune/api/suggestion.rst b/doc/source/tune/api/suggestion.rst index bce657c9ecc5..179a393aaf7a 100644 --- a/doc/source/tune/api/suggestion.rst +++ b/doc/source/tune/api/suggestion.rst @@ -331,6 +331,7 @@ If you are interested in implementing or contributing a new Search Algorithm, pr .. autosummary:: :toctree: doc/ + :template: autosummary/class_with_autosummary.rst Searcher Searcher.suggest From 8a29670024d9973a07ff7216c3d64824c6f7cf22 Mon Sep 17 00:00:00 2001 From: Jiajun Yao Date: Mon, 13 Feb 2023 21:42:02 -0800 Subject: [PATCH 52/55] split Signed-off-by: Jiajun Yao --- doc/source/_templates/autosummary/class.rst | 6 --- .../autosummary/class_with_autosummary.rst | 45 ------------------- .../job-submission/jobs-package-ref.rst | 3 +- doc/source/data/api/dataset.rst | 1 - doc/source/data/api/dataset_context.rst | 1 - doc/source/data/api/grouped_dataset.rst | 1 - doc/source/data/api/random_access_dataset.rst | 1 - .../ray-observability/api/state/cli.rst | 4 +- doc/source/serve/api/python_api.md | 1 - doc/source/tune/api/logging.rst | 1 - doc/source/tune/api/stoppers.rst | 1 - doc/source/tune/api/suggestion.rst | 1 - python/ray/air/execution/resources/request.py | 1 - 13 files changed, 3 insertions(+), 64 deletions(-) delete mode 100644 doc/source/_templates/autosummary/class.rst delete mode 100644 doc/source/_templates/autosummary/class_with_autosummary.rst diff --git a/doc/source/_templates/autosummary/class.rst b/doc/source/_templates/autosummary/class.rst deleted file mode 100644 index 8643ade9ab82..000000000000 --- a/doc/source/_templates/autosummary/class.rst +++ /dev/null @@ -1,6 +0,0 @@ -{{ fullname | escape | underline}} - -.. currentmodule:: {{ module }} - -.. autoclass:: {{ objname }} - :members: diff --git a/doc/source/_templates/autosummary/class_with_autosummary.rst b/doc/source/_templates/autosummary/class_with_autosummary.rst deleted file mode 100644 index b409424e83e9..000000000000 --- a/doc/source/_templates/autosummary/class_with_autosummary.rst +++ /dev/null @@ -1,45 +0,0 @@ -{# - Generating pages for class methods and attributes - significantly increases the doc build time and causes timeouts. - For now, opt in explicitly via `:template: autosummary/class_with_autosummary.rst` -#} -{# - It's a known bug (https://github.com/sphinx-doc/sphinx/issues/9884) - that autosummary will generate warning for inherited instance attributes. - Those warnings will fail our build. - For now, we don't autosummary classes with inherited instance attributes. -#} - -{{ fullname | escape | underline}} - -.. currentmodule:: {{ module }} - -.. autoclass:: {{ objname }} - - {% block methods %} - {% if methods %} - .. rubric:: {{ _('Methods') }} - - .. autosummary:: - :toctree: - - {% for item in methods %} - {{ name }}.{{ item }} - {%- endfor %} - - {% endif %} - {% endblock %} - - {% block attributes %} - {% if attributes %} - .. rubric:: {{ _('Attributes') }} - - .. autosummary:: - :toctree: - - {% for item in attributes %} - {{ name }}.{{ item }} - {%- endfor %} - - {% endif %} - {% endblock %} diff --git a/doc/source/cluster/running-applications/job-submission/jobs-package-ref.rst b/doc/source/cluster/running-applications/job-submission/jobs-package-ref.rst index beaf1127f449..bc610048a718 100644 --- a/doc/source/cluster/running-applications/job-submission/jobs-package-ref.rst +++ b/doc/source/cluster/running-applications/job-submission/jobs-package-ref.rst @@ -8,7 +8,7 @@ Python SDK API Reference For an overview with examples see :ref:`Ray Jobs `. For the CLI reference see :ref:`Ray Job Submission CLI Reference `. - + .. _job-submission-client-ref: JobSubmissionClient @@ -16,7 +16,6 @@ JobSubmissionClient .. autosummary:: :toctree: doc/ - :template: autosummary/class_with_autosummary.rst JobSubmissionClient JobSubmissionClient.submit_job diff --git a/doc/source/data/api/dataset.rst b/doc/source/data/api/dataset.rst index 8bcd4107560c..fe006b68ab4b 100644 --- a/doc/source/data/api/dataset.rst +++ b/doc/source/data/api/dataset.rst @@ -10,7 +10,6 @@ Constructor .. autosummary:: :toctree: doc/ - :template: autosummary/class_with_autosummary.rst Dataset diff --git a/doc/source/data/api/dataset_context.rst b/doc/source/data/api/dataset_context.rst index 2288352d59f3..107f9567dc25 100644 --- a/doc/source/data/api/dataset_context.rst +++ b/doc/source/data/api/dataset_context.rst @@ -10,7 +10,6 @@ Constructor .. autosummary:: :toctree: doc/ - :template: autosummary/class_with_autosummary.rst context.DatasetContext diff --git a/doc/source/data/api/grouped_dataset.rst b/doc/source/data/api/grouped_dataset.rst index 8e496bd92fd3..b426a91aa74d 100644 --- a/doc/source/data/api/grouped_dataset.rst +++ b/doc/source/data/api/grouped_dataset.rst @@ -12,7 +12,6 @@ Constructor .. autosummary:: :toctree: doc/ - :template: autosummary/class_with_autosummary.rst grouped_dataset.GroupedDataset diff --git a/doc/source/data/api/random_access_dataset.rst b/doc/source/data/api/random_access_dataset.rst index f1e1c231f6b2..e3a171af18d2 100644 --- a/doc/source/data/api/random_access_dataset.rst +++ b/doc/source/data/api/random_access_dataset.rst @@ -12,7 +12,6 @@ Constructor .. autosummary:: :toctree: doc/ - :template: autosummary/class_with_autosummary.rst random_access_dataset.RandomAccessDataset diff --git a/doc/source/ray-observability/api/state/cli.rst b/doc/source/ray-observability/api/state/cli.rst index f85874ca83f4..7b38592eb795 100644 --- a/doc/source/ray-observability/api/state/cli.rst +++ b/doc/source/ray-observability/api/state/cli.rst @@ -1,5 +1,5 @@ -State CLI -========= +Ray State CLI +============= .. _state-api-cli-ref: diff --git a/doc/source/serve/api/python_api.md b/doc/source/serve/api/python_api.md index 205f77b24dc3..9b288318842f 100644 --- a/doc/source/serve/api/python_api.md +++ b/doc/source/serve/api/python_api.md @@ -24,7 +24,6 @@ ```{eval-rst} .. autosummary:: :toctree: doc/ - :template: autosummary/class_with_autosummary.rst serve.handle.RayServeHandle serve.handle.RayServeHandle.remote diff --git a/doc/source/tune/api/logging.rst b/doc/source/tune/api/logging.rst index 80e37d77884c..e5eb8732b626 100644 --- a/doc/source/tune/api/logging.rst +++ b/doc/source/tune/api/logging.rst @@ -75,7 +75,6 @@ LoggerCallback Interface .. autosummary:: :toctree: doc/ - :template: autosummary/class_with_autosummary.rst tune.logger.LoggerCallback tune.logger.LoggerCallback.log_trial_start diff --git a/doc/source/tune/api/stoppers.rst b/doc/source/tune/api/stoppers.rst index c1355933cd56..28dfa38d8044 100644 --- a/doc/source/tune/api/stoppers.rst +++ b/doc/source/tune/api/stoppers.rst @@ -23,7 +23,6 @@ Stopper Interface (tune.Stopper) .. autosummary:: :toctree: doc/ - :template: autosummary/class_with_autosummary.rst Stopper Stopper.__call__ diff --git a/doc/source/tune/api/suggestion.rst b/doc/source/tune/api/suggestion.rst index 179a393aaf7a..bce657c9ecc5 100644 --- a/doc/source/tune/api/suggestion.rst +++ b/doc/source/tune/api/suggestion.rst @@ -331,7 +331,6 @@ If you are interested in implementing or contributing a new Search Algorithm, pr .. autosummary:: :toctree: doc/ - :template: autosummary/class_with_autosummary.rst Searcher Searcher.suggest diff --git a/python/ray/air/execution/resources/request.py b/python/ray/air/execution/resources/request.py index 87455a6b3ad8..8a66fb5cfe00 100644 --- a/python/ray/air/execution/resources/request.py +++ b/python/ray/air/execution/resources/request.py @@ -108,7 +108,6 @@ def head_bundle_is_empty(self): @property @DeveloperAPI def head_cpus(self) -> float: - """Returns the number of cpus in the head bundle.""" return 0.0 if self._head_bundle_is_empty else self._bundles[0].get("CPU", 0.0) @property From bba82cf2b532aef63a21a1b92d9a2ef8666d3764 Mon Sep 17 00:00:00 2001 From: Jiajun Yao Date: Mon, 13 Feb 2023 21:43:36 -0800 Subject: [PATCH 53/55] up Signed-off-by: Jiajun Yao --- .../running-applications/job-submission/jobs-package-ref.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/source/cluster/running-applications/job-submission/jobs-package-ref.rst b/doc/source/cluster/running-applications/job-submission/jobs-package-ref.rst index bc610048a718..fa700c94bb0c 100644 --- a/doc/source/cluster/running-applications/job-submission/jobs-package-ref.rst +++ b/doc/source/cluster/running-applications/job-submission/jobs-package-ref.rst @@ -8,7 +8,7 @@ Python SDK API Reference For an overview with examples see :ref:`Ray Jobs `. For the CLI reference see :ref:`Ray Job Submission CLI Reference `. - + .. _job-submission-client-ref: JobSubmissionClient From 7a95a6778c47c30dd10172187422c3c1cef721b3 Mon Sep 17 00:00:00 2001 From: Jiajun Yao Date: Mon, 13 Feb 2023 21:45:47 -0800 Subject: [PATCH 54/55] split Signed-off-by: Jiajun Yao --- doc/source/data/api/dataset_pipeline.rst | 1 - 1 file changed, 1 deletion(-) diff --git a/doc/source/data/api/dataset_pipeline.rst b/doc/source/data/api/dataset_pipeline.rst index 7ec82d999f81..70919f562253 100644 --- a/doc/source/data/api/dataset_pipeline.rst +++ b/doc/source/data/api/dataset_pipeline.rst @@ -10,7 +10,6 @@ Constructor .. autosummary:: :toctree: doc/ - :template: autosummary/class_with_autosummary.rst DatasetPipeline From 575a8dc1d626b154a1fd163a62193e5289b10379 Mon Sep 17 00:00:00 2001 From: Jiajun Yao Date: Mon, 13 Feb 2023 22:04:19 -0800 Subject: [PATCH 55/55] func Signed-off-by: Jiajun Yao --- doc/source/ray-core/actors.rst | 2 +- doc/source/ray-core/configure.rst | 2 +- doc/source/ray-core/examples/monte_carlo_pi.rst | 10 +++++----- doc/source/ray-core/fault_tolerance/actors.rst | 2 +- doc/source/ray-core/fault_tolerance/tasks.rst | 2 +- doc/source/ray-core/objects.rst | 4 ++-- .../patterns/closure-capture-large-objects.rst | 4 ++-- doc/source/ray-core/patterns/limit-pending-tasks.rst | 2 +- doc/source/ray-core/patterns/nested-tasks.rst | 2 +- .../ray-core/patterns/pass-large-arg-by-value.rst | 2 +- doc/source/ray-core/patterns/pipelining.rst | 2 +- doc/source/ray-core/patterns/ray-get-loop.rst | 2 +- .../ray-core/patterns/ray-get-submission-order.rst | 4 ++-- .../ray-core/patterns/ray-get-too-many-objects.rst | 2 +- .../ray-core/patterns/redefine-task-actor-loop.rst | 2 +- doc/source/ray-core/patterns/return-ray-put.rst | 2 +- doc/source/ray-core/patterns/unnecessary-ray-get.rst | 2 +- doc/source/ray-core/scheduling/index.rst | 2 +- doc/source/ray-core/scheduling/resources.rst | 4 ++-- doc/source/ray-core/tasks.rst | 6 +++--- doc/source/ray-core/tasks/using-ray-with-gpus.rst | 4 ++-- doc/source/ray-more-libs/joblib.rst | 2 +- 22 files changed, 33 insertions(+), 33 deletions(-) diff --git a/doc/source/ray-core/actors.rst b/doc/source/ray-core/actors.rst index 83c87c7cc9e2..08509634958e 100644 --- a/doc/source/ray-core/actors.rst +++ b/doc/source/ray-core/actors.rst @@ -337,7 +337,7 @@ By default, Ray actors won't be :ref:`restarted ` and actor tasks won't be retried when actors crash unexpectedly. You can change this behavior by setting ``max_restarts`` and ``max_task_retries`` options -in :meth:`ray.remote() ` and :meth:`.options() `. +in :func:`ray.remote() ` and :meth:`.options() `. See :ref:`Ray fault tolerance ` for more details. FAQ: Actors, Workers and Resources diff --git a/doc/source/ray-core/configure.rst b/doc/source/ray-core/configure.rst index 3287ca3a2903..6a2c5c78272d 100644 --- a/doc/source/ray-core/configure.rst +++ b/doc/source/ray-core/configure.rst @@ -58,7 +58,7 @@ If using the command line, connect to the Ray cluster as follow: .. note:: Ray sets the environment variable ``OMP_NUM_THREADS=`` if ``num_cpus`` is set on - the task/actor via :meth:`ray.remote() ` and :meth:`task.options() `/:meth:`actor.options() `. + the task/actor via :func:`ray.remote() ` and :meth:`task.options() `/:meth:`actor.options() `. Ray sets ``OMP_NUM_THREADS=1`` if ``num_cpus`` is not specified; this is done to avoid performance degradation with many workers (issue #6998). You can also override this by explicitly setting ``OMP_NUM_THREADS`` to override anything Ray sets by default. diff --git a/doc/source/ray-core/examples/monte_carlo_pi.rst b/doc/source/ray-core/examples/monte_carlo_pi.rst index 1aff1234e343..8f3cd1fa23e5 100644 --- a/doc/source/ray-core/examples/monte_carlo_pi.rst +++ b/doc/source/ray-core/examples/monte_carlo_pi.rst @@ -19,7 +19,7 @@ To get started, install Ray via ``pip install -U ray``. See :ref:`Installing Ray Starting Ray ------------ -First, let's include all modules needed for this tutorial and start a local Ray cluster with :meth:`ray.init() `: +First, let's include all modules needed for this tutorial and start a local Ray cluster with :func:`ray.init() `: .. literalinclude:: ../doc_code/monte_carlo_pi.py :language: python @@ -41,7 +41,7 @@ Ray actors are essentially stateful services that anyone with an instance (a han :start-after: __defining_actor_start__ :end-before: __defining_actor_end__ -We define a Ray actor by decorating a normal Python class with :meth:`ray.remote `. +We define a Ray actor by decorating a normal Python class with :func:`ray.remote `. The progress actor has ``report_progress()`` method that will be called by sampling tasks to update their progress individually and ``get_progress()`` method to get the overall progress. @@ -55,7 +55,7 @@ Ray tasks are stateless functions. They execute asynchronously, and run in paral :start-after: __defining_task_start__ :end-before: __defining_task_end__ -To convert a normal Python function as a Ray task, we decorate the function with :meth:`ray.remote `. +To convert a normal Python function as a Ray task, we decorate the function with :func:`ray.remote `. The sampling task takes a progress actor handle as an input and reports progress to it. The above code shows an example of calling actor methods from tasks. @@ -97,7 +97,7 @@ While sampling tasks are running, we can periodically query the progress by call To call an actor method, use ``actor_handle.method.remote()``. This invocation immediately returns an ``ObjectRef`` as a future and then executes the method asynchronously on the remote actor process. -To fetch the actual returned value of ``ObjectRef``, we use the blocking :meth:`ray.get() `. +To fetch the actual returned value of ``ObjectRef``, we use the blocking :func:`ray.get() `. Calculating π ------------- @@ -108,7 +108,7 @@ Finally, we get number of samples inside the circle from the remote sampling tas :start-after: __calculating_pi_start__ :end-before: __calculating_pi_end__ -As we can see from the above code, besides a single ``ObjectRef``, :meth:`ray.get() ` can also take a list of ``ObjectRef`` and return a list of results. +As we can see from the above code, besides a single ``ObjectRef``, :func:`ray.get() ` can also take a list of ``ObjectRef`` and return a list of results. If you run this tutorial, you will see output like: diff --git a/doc/source/ray-core/fault_tolerance/actors.rst b/doc/source/ray-core/fault_tolerance/actors.rst index f4c402c28400..a254fa5fb562 100644 --- a/doc/source/ray-core/fault_tolerance/actors.rst +++ b/doc/source/ray-core/fault_tolerance/actors.rst @@ -24,7 +24,7 @@ After the specified number of restarts, subsequent actor methods will raise a ``RayActorError``. By default, actor tasks execute with at-most-once semantics -(``max_task_retries=0`` in the ``@ray.remote`` :meth:`decorator `). This means that if an +(``max_task_retries=0`` in the ``@ray.remote`` :func:`decorator `). This means that if an actor task is submitted to an actor that is unreachable, Ray will report the error with ``RayActorError``, a Python-level exception that is thrown when ``ray.get`` is called on the future returned by the task. Note that this diff --git a/doc/source/ray-core/fault_tolerance/tasks.rst b/doc/source/ray-core/fault_tolerance/tasks.rst index a7e67aacb59d..f99d934e2463 100644 --- a/doc/source/ray-core/fault_tolerance/tasks.rst +++ b/doc/source/ray-core/fault_tolerance/tasks.rst @@ -71,7 +71,7 @@ If a task is hanging, you may want to cancel the task to continue to make progress. You can do this by calling ``ray.cancel`` on an ``ObjectRef`` returned by the task. By default, this will send a KeyboardInterrupt to the task's worker if it is mid-execution. Passing ``force=True`` to ``ray.cancel`` -will force-exit the worker. See :meth:`the API reference ` for +will force-exit the worker. See :func:`the API reference ` for ``ray.cancel`` for more details. Note that currently, Ray will not automatically retry tasks that have been diff --git a/doc/source/ray-core/objects.rst b/doc/source/ray-core/objects.rst index e90d0d9d4e99..721924c12c2c 100644 --- a/doc/source/ray-core/objects.rst +++ b/doc/source/ray-core/objects.rst @@ -12,7 +12,7 @@ similar. Object refs can be created in two ways. 1. They are returned by remote function calls. - 2. They are returned by :meth:`ray.put() `. + 2. They are returned by :func:`ray.put() `. .. tabbed:: Python @@ -48,7 +48,7 @@ Object refs can be created in two ways. Fetching Object Data -------------------- -You can use the :meth:`ray.get() ` method to fetch the result of a remote object from an object ref. +You can use the :func:`ray.get() ` method to fetch the result of a remote object from an object ref. If the current node's object store does not contain the object, the object is downloaded. .. tabbed:: Python diff --git a/doc/source/ray-core/patterns/closure-capture-large-objects.rst b/doc/source/ray-core/patterns/closure-capture-large-objects.rst index fceea4382451..c51a24f7ecc7 100644 --- a/doc/source/ray-core/patterns/closure-capture-large-objects.rst +++ b/doc/source/ray-core/patterns/closure-capture-large-objects.rst @@ -3,13 +3,13 @@ Anti-pattern: Closure capturing large objects harms performance **TLDR:** Avoid closure capturing large objects in remote functions or classes, use object store instead. -When you define a :meth:`ray.remote ` function or class, +When you define a :func:`ray.remote ` function or class, it is easy to accidentally capture large (more than a few MB) objects implicitly in the definition. This can lead to slow performance or even OOM since Ray is not designed to handle serialized functions or classes that are very large. For such large objects, there are two options to resolve this problem: -- Use :meth:`ray.put() ` to put the large objects in the Ray object store, and then pass object references as arguments to the remote functions or classes (*"better approach #1"* below) +- Use :func:`ray.put() ` to put the large objects in the Ray object store, and then pass object references as arguments to the remote functions or classes (*"better approach #1"* below) - Create the large objects inside the remote functions or classes by passing a lambda method (*"better approach #2"*). This is also the only option for using unserializable objects. diff --git a/doc/source/ray-core/patterns/limit-pending-tasks.rst b/doc/source/ray-core/patterns/limit-pending-tasks.rst index 39e4a4450a2e..8a266990c4d4 100644 --- a/doc/source/ray-core/patterns/limit-pending-tasks.rst +++ b/doc/source/ray-core/patterns/limit-pending-tasks.rst @@ -3,7 +3,7 @@ Pattern: Using ray.wait to limit the number of pending tasks ============================================================ -In this pattern, we use :meth:`ray.wait() ` to limit the number of pending tasks. +In this pattern, we use :func:`ray.wait() ` to limit the number of pending tasks. If we continuously submit tasks faster than their process time, we will accumulate tasks in the pending task queue, which can eventually cause OOM. With ``ray.wait()``, we can apply backpressure and limit the number of pending tasks so that the pending task queue won't grow indefinitely and cause OOM. diff --git a/doc/source/ray-core/patterns/nested-tasks.rst b/doc/source/ray-core/patterns/nested-tasks.rst index 3b0db17703ec..070761ed7720 100644 --- a/doc/source/ray-core/patterns/nested-tasks.rst +++ b/doc/source/ray-core/patterns/nested-tasks.rst @@ -28,7 +28,7 @@ Code example :start-after: __pattern_start__ :end-before: __pattern_end__ -We call :meth:`ray.get() ` after both ``quick_sort_distributed`` function invocations take place. +We call :func:`ray.get() ` after both ``quick_sort_distributed`` function invocations take place. This allows you to maximize parallelism in the workload. See :doc:`ray-get-loop` for more details. Notice in the execution times above that with smaller tasks, the non-distributed version is faster. However, as the task execution diff --git a/doc/source/ray-core/patterns/pass-large-arg-by-value.rst b/doc/source/ray-core/patterns/pass-large-arg-by-value.rst index 58cfe5e3178c..b95b981fdec9 100644 --- a/doc/source/ray-core/patterns/pass-large-arg-by-value.rst +++ b/doc/source/ray-core/patterns/pass-large-arg-by-value.rst @@ -3,7 +3,7 @@ Anti-pattern: Passing the same large argument by value repeatedly harms performance =================================================================================== -**TLDR:** Avoid passing the same large argument by value to multiple tasks, use :meth:`ray.put() ` and pass by reference instead. +**TLDR:** Avoid passing the same large argument by value to multiple tasks, use :func:`ray.put() ` and pass by reference instead. When passing a large argument (>100KB) by value to a task, Ray will implicitly store the argument in the object store and the worker process will fetch the argument to the local object store from the caller's object store before running the task. diff --git a/doc/source/ray-core/patterns/pipelining.rst b/doc/source/ray-core/patterns/pipelining.rst index f4d8393abcaa..cf9e63bdb2ce 100644 --- a/doc/source/ray-core/patterns/pipelining.rst +++ b/doc/source/ray-core/patterns/pipelining.rst @@ -23,5 +23,5 @@ Code example .. literalinclude:: ../doc_code/pattern_pipelining.py In the example above, a worker actor pulls work off of a queue and then does some computation on it. -Without pipelining, we call :meth:`ray.get() ` immediately after requesting a work item, so we block while that RPC is in flight, causing idle CPU time. +Without pipelining, we call :func:`ray.get() ` immediately after requesting a work item, so we block while that RPC is in flight, causing idle CPU time. With pipelining, we instead preemptively request the next work item before processing the current one, so we can use the CPU while the RPC is in flight which increases the CPU utilization. diff --git a/doc/source/ray-core/patterns/ray-get-loop.rst b/doc/source/ray-core/patterns/ray-get-loop.rst index 54c6b5c9de6a..4ecf9666d774 100644 --- a/doc/source/ray-core/patterns/ray-get-loop.rst +++ b/doc/source/ray-core/patterns/ray-get-loop.rst @@ -3,7 +3,7 @@ Anti-pattern: Calling ray.get in a loop harms parallelism ========================================================= -**TLDR:** Avoid calling :meth:`ray.get() ` in a loop since it's a blocking call; use ``ray.get()`` only for the final result. +**TLDR:** Avoid calling :func:`ray.get() ` in a loop since it's a blocking call; use ``ray.get()`` only for the final result. A call to ``ray.get()`` fetches the results of remotely executed functions. However, it is a blocking call, which means that it always waits until the requested result is available. If you call ``ray.get()`` in a loop, the loop will not continue to run until the call to ``ray.get()`` is resolved. diff --git a/doc/source/ray-core/patterns/ray-get-submission-order.rst b/doc/source/ray-core/patterns/ray-get-submission-order.rst index 56ab0f1bc3dc..f0141e2daae3 100644 --- a/doc/source/ray-core/patterns/ray-get-submission-order.rst +++ b/doc/source/ray-core/patterns/ray-get-submission-order.rst @@ -1,12 +1,12 @@ Anti-pattern: Processing results in submission order using ray.get increases runtime ==================================================================================== -**TLDR:** Avoid processing independent results in submission order using :meth:`ray.get() ` since results may be ready in a different order than the submission order. +**TLDR:** Avoid processing independent results in submission order using :func:`ray.get() ` since results may be ready in a different order than the submission order. A batch of tasks is submitted, and we need to process their results individually once they’re done. If each task takes a different amount of time to finish and we process results in submission order, we may waste time waiting for all of the slower (straggler) tasks that were submitted earlier to finish while later faster tasks have already finished. -Instead, we want to process the tasks in the order that they finish using :meth:`ray.wait() ` to speed up total time to completion. +Instead, we want to process the tasks in the order that they finish using :func:`ray.wait() ` to speed up total time to completion. .. figure:: ../images/ray-get-submission-order.svg diff --git a/doc/source/ray-core/patterns/ray-get-too-many-objects.rst b/doc/source/ray-core/patterns/ray-get-too-many-objects.rst index b0b8504e044e..d0f247eebc65 100644 --- a/doc/source/ray-core/patterns/ray-get-too-many-objects.rst +++ b/doc/source/ray-core/patterns/ray-get-too-many-objects.rst @@ -1,7 +1,7 @@ Anti-pattern: Fetching too many objects at once with ray.get causes failure =========================================================================== -**TLDR:** Avoid calling :meth:`ray.get() ` on too many objects since this will lead to heap out-of-memory or object store out-of-space. Instead fetch and process one batch at a time. +**TLDR:** Avoid calling :func:`ray.get() ` on too many objects since this will lead to heap out-of-memory or object store out-of-space. Instead fetch and process one batch at a time. If you have a large number of tasks that you want to run in parallel, trying to do ``ray.get()`` on all of them at once could lead to failure with heap out-of-memory or object store out-of-space since Ray needs to fetch all the objects to the caller at the same time. Instead you should get and process the results one batch at a time. Once a batch is processed, Ray will evict objects in that batch to make space for future batches. diff --git a/doc/source/ray-core/patterns/redefine-task-actor-loop.rst b/doc/source/ray-core/patterns/redefine-task-actor-loop.rst index 3903b20c06b4..c0b2191773b3 100644 --- a/doc/source/ray-core/patterns/redefine-task-actor-loop.rst +++ b/doc/source/ray-core/patterns/redefine-task-actor-loop.rst @@ -3,7 +3,7 @@ Anti-pattern: Redefining the same remote function or class harms performance **TLDR:** Avoid redefining the same remote function or class. -Decorating the same function or class multiple times using the :meth:`ray.remote ` decorator leads to slow performance in Ray. +Decorating the same function or class multiple times using the :func:`ray.remote ` decorator leads to slow performance in Ray. For each Ray remote function or class, Ray will pickle it and upload to GCS. Later on, the worker that runs the task or actor will download and unpickle it. Each decoration of the same function or class generates a new remote function or class from Ray's perspective. diff --git a/doc/source/ray-core/patterns/return-ray-put.rst b/doc/source/ray-core/patterns/return-ray-put.rst index 8cc4f2f56c9f..4afc6b90b4ff 100644 --- a/doc/source/ray-core/patterns/return-ray-put.rst +++ b/doc/source/ray-core/patterns/return-ray-put.rst @@ -1,7 +1,7 @@ Anti-pattern: Returning ray.put() ObjectRefs from a task harms performance and fault tolerance ============================================================================================== -**TLDR:** Avoid calling :meth:`ray.put() ` on task return values and returning the resulting ObjectRefs. +**TLDR:** Avoid calling :func:`ray.put() ` on task return values and returning the resulting ObjectRefs. Instead, return these values directly if possible. Returning ray.put() ObjectRefs are considered anti-patterns for the following reasons: diff --git a/doc/source/ray-core/patterns/unnecessary-ray-get.rst b/doc/source/ray-core/patterns/unnecessary-ray-get.rst index 334f69e10648..c0322c22e190 100644 --- a/doc/source/ray-core/patterns/unnecessary-ray-get.rst +++ b/doc/source/ray-core/patterns/unnecessary-ray-get.rst @@ -3,7 +3,7 @@ Anti-pattern: Calling ray.get unnecessarily harms performance ============================================================= -**TLDR:** Avoid calling :meth:`ray.get() ` unnecessarily for intermediate steps. Work with object references directly, and only call ``ray.get()`` at the end to get the final result. +**TLDR:** Avoid calling :func:`ray.get() ` unnecessarily for intermediate steps. Work with object references directly, and only call ``ray.get()`` at the end to get the final result. When ``ray.get()`` is called, objects must be transferred to the worker/node that calls ``ray.get()``. If you don't need to manipulate the object, you probably don't need to call ``ray.get()`` on it! diff --git a/doc/source/ray-core/scheduling/index.rst b/doc/source/ray-core/scheduling/index.rst index 4158b3e2ccf5..16b28525e83b 100644 --- a/doc/source/ray-core/scheduling/index.rst +++ b/doc/source/ray-core/scheduling/index.rst @@ -31,7 +31,7 @@ If all nodes are infeasible, the task or actor cannot be scheduled until feasibl Scheduling Strategies --------------------- -Tasks or actors support a :meth:`scheduling_strategy ` option to specify the strategy used to decide the best node among feasible nodes. +Tasks or actors support a :func:`scheduling_strategy ` option to specify the strategy used to decide the best node among feasible nodes. Currently the supported strategies are the followings. "DEFAULT" diff --git a/doc/source/ray-core/scheduling/resources.rst b/doc/source/ray-core/scheduling/resources.rst index 42f239706b23..e2455cc27032 100644 --- a/doc/source/ray-core/scheduling/resources.rst +++ b/doc/source/ray-core/scheduling/resources.rst @@ -78,7 +78,7 @@ There are several ways to do that depending on how you start the Ray cluster: .. tabbed:: ray.init() - If you are using :meth:`ray.init() ` to start a single node Ray cluster, you can do the following to manually specify node resources: + If you are using :func:`ray.init() ` to start a single node Ray cluster, you can do the following to manually specify node resources: .. literalinclude:: ../doc_code/resources.py :language: python @@ -136,7 +136,7 @@ The default resource requirements for actors was chosen for historical reasons. It's recommended to always explicitly set ``num_cpus`` for actors to avoid any surprises. If resources are specified explicitly, they are required for both scheduling and running.) -You can also explicitly specify a task's or actor's resource requirements (for example, one task may require a GPU) instead of using default ones via :meth:`ray.remote() ` +You can also explicitly specify a task's or actor's resource requirements (for example, one task may require a GPU) instead of using default ones via :func:`ray.remote() ` and :meth:`task.options() `/:meth:`actor.options() `. .. tabbed:: Python diff --git a/doc/source/ray-core/tasks.rst b/doc/source/ray-core/tasks.rst index 292e270b2c8d..429f7f715445 100644 --- a/doc/source/ray-core/tasks.rst +++ b/doc/source/ray-core/tasks.rst @@ -164,7 +164,7 @@ Waiting for Partial Results --------------------------- Calling **ray.get** on Ray task results will block until the task finished execution. After launching a number of tasks, you may want to know which ones have -finished executing without blocking on all of them. This could be achieved by :meth:`ray.wait() `. The function +finished executing without blocking on all of them. This could be achieved by :func:`ray.wait() `. The function works as follows. .. tabbed:: Python @@ -215,7 +215,7 @@ For tasks that return multiple objects, Ray also supports remote generators that Cancelling tasks ---------------- -Ray tasks can be canceled by calling :meth:`ray.cancel() ` on the returned Object ref. +Ray tasks can be canceled by calling :func:`ray.cancel() ` on the returned Object ref. .. tabbed:: Python @@ -242,7 +242,7 @@ By default, Ray will :ref:`retry ` failed tasks due to system failures and specified application-level failures. You can change this behavior by setting ``max_retries`` and ``retry_exceptions`` options -in :meth:`ray.remote() ` and :meth:`.options() `. +in :func:`ray.remote() ` and :meth:`.options() `. See :ref:`Ray fault tolerance ` for more details. diff --git a/doc/source/ray-core/tasks/using-ray-with-gpus.rst b/doc/source/ray-core/tasks/using-ray-with-gpus.rst index 5200d5a7b31a..c795f3a59913 100644 --- a/doc/source/ray-core/tasks/using-ray-with-gpus.rst +++ b/doc/source/ray-core/tasks/using-ray-with-gpus.rst @@ -40,7 +40,7 @@ and assign GPUs to the task or actor by setting the ``CUDA_VISIBLE_DEVICES`` env :start-after: __get_gpu_ids_start__ :end-before: __get_gpu_ids_end__ -Inside a task or actor, :meth:`ray.get_gpu_ids() ` will return a +Inside a task or actor, :func:`ray.get_gpu_ids() ` will return a list of GPU IDs that are available to the task or actor. Typically, it is not necessary to call ``ray.get_gpu_ids()`` because Ray will automatically set the ``CUDA_VISIBLE_DEVICES`` environment variable, @@ -99,7 +99,7 @@ task tries to use the same GPU. To address the problem, Ray disables the worker process reuse between GPU tasks by default, where the GPU resources is released after the task process exits. Since this adds overhead to GPU task scheduling, you can re-enable worker reuse by setting ``max_calls=0`` -in the :meth:`ray.remote ` decorator. +in the :func:`ray.remote ` decorator. .. literalinclude:: ../doc_code/gpus.py :language: python diff --git a/doc/source/ray-more-libs/joblib.rst b/doc/source/ray-more-libs/joblib.rst index 5eeff08fda62..d41ed8aa4dab 100644 --- a/doc/source/ray-more-libs/joblib.rst +++ b/doc/source/ray-more-libs/joblib.rst @@ -51,7 +51,7 @@ a multi-node Ray cluster instead. with joblib.parallel_backend('ray'): search.fit(digits.data, digits.target) -You can also set the ``ray_remote_args`` argument in ``parallel_backend`` to :meth:`configure +You can also set the ``ray_remote_args`` argument in ``parallel_backend`` to :func:`configure the Ray Actors ` making up the Pool. This can be used to eg. :ref:`assign resources to Actors, such as GPUs `.