Minor: Add additional docstrings to Window function implementations

datafusion/core/src/lib.rs

@@ -40,7 +40,7 @@
 //! # Examples
 //!
 //! The main entry point for interacting with DataFusion is the
-//! [`SessionContext`].
+//! [`SessionContext`]. [`Expr`]s represent expressions such as `a + b`.
 //!
 //! [`SessionContext`]: execution::context::SessionContext
 //!

datafusion/expr/src/window_function.rs

@@ -73,7 +73,9 @@ impl fmt::Display for WindowFunction {
     }
 }
 
-/// An aggregate function that is part of a built-in window function
+/// A [window function] built in to DataFusion
+///
+/// [window function]: https://en.wikipedia.org/wiki/Window_function_(SQL)
 #[derive(Debug, Clone, PartialEq, Eq, Hash, EnumIter)]
 pub enum BuiltInWindowFunction {
     /// number of the current row within its partition, counting from 1

datafusion/physical-expr/src/window/aggregate.rs

@@ -37,12 +37,9 @@ use crate::{
     expressions::PhysicalSortExpr, reverse_order_bys, AggregateExpr, PhysicalExpr,
 };
 
-/// A window expr that takes the form of an aggregate function
-/// Aggregate Window Expressions that have the form
-/// `OVER({ROWS | RANGE| GROUPS} BETWEEN UNBOUNDED PRECEDING AND ...)`
-/// e.g cumulative window frames uses `PlainAggregateWindowExpr`. Where as Aggregate Window Expressions
-/// that have the form `OVER({ROWS | RANGE| GROUPS} BETWEEN M {PRECEDING| FOLLOWING} AND ...)`
-/// e.g sliding window frames uses `SlidingAggregateWindowExpr`.
+/// A window expr that takes the form of an aggregate function.
+///
+/// See comments on [`WindowExpr`] for more details.
 #[derive(Debug)]
 pub struct PlainAggregateWindowExpr {
     aggregate: Arc<dyn AggregateExpr>,

datafusion/physical-expr/src/window/built_in.rs

@@ -39,7 +39,7 @@ use datafusion_common::utils::evaluate_partition_ranges;
 use datafusion_common::{Result, ScalarValue};
 use datafusion_expr::WindowFrame;
 
-/// A window expr that takes the form of a built in window function
+/// A window expr that takes the form of a [`BuiltInWindowFunctionExpr`].
 #[derive(Debug)]
 pub struct BuiltInWindowExpr {
     expr: Arc<dyn BuiltInWindowFunctionExpr>,

datafusion/physical-expr/src/window/built_in_window_function_expr.rs

@@ -24,20 +24,21 @@ use datafusion_common::Result;
 use std::any::Any;
 use std::sync::Arc;
 
-/// A window expression that is a built-in window function.
+/// A window expression that evaluates a window function.
+///
+/// Note that unlike aggregation based window functions, window
+/// functions normally ignore the window frame spec with the exception
+/// of `first_value`, `last_value`, and `nth_value`.
 ///
-/// Note that unlike aggregation based window functions, built-in window functions normally ignore
-/// window frame spec, with the exception of first_value, last_value, and nth_value.
 pub trait BuiltInWindowFunctionExpr: Send + Sync + std::fmt::Debug {
     /// Returns the aggregate expression as [`Any`](std::any::Any) so that it can be
     /// downcast to a specific implementation.
     fn as_any(&self) -> &dyn Any;
 
-    /// the field of the final result of this aggregation.
+    /// The field of the final result of evaluating this window function.
     fn field(&self) -> Result<Field>;
 
-    /// expressions that are passed to the Accumulator.
-    /// Single-column aggregations such as `sum` return a single value, others (e.g. `cov`) return many.
+    /// Expressions that are passed to the [`PartitionEvaluator`].
     fn expressions(&self) -> Vec<Arc<dyn PhysicalExpr>>;
 
     /// Human readable name such as `"MIN(c2)"` or `"RANK()"`. The default
@@ -46,8 +47,9 @@ pub trait BuiltInWindowFunctionExpr: Send + Sync + std::fmt::Debug {
         "BuiltInWindowFunctionExpr: default name"
     }
 
-    /// Evaluate window function arguments against the batch and return
-    /// an array ref. Typically, the resulting vector is a single element vector.
+    /// Evaluate window function's arguments against the batch and
+    /// return an array ref. Typically, the resulting vector is a
+    /// single element vector.
     fn evaluate_args(&self, batch: &RecordBatch) -> Result<Vec<ArrayRef>> {
         self.expressions()
             .iter()
@@ -56,11 +58,13 @@ pub trait BuiltInWindowFunctionExpr: Send + Sync + std::fmt::Debug {
             .collect()
     }
 
-    /// Create built-in window evaluator with a batch
+    /// Create a [`PartitionEvaluator`] to evaluate data on a particular partition.
     fn create_evaluator(&self) -> Result<Box<dyn PartitionEvaluator>>;
 
     /// Construct Reverse Expression that produces the same result
-    /// on a reversed window.  For example `lead(10)` --> `lag(10)`
+    /// on a reversed window. Retuns `None` if not possible.
+    ///
+    /// For example, the reverse of `lead(10)` is `lag(10)`.
     fn reverse_expr(&self) -> Option<Arc<dyn BuiltInWindowFunctionExpr>> {
         None
     }
@@ -69,6 +73,8 @@ pub trait BuiltInWindowFunctionExpr: Send + Sync + std::fmt::Debug {
         false
     }
 
+    /// If returns true, [`Self::create_evaluator`] must implement
+    /// [`PartitionEvaluator::evaluate_inside_range`]
     fn uses_window_frame(&self) -> bool {
         false
     }

datafusion/physical-expr/src/window/mod.rs

@@ -32,6 +32,7 @@ mod window_frame_state;
 pub use aggregate::PlainAggregateWindowExpr;
 pub use built_in::BuiltInWindowExpr;
 pub use built_in_window_function_expr::BuiltInWindowFunctionExpr;
+pub use partition_evaluator::PartitionEvaluator;
 pub use sliding_aggregate::SlidingAggregateWindowExpr;
 pub use window_expr::PartitionBatchState;
 pub use window_expr::PartitionBatches;

datafusion/physical-expr/src/window/partition_evaluator.rs

@@ -25,24 +25,70 @@ use datafusion_common::{DataFusionError, ScalarValue};
 use std::fmt::Debug;
 use std::ops::Range;
 
-/// Partition evaluator
+/// Partition evaluator for Window Functions
+///
+/// An implementation of this trait is created and used for each
+/// partition defined by the OVER clause.
+///
+/// For example, evaluating `window_func(val) OVER (PARTITION BY col)`
+/// on the following data:
+///
+/// ```text
+/// col | val
+/// --- + ----
+///  A  | 1
+///  A  | 1
+///  C  | 2
+///  D  | 3
+///  D  | 3
+/// ```
+///
+/// Will instantiate three `PartitionEvaluator`s, one each for the
+/// partitions defined by `col=A`, `col=B`, and `col=C`.
+///
+/// There are two types of `PartitionEvaluator`:
+///
+/// # Stateless `PartitionEvaluator`
+///
+/// In this case, [`PartitionEvaluator::evaluate`] is called for the
+/// entire partition / window function.
+///
+/// # Stateful `PartitionEvaluator`
+///
+/// This is used for XXXX. In this case YYYYY
+///
 pub trait PartitionEvaluator: Debug + Send {
     /// Whether the evaluator should be evaluated with rank
+    ///
+    /// If `include_rank` is true, then [`Self::evaluate_with_rank`]
+    /// will be called for each partition, which includes the
+    /// `rank`. For example:
+    ///
+    /// ```text
+    /// col | rank
+    /// --- + ----
+    ///  A  | 1
+    ///  A  | 1
+    ///  C  | 2
+    ///  D  | 3
+    ///  D  | 3
+    /// ```
     fn include_rank(&self) -> bool {
         false
     }
 
-    /// Returns state of the Built-in Window Function
+    /// Returns state of the Built-in Window Function (only used for stateful evaluation)
     fn state(&self) -> Result<BuiltinWindowState> {
         // If we do not use state we just return Default
         Ok(BuiltinWindowState::Default)
     }
 
-    /// Updates the internal state for Built-in window function
-    // state is useful to update internal state for Built-in window function.
-    // idx is the index of last row for which result is calculated.
-    // range_columns is the result of order by column values. It is used to calculate rank boundaries
-    // sort_partition_points is the boundaries of each rank in the range_column. It is used to update rank.
+    /// Updates the internal state for Built-in window function, if desired.
+    ///
+    /// `state`: is useful to update internal state for Built-in window function.
+    /// `idx`: is the index of last row for which result is calculated.
+    /// `range_columns`: is the result of order by column values. It is used to calculate rank boundaries
+    /// `sort_partition_points`: is the boundaries of each rank in the range_column. It is used to update rank.
     fn update_state(
         &mut self,
         _state: &WindowAggState,
@@ -54,15 +100,17 @@ pub trait PartitionEvaluator: Debug + Send {
         Ok(())
     }
 
+    /// Sets the internal state for Built-in window function, if supported
     fn set_state(&mut self, _state: &BuiltinWindowState) -> Result<()> {
         Err(DataFusionError::NotImplemented(
             "set_state is not implemented for this window function".to_string(),
         ))
     }
 
     /// Gets the range where Built-in window function result is calculated.
-    // idx is the index of last row for which result is calculated.
-    // n_rows is the number of rows of the input record batch (Used during bound check)
+    ///
+    /// `idx`: is the index of last row for which result is calculated.
+    /// `n_rows`: is the number of rows of the input record batch (Used during bound check)
     fn get_range(&self, _idx: usize, _n_rows: usize) -> Result<Range<usize>> {
         Err(DataFusionError::NotImplemented(
             "get_range is not implemented for this window function".to_string(),
@@ -83,7 +131,9 @@ pub trait PartitionEvaluator: Debug + Send {
         ))
     }
 
-    /// evaluate the partition evaluator against the partition but with rank
+    /// Evaluate the partition evaluator against the partition but with rank
+    ///
+    /// See [`Self::include_rank`] for more details
     fn evaluate_with_rank(
         &self,
         _num_rows: usize,

datafusion/physical-expr/src/window/sliding_aggregate.rs

@@ -36,12 +36,10 @@ use crate::{
     expressions::PhysicalSortExpr, reverse_order_bys, AggregateExpr, PhysicalExpr,
 };
 
-/// A window expr that takes the form of an aggregate function
-/// Aggregate Window Expressions that have the form
-/// `OVER({ROWS | RANGE| GROUPS} BETWEEN UNBOUNDED PRECEDING AND ...)`
-/// e.g cumulative window frames uses `PlainAggregateWindowExpr`. Where as Aggregate Window Expressions
-/// that have the form `OVER({ROWS | RANGE| GROUPS} BETWEEN M {PRECEDING| FOLLOWING} AND ...)`
-/// e.g sliding window frames uses `SlidingAggregateWindowExpr`.
+/// A window expr that takes the form of an aggregate function that
+/// can be incrementally computed over sliding windows.
+///
+/// See comments on [`WindowExpr`] for more details.
 #[derive(Debug)]
 pub struct SlidingAggregateWindowExpr {
     aggregate: Arc<dyn AggregateExpr>,
@@ -72,10 +70,11 @@ impl SlidingAggregateWindowExpr {
     }
 }
 
-/// peer based evaluation based on the fact that batch is pre-sorted given the sort columns
-/// and then per partition point we'll evaluate the peer group (e.g. SUM or MAX gives the same
-/// results for peers) and concatenate the results.
-
+/// Incrementally update window function using the fact that batch is
+/// pre-sorted given the sort columns and then per partition point.
+///
+/// Evaluates the peer group (e.g. `SUM` or `MAX` gives the same results
+/// for peers) and concatenate the results.
 impl WindowExpr for SlidingAggregateWindowExpr {
     /// Return a reference to Any that can be used for downcasting
     fn as_any(&self) -> &dyn Any {

datafusion/physical-expr/src/window/window_expr.rs

@@ -32,8 +32,28 @@ use std::fmt::Debug;
 use std::ops::Range;
 use std::sync::Arc;
 
-/// A window expression that:
-/// * knows its resulting field
+/// Common trait for [window function] implementations,
+///
+/// Aggregate Window Expressions that have the form
+///
+/// ```text
+/// OVER({ROWS | RANGE| GROUPS} BETWEEN UNBOUNDED PRECEDING AND ...)
+/// ```
+///
+/// e.g cumulative window frames uses `PlainAggregateWindowExpr`.
+///
+/// Aggregate Window Expressions that have the form
+///
+/// ```text
+/// OVER({ROWS | RANGE| GROUPS} BETWEEN M {PRECEDING| FOLLOWING} AND ...)
+/// ```
+///
+/// e.g sliding window frames use [`SlidingAggregateWindowExpr`].
+///
+///
+/// [window function]: https://en.wikipedia.org/wiki/Window_function_(SQL)
+/// [`PlainAggregateWindowExpr`]: crate::window::PlainAggregateWindowExpr
+/// [`SlidingAggregateWindowExpr`]: crate::window::SlidingAggregateWindowExpr
 pub trait WindowExpr: Send + Sync + Debug {
     /// Returns the window expression as [`Any`](std::any::Any) so that it can be
     /// downcast to a specific implementation.
@@ -123,7 +143,7 @@ pub trait WindowExpr: Send + Sync + Debug {
     fn get_reverse_expr(&self) -> Option<Arc<dyn WindowExpr>>;
 }
 
-/// Trait for different `AggregateWindowExpr`s (`PlainAggregateWindowExpr`, `SlidingAggregateWindowExpr`)
+/// Extension trait that adds common functionality to [`AggregateWindowExpr`]s
 pub trait AggregateWindowExpr: WindowExpr {
     /// Get the accumulator for the window expression. Note that distinct
     /// window expressions may return distinct accumulators; e.g. sliding