Sort envs returned by REST API by current build's scheduled_on time

conda-store-server/conda_store_server/_internal/schema.py

@@ -205,7 +205,7 @@ class Environment(BaseModel):
     id: int
     namespace: Namespace
     name: str
-    current_build_id: int
+    current_build_id: Optional[int]
     current_build: Optional[Build]
 
     description: Optional[str]
@@ -648,6 +648,14 @@ class APIPaginatedResponse(APIResponse):
     count: int
 
 
+class APICursorPaginatedResponse(BaseModel):
+    data: Optional[Any]
+    status: APIStatus
+    message: Optional[str]
+    cursor: Optional[str]
+    count: int
+
+
 class APIAckResponse(BaseModel):
     status: APIStatus
     message: Optional[str]
@@ -712,7 +720,7 @@ class APIDeleteNamespaceRole(BaseModel):
 
 
 # GET /api/v1/environment
-class APIListEnvironment(APIPaginatedResponse):
+class APIListEnvironment(APICursorPaginatedResponse):
     data: List[Environment]
 
 

conda-store-server/conda_store_server/_internal/server/views/api.py

@@ -14,11 +14,37 @@
 from conda_store_server import __version__, api, app
 from conda_store_server._internal import orm, schema, utils
 from conda_store_server._internal.environment import filter_environments
-from conda_store_server._internal.schema import AuthenticationToken, Permissions
+from conda_store_server._internal.schema import (
+    AuthenticationToken,
+    Permissions,
+)
 from conda_store_server._internal.server import dependencies
+from conda_store_server._internal.server.views.pagination import (
+    Cursor,
+    CursorPaginatedArgs,
+    OrderingMetadata,
+    paginate,
+)
 from conda_store_server.server.auth import Authentication
 
 
+def get_cursor(encoded_cursor: Optional[str] = None) -> Cursor:
+    return Cursor.load(encoded_cursor)
+
+
+def get_cursor_paginated_args(
+    order: Optional[str] = None,
+    limit: Optional[int] = None,
+    sort_by: List[str] = Query([]),
+    server=Depends(dependencies.get_server),
+) -> CursorPaginatedArgs:
+    return CursorPaginatedArgs(
+        limit=server.max_page_size if limit is None else limit,
+        order=order,
+        sort_by=sort_by,
+    )
+
+
 class PaginatedArgs(TypedDict):
     """Dictionary type holding information about paginated requests."""
 
@@ -631,15 +657,16 @@ async def api_list_environments(
     auth: Authentication = Depends(dependencies.get_auth),
     conda_store: app.CondaStore = Depends(dependencies.get_conda_store),
     entity: AuthenticationToken = Depends(dependencies.get_entity),
-    paginated_args: PaginatedArgs = Depends(get_paginated_args),
+    paginated_args: CursorPaginatedArgs = Depends(get_cursor_paginated_args),
+    cursor: Cursor = Depends(get_cursor),
     artifact: Optional[schema.BuildArtifactType] = None,
     jwt: Optional[str] = None,
     name: Optional[str] = None,
     namespace: Optional[str] = None,
     packages: Optional[List[str]] = Query([]),
     search: Optional[str] = None,
     status: Optional[schema.BuildStatus] = None,
-):
+) -> schema.APIListEnvironment:
     """Retrieve a list of environments.
 
     Parameters
@@ -650,7 +677,7 @@ async def api_list_environments(
         the request
     entity : AuthenticationToken
         Token of the user making the request
-    paginated_args : PaginatedArgs
+    paginated_args : CursorPaginatedArgs
         Arguments for controlling pagination of the response
     conda_store : app.CondaStore
         The running conda store application
@@ -674,9 +701,11 @@ async def api_list_environments(
 
     Returns
     -------
-    Dict
-        Paginated JSON response containing the requested environments
-
+    schema.APIListEnvironment
+        Paginated JSON response containing the requested environments. Results are sorted by each
+        envrionment's build's scheduled_on time to ensure all results are returned when iterating
+        over pages in systems where the number of environments is changing while results are being
+        requested; see https://github.com/conda-incubator/conda-store/issues/859 for context
     """
     with conda_store.get_db() as db:
         if jwt:
@@ -687,7 +716,7 @@ async def api_list_environments(
         else:
             role_bindings = None
 
-        orm_environments = api.list_environments(
+        query = api.list_environments(
             db,
             search=search,
             namespace=namespace,
@@ -700,21 +729,27 @@ async def api_list_environments(
         )
 
         # Filter by environments that the user who made the query has access to
-        orm_environments = filter_environments(
-            query=orm_environments,
+        query = filter_environments(
+            query=query,
             role_bindings=auth.entity_bindings(entity),
         )
 
-        return paginated_api_response(
-            orm_environments,
-            paginated_args,
-            schema.Environment,
-            exclude={"current_build"},
-            allowed_sort_bys={
-                "namespace": orm.Namespace.name,
-                "name": orm.Environment.name,
-            },
-            default_sort_by=["namespace", "name"],
+        paginated, next_cursor = paginate(
+            query=query,
+            ordering_metadata=OrderingMetadata(
+                order_names=["namespace", "name"],
+                column_names=["namespace.name", "name"],
+            ),
+            cursor=cursor,
+            order_by=paginated_args["sort_by"],
+            limit=paginated_args["limit"],
+        )
+
+        return schema.APIListEnvironment(
+            data=paginated,
+            status="ok",
+            cursor=None if next_cursor is None else next_cursor.dump(),
+            count=1000,
         )
 
 

conda-store-server/conda_store_server/_internal/server/views/pagination.py

@@ -0,0 +1,216 @@
+from __future__ import annotations
+
+import base64
+import operator
+from typing import Any, TypedDict
+
+import pydantic
+from fastapi import HTTPException
+from sqlalchemy import asc, desc, tuple_
+from sqlalchemy.orm import Query as SqlQuery
+from sqlalchemy.sql.expression import ColumnClause
+
+
+class Cursor(pydantic.BaseModel):
+    last_id: int | None = 0
+    count: int | None = None
+
+    # List query parameters to order by, and the last value of the ordered attribute
+    # {
+    #   'namespace': 'foo',
+    #   'environment': 'bar',
+    # }
+    last_value: dict[str, str] | None = {}
+
+    def dump(self) -> str:
+        return base64.b64encode(self.model_dump_json())
+
+    @classmethod
+    def load(cls, data: str | None = None) -> Cursor | None:
+        if data is None:
+            return None
+        return cls.from_json(base64.b64decode(data))
+
+    def get_last_values(self, order_names: list[str]) -> list[Any]:
+        if order_names:
+            return [self.last_value[name] for name in order_names]
+        else:
+            return []
+
+
+def paginate(
+    query: SqlQuery,
+    ordering_metadata: OrderingMetadata,
+    cursor: Cursor | None = None,
+    order_by: list[str] | None = None,
+    # valid_order_by: dict[str, str] | None = None,
+    order: str = "asc",
+    limit: int = 10,
+) -> tuple[SqlQuery, Cursor]:
+    """Paginate the query using the cursor and the requested sort_bys.
+
+    This function assumes that the first column of the query contains
+    the type whose ID should be used to sort the results.
+
+    Additionally, with cursor pagination all keys used to order the results
+    must be included in the call to query.filter().
+
+    https://medium.com/@george_16060/cursor-based-pagination-with-arbitrary-ordering-b4af6d5e22db
+
+    Parameters
+    ----------
+    query : SqlQuery
+        Query containing database results to paginate
+    valid_order_by : dict[str, str] | None
+        Mapping between valid names to order by and the column names on the orm object they apply to
+    cursor : Cursor | None
+        Cursor object containing information about the last item on the previous page.
+        If None, the first page is returned.
+    order_by : list[str] | None
+        List of sort_by query parameters
+
+    Returns
+    -------
+    tuple[SqlQuery, Cursor]
+        Query containing the paginated results, and Cursor for retrieving
+        the next page
+    """
+    if order_by is None:
+        order_by = []
+
+    if order == "asc":
+        comparison = operator.gt
+        order_func = asc
+    elif order == "desc":
+        comparison = operator.lt
+        order_func = desc
+    else:
+        raise HTTPException(
+            status_code=400,
+            detail=f"Invalid query parameter: order = {order}; must be one of ['asc', 'desc']",
+        )
+
+    # Get the python type of the objects being queried
+    queried_type = query.column_descriptions[0]["type"]
+    columns = ordering_metadata.get_requested_columns(order_by)
+
+    # If there's a cursor already, use the last attributes to filter
+    # the results by (*attributes, id) >/< (*last_values, last_id)
+    # Order by desc or asc
+    if cursor is not None:
+        last_values = cursor.get_last_values(order_by)
+        query = query.filter(
+            comparison(
+                tuple_(*columns, queried_type.id),
+                (*last_values, cursor.last_id),
+            )
+        )
+
+    query = query.order_by(
+        *[order_func(col) for col in columns], order_func(queried_type.id)
+    )
+    data = query.limit(limit).all()
+    count = query.count()
+
+    if count > 0:
+        last_result = data[-1]
+        last_value = ordering_metadata.get_attr_values(last_result, order_by)
+
+        next_cursor = Cursor(
+            last_id=data[-1].id, last_value=last_value, count=query.count()
+        )
+    else:
+        next_cursor = None
+
+    return (data, next_cursor)
+
+
+class CursorPaginatedArgs(TypedDict):
+    limit: int
+    order: str
+    sort_by: list[str]
+
+
+class OrderingMetadata:
+    def __init__(
+        self,
+        order_names: list[str] | None = None,
+        column_names: list[str] | None = None,
+    ):
+        self.order_names = order_names
+        self.column_names = column_names
+
+    def validate(self, model: Any):
+        if len(self.order_names) != len(self.column_names):
+            raise ValueError(
+                "Each name of a valid ordering available to the order_by query parameter"
+                "must have an associated column name to select in the table."
+            )
+
+        for col in self.column_names:
+            if not hasattr(model, col):
+                raise ValueError(f"No column named {col} found on model {model}.")
+
+    def get_requested_columns(
+        self,
+        order_by: list[str] | None = None,
+    ) -> list[ColumnClause]:
+        """Get a list of sqlalchemy columns requested by the value of the order_by query param.
+
+        Parameters
+        ----------
+        order_by : list[str] | None
+            If specified, this should be a subset of self.order_names. If none, an
+            empty list is returned.
+
+        Returns
+        -------
+        list[ColumnClause]
+            A list of sqlalchemy columns corresponding to the order_by values passed
+            as a query parameter
+        """
+        columns = []
+        if order_by:
+            for order_name in order_by:
+                idx = self.order_names.index(order_name)
+                columns.append(self.column_names[idx])
+
+        return columns
+
+    def get_attr_values(
+        self,
+        obj: Any,
+        order_by: list[str] | None = None,
+    ) -> dict[str, Any]:
+        """Using the order_by values, get the corresponding attribute values on obj.
+
+        Parameters
+        ----------
+        obj : Any
+            sqlalchemy model containing attribute names that are contained in
+            `self.column_names`
+        order_by : list[str] | None
+            Values that the user wants to order by; these are used to look up the corresponding
+            column names that are used to access the attributes of `obj`.
+
+        Returns
+        -------
+        dict[str, Any]
+            A mapping between the `order_by` values and the attribute values on `obj`
+
+        """
+        values = {}
+        for order_name in order_by:
+            idx = self.order_names.index(order_name)
+            values[order_name] = get_nested_attribute(obj, self.column_names[idx])
+
+        return values
+
+
+def get_nested_attribute(obj: Any, attr: str) -> Any:
+    attribute, *rest = attr.split(".")
+    while len(rest) > 0:
+        obj = getattr(obj, attribute)
+        attribute, *rest = rest
+
+    return getattr(obj, attribute)

conda-store-server/pyproject.toml

@@ -124,9 +124,6 @@ user-journey-test = ["pytest -m user_journey"]
 conda-store-server = "conda_store_server._internal.server.__main__:main"
 conda-store-worker = "conda_store_server._internal.worker.__main__:main"
 
-[tool.black]
-line-length = 88
-
 [tool.isort]
 lines_between_types = 1
 lines_after_imports = 2
@@ -141,7 +138,6 @@ exclude = [
 [tool.ruff.lint]
 ignore = [
     "E501", # line-length
-
     "ANN001",	# missing-type-function-argument
     "ANN002",	# missing-type-args
     "ANN003",	# missing-type-kwargs