Skip to content

Add support for list, describe, and delete namespaces in grpc #517

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 4 commits into from
Jun 27, 2025
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion codegen/apis
Submodule apis updated from 7e21ca to 827d26
2 changes: 1 addition & 1 deletion codegen/buf.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -9,4 +9,4 @@ breaking:
deps:
- buf.build/googleapis/googleapis
modules:
- path: apis/_build/2025-01
- path: apis/_build/2025-04
11 changes: 11 additions & 0 deletions docs/grpc.rst
Original file line number Diff line number Diff line change
Expand Up @@ -80,3 +80,14 @@ Vectors
.. automethod:: pinecone.grpc::GRPCIndex.list

.. automethod:: pinecone.grpc::GRPCIndex.list_paginated

Namespaces
----------

.. automethod:: pinecone.grpc::GRPCIndex.list_namespaces

.. automethod:: pinecone.grpc::GRPCIndex.list_namespaces_paginated

.. automethod:: pinecone.grpc::GRPCIndex.describe_namespace

.. automethod:: pinecone.grpc::GRPCIndex.delete_namespace

Large diffs are not rendered by default.

Original file line number Diff line number Diff line change
Expand Up @@ -235,6 +235,42 @@ class NamespaceSummary(_message.Message):
vector_count: int
def __init__(self, vector_count: _Optional[int] = ...) -> None: ...

class ListNamespacesRequest(_message.Message):
__slots__ = ("pagination_token", "limit")
PAGINATION_TOKEN_FIELD_NUMBER: _ClassVar[int]
LIMIT_FIELD_NUMBER: _ClassVar[int]
pagination_token: str
limit: int
def __init__(self, pagination_token: _Optional[str] = ..., limit: _Optional[int] = ...) -> None: ...

class ListNamespacesResponse(_message.Message):
__slots__ = ("namespaces", "pagination")
NAMESPACES_FIELD_NUMBER: _ClassVar[int]
PAGINATION_FIELD_NUMBER: _ClassVar[int]
namespaces: _containers.RepeatedCompositeFieldContainer[NamespaceDescription]
pagination: Pagination
def __init__(self, namespaces: _Optional[_Iterable[_Union[NamespaceDescription, _Mapping]]] = ..., pagination: _Optional[_Union[Pagination, _Mapping]] = ...) -> None: ...

class DescribeNamespaceRequest(_message.Message):
__slots__ = ("namespace",)
NAMESPACE_FIELD_NUMBER: _ClassVar[int]
namespace: str
def __init__(self, namespace: _Optional[str] = ...) -> None: ...

class NamespaceDescription(_message.Message):
__slots__ = ("name", "record_count")
NAME_FIELD_NUMBER: _ClassVar[int]
RECORD_COUNT_FIELD_NUMBER: _ClassVar[int]
name: str
record_count: int
def __init__(self, name: _Optional[str] = ..., record_count: _Optional[int] = ...) -> None: ...

class DeleteNamespaceRequest(_message.Message):
__slots__ = ("namespace",)
NAMESPACE_FIELD_NUMBER: _ClassVar[int]
namespace: str
def __init__(self, namespace: _Optional[str] = ...) -> None: ...

class DescribeIndexStatsResponse(_message.Message):
__slots__ = ("namespaces", "dimension", "index_fullness", "total_vector_count", "metric", "vector_type")
class NamespacesEntry(_message.Message):
Expand Down

Large diffs are not rendered by default.

5 changes: 4 additions & 1 deletion pinecone/grpc/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -51,12 +51,14 @@

from pinecone.db_data.dataclasses import Vector, SparseValues

from pinecone.core.grpc.protos.db_data_2025_01_pb2 import (
from pinecone.core.grpc.protos.db_data_2025_04_pb2 import (
Vector as GRPCVector,
SparseValues as GRPCSparseValues,
DeleteResponse as GRPCDeleteResponse,
)

from pinecone.core.openapi.db_data.models import ListNamespacesResponse

__all__ = [
"GRPCIndex",
"PineconeGRPC",
Expand All @@ -67,4 +69,5 @@
"Vector",
"SparseValues",
"PineconeGrpcFuture",
"ListNamespacesResponse",
]
2 changes: 2 additions & 0 deletions pinecone/grpc/grpc_runner.py
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@
from pinecone.exceptions.exceptions import PineconeException
from grpc import CallCredentials, Compression
from google.protobuf.message import Message
from pinecone.openapi_support.api_version import API_VERSION


class GrpcRunner:
Expand All @@ -21,6 +22,7 @@ def __init__(self, index_name: str, config: Config, grpc_config: GRPCClientConfi
"api-key": config.api_key,
"service-name": index_name,
"client-version": CLIENT_VERSION,
"x-pinecone-api-version": API_VERSION,
}
if self.grpc_client_config.additional_metadata:
self.fixed_metadata.update(self.grpc_client_config.additional_metadata)
Expand Down
150 changes: 147 additions & 3 deletions pinecone/grpc/index_grpc.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@
from google.protobuf import json_format

from pinecone.utils.tqdm import tqdm
from pinecone.utils import require_kwargs
from concurrent.futures import as_completed, Future


Expand All @@ -15,6 +16,8 @@
parse_upsert_response,
parse_update_response,
parse_delete_response,
parse_namespace_description,
parse_list_namespaces_response,
)
from .vector_factory_grpc import VectorFactoryGRPC
from .sparse_values_factory import SparseValuesFactory
Expand All @@ -23,9 +26,11 @@
FetchResponse,
QueryResponse,
IndexDescription as DescribeIndexStatsResponse,
NamespaceDescription,
ListNamespacesResponse,
)
from pinecone.db_control.models.list_response import ListResponse as SimpleListResponse, Pagination
from pinecone.core.grpc.protos.db_data_2025_01_pb2 import (
from pinecone.core.grpc.protos.db_data_2025_04_pb2 import (
Vector as GRPCVector,
QueryVector as GRPCQueryVector,
UpsertRequest,
Expand All @@ -39,10 +44,13 @@
DeleteResponse,
UpdateResponse,
SparseValues as GRPCSparseValues,
DescribeNamespaceRequest,
DeleteNamespaceRequest,
ListNamespacesRequest,
)
from pinecone import Vector, SparseValues
from pinecone.db_data.query_results_aggregator import QueryNamespacesResults, QueryResultsAggregator
from pinecone.core.grpc.protos.db_data_2025_01_pb2_grpc import VectorServiceStub
from pinecone.core.grpc.protos.db_data_2025_04_pb2_grpc import VectorServiceStub
from .base import GRPCIndexBase
from .future import PineconeGrpcFuture
from ..db_data.types import (
Expand All @@ -54,7 +62,7 @@
)


__all__ = ["GRPCIndex", "GRPCVector", "GRPCQueryVector", "GRPCSparseValues"]
__all__ = ["GRPCIndex", "GRPCVector", "GRPCQueryVector", "GRPCSparseValues", "NamespaceDescription", "ListNamespacesResponse"]

_logger = logging.getLogger(__name__)
""" :meta private: """
Expand Down Expand Up @@ -681,6 +689,142 @@ def describe_index_stats(
json_response = json_format.MessageToDict(response)
return parse_stats_response(json_response)

@require_kwargs
def describe_namespace(
self, namespace: str, **kwargs
) -> NamespaceDescription:
"""
The describe_namespace operation returns information about a specific namespace,
including the total number of vectors in the namespace.

Examples:

.. code-block:: python

>>> index.describe_namespace(namespace='my_namespace')

Args:
namespace (str): The namespace to describe.

Returns: NamespaceDescription object which contains information about the namespace.
"""
timeout = kwargs.pop("timeout", None)
request = DescribeNamespaceRequest(namespace=namespace)
response = self.runner.run(self.stub.DescribeNamespace, request, timeout=timeout)
return parse_namespace_description(response)

@require_kwargs
def delete_namespace(
self, namespace: str, **kwargs
) -> Dict[str, Any]:
"""
The delete_namespace operation deletes a namespace from an index.
This operation is irreversible and will permanently delete all data in the namespace.

Examples:

.. code-block:: python

>>> index.delete_namespace(namespace='my_namespace')

Args:
namespace (str): The namespace to delete.

Returns: Empty dictionary indicating successful deletion.
"""
timeout = kwargs.pop("timeout", None)
request = DeleteNamespaceRequest(namespace=namespace)
response = self.runner.run(self.stub.DeleteNamespace, request, timeout=timeout)
return parse_delete_response(response)

@require_kwargs
def list_namespaces_paginated(
self,
limit: Optional[int] = None,
pagination_token: Optional[str] = None,
**kwargs,
) -> ListNamespacesResponse:
"""
The list_namespaces_paginated operation returns a list of all namespaces in a serverless index.
It returns namespaces in a paginated form, with a pagination token to fetch the next page of results.

Examples:

.. code-block:: python

>>> results = index.list_namespaces_paginated(limit=10)
>>> [ns.name for ns in results.namespaces]
['namespace1', 'namespace2', 'namespace3']
>>> results.pagination.next
eyJza2lwX3Bhc3QiOiI5OTMiLCJwcmVmaXgiOiI5OSJ9
>>> next_results = index.list_namespaces_paginated(limit=10, pagination_token=results.pagination.next)

Args:
limit (Optional[int]): The maximum number of namespaces to return. If unspecified, the server will use a default value. [optional]
pagination_token (Optional[str]): A token needed to fetch the next page of results. This token is returned
in the response if additional results are available. [optional]

Returns: ListNamespacesResponse object which contains the list of namespaces and pagination information.
"""
args_dict = self._parse_non_empty_args(
[
("limit", limit),
("pagination_token", pagination_token),
]
)
timeout = kwargs.pop("timeout", None)
request = ListNamespacesRequest(**args_dict, **kwargs)
response = self.runner.run(self.stub.ListNamespaces, request, timeout=timeout)
return parse_list_namespaces_response(response)

@require_kwargs
def list_namespaces(self, limit: Optional[int] = None, **kwargs):
"""
The list_namespaces operation accepts all of the same arguments as list_namespaces_paginated, and returns a generator that yields
each namespace. It automatically handles pagination tokens on your behalf.

Args:
limit (Optional[int]): The maximum number of namespaces to fetch in each network call. If unspecified, the server will use a default value. [optional]

Returns:
Returns a generator that yields each namespace. It automatically handles pagination tokens on your behalf so you can
easily iterate over all results. The ``list_namespaces`` method accepts all of the same arguments as list_namespaces_paginated

Examples:

.. code-block:: python

>>> for namespace in index.list_namespaces():
>>> print(namespace.name)
namespace1
namespace2
namespace3

You can convert the generator into a list by wrapping the generator in a call to the built-in ``list`` function:

.. code-block:: python

namespaces = list(index.list_namespaces())

You should be cautious with this approach because it will fetch all namespaces at once, which could be a large number
of network calls and a lot of memory to hold the results.
"""
done = False
while not done:
try:
results = self.list_namespaces_paginated(limit=limit, **kwargs)
except Exception as e:
raise e

if results.namespaces and len(results.namespaces) > 0:
for namespace in results.namespaces:
yield namespace

if results.pagination and results.pagination.next:
kwargs.update({"pagination_token": results.pagination.next})
else:
done = True

@staticmethod
def _parse_non_empty_args(args: List[Tuple[str, Any]]) -> Dict[str, Any]:
return {arg_name: val for arg_name, val in args if val is not None}
2 changes: 1 addition & 1 deletion pinecone/grpc/sparse_values_factory.py
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@
from ..db_data import SparseValuesTypeError, SparseValuesMissingKeysError
from ..db_data.types import SparseVectorTypedDict

from pinecone.core.grpc.protos.db_data_2025_01_pb2 import SparseValues as GRPCSparseValues
from pinecone.core.grpc.protos.db_data_2025_04_pb2 import SparseValues as GRPCSparseValues
from pinecone.core.openapi.db_data.models import SparseValues as OpenApiSparseValues
from pinecone import SparseValues

Expand Down
37 changes: 37 additions & 0 deletions pinecone/grpc/utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,9 @@
IndexDescription as DescribeIndexStatsResponse,
UpsertResponse,
NamespaceSummary,
NamespaceDescription,
ListNamespacesResponse,
Pagination,
)
from pinecone.db_data.dataclasses import FetchResponse

Expand Down Expand Up @@ -126,3 +129,37 @@ def parse_stats_response(response: dict):
total_vector_count=total_vector_count,
_check_type=False,
)


def parse_namespace_description(response: Message) -> NamespaceDescription:
json_response = json_format.MessageToDict(response)
return NamespaceDescription(
name=json_response.get("name", ""),
record_count=json_response.get("recordCount", 0),
_check_type=False,
)


def parse_list_namespaces_response(response: Message) -> ListNamespacesResponse:
json_response = json_format.MessageToDict(response)

namespaces = []
for ns in json_response.get("namespaces", []):
namespaces.append(NamespaceDescription(
name=ns.get("name", ""),
record_count=ns.get("recordCount", 0),
_check_type=False,
))

pagination = None
if "pagination" in json_response and json_response["pagination"]:
pagination = Pagination(
next=json_response["pagination"].get("next", ""),
_check_type=False,
)

return ListNamespacesResponse(
namespaces=namespaces,
pagination=pagination,
_check_type=False,
)
2 changes: 1 addition & 1 deletion pinecone/grpc/vector_factory_grpc.py
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@
from ..db_data.types import VectorTuple, VectorTypedDict
from .sparse_values_factory import SparseValuesFactory

from pinecone.core.grpc.protos.db_data_2025_01_pb2 import (
from pinecone.core.grpc.protos.db_data_2025_04_pb2 import (
Vector as GRPCVector,
SparseValues as GRPCSparseValues,
)
Expand Down
6 changes: 0 additions & 6 deletions tests/integration/data/test_namespace.py
Original file line number Diff line number Diff line change
@@ -1,9 +1,6 @@
import os
import time
import logging

import pytest

from pinecone import NamespaceDescription

logger = logging.getLogger(__name__)
Expand Down Expand Up @@ -43,9 +40,6 @@ def delete_all_namespaces(index):
except Exception as e:
logger.error(f"Error in delete_all_namespaces: {e}")

@pytest.mark.skipif(
os.getenv("USE_GRPC") == "true", reason="Disable until grpc namespaces support is added"
)
class TestNamespaceOperations:
def test_describe_namespace(self, idx):
"""Test describing a namespace"""
Expand Down
4 changes: 2 additions & 2 deletions tests/integration/data_grpc_futures/stub_backend.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,8 +2,8 @@
import grpc
import logging
from concurrent import futures
import pinecone.core.grpc.protos.db_data_2025_01_pb2 as pb2
import pinecone.core.grpc.protos.db_data_2025_01_pb2_grpc as pb2_grpc
import pinecone.core.grpc.protos.db_data_2025_04_pb2 as pb2
import pinecone.core.grpc.protos.db_data_2025_04_pb2_grpc as pb2_grpc

logger = logging.getLogger(__name__)

Expand Down
Loading
Loading