From e8b69363e0b6756d08331afa64b239b1d0c78d40 Mon Sep 17 00:00:00 2001 From: Jen Hamon Date: Fri, 11 Apr 2025 12:34:30 -0400 Subject: [PATCH 1/4] Refactor PluginAware to do lazy loading --- pinecone/control/pinecone.py | 7 +- pinecone/data/features/inference/inference.py | 4 +- pinecone/data/index.py | 6 +- pinecone/utils/plugin_aware.py | 92 +++++++++++++++++-- 4 files changed, 91 insertions(+), 18 deletions(-) diff --git a/pinecone/control/pinecone.py b/pinecone/control/pinecone.py index f3c8f404..17b3d953 100644 --- a/pinecone/control/pinecone.py +++ b/pinecone/control/pinecone.py @@ -43,7 +43,7 @@ """ @private """ -class Pinecone(PineconeDBControlInterface, PluginAware): +class Pinecone(PluginAware, PineconeDBControlInterface): """ A client for interacting with Pinecone's vector database. @@ -107,9 +107,8 @@ def __init__( self.index_host_store = IndexHostStore() """ @private """ - self.load_plugins( - config=self.config, openapi_config=self.openapi_config, pool_threads=self.pool_threads - ) + # Initialize PluginAware first, which will then call PineconeDBControlInterface.__init__ + super().__init__() @property def inference(self): diff --git a/pinecone/data/features/inference/inference.py b/pinecone/data/features/inference/inference.py index 71ada564..9ab34e33 100644 --- a/pinecone/data/features/inference/inference.py +++ b/pinecone/data/features/inference/inference.py @@ -63,9 +63,7 @@ def __init__(self, config, openapi_config, **kwargs) -> None: api_version=API_VERSION, ) - self.load_plugins( - config=self.config, openapi_config=self.openapi_config, pool_threads=self.pool_threads - ) + super().__init__() # Initialize PluginAware def embed( self, diff --git a/pinecone/data/index.py b/pinecone/data/index.py index ebd5cecd..a228bfbe 100644 --- a/pinecone/data/index.py +++ b/pinecone/data/index.py @@ -55,7 +55,7 @@ def parse_query_response(response: QueryResponse): return response -class Index(IndexInterface, ImportFeatureMixin, PluginAware): +class Index(PluginAware, IndexInterface, ImportFeatureMixin): """ A client for interacting with a Pinecone index via REST API. For improved performance, use the Pinecone GRPC index client. @@ -101,10 +101,6 @@ def __init__( # Pass the same api_client to the ImportFeatureMixin super().__init__(api_client=self._api_client) - self.load_plugins( - config=self.config, openapi_config=self.openapi_config, pool_threads=self.pool_threads - ) - def _openapi_kwargs(self, kwargs: Dict[str, Any]) -> Dict[str, Any]: return filter_dict(kwargs, OPENAPI_ENDPOINT_PARAMS) diff --git a/pinecone/utils/plugin_aware.py b/pinecone/utils/plugin_aware.py index ce1e4b87..a99223e2 100644 --- a/pinecone/utils/plugin_aware.py +++ b/pinecone/utils/plugin_aware.py @@ -1,8 +1,8 @@ +from typing import Any from .setup_openapi_client import build_plugin_setup_client from pinecone.config import Config from pinecone.openapi_support.configuration import Configuration as OpenApiConfig - from pinecone_plugin_interface import load_and_install as install_plugins import logging @@ -11,17 +11,97 @@ class PluginAware: + """ + Base class for classes that support plugin loading. + + This class provides functionality to lazily load plugins when they are first accessed. + Subclasses must set the following attributes before calling super().__init__(): + - config: Config + - openapi_config: OpenApiConfig + - pool_threads: int + """ + + def __init__(self, *args: Any, **kwargs: Any) -> None: + """ + Initialize the PluginAware class. + + Args: + *args: Variable length argument list. + **kwargs: Arbitrary keyword arguments. + + Raises: + AttributeError: If required attributes are not set in the subclass. + """ + logger.debug("PluginAware __init__ called for %s", self.__class__.__name__) + + # Check for required attributes after super().__init__ has been called + missing_attrs = [] + if not hasattr(self, "config"): + missing_attrs.append("config") + if not hasattr(self, "openapi_config"): + missing_attrs.append("openapi_config") + if not hasattr(self, "pool_threads"): + missing_attrs.append("pool_threads") + + if missing_attrs: + raise AttributeError( + f"PluginAware class requires the following attributes: {', '.join(missing_attrs)}. " + f"These must be set in the {self.__class__.__name__} class's __init__ method " + f"before calling super().__init__()." + ) + + self._plugins_loaded = False + """ @private """ + + def __getattr__(self, name: str) -> Any: + """ + Called when an attribute is not found through the normal lookup process. + This allows for lazy loading of plugins when they are first accessed. + + Args: + name: The name of the attribute being accessed. + + Returns: + The requested attribute. + + Raises: + AttributeError: If the attribute cannot be found after loading plugins. + """ + if not self._plugins_loaded: + logger.debug("Loading plugins for %s", self.__class__.__name__) + self.load_plugins( + config=self.config, + openapi_config=self.openapi_config, + pool_threads=self.pool_threads, + ) + self._plugins_loaded = True + try: + return object.__getattribute__(self, name) + except AttributeError: + pass + + raise AttributeError(f"'{self.__class__.__name__}' object has no attribute '{name}'") + def load_plugins( self, config: Config, openapi_config: OpenApiConfig, pool_threads: int ) -> None: - """@private""" + """ + Load plugins for the parent class. + + Args: + config: The Pinecone configuration. + openapi_config: The OpenAPI configuration. + pool_threads: The number of threads in the pool. + """ try: - # I don't expect this to ever throw, but wrapping this in a - # try block just in case to make sure a bad plugin doesn't - # halt client initialization. + # Build the OpenAPI client for plugin setup openapi_client_builder = build_plugin_setup_client( config=config, openapi_config=openapi_config, pool_threads=pool_threads ) + # Install plugins install_plugins(self, openapi_client_builder) + logger.debug("Plugins loaded successfully for %s", self.__class__.__name__) + except ImportError as e: + logger.warning("Failed to import plugin module: %s", e) except Exception as e: - logger.error(f"Error loading plugins: {e}") + logger.error("Error loading plugins: %s", e, exc_info=True) From 7419a5824581bbe11ec21f6a4eb298ef21076d29 Mon Sep 17 00:00:00 2001 From: Jen Hamon Date: Fri, 11 Apr 2025 12:55:16 -0400 Subject: [PATCH 2/4] Fix unit test --- tests/unit/test_control.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/tests/unit/test_control.py b/tests/unit/test_control.py index c0b909dd..ad3b2872 100644 --- a/tests/unit/test_control.py +++ b/tests/unit/test_control.py @@ -77,9 +77,12 @@ def index_list_response(): class TestControl: - def test_plugins_are_installed(self): + def test_plugins_are_lazily_loaded(self): with patch.object(PluginAware, "load_plugins") as mock_install_plugins: - Pinecone(api_key="asdf") + pc = Pinecone(api_key="asdf") + mock_install_plugins.assert_not_called() + with pytest.raises(AttributeError): + pc.foo() # Accessing a non-existent attribute should raise an AttributeError after PluginAware installs any applicable plugins mock_install_plugins.assert_called_once() def test_default_host(self): From f5dc7280d92aa75063c65ff2ec641c8ca22098f9 Mon Sep 17 00:00:00 2001 From: Jen Hamon Date: Mon, 14 Apr 2025 11:13:13 -0400 Subject: [PATCH 3/4] Add unit tests for PluginAware --- pinecone/utils/plugin_aware.py | 35 ++++++++++++++++------- tests/unit/test_plugin_aware.py | 49 +++++++++++++++++++++++++++++++++ 2 files changed, 74 insertions(+), 10 deletions(-) create mode 100644 tests/unit/test_plugin_aware.py diff --git a/pinecone/utils/plugin_aware.py b/pinecone/utils/plugin_aware.py index a99223e2..8410397a 100644 --- a/pinecone/utils/plugin_aware.py +++ b/pinecone/utils/plugin_aware.py @@ -34,6 +34,9 @@ def __init__(self, *args: Any, **kwargs: Any) -> None: """ logger.debug("PluginAware __init__ called for %s", self.__class__.__name__) + self._plugins_loaded = False + """ @private """ + # Check for required attributes after super().__init__ has been called missing_attrs = [] if not hasattr(self, "config"): @@ -50,9 +53,6 @@ def __init__(self, *args: Any, **kwargs: Any) -> None: f"before calling super().__init__()." ) - self._plugins_loaded = False - """ @private """ - def __getattr__(self, name: str) -> Any: """ Called when an attribute is not found through the normal lookup process. @@ -67,17 +67,32 @@ def __getattr__(self, name: str) -> Any: Raises: AttributeError: If the attribute cannot be found after loading plugins. """ + # Check if this is one of the required attributes that should be set by subclasses + required_attrs = ["config", "openapi_config", "pool_threads"] + if name in required_attrs: + raise AttributeError( + f"'{self.__class__.__name__}' object has no attribute '{name}'. " + f"This attribute must be set in the subclass's __init__ method " + f"before calling super().__init__()." + ) + if not self._plugins_loaded: logger.debug("Loading plugins for %s", self.__class__.__name__) - self.load_plugins( - config=self.config, - openapi_config=self.openapi_config, - pool_threads=self.pool_threads, - ) - self._plugins_loaded = True + # Use object.__getattribute__ to avoid triggering __getattr__ again try: - return object.__getattribute__(self, name) + config = object.__getattribute__(self, "config") + openapi_config = object.__getattribute__(self, "openapi_config") + pool_threads = object.__getattribute__(self, "pool_threads") + self.load_plugins( + config=config, openapi_config=openapi_config, pool_threads=pool_threads + ) + self._plugins_loaded = True + try: + return object.__getattribute__(self, name) + except AttributeError: + pass except AttributeError: + # If we can't get the required attributes, we can't load plugins pass raise AttributeError(f"'{self.__class__.__name__}' object has no attribute '{name}'") diff --git a/tests/unit/test_plugin_aware.py b/tests/unit/test_plugin_aware.py new file mode 100644 index 00000000..7f4329d1 --- /dev/null +++ b/tests/unit/test_plugin_aware.py @@ -0,0 +1,49 @@ +import pytest +from pinecone.utils.plugin_aware import PluginAware +from pinecone.config import Config +from pinecone.openapi_support.configuration import Configuration as OpenApiConfig + + +class TestPluginAware: + def test_errors_when_required_attributes_are_missing(self): + class Foo(PluginAware): + def __init__(self): + # does not set config, openapi_config, or pool_threads + super().__init__() + + with pytest.raises(AttributeError) as e: + Foo() + + assert "config" in str(e.value) + assert "openapi_config" in str(e.value) + assert "pool_threads" in str(e.value) + + def test_correctly_raise_attribute_errors(self): + class Foo(PluginAware): + def __init__(self): + self.config = Config() + self.openapi_config = OpenApiConfig() + self.pool_threads = 1 + + super().__init__() + + foo = Foo() + + with pytest.raises(AttributeError) as e: + foo.bar() + + assert "bar" in str(e.value) + + def test_plugins_are_lazily_loaded(self): + class Pinecone(PluginAware): + def __init__(self): + self.config = Config() + self.openapi_config = OpenApiConfig() + self.pool_threads = 10 + + super().__init__() + + pc = Pinecone() + assert "assistant" not in dir(pc) + + assert pc.assistant is not None From 4d720f2ed695be2279230277556fb317986db105 Mon Sep 17 00:00:00 2001 From: Jen Hamon Date: Mon, 14 Apr 2025 11:32:28 -0400 Subject: [PATCH 4/4] Add assistant plugin to dev deps --- poetry.lock | 25 ++++++++++++++++++++----- pyproject.toml | 1 + 2 files changed, 21 insertions(+), 5 deletions(-) diff --git a/poetry.lock b/poetry.lock index 427dc1e2..fb037257 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1001,13 +1001,13 @@ files = [ [[package]] name = "packaging" -version = "23.2" +version = "24.2" description = "Core utilities for Python packages" optional = false -python-versions = ">=3.7" +python-versions = ">=3.8" files = [ - {file = "packaging-23.2-py3-none-any.whl", hash = "sha256:8c491190033a9af7e1d931d0b5dacc2ef47509b34dd0de67ed209b5203fc88c7"}, - {file = "packaging-23.2.tar.gz", hash = "sha256:048fb0e9405036518eaaf48a55953c750c11e1a1b68e0dd1a9d62ed0c092cfc5"}, + {file = "packaging-24.2-py3-none-any.whl", hash = "sha256:09abb1bccd265c01f4a3aa3f7a7db064b36514d2cba19a2f694fe6150451a759"}, + {file = "packaging-24.2.tar.gz", hash = "sha256:c228a6dc5e932d346bc5739379109d49e8853dd8223571c7c5b55260edc0b97f"}, ] [[package]] @@ -1130,6 +1130,21 @@ pygments = ">=2.12.0" [package.extras] dev = ["hypothesis", "mypy", "pdoc-pyo3-sample-library (==1.0.11)", "pygments (>=2.14.0)", "pytest", "pytest-cov", "pytest-timeout", "ruff", "tox", "types-pygments"] +[[package]] +name = "pinecone-plugin-assistant" +version = "1.6.0" +description = "Assistant plugin for Pinecone SDK" +optional = false +python-versions = "<4.0,>=3.9" +files = [ + {file = "pinecone_plugin_assistant-1.6.0-py3-none-any.whl", hash = "sha256:d742273d136fba66d020f1af01af2c6bfbc802f7ff9ddf46c590b7ea26932175"}, + {file = "pinecone_plugin_assistant-1.6.0.tar.gz", hash = "sha256:b7c531743f87269ba567dd6084b1464b62636a011564d414bc53147571b2f2c1"}, +] + +[package.dependencies] +packaging = ">=24.2,<25.0" +requests = ">=2.32.3,<3.0.0" + [[package]] name = "pinecone-plugin-interface" version = "0.0.7" @@ -1899,4 +1914,4 @@ grpc = ["googleapis-common-protos", "grpcio", "grpcio", "grpcio", "lz4", "protob [metadata] lock-version = "2.0" python-versions = "^3.9" -content-hash = "8a10046c5826a9773836e6b3ee50271bb0077d0faf32d709f1e65c4bb1fc53ea" +content-hash = "6e2107c224f622bcd0492b87d8a92f36318d9487af485e766b0e944e378e083a" diff --git a/pyproject.toml b/pyproject.toml index 0525d08d..ff491308 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -94,6 +94,7 @@ urllib3_mock = "0.3.3" responses = ">=0.8.1" ruff = "^0.9.3" beautifulsoup4 = "^4.13.3" +pinecone-plugin-assistant = "^1.6.0" [tool.poetry.extras]