diff --git a/.github/workflows/common.yml b/.github/workflows/common.yml
index 34453ad0daa..438371f55b9 100644
--- a/.github/workflows/common.yml
+++ b/.github/workflows/common.yml
@@ -102,6 +102,10 @@ jobs:
           export E2B_API_KEY=${{ secrets.E2B_API_KEY_STAGING }}
 
           tox -e test -- -m 'not e2e and not swe'
+      - name: Plugin tests
+        if: matrix.python-version == '3.10'
+        run: |
+          tox -e plugins
       - if: matrix.python-version == '3.10'
         name: Upload test results to Codecov
         uses: codecov/test-results-action@v1
diff --git a/python/composio/tools/local/codeanalysis/embedder.py b/python/composio/tools/local/codeanalysis/embedder.py
index 448967b109d..ce61ca273ba 100644
--- a/python/composio/tools/local/codeanalysis/embedder.py
+++ b/python/composio/tools/local/codeanalysis/embedder.py
@@ -2,7 +2,6 @@
 from typing import Any, Dict, List
 
 from deeplake.core.vectorstore.deeplake_vectorstore import DeepLakeVectorStore
-from sentence_transformers import SentenceTransformer
 
 from composio.tools.local.codeanalysis.constants import (
     CODE_MAP_CACHE,
@@ -49,6 +48,9 @@ def get_vector_store(repo_name: str, overwrite: bool = True) -> DeepLakeVectorSt
 
 class Embedding:
     def __init__(self):
+        # pylint: disable=import-outside-toplevel
+        from sentence_transformers import SentenceTransformer
+
         self.model = SentenceTransformer(EMBEDDER)
 
     def compute(self, texts: List[str]) -> List[List[float]]:
diff --git a/python/tests/test_tools/test_plugins.py b/python/tests/test_tools/test_plugins.py
new file mode 100644
index 00000000000..52f9265fb9c
--- /dev/null
+++ b/python/tests/test_tools/test_plugins.py
@@ -0,0 +1,58 @@
+import composio_openai
+
+
+def test_openai_toolset():
+    toolset = composio_openai.ComposioToolSet()
+
+    @composio_openai.action(toolname="my_tool")
+    def create_draft(
+        message_body: str,
+        thread_id: str | None = None,
+    ) -> dict:
+        """
+        Create a draft reply to a specific Gmail thread
+
+        :param thread_id: The ID of the thread to which the reply is to be drafted
+        :param message_body: The content of the draft reply
+        :return draft: The created draft details
+        """
+        _ = message_body, thread_id
+        return {}
+
+    tools = toolset.get_tools(actions=[create_draft])
+    assert tools == [
+        {
+            "function": {
+                "description": "Create a draft reply to a specific Gmail thread",
+                "name": "MYTOOL_CREATE_DRAFT",
+                "parameters": {
+                    "properties": {
+                        "message_body": {
+                            "description": "The content of the draft reply. Please provide a "
+                            "value of type string. This parameter is required.",
+                            "title": "Message Body",
+                            "type": "string",
+                        },
+                        "thread_id": {
+                            "description": "The ID of the thread to which the reply is to be "
+                            "drafted. Please provide a value of type string.",
+                            "title": "Thread Id",
+                            # TODO: this seems wrong, why both type and anyof
+                            "type": "string",
+                            "default": None,
+                            "anyOf": [
+                                {"type": "string"},
+                                {"type": "null"},
+                            ],
+                        },
+                    },
+                    "required": [
+                        "message_body",
+                    ],
+                    "title": "CreateDraftRequest",
+                    "type": "object",
+                },
+            },
+            "type": "function",
+        },
+    ]
diff --git a/python/tox.ini b/python/tox.ini
index 3edce8485e2..68ea9aa009d 100644
--- a/python/tox.ini
+++ b/python/tox.ini
@@ -121,17 +121,29 @@ commands =
 
     ; TODO: Extract plugin tests separately
     ; Installing separately because of the dependency conflicts
-    uv pip install plugins/langchain --no-deps
+    uv pip install plugins/langchain
 
-    pytest -vvv -rfE --doctest-modules composio/ tests/ swe/tests --junitxml=junit.xml --cov=composio --cov=examples --cov=swe --cov-report=html --cov-report=xml --cov-report=term --cov-report=term-missing --cov-config=.coveragerc {posargs}
+    pytest -vvv -rfE --doctest-modules composio/ tests/ swe/tests --ignore tests/test_tools/test_plugins.py --junitxml=junit.xml --cov=composio --cov=examples --cov=swe --cov-report=html --cov-report=xml --cov-report=term --cov-report=term-missing --cov-config=.coveragerc {posargs}
 
-    ; uv pip install plugins/autogen
-    ; uv pip install plugins/claude
-    ; uv pip install plugins/crew_ai
-    ; uv pip install plugins/griptape
-    ; uv pip install plugins/julep
-    ; uv pip install plugins/lyzr
-    ; uv pip install plugins/openai
+
+[testenv:plugins]
+setenv =
+    CI={env:CI}
+    COMPOSIO_API_KEY={env:COMPOSIO_API_KEY}
+    COMPOSIO_BASE_URL={env:COMPOSIO_BASE_URL}
+deps = 
+    .
+    plugins/autogen
+    plugins/claude
+    plugins/crew_ai
+    plugins/griptape
+    plugins/julep
+    plugins/langchain
+    plugins/langgraph
+    plugins/llamaindex
+    plugins/openai
+commands =
+    pytest -vv tests/test_tools/test_plugins.py {posargs}
 
 ; Linter config