community: add MarkItDown document loader

libs/community/extended_testing_deps.txt

@@ -44,6 +44,7 @@ langchain_openai>=0.2.1
 litellm>=1.30,<=1.39.5
 lxml>=4.9.3,<6.0
 markdownify>=0.11.6,<0.12
+markitdown>=0.0.1a3
 motor>=3.3.1,<4
 msal>=1.25.0,<2
 mwparserfromhell>=0.6.4,<0.7

libs/community/langchain_community/document_loaders/__init__.py

@@ -278,6 +278,9 @@
     from langchain_community.document_loaders.markdown import (
         UnstructuredMarkdownLoader,
     )
+    from langchain_community.document_loaders.markitdown import (
+        MarkItDownLoader,
+    )
     from langchain_community.document_loaders.mastodon import (
         MastodonTootsLoader,
     )
@@ -628,6 +631,7 @@
     "LLMSherpaFileLoader": "langchain_community.document_loaders.llmsherpa",
     "MHTMLLoader": "langchain_community.document_loaders.mhtml",
     "MWDumpLoader": "langchain_community.document_loaders.mediawikidump",
+    "MarkItDownLoader": "langchain_community.document_loaders.markitdown",
     "MastodonTootsLoader": "langchain_community.document_loaders.mastodon",
     "MathpixPDFLoader": "langchain_community.document_loaders.pdf",
     "MaxComputeLoader": "langchain_community.document_loaders.max_compute",
@@ -833,6 +837,7 @@ def __getattr__(name: str) -> Any:
     "LakeFSLoader",
     "LarkSuiteDocLoader",
     "LLMSherpaFileLoader",
+    "MarkItDownLoader",
     "MastodonTootsLoader",
     "MHTMLLoader",
     "MWDumpLoader",

libs/community/langchain_community/document_loaders/markitdown.py

@@ -0,0 +1,30 @@
+"""Loader that uses MarkItDown to load files."""
+
+from collections.abc import Iterator
+from typing import Any, Dict, Union
+
+import requests
+from langchain_core.documents import Document
+from markitdown import MarkItDown
+
+from langchain_community.document_loaders.base import BaseLoader
+
+
+class MarkItDownLoader(BaseLoader):
+    """Loader using MarkItDown to load files."""
+
+    def __init__(
+        self, source: Union[str, requests.Response], **markitdown_kwargs: Dict[str, Any]
+    ):
+        self.client = MarkItDown(**markitdown_kwargs)
+        self.source = source
+        super().__init__()
+
+    def lazy_load(self) -> Iterator[Document]:
+        """A lazy loader loading files as Markdown documents."""
+        result = self.client.convert(self.source)
+        page_content = result.__dict__.pop("text_content")
+        metadata = result.__dict__
+        if "source" not in metadata:
+            metadata["source"] = self.source
+        yield Document(page_content=page_content, metadata=metadata)

libs/community/tests/integration_tests/document_loaders/test_markitdown_loader.py

@@ -0,0 +1,17 @@
+from pathlib import Path
+
+import pytest
+
+from langchain_community.document_loaders import MarkItDownLoader
+
+
+@pytest.mark.requires("markitdown")
+def test_markitdown_loader() -> None:
+    """Test MarkItDown loader."""
+    file_path = Path(__file__).parent.parent / "examples/example.html"
+    loader = MarkItDownLoader(str(file_path))
+    docs = loader.load()
+
+    assert len(docs) == 1
+
+    assert docs[0].page_content

libs/community/tests/unit_tests/document_loaders/test_markitdown_loader.py

@@ -0,0 +1,17 @@
+from pathlib import Path
+
+import pytest
+
+from langchain_community.document_loaders import MarkItDownLoader
+
+
+@pytest.mark.requires("markitdown")
+def test_markitdown_loader() -> None:
+    """Test MarkItDown loader."""
+    file_path = Path(__file__).parent.parent / "examples/example.html"
+    loader = MarkItDownLoader(str(file_path))
+    docs = loader.load()
+
+    assert len(docs) == 1
+
+    assert docs[0].page_content

libs/langchain/langchain/document_loaders/__init__.py

@@ -97,6 +97,7 @@
         JSONLoader,
         LakeFSLoader,
         LarkSuiteDocLoader,
+        MarkItDownLoader,
         MastodonTootsLoader,
         MathpixPDFLoader,
         MaxComputeLoader,
@@ -281,6 +282,7 @@
     "LakeFSLoader": "langchain_community.document_loaders",
     "MHTMLLoader": "langchain_community.document_loaders",
     "MWDumpLoader": "langchain_community.document_loaders",
+    "MarkItDownLoader": "langchain_community.document_loaders",
     "MastodonTootsLoader": "langchain_community.document_loaders",
     "MathpixPDFLoader": "langchain_community.document_loaders",
     "MaxComputeLoader": "langchain_community.document_loaders",

libs/langchain/tests/unit_tests/document_loaders/test_imports.py

@@ -83,6 +83,7 @@
     "LakeFSLoader",
     "MHTMLLoader",
     "MWDumpLoader",
+    "MarkItDownLoader",
     "MastodonTootsLoader",
     "MathpixPDFLoader",
     "MaxComputeLoader",