Fix/docx loader

app/loaders/__init__.py

@@ -1,6 +1,6 @@
 
 import os
-from app.loaders.loaders import (DataLoader, txt_loader, pdf_loader, docx_loader, xlsx_loader, DataLoaderCls, PDFLoader, DocxLoader, TxtLoader)
+from app.loaders.loaders import (DataLoader, txt_loader, pdf_loader, docx_loader, xlsx_loader, DataLoaderCls, PDFLoader, DocxLoader, TxtLoader, XlsxLoader)
 from langchain.schema.document import Document
 from typing import List
 from app.text_splitters import ITextSplitter
@@ -18,6 +18,8 @@
  'doc': DocxLoader,
  'docx': DocxLoader,
  'txt': TxtLoader,
+ 'xlsx': XlsxLoader,
+ 'xls': XlsxLoader,
 }
 
 

app/loaders/loaders.py

@@ -130,7 +130,7 @@ def pdf_loader(file: str) -> Callable:
 
 class DocxLoader(DocumentLoader):
  def __init__(self, file:str) -> None:
- self.loader = UnstructuredWordDocumentLoader(file)
+ self.loader = Docx2txtLoader(file)
 
  def load(self) -> List[Document]:
  return self.loader.load_and_split()
@@ -164,3 +164,33 @@ def xlsx_loader(file: str) -> Callable:
  """Loads .xlsx and .xls files"""
  loader = UnstructuredExcelLoader(file, mode="elements")
  return loader.load()
+
+from urllib.request import urlretrieve
+from urllib.parse import urlparse
+class XlsxLoader(DocumentLoader):
+ def __init__(self, file:str) -> None:
+ tmp_file, _ = self._get_temp_file(file)
+ self.loader = UnstructuredExcelLoader(tmp_file, mode="single")
+
+ def _get_temp_file(self, file_url: str):
+ result = urlparse(file_url)
+ filename = result.path.strip("/")
+ file_path, message = urlretrieve(file_url, f"/tmp/{filename}")
+ return file_path, message
+
+
+ def load(self) -> List[Document]:
+ return self.loader.load_and_split()
+
+ def load_and_split_text(self, text_splitter: ITextSplitter) -> List[Document]:
+ pages = self.load()
+ split_pages = []
+ for page in pages:
+ page_content = page.page_content.lower()
+ metadatas = page.metadata
+ metadatas.update({"full_page": page_content})
+
+ text_chunks = text_splitter.split_text(page_content)
+ for chunk in text_chunks:
+ split_pages.append(Document(page_content=chunk, metadata=metadatas))
+ return split_pages