llamindex_store_get_data_from_chromadb.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# llama index to store data to Chroma vector Database local\n",
    "from llama_index.embeddings.huggingface import HuggingFaceEmbedding\n",
    "from llama_index.core import VectorStoreIndex, SimpleDirectoryReader\n",
    "from llama_index.core import Settings\n",
    "from llama_index.vector_stores.chroma import ChromaVectorStore\n",
    "from llama_index.core import StorageContext\n",
    "from llama_index.embeddings.huggingface import HuggingFaceEmbedding\n",
    "from IPython.display import Markdown, display\n",
    "import chromadb\n",
    "from llama_index.core.node_parser import (\n",
    "    SentenceSplitter\n",
    ")\n",
    "\n",
    "# define embedding function\n",
    "embed_model = HuggingFaceEmbedding(model_name=\"BAAI/bge-base-en-v1.5\")\n",
    "\n",
    "Settings.embed_model = embed_model\n",
    "Settings.chunk_size = 512\n",
    "Settings.chunk_overlap = 64\n",
    "documents = SimpleDirectoryReader(\"./data\").load_data()\n",
    "\n",
    "splitter = SentenceSplitter()\n",
    "\n",
    "#nodes = splitter.get_nodes_from_documents(documents)\n",
    "#splitter = SemanticSplitterNodeParser(buffer_size=1, breakpoint_percentile_threshold=95, embed_model=embed_model)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Index data and store to Chroma DB"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<llama_index.core.indices.vector_store.base.VectorStoreIndex at 0x17693298880>"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "db = chromadb.PersistentClient(path=\"./chroma_db\")\n",
    "chroma_collection = db.get_or_create_collection(\"DB_collection\")\n",
    "\n",
    "# set up ChromaVectorStore and load in data\n",
    "vector_store = ChromaVectorStore(chroma_collection=chroma_collection)\n",
    "storage_context = StorageContext.from_defaults(vector_store=vector_store)\n",
    "\n",
    "\n",
    "#index = VectorStoreIndex.from_documents(\n",
    "#    documents, storage_context=storage_context, embed_model=embed_model\n",
    "#)\n",
    "VectorStoreIndex.from_documents(documents, storage_context=storage_context, embed_model=embed_model, transformations=[splitter])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "llamindex to read chroma db data from chroma db"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<llama_index.core.indices.vector_store.retrievers.retriever.VectorIndexRetriever at 0x1769f495930>"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# llamindex to read chroma db data from disk\n",
    "db2 = chromadb.PersistentClient(path=\"./chroma_db\")\n",
    "\n",
    "chroma_collection = db2.get_or_create_collection(\"DB_collection\")\n",
    "\n",
    "vector_store= ChromaVectorStore(chroma_collection=chroma_collection)\n",
    "\n",
    "vector_store_llamindex = VectorStoreIndex.from_vector_store(\n",
    "    vector_store,embed_model=embed_model\n",
    ")\n",
    "vector_store_llamindex.as_retriever()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Response(response=\"Aparna is Naresh Dulam's spouse.\", source_nodes=[NodeWithScore(node=TextNode(id_='9eed7949-0f15-4937-8dcd-46f1aae083dc', embedding=None, metadata={'page_label': '1', 'file_name': 'Naresh_Dulam.pdf', 'file_path': 'd:\\\\Data\\\\Cloud & AI\\\\CODE-PROJECTS\\\\llamaindex-llama2\\\\data\\\\Naresh_Dulam.pdf', 'file_type': 'application/pdf', 'file_size': 79570, 'creation_date': '2024-04-03', 'last_modified_date': '2024-04-06'}, excluded_embed_metadata_keys=['file_name', 'file_type', 'file_size', 'creation_date', 'last_modified_date', 'last_accessed_date'], excluded_llm_metadata_keys=['file_name', 'file_type', 'file_size', 'creation_date', 'last_modified_date', 'last_accessed_date'], relationships={<NodeRelationship.SOURCE: '1'>: RelatedNodeInfo(node_id='ce174276-a87c-45f1-99da-0325629b1b67', node_type=<ObjectType.DOCUMENT: '4'>, metadata={'page_label': '1', 'file_name': 'Naresh_Dulam.pdf', 'file_path': 'd:\\\\Data\\\\Cloud & AI\\\\CODE-PROJECTS\\\\llamaindex-llama2\\\\data\\\\Naresh_Dulam.pdf', 'file_type': 'application/pdf', 'file_size': 79570, 'creation_date': '2024-04-03', 'last_modified_date': '2024-04-06'}, hash='2acfa7f4e66b2d4a5990fddbe022b99f52ede3a7bc7294c5891c4477e26f9efa'), <NodeRelationship.PREVIOUS: '2'>: RelatedNodeInfo(node_id='3d079b64-bc77-4d84-b326-3ee958e05169', node_type=<ObjectType.TEXT: '1'>, metadata={'page_label': '70', 'file_name': 'IPL.pdf', 'file_path': 'd:\\\\Data\\\\Cloud & AI\\\\CODE-PROJECTS\\\\llamaindex-llama2\\\\data\\\\IPL.pdf', 'file_type': 'application/pdf', 'file_size': 1196399, 'creation_date': '2024-04-03', 'last_modified_date': '2024-04-02'}, hash='3fd29f3815bd385b449ad66521865123e406b4a12115b79ad52f4d394fa58bd8'), <NodeRelationship.NEXT: '3'>: RelatedNodeInfo(node_id='1956fd6b-32d9-4a25-87d3-4a2e3a501f22', node_type=<ObjectType.TEXT: '1'>, metadata={}, hash='af1517dacef2ec2e7ff29b3bff4194260fbf212c6e454d7f18ee992439ee22d1')}, text='Naresh Dulam – born and brought up in village Burugupally, District Karimnagar, Telangana state.  \\nHe completed his schooling in his village private school in Telugu medium. He had scored 534 marks  \\nin his 10th class and joined for Master Jr. College for intermediate and completed it in 2006 with  \\nmarks of 955 and EMCET rank of 17k. He then joined for bachelor’s degree in Jyothismathi engineering \\ncollege, Karimnagar, passed with  \\n69%.  \\nAfter the engineering, he joined Infor global  private  ltd as R&D engineer in Supply Chain Execution  \\nproduct development, worked as part COE team to product enhancements and maintenance.   \\nLater he joined in Oracle India private  ltd as Application Developer in developing product of Fusion GRC  \\nfor 16 months before he moved to USA.   \\nIn USA worked as big data analytics at various roles in T -Mobile, ATT, Bank of America and JP  \\nMorgan chase.   \\nHe married Aparna Balagoni and they have two children of Meenakshi Dulam, 5yr old and  \\nKesarinandan Dulam 2 yrs. old.   \\nNaresh is currently living in McKinney Texas and working at JP Morgan chase.  \\nAs the VP of Software Engineering at JPMorgan Chase, I oversee the design and implementation of  \\nenterprise -level Data Lakes/Mesh across various cloud platforms and technologies. With more than  \\n10 years of experience in the software industry, I have a strong background in data engineering and  \\narchitecture, software infrastructure and product management, and enterprise software  \\ndevelopment.  \\n  \\nI am skilled in AWS and Azure services, Big Data technologies, and programming languages, as well  \\nas AI/ML frameworks and tools. I work with cross -functional teams to optimize data architecture and  \\nstorage solutions, resulting in cost -effective and scalable data ecosystems. I am also passionate  \\nabout mentoring and sharing my knowledge and expertise with others, as well as learning new  \\ntechnologies and trends in the data and cloud space. My goal is to drive data -driven excellence and  \\ninnovation in every project I undertake.  \\n  \\nHe loves teaching and he runs a YouTube channel on latest Cloud Analytics, Vector databases and', start_char_idx=0, end_char_idx=2150, text_template='{metadata_str}\\n\\n{content}', metadata_template='{key}: {value}', metadata_seperator='\\n'), score=0.3011274456892524), NodeWithScore(node=TextNode(id_='8d90c342-e77c-4fff-a3cd-b7b0075e197f', embedding=None, metadata={'page_label': '1', 'file_name': 'Naresh_Dulam.pdf', 'file_path': 'd:\\\\Data\\\\Cloud & AI\\\\CODE-PROJECTS\\\\llamaindex-llama2\\\\data\\\\Naresh_Dulam.pdf', 'file_type': 'application/pdf', 'file_size': 62412, 'creation_date': '2024-04-03', 'last_modified_date': '2024-04-03'}, excluded_embed_metadata_keys=['file_name', 'file_type', 'file_size', 'creation_date', 'last_modified_date', 'last_accessed_date'], excluded_llm_metadata_keys=['file_name', 'file_type', 'file_size', 'creation_date', 'last_modified_date', 'last_accessed_date'], relationships={<NodeRelationship.SOURCE: '1'>: RelatedNodeInfo(node_id='72d011ba-3789-4f68-82d1-3682004b673c', node_type=<ObjectType.DOCUMENT: '4'>, metadata={'page_label': '1', 'file_name': 'Naresh_Dulam.pdf', 'file_path': 'd:\\\\Data\\\\Cloud & AI\\\\CODE-PROJECTS\\\\llamaindex-llama2\\\\data\\\\Naresh_Dulam.pdf', 'file_type': 'application/pdf', 'file_size': 62412, 'creation_date': '2024-04-03', 'last_modified_date': '2024-04-03'}, hash='1bdde0b9fc6e37710f7558c3f005f45a56b8922385e79b9397cbf09ac7e9b13f'), <NodeRelationship.PREVIOUS: '2'>: RelatedNodeInfo(node_id='08361d69-b10a-4898-b690-bd18547503bd', node_type=<ObjectType.TEXT: '1'>, metadata={'page_label': '70', 'file_name': 'IPL.pdf', 'file_path': 'd:\\\\Data\\\\Cloud & AI\\\\CODE-PROJECTS\\\\llamaindex-llama2\\\\data\\\\IPL.pdf', 'file_type': 'application/pdf', 'file_size': 1196399, 'creation_date': '2024-04-03', 'last_modified_date': '2024-04-02'}, hash='3fd29f3815bd385b449ad66521865123e406b4a12115b79ad52f4d394fa58bd8'), <NodeRelationship.NEXT: '3'>: RelatedNodeInfo(node_id='582a1810-61f6-423a-817a-7b845cf32e9c', node_type=<ObjectType.TEXT: '1'>, metadata={}, hash='3a2f2dae9fedea2221453ed26d902b013750c0c91e8185071ab3c51f0c0acce8')}, text='Naresh Dulam  – born and brought up in village Burugu pally , Dist rict Karimnagar, Tela ngana state.  \\nHe completed hi s school ing in his v illage pr ivate school in Telugu  medium. He had scored 534 marks \\nin his 1 0th class and joined for Master Jr. College for intermediate  and completed it in 20 06 w ith \\nmarks  of 955 and E MCET ran k of 17k.  \\nHe the n joined for bachelor’s  degree in Jyothismathi engineering college , Karim nagar, passed  with \\n69%.  \\nAfter the  engineering, he joined Infor global pvt  ltd as R&D engineer in Supply Chain Exec ution \\nproduc t development , worked as part COE team to product enhanc ement s and maintenance.  \\nLater he joined in Oracle India  pvt ltd as Application Developer in developing product of Fusion GRC \\nfor 16 m onths before he moved to USA.  \\nIn USA worked as big data analy tics at vari ous roles in T -Mobile, ATT, Bank of Americ a and J P \\nMorgan chase.  \\nHe married Aparna Balagoni  and they have tw o children of Meenaks hi Du lam, 5yr old and \\nKesarina ndan Dulam 2 yrs. old.  \\nNaresh is current ly living in McKinney  Texas  and work ing at JP  Morgan chase.  \\nAs the VP of Software Engineering at JPMorgan Chase, I oversee the design and implementation of \\nenterprise -level Data Lakes/Mesh across various cloud platforms and technologies. With more than \\n10 years of experience in the software industry, I have a stron g background in data engineering and \\narchitecture, software infrastructure and product management, and enterprise software \\ndevelopment . \\n \\nI am skilled in AWS and Azure services, Big Data technologies, and programming languages, as well \\nas AI/ML frameworks and tools. I work with cross -functional teams to optimize data architecture and \\nstorage solutions, resulting in cost -effective and scalabl e data ecosystems. I am also passionate \\nabout mentoring and sharing my knowledge and expertise with others, as well as learning new \\ntechnologies and trends in the data and cloud space. My goal is to drive data -driven excellence and \\ninnovation in every proj ect I undertake . \\n \\nHe loves teaching and he runs a YouTube  channel on latest Cloud Anal ytics, Vector databases and \\nGenerative AI applications.  \\nHe had certifications of  2 AWS, 3 K8S, 1 Snowflake , 1 Azure.  \\nHis notable pr ojects are JADE Framework,  Spark ETL,  Data Lake House, Cloud Native Data Analytics \\nPipeline , Metadata driven Kafka  inges tion, Authored attributes.  \\nHe loves  doing different things depends on situations like watching, reading, sleeping etc.', start_char_idx=0, end_char_idx=2527, text_template='{metadata_str}\\n\\n{content}', metadata_template='{key}: {value}', metadata_seperator='\\n'), score=0.2969991228701457)], metadata={'9eed7949-0f15-4937-8dcd-46f1aae083dc': {'page_label': '1', 'file_name': 'Naresh_Dulam.pdf', 'file_path': 'd:\\\\Data\\\\Cloud & AI\\\\CODE-PROJECTS\\\\llamaindex-llama2\\\\data\\\\Naresh_Dulam.pdf', 'file_type': 'application/pdf', 'file_size': 79570, 'creation_date': '2024-04-03', 'last_modified_date': '2024-04-06'}, '8d90c342-e77c-4fff-a3cd-b7b0075e197f': {'page_label': '1', 'file_name': 'Naresh_Dulam.pdf', 'file_path': 'd:\\\\Data\\\\Cloud & AI\\\\CODE-PROJECTS\\\\llamaindex-llama2\\\\data\\\\Naresh_Dulam.pdf', 'file_type': 'application/pdf', 'file_size': 62412, 'creation_date': '2024-04-03', 'last_modified_date': '2024-04-03'}})"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "vector_store_llamindex.as_query_engine().query(\"Who is Naresh Dulam?\")"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": ".venv",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.11"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}