find_keywords.py

# -*- coding: utf-8 -*-
"""find_keywords.ipynb

Automatically generated by Colab.

Original file is located at
    https://colab.research.google.com/drive/1iOmS_vGN_3AMFqEUEdvGpvDcSLH0Ofvd
"""

!pip install sentence_transformers

from sentence_transformers import SentenceTransformer

# 1. Load a pretrained Sentence Transformer model
model = SentenceTransformer("all-MiniLM-L6-v2")

# The sentences to encode
sentences = [
    "Yeşil Enerji Yatırım A.Ş.",
    "Dönem zararı olması",
    "Temettü verilmesi",
]

# 2. Calculate embeddings by calling model.encode()
embeddings = model.encode(sentences)
print(embeddings.shape)
# [3, 384]

# 3. Calculate the embedding similarities
similarities = model.similarity(embeddings, embeddings)
print(similarities)
# tensor([[1.0000, 0.6660, 0.1046],
#         [0.6660, 1.0000, 0.1411],
#         [0.1046, 0.1411, 1.0000]])

from sentence_transformers import SentenceTransformer

# 1. Load a pretrained Sentence Transformer model
model = SentenceTransformer("multi-qa-mpnet-base-dot-v1")

# The sentences to encode
sentences = [
    "Yeşil Gayrimenkul Yatırım Ortaklığı Anonim Şirketi (“Şirket”),İhlas Gayrimenkul Yatırım Ortaklığı A.Ş. adıyla 1997 yılında İstanbul’da kurulmuştur. 26 Aralık 2006 tarih ve 26 No’lu Yönetim Kurulu kararına göre Şirket’in “İhlas Gayrimenkul Yatırım Ortaklığı Anonim Şirketi” olan unvanının “Y ve Y Gayrimenkul Yatırım Ortaklığı Anonim Şirketi” olarak değiştirilmesine karar verilmiş ve bu karar 29 Ocak 2007 tarihinde İstanbul Ticaret Memurluğu’nda tescil ettirilmiştir. 17 Mart 2011 tarihinde yapılan Olağan Genel kurul toplantısında alınan karara göre Şirket’in “Y ve Y Gayrimenkul Yatırım Ortaklığı Anonim Şirketi” olan unvanının “Yeşil Gayrimenkul Yatırım Ortaklığı Anonim Şirketi” olarak değiştirilmesine karar verilmiş ve bu karar 30 Mart 2011 tarih ve 7783 sayılı Türkiye Ticaret Sicili Gazetesi’nde ilan edilmiştir.",
    "15.06.2023 tarihinde gerçekleştirilen 2022 yılı Olağan Genel Kurul Toplantısı’nda Yönetim Kurulu üyelerine TTK'nın“Şirketle işlem yapma, şirkete borçlanma yasağı” başlıklı 395. ve “Rekabet Yasağı” başlıklı 396. maddeleri uyarınca gerekli iznin verilmesi hususu görüşülmüştür. Yapılan oylama ile maddelerdeki hususlar hakkında yönetim kurulu üyelerine oy birliği ile izin verilmiştir.",
    "Dönem zararı olması",
    "Temettü verilmesi",
    "Sürdürülebilirlik",
    "İsim değişikliği"
]

# 2. Calculate embeddings by calling model.encode()
embeddings = model.encode(sentences)
print(embeddings.shape)
# [3, 384]

# 3. Calculate the embedding similarities
similarities = model.similarity(embeddings, embeddings)
print(similarities)

from sentence_transformers import SentenceTransformer

model = SentenceTransformer("multi-qa-mpnet-base-dot-v1")

query_embedding = model.encode("Dönem zararı, bir işletmenin belirli bir dönemde (genellikle bir yıl) gelirlerinin giderlerinden düşük olması sonucu ortaya çıkan finansal kaybı ifade eder. Yani, işletmenin o dönemde kâr edemeyip zarar ettiğini gösterir.")
passage_embeddings = model.encode([
    "Yeşil Gayrimenkul Yatırım Ortaklığı Anonim Şirketi (“Şirket”),İhlas Gayrimenkul Yatırım Ortaklığı A.Ş. adıyla 1997 yılında İstanbul’da kurulmuştur. 26 Aralık 2006 tarih ve 26 No’lu Yönetim Kurulu kararına göre Şirket’in “İhlas Gayrimenkul Yatırım Ortaklığı Anonim Şirketi” olan unvanının “Y ve Y Gayrimenkul Yatırım Ortaklığı Anonim Şirketi” olarak değiştirilmesine karar verilmiş ve bu karar 29 Ocak 2007 tarihinde İstanbul Ticaret Memurluğu’nda tescil ettirilmiştir. 17 Mart 2011 tarihinde yapılan Olağan Genel kurul toplantısında alınan karara göre Şirket’in “Y ve Y Gayrimenkul Yatırım Ortaklığı Anonim Şirketi” olan unvanının “Yeşil Gayrimenkul Yatırım Ortaklığı Anonim Şirketi” olarak değiştirilmesine karar verilmiş ve bu karar 30 Mart 2011 tarih ve 7783 sayılı Türkiye Ticaret Sicili Gazetesi’nde ilan edilmiştir.",
    "15.06.2023 tarihinde gerçekleştirilen 2022 yılı Olağan Genel Kurul Toplantısı’nda Yönetim Kurulu üyelerine TTK'nın“Şirketle işlem yapma, şirkete borçlanma yasağı” başlıklı 395. ve “Rekabet Yasağı” başlıklı 396. maddeleri uyarınca gerekli iznin verilmesi hususu görüşülmüştür. Yapılan oylama ile maddelerdeki hususlar hakkında yönetim kurulu üyelerine oy birliği ile izin verilmiştir.",
    "Yeşil GYO, gayrimenkul yatırım ortaklığı sektöründe faaliyet göstermektedir.  Türkiye’de Gayrimenkul Yatırım Ortaklıkları ilk olarak 1995 yılında kurulmuştur. 1997 yılında Sermaye Piyasası Kurulu düzenlemeleri gereğince halka açılıp BİST’e kote olmuşlardır. Ayrıca 2023 senesine bakıldığında son çeyrek zarar bulunmaktadır."
])

similarity = model.similarity(query_embedding, passage_embeddings)
print(similarity)
# => tensor([[25.4116, 26.8643, 24.4806]])







from langchain.document_loaders import PyPDFLoader
from sentence_transformers import SentenceTransformer
from langchain.text_splitter import RecursiveCharacterTextSplitter
import os

# Step 1: Load the PDF and extract text
def load_pdf(file_path):
    loader = PyPDFLoader(file_path)
    documents = loader.load()  # Load the text as documents
    return documents

# Step 2: Initialize the MiniLM embedding model
embedding_model = SentenceTransformer('sentence-transformers/all-MiniLM-L6-v2')

# Step 3: Create a function to chunk the document text semantically
def create_semantic_chunks(documents, chunk_size=1000, chunk_overlap=100):
    text_splitter = RecursiveCharacterTextSplitter(
        chunk_size=chunk_size,
        chunk_overlap=chunk_overlap,
        separators=["\n\n", ".", "?", "!", ";"]  # Customize the split points
    )
    # Split the text into chunks
    chunks = text_splitter.split_documents(documents)
    return chunks

# Step 4: Main function to process the PDF and generate semantic chunks
def process_pdf_for_semantic_chunks(pdf_path):
    # Load PDF
    documents = load_pdf(pdf_path)

    # Chunk the text
    chunks = create_semantic_chunks(documents)

    # Print out the first few chunks as a sample
    for i, chunk in enumerate(chunks[:5]):  # Limit to first 5 chunks for display
        print(f"Chunk {i+1}:\n{chunk.page_content}\n")

# Step 5: Provide the PDF path and process
pdf_path = "/mnt/data/YGYO 2023 YILLIK FAAL_YET RAPORU.pdf"
process_pdf_for_semantic_chunks(pdf_path)

! pip install langchain sentence_transformers langchain-community sentence_transformers pypdf

from langchain.document_loaders import PyPDFLoader
from sentence_transformers import SentenceTransformer
from langchain.text_splitter import RecursiveCharacterTextSplitter
import os

# Step 1: Load the PDF and extract text
def load_pdf(file_path):
    loader = PyPDFLoader(file_path)
    documents = loader.load()  # Load the text as documents
    return documents

# Step 2: Initialize the MiniLM embedding model
embedding_model = SentenceTransformer("multi-qa-mpnet-base-dot-v1")

# Step 3: Create a function to chunk the document text semantically
def create_semantic_chunks(documents, chunk_size=1000, chunk_overlap=100):
    text_splitter = RecursiveCharacterTextSplitter(
        chunk_size=chunk_size,
        chunk_overlap=chunk_overlap,
        separators=["\n\n", ".", "?", "!", ";"]  # Customize the split points
    )
    # Split the text into chunks
    chunks = text_splitter.split_documents(documents)
    return chunks

# Step 4: Main function to process the PDF and generate semantic chunks
def process_pdf_for_semantic_chunks(pdf_path):
    # Load PDF
    documents = load_pdf(pdf_path)

    # Chunk the text
    chunks = create_semantic_chunks(documents)

    # Print out the first few chunks as a sample
    for i, chunk in enumerate(chunks[:30]):  # Limit to first 5 chunks for display
        print(f"Chunk {i+1}:\n{chunk.page_content}\n")

# Step 5: Provide the PDF path and process
pdf_path = "/content/YGYO 2023 YILLIK FAAL_YET RAPORU.pdf"
process_pdf_for_semantic_chunks(pdf_path)

import numpy as np
from sklearn.metrics.pairwise import cosine_similarity
from sentence_transformers import SentenceTransformer
from langchain.document_loaders import PyPDFLoader
from langchain.text_splitter import RecursiveCharacterTextSplitter

# Step 1: Load PDF and extract text
def load_pdf(file_path):
    loader = PyPDFLoader(file_path)
    documents = loader.load()  # Load text as documents
    return documents

# Step 2: Initialize the MiniLM embedding model
embedding_model = SentenceTransformer("multi-qa-mpnet-base-dot-v1")

# Step 3: Function to get embeddings for text chunks
def get_embeddings(text_chunks):
    return embedding_model.encode(text_chunks)

# Step 4: Compute similarity between chunks based on embeddings
def compute_similarity(embeddings):
    return cosine_similarity(embeddings)

# Step 5: Create custom semantic chunks based on embedding similarity
def create_custom_semantic_chunks(documents, threshold=0.85):
    # Step 5a: Split the document into sentences/paragraphs
    text_splitter = RecursiveCharacterTextSplitter(chunk_size=500, chunk_overlap=0, separators=["\n\n", ".", "?", "!"])
    text_chunks = text_splitter.split_documents(documents)

    # Step 5b: Extract the actual text from chunks
    chunk_texts = [chunk.page_content for chunk in text_chunks]

    # Step 5c: Generate embeddings for each chunk
    embeddings = get_embeddings(chunk_texts)

    # Step 5d: Compute similarity between chunks
    similarity_matrix = compute_similarity(embeddings)

    # Step 5e: Group chunks based on similarity
    custom_chunks = []
    current_chunk = chunk_texts[0]

    for i in range(1, len(chunk_texts)):
        if similarity_matrix[i-1, i] >= threshold:
            # If the similarity is above the threshold, merge chunks
            current_chunk += " " + chunk_texts[i]
        else:
            # Otherwise, start a new chunk
            custom_chunks.append(current_chunk)
            current_chunk = chunk_texts[i]

    # Append the last chunk
    custom_chunks.append(current_chunk)

    return custom_chunks

# Step 6: Main function to process the PDF and generate semantic chunks
def process_pdf_for_semantic_chunks(pdf_path, threshold=0.85):
    # Load PDF document
    documents = load_pdf(pdf_path)

    # Create custom semantic chunks
    semantic_chunks = create_custom_semantic_chunks(documents, threshold)

    # Print first few chunks for demonstration
    for i, chunk in enumerate(semantic_chunks):
        print(f"Semantic Chunk {i+1}:\n{chunk}\n")

# Step 7: Provide the PDF path and process
pdf_path = "/content/YGYO 2023 YILLIK FAAL_YET RAPORU.pdf"  # Replace with your PDF file path
process_pdf_for_semantic_chunks(pdf_path)

import numpy as np
from sklearn.metrics.pairwise import cosine_similarity
from sentence_transformers import SentenceTransformer
from langchain.document_loaders import PyPDFLoader
from langchain.text_splitter import RecursiveCharacterTextSplitter

# Step 1: Load PDF and extract text
def load_pdf(file_path):
    loader = PyPDFLoader(file_path)
    documents = loader.load()  # Load text as documents
    return documents

# Step 2: Initialize the MiniLM embedding model
embedding_model = SentenceTransformer("multi-qa-mpnet-base-dot-v1")

# Step 3: Function to get embeddings for text chunks
def get_embeddings(text_chunks):
    return embedding_model.encode(text_chunks)

# Step 4: Function to compute similarity between embeddings
def compute_similarity(embedding_a, embedding_b):
    return cosine_similarity([embedding_a], [embedding_b])[0][0]

# Step 5: Create custom semantic chunks based on embedding similarity
def create_custom_semantic_chunks(documents):
    text_splitter = RecursiveCharacterTextSplitter(
        chunk_size=500,
        chunk_overlap=0,
        separators=["\n\n", ".", "?", "!"]
    )
    text_chunks = text_splitter.split_documents(documents)

    # Extract the actual text from chunks
    chunk_texts = [chunk.page_content for chunk in text_chunks]

    return chunk_texts

# Step 6: Calculate the most similar chunk to the keyword list
def find_most_similar_chunk(chunks, keyword_list):
    # Get embeddings for each chunk
    chunk_embeddings = get_embeddings(chunks)

    # Combine keywords into a single string and calculate its embedding
    keywords_combined = " ".join(keyword_list)
    keyword_embedding = get_embeddings([keywords_combined])[0]

    # Compute similarity between each chunk and the keyword embedding
    similarities = [compute_similarity(keyword_embedding, chunk_embedding) for chunk_embedding in chunk_embeddings]

    # Find the index of the most similar chunk
    most_similar_index = np.argmax(similarities)

    return most_similar_index, chunks[most_similar_index], similarities[most_similar_index]

# Step 7: Main function to process the PDF, find and print the most similar chunk
def process_pdf_and_find_similar_chunk(pdf_path, keyword_list):
    # Load PDF and chunk it
    documents = load_pdf(pdf_path)
    chunks = create_custom_semantic_chunks(documents)

    # Find the most similar chunk
    most_similar_index, most_similar_chunk, similarity_score = find_most_similar_chunk(chunks, keyword_list)

    # Print the most similar chunk and its similarity score
    print(f"Most similar chunk (index {most_similar_index}) with similarity score {similarity_score}:\n{most_similar_chunk}\n")

# Step 8: Provide the PDF path and keyword list, then process
pdf_path = "/content/YGYO 2023 YILLIK FAAL_YET RAPORU.pdf"  # Replace with your PDF file path
keyword_list = ["temettü", "kar", "zarar", "sermaye artırımı"]  # Define your keywords

process_pdf_and_find_similar_chunk(pdf_path, keyword_list)

# Step 6: Find the most similar chunk for each keyword
def find_most_similar_chunk_for_keywords(chunks, keyword_list):
    # Get embeddings for each chunk
    chunk_embeddings = get_embeddings(chunks)

    # Dictionary to store the most similar chunk for each keyword
    most_similar_for_keywords = {}

    # For each keyword, find the chunk with the highest similarity
    for keyword in keyword_list:
        keyword_embedding = get_embeddings([keyword])[0]

        # Compute similarity for each chunk with the current keyword
        similarities = [compute_similarity(keyword_embedding, chunk_embedding) for chunk_embedding in chunk_embeddings]

        # Find the index of the most similar chunk for this keyword
        most_similar_index = np.argmax(similarities)

        # Store the most similar chunk and similarity score for this keyword
        most_similar_for_keywords[keyword] = {
            "chunk": chunks[most_similar_index],
            "similarity_score": similarities[most_similar_index]
        }

    return most_similar_for_keywords



# Step 7: Main function to process the PDF, find and print the most similar chunk for each keyword
def process_pdf_and_find_similar_chunks_for_keywords(pdf_path, keyword_list):
    # Load PDF and chunk it
    documents = load_pdf(pdf_path)
    chunks = create_custom_semantic_chunks(documents)

    # Find the most similar chunk for each keyword
    most_similar_chunks = find_most_similar_chunk_for_keywords(chunks, keyword_list)

    # Print the most similar chunk and its similarity score for each keyword
    for keyword, result in most_similar_chunks.items():
        print(f"Keyword: {keyword}")
        print(f"Most similar chunk (with similarity score {result['similarity_score']}):\n{result['chunk']}\n")


# Step 8: Provide the PDF path and keyword list, then process
pdf_path = "/content/YGYO 2023 YILLIK FAAL_YET RAPORU.pdf"  # Replace with your PDF file path
keyword_list = ["temettü", "kar", "zarar", "sermaye artırımı"]  # Define your keywords

process_pdf_and_find_similar_chunks_for_keywords(pdf_path, keyword_list)

from sklearn.neighbors import KDTree
import numpy as np
from sentence_transformers import SentenceTransformer
from langchain.document_loaders import PyPDFLoader
from langchain.text_splitter import RecursiveCharacterTextSplitter

# Step 1: Load PDF and extract text
def load_pdf(file_path):
    loader = PyPDFLoader(file_path)
    documents = loader.load()  # Load text as documents
    return documents

# Step 2: Initialize the MiniLM embedding model
embedding_model = SentenceTransformer("multi-qa-mpnet-base-dot-v1")

# Step 3: Function to chunk text from the PDF
def create_semantic_chunks(documents, chunk_size=500):
    text_splitter = RecursiveCharacterTextSplitter(
        chunk_size=chunk_size,
        chunk_overlap=0,
        separators=["\n\n", ".", "?", "!"]  # Splitting by common sentence delimiters
    )
    text_chunks = text_splitter.split_documents(documents)

    # Extract the actual text from chunks
    chunk_texts = [chunk.page_content for chunk in text_chunks]

    return chunk_texts

# Step 4: Function to get embeddings for text chunks
def get_embeddings(text_chunks):
    return embedding_model.encode(text_chunks)

# Step 5: Generate KDTree from chunk embeddings
def build_kdtree(embeddings):
    kdtree = KDTree(embeddings, leaf_size=2, metric='euclidean')
    return kdtree

# Step 6: Find the most similar chunk using KDTree
def find_similar_chunk_kdtree(kdtree, query_embedding, chunk_embeddings, chunks, k=1):
    distances, indices = kdtree.query([query_embedding], k=k)  # Query the KDTree
    return [(chunks[index], distances[0][i]) for i, index in enumerate(indices[0])]

# Step 7: Process chunks, keywords, and find most similar chunks
def find_most_similar_chunk_kdtree(chunks, keyword_list):
    # Get embeddings for chunks
    chunk_embeddings = get_embeddings(chunks)

    # Build KDTree
    kdtree = build_kdtree(chunk_embeddings)

    results = {}

    # Find the most similar chunk for each keyword
    for keyword in keyword_list:
        keyword_embedding = embedding_model.encode([keyword])[0]

        # Query KDTree for the nearest chunk to the keyword
        similar_chunks = find_similar_chunk_kdtree(kdtree, keyword_embedding, chunk_embeddings, chunks, k=1)

        # Store result
        results[keyword] = similar_chunks[0]  # Get the closest chunk and its distance

    return results

# Step 8: Main function to process the PDF, chunk the text, and find similar chunks for each keyword
def process_pdf_and_find_similar_chunks_for_keywords(pdf_path, keyword_list):
    # Load PDF document
    documents = load_pdf(pdf_path)

    # Create semantic chunks from the document text
    chunks = create_semantic_chunks(documents)

    # Find the most similar chunk for each keyword
    most_similar_chunks = find_most_similar_chunk_kdtree(chunks, keyword_list)

    # Print the most similar chunk and its similarity score for each keyword
    for keyword, (chunk, distance) in most_similar_chunks.items():
        print(f"Keyword: {keyword}")
        print(f"Most similar chunk (with similarity score {distance}):\n{chunk}\n")

# Step 9: Provide the PDF path and keyword list, then process
pdf_path = "/content/YGYO 2023 YILLIK FAAL_YET RAPORU.pdf" # Replace with your PDF file path
keyword_list = ["temettü", "kar", "zarar", "sermaye artırımı"]  # Define your keywords

process_pdf_and_find_similar_chunks_for_keywords(pdf_path, keyword_list)

!pip install faiss-cpu

import faiss
import numpy as np
from sentence_transformers import SentenceTransformer
from langchain.document_loaders import PyPDFLoader
from langchain.text_splitter import RecursiveCharacterTextSplitter

# Step 1: Load PDF and extract text
def load_pdf(file_path):
    loader = PyPDFLoader(file_path)
    documents = loader.load()  # Load text as documents
    return documents

# Step 2: Initialize the MiniLM embedding model
embedding_model = SentenceTransformer("multi-qa-mpnet-base-dot-v1")

# Step 3: Function to chunk text from the PDF
def create_semantic_chunks(documents, chunk_size=500):
    text_splitter = RecursiveCharacterTextSplitter(
        chunk_size=chunk_size,
        chunk_overlap=0,
        separators=["\n\n", ".", "?", "!"]  # Splitting by common sentence delimiters
    )
    text_chunks = text_splitter.split_documents(documents)

    # Extract the actual text from chunks
    chunk_texts = [chunk.page_content for chunk in text_chunks]

    return chunk_texts

# Step 4: Function to get embeddings for text chunks
def get_embeddings(text_chunks):
    return embedding_model.encode(text_chunks)

# Step 5: Build FAISS Index from chunk embeddings
def build_faiss_index(embeddings):
    dimension = embeddings.shape[1]
    index = faiss.IndexFlatL2(dimension)  # Use L2 (Euclidean) distance for FAISS
    index.add(embeddings)  # Add embeddings to the FAISS index
    return index

# Step 6: Find the most similar chunk using FAISS
def find_similar_chunk_faiss(index, query_embedding, chunk_embeddings, chunks, k=1):
    query_embedding = np.expand_dims(query_embedding, axis=0)  # Reshape query for FAISS
    distances, indices = index.search(query_embedding, k)  # Perform FAISS search
    return [(chunks[index], distances[0][i]) for i, index in enumerate(indices[0])]

# Step 7: Process chunks, keywords, and find most similar chunks
def find_most_similar_chunk_faiss(chunks, keyword_list):
    # Get embeddings for chunks
    chunk_embeddings = np.array(get_embeddings(chunks))

    # Build FAISS index
    faiss_index = build_faiss_index(chunk_embeddings)

    results = {}

    # Find the most similar chunk for each keyword
    for keyword in keyword_list:
        keyword_embedding = embedding_model.encode([keyword])[0]

        # Query FAISS for the nearest chunk to the keyword
        similar_chunks = find_similar_chunk_faiss(faiss_index, keyword_embedding, chunk_embeddings, chunks, k=1)

        # Store result
        results[keyword] = similar_chunks[0]  # Get the closest chunk and its distance

    return results

# Step 8: Main function to process the PDF, chunk the text, and find similar chunks for each keyword
def process_pdf_and_find_similar_chunks_for_keywords(pdf_path, keyword_list):
    # Load PDF document
    documents = load_pdf(pdf_path)

    # Create semantic chunks from the document text
    chunks = create_semantic_chunks(documents)

    # Find the most similar chunk for each keyword
    most_similar_chunks = find_most_similar_chunk_faiss(chunks, keyword_list)

    # Print the most similar chunk and its similarity score for each keyword
    for keyword, (chunk, distance) in most_similar_chunks.items():
        print(f"Keyword: {keyword}")
        print(f"Most similar chunk (with similarity score {distance}):\n{chunk}\n")

# Step 9: Provide the PDF path and keyword list, then process
pdf_path = "/content/YGYO 2023 YILLIK FAAL_YET RAPORU.pdf"  # Replace with your PDF file path
keyword_list = ["temettü", "kar", "zarar", "sermaye artırımı"]  # Define your keywords

process_pdf_and_find_similar_chunks_for_keywords(pdf_path, keyword_list)