replace the column name to get the correct column to cosine distance,…

… the correct column is TEXT_CLEAN.
rafatieppo · Nov 22, 2024 · 234335c · 234335c
1 parent f6f6452
commit 234335c
Showing 1 changed file with 1 addition and 1 deletion.
diff --git a/resources/tidydata_uniq_titles.py b/resources/tidydata_uniq_titles.py
@@ -55,7 +55,7 @@ def drop_similar_rows(df, column, threshold):
 
     # Convert the text data to TF-IDF features
     vectorizer = TfidfVectorizer()
-    tfidf_matrix = vectorizer.fit_transform(dfdata[column])
+    tfidf_matrix = vectorizer.fit_transform(dfdata['TEXT_CLEAN'])
     # Compute cosine similarity matrix
     cosine_sim = cosine_similarity(tfidf_matrix)
     # Track rows to drop