using set

Author: Dishant1804

backend/apps/ai/common/base/chunk_command.py

@@ -58,7 +58,7 @@ def process_chunks_batch(self, entities: list[Model]) -> int:
                     continue
 
                 chunk_texts = Chunk.split_text(full_content)
-                unique_chunk_texts = list(dict.fromkeys(chunk_texts))
+                unique_chunk_texts = list(set(chunk_texts))
 
                 if not unique_chunk_texts:
                     self.stdout.write(f"No chunks created for {self.entity_name} {entity_key}")

backend/tests/apps/ai/common/base/chunk_command_test.py

@@ -221,12 +221,11 @@ def test_process_chunks_batch_success(
             result = command.process_chunks_batch([mock_entity])
 
             assert result == 1
-            mock_create_chunks.assert_called_once_with(
-                chunk_texts=["chunk1", "chunk2", "chunk3"],
-                context=mock_context,
-                openai_client=command.openai_client,
-                save=False,
-            )
+            _, kwargs = mock_create_chunks.call_args
+            assert set(kwargs["chunk_texts"]) == {"chunk1", "chunk2", "chunk3"}
+            assert kwargs["context"] == mock_context
+            assert kwargs["openai_client"] == command.openai_client
+            assert kwargs["save"] is False
             mock_bulk_save.assert_called_once_with(mock_chunks)
             mock_write.assert_has_calls(
                 [
@@ -461,12 +460,11 @@ def test_process_chunks_batch_with_duplicates(
 
             assert result == 1
             mock_split_text.assert_called_once()
-            mock_create_chunks.assert_called_once_with(
-                chunk_texts=["chunk1", "chunk2", "chunk3"],
-                context=mock_context,
-                openai_client=command.openai_client,
-                save=False,
-            )
+            _, kwargs = mock_create_chunks.call_args
+            assert set(kwargs["chunk_texts"]) == {"chunk1", "chunk2", "chunk3"}
+            assert kwargs["context"] == mock_context
+            assert kwargs["openai_client"] == command.openai_client
+            assert kwargs["save"] is False
             mock_bulk_save.assert_called_once_with(mock_chunks)
 
     def test_process_chunks_batch_whitespace_only_content(