diff --git a/src/khoj/processor/conversation/offline/chat_model.py b/src/khoj/processor/conversation/offline/chat_model.py index 8361650cf..c366b1fdc 100644 --- a/src/khoj/processor/conversation/offline/chat_model.py +++ b/src/khoj/processor/conversation/offline/chat_model.py @@ -171,9 +171,10 @@ def converse_offline( ) # Setup Prompt with Primer or Conversation History + current_date = datetime.now().strftime("%Y-%m-%d") messages = generate_chatml_messages_with_context( conversation_primer, - prompts.system_prompt_message_gpt4all, + prompts.system_prompt_message_gpt4all.format(current_date=current_date), conversation_log, model_name=model, max_prompt_size=max_prompt_size, @@ -198,7 +199,7 @@ def llm_thread(g, messages: List[ChatMessage], model: Any): for message in conversation_history ] - stop_words = [""] + stop_words = ["", "INST]", "Notes:"] chat_history = "".join(formatted_messages) templated_system_message = prompts.system_prompt_gpt4all.format(message=system_message.content) templated_user_message = prompts.user_message_gpt4all.format(message=user_message.content) diff --git a/src/khoj/processor/conversation/prompts.py b/src/khoj/processor/conversation/prompts.py index 67d8725f6..9c0093268 100644 --- a/src/khoj/processor/conversation/prompts.py +++ b/src/khoj/processor/conversation/prompts.py @@ -5,7 +5,7 @@ personality = PromptTemplate.from_template( """ You are Khoj, a smart, inquisitive and helpful personal assistant. -Use your general knowledge and the past conversation with the user as context to inform your responses. +Use your general knowledge and past conversation with the user as context to inform your responses. You were created by Khoj Inc. with the following capabilities: - You *CAN REMEMBER ALL NOTES and PERSONAL INFORMATION FOREVER* that the user ever shares with you. @@ -48,9 +48,17 @@ ## Conversation Prompts for GPT4All Models ## -- -system_prompt_message_gpt4all = f"""You are Khoj, a smart, inquisitive and helpful personal assistant. -Using your general knowledge and our past conversations as context, answer the following question. -If you do not know the answer, say 'I don't know.'""" +system_prompt_message_gpt4all = PromptTemplate.from_template( + """ +You are Khoj, a smart, inquisitive and helpful personal assistant. +- Use your general knowledge and past conversation with the user as context to inform your responses. +- If you do not know the answer, say 'I don't know.' +- Think step-by-step and ask questions to get the necessary information to answer the user's question. +- Do not print verbatim Notes unless necessary. + +Today is {current_date} in UTC. + """.strip() +) system_prompt_message_extract_questions_gpt4all = f"""You are Khoj, a kind and intelligent personal assistant. When the user asks you a question, you ask follow-up questions to clarify the necessary information you need in order to answer from the user's perspective. - Write the question as if you can search for the answer on the user's personal notes. diff --git a/tests/test_gpt4all_chat_actors.py b/tests/test_gpt4all_chat_actors.py index 30209b81b..3b1d72491 100644 --- a/tests/test_gpt4all_chat_actors.py +++ b/tests/test_gpt4all_chat_actors.py @@ -41,7 +41,7 @@ def loaded_model(): # ---------------------------------------------------------------------------------------------------- @pytest.mark.xfail(reason="Search actor isn't very date aware nor capable of formatting") @pytest.mark.chatquality -@freeze_time("1984-04-02") +@freeze_time("1984-04-02", ignore=["transformers"]) def test_extract_question_with_date_filter_from_relative_day(loaded_model): # Act response = extract_questions_offline("Where did I go for dinner yesterday?", loaded_model=loaded_model) @@ -61,7 +61,7 @@ def test_extract_question_with_date_filter_from_relative_day(loaded_model): # ---------------------------------------------------------------------------------------------------- @pytest.mark.xfail(reason="Search actor still isn't very date aware nor capable of formatting") @pytest.mark.chatquality -@freeze_time("1984-04-02") +@freeze_time("1984-04-02", ignore=["transformers"]) def test_extract_question_with_date_filter_from_relative_month(loaded_model): # Act response = extract_questions_offline("Which countries did I visit last month?", loaded_model=loaded_model) @@ -83,7 +83,7 @@ def test_extract_question_with_date_filter_from_relative_month(loaded_model): # ---------------------------------------------------------------------------------------------------- @pytest.mark.xfail(reason="Chat actor still isn't very date aware nor capable of formatting") @pytest.mark.chatquality -@freeze_time("1984-04-02") +@freeze_time("1984-04-02", ignore=["transformers"]) def test_extract_question_with_date_filter_from_relative_year(): # Act response = extract_questions_offline("Which countries have I visited this year?") @@ -102,7 +102,7 @@ def test_extract_question_with_date_filter_from_relative_year(): # ---------------------------------------------------------------------------------------------------- @pytest.mark.chatquality -@freeze_time("1984-04-02") +@freeze_time("1984-04-02", ignore=["transformers"]) def test_extract_question_includes_root_question(loaded_model): # Act response = extract_questions_offline("Which countries have I visited this year?", loaded_model=loaded_model) diff --git a/tests/test_gpt4all_chat_director.py b/tests/test_gpt4all_chat_director.py index dd9b2d21e..48520cada 100644 --- a/tests/test_gpt4all_chat_director.py +++ b/tests/test_gpt4all_chat_director.py @@ -298,14 +298,16 @@ def test_answer_not_known_using_notes_command(client_offline_chat, default_user2 # ---------------------------------------------------------------------------------------------------- -@pytest.mark.xfail(AssertionError, reason="Chat director not capable of answering time aware questions yet") @pytest.mark.chatquality @pytest.mark.django_db(transaction=True) -@freeze_time("2023-04-01") +@freeze_time("2023-04-01", ignore=["transformers"]) def test_answer_requires_current_date_awareness(client_offline_chat): "Chat actor should be able to answer questions relative to current date using provided notes" + # Arrange + query = urllib.parse.quote("Where did I have lunch today?") + # Act - response = client_offline_chat.get(f'/api/chat?q="Where did I have lunch today?"&stream=true') + response = client_offline_chat.get(f"/api/chat?q={query}&stream=true") response_message = response.content.decode("utf-8") # Assert @@ -320,7 +322,7 @@ def test_answer_requires_current_date_awareness(client_offline_chat): @pytest.mark.xfail(AssertionError, reason="Chat director not capable of answering this question yet") @pytest.mark.chatquality @pytest.mark.django_db(transaction=True) -@freeze_time("2023-04-01") +@freeze_time("2023-04-01", ignore=["transformers"]) def test_answer_requires_date_aware_aggregation_across_provided_notes(client_offline_chat): "Chat director should be able to answer questions that require date aware aggregation across multiple notes" # Act @@ -333,7 +335,6 @@ def test_answer_requires_date_aware_aggregation_across_provided_notes(client_off # ---------------------------------------------------------------------------------------------------- -@pytest.mark.xfail(AssertionError, reason="Chat director not capable of answering this question yet") @pytest.mark.chatquality @pytest.mark.django_db(transaction=True) def test_answer_general_question_not_in_chat_history_or_retrieved_content(client_offline_chat, default_user2): @@ -425,7 +426,6 @@ def test_answer_chat_history_very_long(client_offline_chat, default_user2): # ---------------------------------------------------------------------------------------------------- -@pytest.mark.xfail(AssertionError, reason="Chat director not capable of answering this question yet") @pytest.mark.chatquality @pytest.mark.django_db(transaction=True) def test_answer_requires_multiple_independent_searches(client_offline_chat): diff --git a/tests/test_openai_chat_actors.py b/tests/test_openai_chat_actors.py index 46f63d0ec..201e3fef1 100644 --- a/tests/test_openai_chat_actors.py +++ b/tests/test_openai_chat_actors.py @@ -22,7 +22,7 @@ # Test # ---------------------------------------------------------------------------------------------------- @pytest.mark.chatquality -@freeze_time("1984-04-02") +@freeze_time("1984-04-02", ignore=["transformers"]) def test_extract_question_with_date_filter_from_relative_day(): # Act response = extract_questions("Where did I go for dinner yesterday?") @@ -41,7 +41,7 @@ def test_extract_question_with_date_filter_from_relative_day(): # ---------------------------------------------------------------------------------------------------- @pytest.mark.chatquality -@freeze_time("1984-04-02") +@freeze_time("1984-04-02", ignore=["transformers"]) def test_extract_question_with_date_filter_from_relative_month(): # Act response = extract_questions("Which countries did I visit last month?") @@ -56,7 +56,7 @@ def test_extract_question_with_date_filter_from_relative_month(): # ---------------------------------------------------------------------------------------------------- @pytest.mark.chatquality -@freeze_time("1984-04-02") +@freeze_time("1984-04-02", ignore=["transformers"]) def test_extract_question_with_date_filter_from_relative_year(): # Act response = extract_questions("Which countries have I visited this year?") diff --git a/tests/test_openai_chat_director.py b/tests/test_openai_chat_director.py index fc92d4b70..ce5897e6d 100644 --- a/tests/test_openai_chat_director.py +++ b/tests/test_openai_chat_director.py @@ -277,7 +277,7 @@ def test_answer_not_known_using_notes_command(chat_client_no_background, default @pytest.mark.xfail(AssertionError, reason="Chat director not capable of answering time aware questions yet") @pytest.mark.django_db(transaction=True) @pytest.mark.chatquality -@freeze_time("2023-04-01") +@freeze_time("2023-04-01", ignore=["transformers"]) def test_answer_requires_current_date_awareness(chat_client): "Chat actor should be able to answer questions relative to current date using provided notes" # Act @@ -295,11 +295,10 @@ def test_answer_requires_current_date_awareness(chat_client): # ---------------------------------------------------------------------------------------------------- @pytest.mark.django_db(transaction=True) @pytest.mark.chatquality -@freeze_time("2023-04-01") +@freeze_time("2023-04-01", ignore=["transformers"]) def test_answer_requires_date_aware_aggregation_across_provided_notes(chat_client): "Chat director should be able to answer questions that require date aware aggregation across multiple notes" # Act - response = chat_client.get(f'/api/chat?q="How much did I spend on dining this year?"&stream=true') response_message = response.content.decode("utf-8")