From ce46f3848b6594f595eb0b89deef5dde719a94b0 Mon Sep 17 00:00:00 2001 From: chjinche <49483542+chjinche@users.noreply.github.com> Date: Mon, 26 Feb 2024 19:08:41 +0800 Subject: [PATCH] [Bugfix] Update expected error message of 'test_completion_with_chat_model' (#2115) # Description Previous AOAI error message ""logprobs, best_of and echo parameters are not available"" It changes to "**best_of and echo** parameters are not available on gpt-35-turbo model. Please remove the parameter and try again. For more details, see https://go.microsoft.com/fwlink/?linkid=2227346.' ![image](https://github.com/microsoft/promptflow/assets/49483542/46fbc2c4-739a-497e-94d9-b89eaedfa193) # All Promptflow Contribution checklist: - [x] **The pull request does not introduce [breaking changes].** - [ ] **CHANGELOG is updated for new features, bug fixes or other significant changes.** - [x] **I have read the [contribution guidelines](../CONTRIBUTING.md).** - [ ] **Create an issue and link to the pull request to get dedicated review from promptflow team. Learn more: [suggested workflow](../CONTRIBUTING.md#suggested-workflow).** ## General Guidelines and Best Practices - [x] Title of the pull request is clear and informative. - [x] There are a small number of commits, each of which have an informative message. This means that previously merged commits do not appear in the history of the PR. For more information on cleaning up the commits in your PR, [see this page](https://github.com/Azure/azure-powershell/blob/master/documentation/development-docs/cleaning-up-commits.md). ### Testing Guidelines - [ ] Pull request includes test coverage for the included changes. --- .../promptflow/tools/exception.py | 21 ++++++++++++------- src/promptflow-tools/tests/test_aoai_gptv.py | 18 ++++++++++++---- 2 files changed, 27 insertions(+), 12 deletions(-) diff --git a/src/promptflow-tools/promptflow/tools/exception.py b/src/promptflow-tools/promptflow/tools/exception.py index 3c098aae186..260684f9a29 100644 --- a/src/promptflow-tools/promptflow/tools/exception.py +++ b/src/promptflow-tools/promptflow/tools/exception.py @@ -7,12 +7,15 @@ def to_openai_error_message(e: Exception) -> str: ex_type = type(e).__name__ - if str(e) == "": + error_message = str(e) + # https://learn.microsoft.com/en-gb/azure/ai-services/openai/reference + params_chat_model_cannot_accept = ["best_of", "echo", "logprobs"] + if error_message == "": msg = "The api key is invalid or revoked. " \ "You can correct or regenerate the api key of your connection." return f"OpenAI API hits {ex_type}: {msg}" # for models that do not support the `functions` parameter. - elif "Unrecognized request argument supplied: functions" in str(e): + elif "Unrecognized request argument supplied: functions" in error_message: msg = "Current model does not support the `functions` parameter. If you are using openai connection, then " \ "please use gpt-3.5-turbo, gpt-4, gpt-4-32k, gpt-3.5-turbo-0613 or gpt-4-0613. You can refer to " \ "https://platform.openai.com/docs/guides/gpt/function-calling. If you are using azure openai " \ @@ -21,15 +24,16 @@ def to_openai_error_message(e: Exception) -> str: "'2023-07-01-preview'. You can refer to " \ "https://learn.microsoft.com/en-us/azure/ai-services/openai/how-to/function-calling." return f"OpenAI API hits {ex_type}: {msg}" - elif "The completion operation does not work with the specified model" in str(e) or \ - "logprobs, best_of and echo parameters are not available" in str(e): + elif "The completion operation does not work with the specified model" in error_message or \ + ("parameters are not available" in error_message + and any(param in error_message for param in params_chat_model_cannot_accept)): msg = "The completion operation does not work with the current model. " \ "Completion API is a legacy api and is going to be deprecated soon. " \ "Please change to use Chat API for current model. " \ "You could refer to guideline at https://aka.ms/pfdoc/chat-prompt " \ "or view the samples in our gallery that contain 'Chat' in the name." return f"OpenAI API hits {ex_type}: {msg}" - elif "Invalid content type. image_url is only supported by certain models" in str(e): + elif "Invalid content type. image_url is only supported by certain models" in error_message: msg = "Current model does not support the image input. If you are using openai connection, then please use " \ "gpt-4-vision-preview. You can refer to https://platform.openai.com/docs/guides/vision." \ "If you are using azure openai connection, then please first go to your Azure OpenAI resource, " \ @@ -37,8 +41,9 @@ def to_openai_error_message(e: Exception) -> str: "model version \"vision-preview\". You can refer to " \ "https://learn.microsoft.com/en-us/azure/ai-services/openai/how-to/gpt-with-vision" return f"OpenAI API hits {ex_type}: {msg}" - elif ("\'response_format\' of type" in str(e) and "is not supported with this model." in str(e))\ - or ("Additional properties are not allowed" in str(e) and "unexpected) - \'response_format\'" in str(e)): + elif ("\'response_format\' of type" in error_message and "is not supported with this model." in error_message)\ + or ("Additional properties are not allowed" in error_message + and "unexpected) - \'response_format\'" in error_message): msg = "The response_format parameter needs to be a dictionary such as {\"type\": \"text\"}. " \ "The value associated with the type key should be either 'text' or 'json_object' " \ "If you are using openai connection, you can only set response_format to { \"type\": \"json_object\" } " \ @@ -49,7 +54,7 @@ def to_openai_error_message(e: Exception) -> str: "https://learn.microsoft.com/en-us/azure/ai-services/openai/how-to/json-mode?tabs=python." return f"OpenAI API hits {ex_type}: {msg}" else: - return f"OpenAI API hits {ex_type}: {str(e)} [{openai_error_code_ref_message}]" + return f"OpenAI API hits {ex_type}: {error_message} [{openai_error_code_ref_message}]" class WrappedOpenAIError(UserErrorException): diff --git a/src/promptflow-tools/tests/test_aoai_gptv.py b/src/promptflow-tools/tests/test_aoai_gptv.py index cee7b140506..5f728a7c90a 100644 --- a/src/promptflow-tools/tests/test_aoai_gptv.py +++ b/src/promptflow-tools/tests/test_aoai_gptv.py @@ -1,10 +1,6 @@ import pytest from unittest.mock import patch -from azure.ai.ml._azure_environments import AzureEnvironments -from promptflow.azure.operations._arm_connection_operations import \ - ArmConnectionOperations, OpenURLFailedUserError - from promptflow.tools.aoai_gpt4v import AzureOpenAI, ListDeploymentsError, ParseConnectionError, \ _parse_resource_id, list_deployment_names, GPT4V_VERSION @@ -45,6 +41,8 @@ def azure_openai_provider(azure_open_ai_connection) -> AzureOpenAI: def mock_build_connection_dict_func1(**kwargs): + from promptflow.azure.operations._arm_connection_operations import OpenURLFailedUserError + raise OpenURLFailedUserError @@ -85,6 +83,8 @@ def test_parse_resource_id_with_error(resource_id, error_message): def test_list_deployment_names_with_conn_error(monkeypatch): + from promptflow.azure.operations._arm_connection_operations import ArmConnectionOperations + monkeypatch.setattr( ArmConnectionOperations, "_build_connection_dict", @@ -100,6 +100,8 @@ def test_list_deployment_names_with_conn_error(monkeypatch): def test_list_deployment_names_with_wrong_connection_id(monkeypatch): + from promptflow.azure.operations._arm_connection_operations import ArmConnectionOperations + monkeypatch.setattr( ArmConnectionOperations, "_build_connection_dict", @@ -115,6 +117,8 @@ def test_list_deployment_names_with_wrong_connection_id(monkeypatch): def test_list_deployment_names_with_permission_issue(monkeypatch): + from promptflow.azure.operations._arm_connection_operations import ArmConnectionOperations + monkeypatch.setattr( ArmConnectionOperations, "_build_connection_dict", @@ -133,6 +137,9 @@ def test_list_deployment_names_with_permission_issue(monkeypatch): def test_list_deployment_names(monkeypatch): + from promptflow.azure.operations._arm_connection_operations import ArmConnectionOperations + from azure.ai.ml._azure_environments import AzureEnvironments + monkeypatch.setattr( ArmConnectionOperations, "_build_connection_dict", @@ -160,6 +167,9 @@ def test_list_deployment_names(monkeypatch): def test_list_deployment_names_sovereign_credential(monkeypatch): + from promptflow.azure.operations._arm_connection_operations import ArmConnectionOperations + from azure.ai.ml._azure_environments import AzureEnvironments + monkeypatch.setattr( ArmConnectionOperations, "_build_connection_dict",