From 9082a952370024cb91af7eb2bc5fa4ccedd7c298 Mon Sep 17 00:00:00 2001 From: Richard Park Date: Wed, 26 Jul 2023 21:13:43 -0700 Subject: [PATCH 1/9] Attempting to enable live testing --- sdk/ai/azopenai/ci.yml | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/sdk/ai/azopenai/ci.yml b/sdk/ai/azopenai/ci.yml index 143bc1e6c2fd..83ac33d6e959 100644 --- a/sdk/ai/azopenai/ci.yml +++ b/sdk/ai/azopenai/ci.yml @@ -26,3 +26,32 @@ stages: - template: /eng/pipelines/templates/jobs/archetype-sdk-client.yml parameters: ServiceDirectory: "ai/azopenai" + EnvVars: + AZURE_TEST_RUN_LIVE: 'true' # use when utilizing the New-TestResources Script + + # these come from our keyvault (TestSecrets-openai) + AZURE_CLIENT_ID: $(openai-client-id) + AZURE_CLIENT_SECRET: $(openai-client-secret) + AZURE_TENANT_ID: $(openai-tenant-id) + + # Azure OpenAI + AOAI_ENDPOINT: $(AOAI-ENDPOINT) + AOAI_API_KEY: $(AOAI-API-KEY) + AOAI_CHAT_COMPLETIONS_MODEL_DEPLOYMENT: $(AOAI-CHAT-COMPLETIONS-MODEL-DEPLOYMENT) + AOAI_COMPLETIONS_MODEL_DEPLOYMENT: $(AOAI-COMPLETIONS-MODEL-DEPLOYMENT) + AOAI_EMBEDDINGS_MODEL_DEPLOYMENT: $(AOAI-EMBEDDINGS-MODEL-DEPLOYMENT) + + # Azure OpenAI "Canary" + AOAI_COMPLETIONS_MODEL_DEPLOYMENT_CANARY: $(AOAI-COMPLETIONS-MODEL-DEPLOYMENT-CANARY) + AOAI_API_KEY_CANARY: $(AOAI-API-KEY-CANARY) + AOAI_EMBEDDINGS_MODEL_DEPLOYMENT_CANARY: $(AOAI-EMBEDDINGS-MODEL-DEPLOYMENT-CANARY) + AOAI_CHAT_COMPLETIONS_MODEL_DEPLOYMENT_CANARY: $(AOAI-CHAT-COMPLETIONS-MODEL-DEPLOYMENT-CANARY) + AOAI_ENDPOINT_CANARY: $(AOAI-ENDPOINT-CANARY) + + # OpenAI + OPENAI_API_KEY: $(OPENAI-API-KEY) + OPENAI_ENDPOINT: $(OPENAI-ENDPOINT) + OPENAI_EMBEDDINGS_MODEL: $(OPENAI-EMBEDDINGS-MODEL) + OPENAI_CHAT_COMPLETIONS_MODEL: $(OPENAI-CHAT-COMPLETIONS-MODEL) + OPENAI_COMPLETIONS_MODEL: $(OPENAI-COMPLETIONS-MODEL) + \ No newline at end of file From e4ae897954408aa40945f1cf81cd489d660ea264 Mon Sep 17 00:00:00 2001 From: Richard Park Date: Wed, 26 Jul 2023 21:15:53 -0700 Subject: [PATCH 2/9] Add in RunLiveTests --- sdk/ai/azopenai/ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/sdk/ai/azopenai/ci.yml b/sdk/ai/azopenai/ci.yml index 83ac33d6e959..eda819ccf0f3 100644 --- a/sdk/ai/azopenai/ci.yml +++ b/sdk/ai/azopenai/ci.yml @@ -26,6 +26,7 @@ stages: - template: /eng/pipelines/templates/jobs/archetype-sdk-client.yml parameters: ServiceDirectory: "ai/azopenai" + RunLiveTests: true EnvVars: AZURE_TEST_RUN_LIVE: 'true' # use when utilizing the New-TestResources Script From a349a5c06ec3d87fa38644893e283766f6b3ea3d Mon Sep 17 00:00:00 2001 From: Richard Park Date: Wed, 26 Jul 2023 21:33:44 -0700 Subject: [PATCH 3/9] It's okay if we don't load a .env file if the variables are already in the environment. Just let the loader determine it. --- sdk/ai/azopenai/client_shared_test.go | 1 - 1 file changed, 1 deletion(-) diff --git a/sdk/ai/azopenai/client_shared_test.go b/sdk/ai/azopenai/client_shared_test.go index bca594c8fe9a..c35b064ca822 100644 --- a/sdk/ai/azopenai/client_shared_test.go +++ b/sdk/ai/azopenai/client_shared_test.go @@ -111,7 +111,6 @@ func initEnvVars() { } else { if err := godotenv.Load(); err != nil { fmt.Printf("Failed to load .env file: %s\n", err) - os.Exit(1) } azureOpenAI = newTestVars("AOAI", false) From ff4e6cf0da51c461077652db4689098d157eaa92 Mon Sep 17 00:00:00 2001 From: Richard Park Date: Wed, 26 Jul 2023 22:20:40 -0700 Subject: [PATCH 4/9] Prompt and content filter annotations are coming back propery now --- sdk/ai/azopenai/client_completions_test.go | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/sdk/ai/azopenai/client_completions_test.go b/sdk/ai/azopenai/client_completions_test.go index 41e3191b73d4..97e237fadd17 100644 --- a/sdk/ai/azopenai/client_completions_test.go +++ b/sdk/ai/azopenai/client_completions_test.go @@ -67,6 +67,13 @@ func testGetCompletions(t *testing.T, client *azopenai.Client, isAzure bool) { }, } + if isAzure { + want.Choices[0].ContentFilterResults = (*azopenai.ChoiceContentFilterResults)(safeContentFilter) + want.PromptAnnotations = []azopenai.PromptFilterResult{ + {PromptIndex: to.Ptr[int32](0), ContentFilterResults: (*azopenai.PromptFilterResultContentFilterResults)(safeContentFilter)}, + } + } + want.ID = resp.Completions.ID want.Created = resp.Completions.Created From 51c3e46d5291e2d61c64b7453c35b3ebdc91fa3c Mon Sep 17 00:00:00 2001 From: Richard Park Date: Wed, 26 Jul 2023 22:23:32 -0700 Subject: [PATCH 5/9] Fixing one of the tests to be less picky about the implementation details of a model --- sdk/ai/azopenai/custom_client_test.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/sdk/ai/azopenai/custom_client_test.go b/sdk/ai/azopenai/custom_client_test.go index 8dc93029ae7e..9bd95400407a 100644 --- a/sdk/ai/azopenai/custom_client_test.go +++ b/sdk/ai/azopenai/custom_client_test.go @@ -144,7 +144,11 @@ func testGetCompletionsStream(t *testing.T, client *azopenai.Client, tv testVars const want = "\n\nAzure OpenAI is a platform from Microsoft that provides access to OpenAI's artificial intelligence (AI) technologies. It enables developers to build, train, and deploy AI models in the cloud. Azure OpenAI provides access to OpenAI's powerful AI technologies, such as GPT-3, which can be used to create natural language processing (NLP) applications, computer vision models, and reinforcement learning models." require.Equal(t, want, got) - require.Equal(t, 86, eventCount) + + // there's no strict requirement of how the response is streamed so just + // choosing something that's reasonable but will be lower than typical usage + // (which is usually somewhere around the 80s). + require.GreaterOrEqual(t, eventCount, 50) } func TestClient_GetCompletions_Error(t *testing.T) { From fc9c917b733011442bd04c77ef3b3703f456e531 Mon Sep 17 00:00:00 2001 From: Richard Park Date: Thu, 27 Jul 2023 15:11:39 -0700 Subject: [PATCH 6/9] We can't consider these results to be deterministic. --- sdk/ai/azopenai/client_embeddings_test.go | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/sdk/ai/azopenai/client_embeddings_test.go b/sdk/ai/azopenai/client_embeddings_test.go index b9d70961ce8d..18ba2f5dd51e 100644 --- a/sdk/ai/azopenai/client_embeddings_test.go +++ b/sdk/ai/azopenai/client_embeddings_test.go @@ -90,10 +90,8 @@ func testGetEmbeddings(t *testing.T, client *azopenai.Client, modelOrDeploymentI t.Errorf("Client.GetEmbeddings() error = %v, wantErr %v", err, tt.wantErr) return } - if len(got.Embeddings.Data[0].Embedding) != 4096 { - t.Errorf("Client.GetEmbeddings() len(Data) want 4096, got %d", len(got.Embeddings.Data)) - return - } + + require.NotEmpty(t, got.Embeddings.Data[0].Embedding) }) } } From c677081457896ae1daedca0939ade54a7a11fd49 Mon Sep 17 00:00:00 2001 From: Richard Park Date: Thu, 27 Jul 2023 16:24:20 -0700 Subject: [PATCH 7/9] Rerecord (and updated associated stuff) --- sdk/ai/azopenai/assets.json | 4 ++-- sdk/ai/azopenai/client_shared_test.go | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/sdk/ai/azopenai/assets.json b/sdk/ai/azopenai/assets.json index 80e73970708d..88dadfd79f71 100644 --- a/sdk/ai/azopenai/assets.json +++ b/sdk/ai/azopenai/assets.json @@ -1,6 +1,6 @@ { "AssetsRepo": "Azure/azure-sdk-assets", "AssetsRepoPrefixPath": "go", - "TagPrefix": "go/cognitiveservices/azopenai", - "Tag": "go/cognitiveservices/azopenai_8fdad86997" + "TagPrefix": "go/ai/azopenai", + "Tag": "go/ai/azopenai_2bf13bba09" } diff --git a/sdk/ai/azopenai/client_shared_test.go b/sdk/ai/azopenai/client_shared_test.go index c35b064ca822..052b7c30c975 100644 --- a/sdk/ai/azopenai/client_shared_test.go +++ b/sdk/ai/azopenai/client_shared_test.go @@ -123,7 +123,7 @@ func newRecordingTransporter(t *testing.T) policy.Transporter { transport, err := recording.NewRecordingHTTPClient(t, nil) require.NoError(t, err) - err = recording.Start(t, "sdk/cognitiveservices/azopenai/testdata", nil) + err = recording.Start(t, "sdk/ai/azopenai/testdata", nil) require.NoError(t, err) if recording.GetRecordMode() != recording.PlaybackMode { From 05382990adf0d5169e6a35f724761cbf7f56b380 Mon Sep 17 00:00:00 2001 From: Richard Park Date: Thu, 27 Jul 2023 17:12:11 -0700 Subject: [PATCH 8/9] Match the model were using in the recordings. --- sdk/ai/azopenai/client_shared_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sdk/ai/azopenai/client_shared_test.go b/sdk/ai/azopenai/client_shared_test.go index 052b7c30c975..e40a40208b54 100644 --- a/sdk/ai/azopenai/client_shared_test.go +++ b/sdk/ai/azopenai/client_shared_test.go @@ -106,7 +106,7 @@ func initEnvVars() { azureOpenAI.ChatCompletions = "gpt-4-0613" openAI.ChatCompletions = "gpt-4-0613" - openAI.Embeddings = "text-similarity-curie-001" + openAI.Embeddings = "text-embedding-ada-002" azureOpenAI.Embeddings = "embedding" } else { if err := godotenv.Load(); err != nil { From 7873f800a7f5f10130d8781325bd7ec5eba0751a Mon Sep 17 00:00:00 2001 From: Richard Park Date: Fri, 28 Jul 2023 10:49:16 -0700 Subject: [PATCH 9/9] Fixing last newline. --- sdk/ai/azopenai/ci.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/sdk/ai/azopenai/ci.yml b/sdk/ai/azopenai/ci.yml index eda819ccf0f3..2bfcd1c974d5 100644 --- a/sdk/ai/azopenai/ci.yml +++ b/sdk/ai/azopenai/ci.yml @@ -55,4 +55,3 @@ stages: OPENAI_EMBEDDINGS_MODEL: $(OPENAI-EMBEDDINGS-MODEL) OPENAI_CHAT_COMPLETIONS_MODEL: $(OPENAI-CHAT-COMPLETIONS-MODEL) OPENAI_COMPLETIONS_MODEL: $(OPENAI-COMPLETIONS-MODEL) - \ No newline at end of file