From 4e0f70c4a6c70b89bbf0bfa98310643371a59cf8 Mon Sep 17 00:00:00 2001 From: happy-qiao Date: Tue, 27 Aug 2024 21:00:27 +0000 Subject: [PATCH 1/8] Add corpora_test for local tokenizer --- vertexai/genai/tokenizer/corpora_test.go | 274 +++++++++++++++++++++++ 1 file changed, 274 insertions(+) create mode 100644 vertexai/genai/tokenizer/corpora_test.go diff --git a/vertexai/genai/tokenizer/corpora_test.go b/vertexai/genai/tokenizer/corpora_test.go new file mode 100644 index 000000000000..0a5cb6b41f8a --- /dev/null +++ b/vertexai/genai/tokenizer/corpora_test.go @@ -0,0 +1,274 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package tokenizer + +import ( + "archive/zip" + "bytes" + "context" + "fmt" + "io" + "log" + "net/http" + "regexp" + "strings" + "testing" + + "cloud.google.com/go/vertexai/genai" + "golang.org/x/text/encoding" + "golang.org/x/text/encoding/charmap" + "golang.org/x/text/encoding/japanese" + "golang.org/x/text/encoding/simplifiedchinese" + "golang.org/x/text/transform" +) + +// FileInfo holds the name and content of a file in the zip archive +type CorporaInfo struct { + Name string + Content []byte +} + +// fileGenerator is a generator function that returns a channel to iterate over files in the zip archive +func CorporaGenerator(url string) (<-chan CorporaInfo, <-chan error) { + out := make(chan CorporaInfo) + errCh := make(chan error, 1) + + go func() { + defer close(out) + defer close(errCh) + + // Download the zip file + resp, err := http.Get(url) + if err != nil { + errCh <- fmt.Errorf("error downloading file: %v", err) + return + } + defer resp.Body.Close() + + // Read the content of the response body + body, err := io.ReadAll(resp.Body) + if err != nil { + errCh <- fmt.Errorf("error reading response body: %v", err) + return + } + + // Create a zip reader from the downloaded content + zipReader, err := zip.NewReader(bytes.NewReader(body), int64(len(body))) + if err != nil { + errCh <- fmt.Errorf("error creating zip reader: %v", err) + return + } + + // Iterate over each file in the zip archive + for _, file := range zipReader.File { + fileReader, err := file.Open() + if err != nil { + errCh <- fmt.Errorf("error opening file: %v", err) + continue + } + + // Check if the file is a text file + if !file.FileInfo().IsDir() && file.FileInfo().Mode().IsRegular() { + content, err := io.ReadAll(fileReader) + if err != nil { + errCh <- fmt.Errorf("error reading file: %v", err) + fileReader.Close() + continue + } + fileReader.Close() + + out <- CorporaInfo{ + Name: file.Name[len("udhr/"):], + Content: content, + } + } else { + fileReader.Close() + } + } + }() + + return out, errCh +} + +// UdhrCorpusReader struct to hold encoding patterns and skip set +type UdhrCorpusReader struct { + Encodings []EncodingPattern + Skip map[string]bool +} + +// EncodingPattern struct to hold regex pattern and corresponding encoding +type EncodingPattern struct { + Pattern *regexp.Regexp + Encoding encoding.Encoding +} + +// NewUdhrCorpusReader initializes a new UdhrCorpusReader with encoding patterns and skip set +func NewUdhrCorpusReader() *UdhrCorpusReader { + encodings := []EncodingPattern{ + {Pattern: regexp.MustCompile(".*-Latin1$"), Encoding: charmap.ISO8859_1}, + {Pattern: regexp.MustCompile(".*-Hebrew$"), Encoding: charmap.ISO8859_8}, + {Pattern: regexp.MustCompile(".*-Arabic$"), Encoding: charmap.Windows1256}, + {Pattern: regexp.MustCompile("Czech_Cesky-UTF8"), Encoding: charmap.Windows1250}, + {Pattern: regexp.MustCompile("Polish-Latin2"), Encoding: charmap.Windows1250}, + {Pattern: regexp.MustCompile("Polish_Polski-Latin2"), Encoding: charmap.Windows1250}, + {Pattern: regexp.MustCompile(".*-Cyrillic$"), Encoding: charmap.Windows1251}, + {Pattern: regexp.MustCompile(".*-SJIS$"), Encoding: japanese.ShiftJIS}, + {Pattern: regexp.MustCompile(".*-GB2312$"), Encoding: simplifiedchinese.HZGB2312}, + {Pattern: regexp.MustCompile(".*-Latin2$"), Encoding: charmap.ISO8859_2}, + {Pattern: regexp.MustCompile(".*-Greek$"), Encoding: charmap.ISO8859_7}, + {Pattern: regexp.MustCompile(".*-UTF8$"), Encoding: encoding.Nop}, // No transformation needed + {Pattern: regexp.MustCompile("Amahuaca"), Encoding: charmap.ISO8859_1}, + {Pattern: regexp.MustCompile("Turkish_Turkce-Turkish"), Encoding: charmap.ISO8859_9}, + {Pattern: regexp.MustCompile("Lithuanian_Lietuviskai-Baltic"), Encoding: charmap.ISO8859_4}, + {Pattern: regexp.MustCompile("Japanese_Nihongo-EUC"), Encoding: japanese.EUCJP}, + {Pattern: regexp.MustCompile(`Abkhaz\-Cyrillic\+Abkh`), Encoding: charmap.Windows1251}, + } + + skip := map[string]bool{ + "Burmese_Myanmar-UTF8": true, + "Japanese_Nihongo-JIS": true, + "Chinese_Mandarin-HZ": true, + "Chinese_Mandarin-UTF8": true, + "Gujarati-UTF8": true, + "Hungarian_Magyar-Unicode": true, + "Lao-UTF8": true, + "Magahi-UTF8": true, + "Marathi-UTF8": true, + "Tamil-UTF8": true, + "Vietnamese-VPS": true, + "Vietnamese-VIQR": true, + "Vietnamese-TCVN": true, + "Magahi-Agra": true, + "Bhojpuri-Agra": true, + "Esperanto-T61": true, + "Burmese_Myanmar-WinResearcher": true, + "Armenian-DallakHelv": true, + "Tigrinya_Tigrigna-VG2Main": true, + "Amharic-Afenegus6..60375": true, + "Navaho_Dine-Navajo-Navaho-font": true, + "Azeri_Azerbaijani_Cyrillic-Az.Times.Cyr.Normal0117": true, + "Azeri_Azerbaijani_Latin-Az.Times.Lat0117": true, + "Czech-Latin2-err": true, + "Russian_Russky-UTF8~": true, + } + + return &UdhrCorpusReader{ + Encodings: encodings, + Skip: skip, + } +} + +// GetEncoding returns the encoding for a given filename based on patterns +func (ucr *UdhrCorpusReader) GetEncoding(filename string) (encoding.Encoding, bool) { + for _, pattern := range ucr.Encodings { + if pattern.Pattern.MatchString(filename) { + return pattern.Encoding, true + } + } + return nil, false +} + +// ShouldSkip checks if the file should be skipped +func (ucr *UdhrCorpusReader) ShouldSkip(filename string) bool { + return ucr.Skip[filename] +} + +// DecodeBytes converts a byte slice from a specified encoding to a UTF-8 string +func DecodeBytes(enc encoding.Encoding, data []byte) (string, error) { + if enc == encoding.Nop { + return string(data), nil + } + + if enc == nil { + return "", fmt.Errorf("unsupported encoding or custom handling required") + } + + decoder := enc.NewDecoder() + reader := transform.NewReader(strings.NewReader(string(data)), decoder) + decodedBytes, err := io.ReadAll(reader) + if err != nil { + return "", fmt.Errorf("error decoding data: %v", err) + } + + return string(decodedBytes), nil +} + +const defaultModel = "gemini-1.0-pro" +const defaultLocation = "us-central1" + +func TestCountTokensWithCorpora(t *testing.T) { + // projectID := os.Getenv("VERTEX_PROJECT_ID") + // if testing.Short() { + // t.Skip("skipping live test in -short mode") + // } + + // if projectID == "" { + // t.Skip("set a VERTEX_PROJECT_ID env var to run live tests") + // } + projectID := "vertexsdk" + ctx := context.Background() + client, err := genai.NewClient(ctx, projectID, defaultLocation) + if err != nil { + t.Fatal(err) + } + defer client.Close() + model := client.GenerativeModel(defaultModel) + + t.Run("RemoteAndLocalCountTokensTest", func(t *testing.T) { + corporaUrl := "https://raw.githubusercontent.com/nltk/nltk_data/gh-pages/packages/corpora/udhr.zip" + fileCh, errCh := CorporaGenerator(corporaUrl) + ucr := NewUdhrCorpusReader() + + // Iterate over files generated by the generator function + for fileInfo := range fileCh { + if ucr.ShouldSkip(fileInfo.Name) { + fmt.Printf("Skipping file: %s\n", fileInfo.Name) + continue + } + + enc, found := ucr.GetEncoding(fileInfo.Name) + if !found { + fmt.Printf("No encoding found for file: %s\n", fileInfo.Name) + continue + } + + decodedContent, err := DecodeBytes(enc, fileInfo.Content) + if err != nil { + log.Fatalf("Failed to decode bytes: %v", err) + } + + tok, err := New("gemini-1.5-flash") + if err != nil { + log.Fatal(err) + } + + localNtoks, err := tok.CountTokens(genai.Text(decodedContent)) + if err != nil { + log.Fatal(err) + } + remoteNtoks, err := model.CountTokens(ctx, genai.Text(decodedContent)) + if err != nil { + log.Fatal(fileInfo.Name, err) + } + if localNtoks.TotalTokens != remoteNtoks.TotalTokens { + t.Errorf("expected %d(remote count-token results), but got %d(local count-token results)", remoteNtoks, localNtoks) + } + + } + + if err := <-errCh; err != nil { + fmt.Println("Error:", err) + } + }) +} From a86eec056a12fddec3fffecf07652d77fecf35fc Mon Sep 17 00:00:00 2001 From: happy-qiao Date: Fri, 30 Aug 2024 11:57:59 -0700 Subject: [PATCH 2/8] fix vet --- internal/generated/snippets/go.mod | 238 ++++++++++++++--------------- internal/generated/snippets/go.sum | 56 +++++-- vertexai/go.mod | 2 +- 3 files changed, 161 insertions(+), 135 deletions(-) diff --git a/internal/generated/snippets/go.mod b/internal/generated/snippets/go.mod index f6976c1cea3e..a466a698a6c4 100644 --- a/internal/generated/snippets/go.mod +++ b/internal/generated/snippets/go.mod @@ -4,135 +4,135 @@ go 1.21 require ( cloud.google.com/go v0.115.1 - cloud.google.com/go/accessapproval v1.7.11 - cloud.google.com/go/accesscontextmanager v1.8.12 + cloud.google.com/go/accessapproval v1.8.0 + cloud.google.com/go/accesscontextmanager v1.9.0 cloud.google.com/go/advisorynotifications v0.0.0-00010101000000-000000000000 cloud.google.com/go/aiplatform v1.68.0 cloud.google.com/go/alloydb v0.0.0-00010101000000-000000000000 - cloud.google.com/go/analytics v0.23.6 - cloud.google.com/go/apigateway v1.6.11 - cloud.google.com/go/apigeeconnect v1.6.11 - cloud.google.com/go/apigeeregistry v0.8.9 + cloud.google.com/go/analytics v0.25.0 + cloud.google.com/go/apigateway v1.7.0 + cloud.google.com/go/apigeeconnect v1.7.0 + cloud.google.com/go/apigeeregistry v0.9.0 cloud.google.com/go/apikeys v0.6.0 - cloud.google.com/go/appengine v1.8.11 - cloud.google.com/go/area120 v0.8.11 - cloud.google.com/go/artifactregistry v1.14.13 - cloud.google.com/go/asset v1.19.5 - cloud.google.com/go/assuredworkloads v1.11.11 - cloud.google.com/go/automl v1.13.11 - cloud.google.com/go/baremetalsolution v1.2.10 - cloud.google.com/go/batch v1.9.2 - cloud.google.com/go/beyondcorp v1.0.10 + cloud.google.com/go/appengine v1.9.0 + cloud.google.com/go/area120 v0.9.0 + cloud.google.com/go/artifactregistry v1.15.0 + cloud.google.com/go/asset v1.20.0 + cloud.google.com/go/assuredworkloads v1.12.0 + cloud.google.com/go/automl v1.14.0 + cloud.google.com/go/baremetalsolution v1.3.0 + cloud.google.com/go/batch v1.10.0 + cloud.google.com/go/beyondcorp v1.1.0 cloud.google.com/go/bigquery v1.62.0 - cloud.google.com/go/billing v1.18.9 - cloud.google.com/go/binaryauthorization v1.8.7 - cloud.google.com/go/certificatemanager v1.8.5 - cloud.google.com/go/channel v1.17.11 - cloud.google.com/go/cloudbuild v1.16.5 - cloud.google.com/go/clouddms v1.7.10 - cloud.google.com/go/cloudtasks v1.12.12 - cloud.google.com/go/compute v1.27.4 + cloud.google.com/go/billing v1.19.0 + cloud.google.com/go/binaryauthorization v1.9.0 + cloud.google.com/go/certificatemanager v1.9.0 + cloud.google.com/go/channel v1.18.0 + cloud.google.com/go/cloudbuild v1.17.0 + cloud.google.com/go/clouddms v1.8.0 + cloud.google.com/go/cloudtasks v1.13.0 + cloud.google.com/go/compute v1.28.0 cloud.google.com/go/confidentialcomputing v0.1.0 - cloud.google.com/go/contactcenterinsights v1.13.6 - cloud.google.com/go/container v1.38.0 - cloud.google.com/go/containeranalysis v0.12.1 - cloud.google.com/go/datacatalog v1.21.1 - cloud.google.com/go/dataflow v0.9.11 - cloud.google.com/go/dataform v0.9.8 - cloud.google.com/go/datafusion v1.7.11 - cloud.google.com/go/datalabeling v0.8.11 - cloud.google.com/go/dataplex v1.18.2 - cloud.google.com/go/dataproc/v2 v2.5.3 - cloud.google.com/go/dataqna v0.8.11 - cloud.google.com/go/datastore v1.17.1 - cloud.google.com/go/datastream v1.10.10 - cloud.google.com/go/deploy v1.21.0 - cloud.google.com/go/dialogflow v1.55.0 + cloud.google.com/go/contactcenterinsights v1.14.0 + cloud.google.com/go/container v1.39.0 + cloud.google.com/go/containeranalysis v0.13.0 + cloud.google.com/go/datacatalog v1.22.0 + cloud.google.com/go/dataflow v0.10.0 + cloud.google.com/go/dataform v0.10.0 + cloud.google.com/go/datafusion v1.8.0 + cloud.google.com/go/datalabeling v0.9.0 + cloud.google.com/go/dataplex v1.19.0 + cloud.google.com/go/dataproc/v2 v2.6.0 + cloud.google.com/go/dataqna v0.9.0 + cloud.google.com/go/datastore v1.19.0 + cloud.google.com/go/datastream v1.11.0 + cloud.google.com/go/deploy v1.22.0 + cloud.google.com/go/dialogflow v1.57.0 cloud.google.com/go/discoveryengine v0.0.0-00010101000000-000000000000 - cloud.google.com/go/dlp v1.16.0 - cloud.google.com/go/documentai v1.31.0 - cloud.google.com/go/domains v0.9.11 - cloud.google.com/go/edgecontainer v1.2.5 + cloud.google.com/go/dlp v1.18.0 + cloud.google.com/go/documentai v1.33.0 + cloud.google.com/go/domains v0.10.0 + cloud.google.com/go/edgecontainer v1.3.0 cloud.google.com/go/errorreporting v0.3.1 - cloud.google.com/go/essentialcontacts v1.6.12 - cloud.google.com/go/eventarc v1.13.10 - cloud.google.com/go/filestore v1.8.7 + cloud.google.com/go/essentialcontacts v1.7.0 + cloud.google.com/go/eventarc v1.14.0 + cloud.google.com/go/filestore v1.9.0 cloud.google.com/go/firestore v1.16.0 - cloud.google.com/go/functions v1.16.6 - cloud.google.com/go/gkebackup v1.5.4 - cloud.google.com/go/gkeconnect v0.8.11 - cloud.google.com/go/gkehub v0.14.11 - cloud.google.com/go/gkemulticloud v1.2.4 - cloud.google.com/go/gsuiteaddons v1.6.11 - cloud.google.com/go/iam v1.1.13 - cloud.google.com/go/iap v1.9.10 - cloud.google.com/go/ids v1.4.11 - cloud.google.com/go/iot v1.7.11 - cloud.google.com/go/kms v1.18.5 - cloud.google.com/go/language v1.13.0 - cloud.google.com/go/lifesciences v0.9.11 + cloud.google.com/go/functions v1.19.0 + cloud.google.com/go/gkebackup v1.6.0 + cloud.google.com/go/gkeconnect v0.10.0 + cloud.google.com/go/gkehub v0.15.0 + cloud.google.com/go/gkemulticloud v1.3.0 + cloud.google.com/go/gsuiteaddons v1.7.0 + cloud.google.com/go/iam v1.2.0 + cloud.google.com/go/iap v1.10.0 + cloud.google.com/go/ids v1.5.0 + cloud.google.com/go/iot v1.8.0 + cloud.google.com/go/kms v1.19.0 + cloud.google.com/go/language v1.14.0 + cloud.google.com/go/lifesciences v0.10.0 cloud.google.com/go/logging v1.11.0 - cloud.google.com/go/longrunning v0.5.12 - cloud.google.com/go/managedidentities v1.6.11 - cloud.google.com/go/maps v1.11.6 - cloud.google.com/go/mediatranslation v0.8.11 - cloud.google.com/go/memcache v1.10.11 - cloud.google.com/go/metastore v1.13.10 - cloud.google.com/go/monitoring v1.20.4 - cloud.google.com/go/networkconnectivity v1.14.10 - cloud.google.com/go/networkmanagement v1.13.6 - cloud.google.com/go/networksecurity v0.9.11 - cloud.google.com/go/notebooks v1.11.9 - cloud.google.com/go/optimization v1.6.9 - cloud.google.com/go/orchestration v1.9.6 - cloud.google.com/go/orgpolicy v1.12.8 - cloud.google.com/go/osconfig v1.13.3 - cloud.google.com/go/oslogin v1.13.7 - cloud.google.com/go/phishingprotection v0.8.11 - cloud.google.com/go/policytroubleshooter v1.10.9 - cloud.google.com/go/privatecatalog v0.9.11 + cloud.google.com/go/longrunning v0.6.0 + cloud.google.com/go/managedidentities v1.7.0 + cloud.google.com/go/maps v1.12.0 + cloud.google.com/go/mediatranslation v0.9.0 + cloud.google.com/go/memcache v1.11.0 + cloud.google.com/go/metastore v1.14.0 + cloud.google.com/go/monitoring v1.21.0 + cloud.google.com/go/networkconnectivity v1.15.0 + cloud.google.com/go/networkmanagement v1.14.0 + cloud.google.com/go/networksecurity v0.10.0 + cloud.google.com/go/notebooks v1.12.0 + cloud.google.com/go/optimization v1.7.0 + cloud.google.com/go/orchestration v1.10.0 + cloud.google.com/go/orgpolicy v1.13.0 + cloud.google.com/go/osconfig v1.14.0 + cloud.google.com/go/oslogin v1.14.0 + cloud.google.com/go/phishingprotection v0.9.0 + cloud.google.com/go/policytroubleshooter v1.11.0 + cloud.google.com/go/privatecatalog v0.10.0 cloud.google.com/go/pubsub v1.42.0 cloud.google.com/go/pubsublite v1.8.2 - cloud.google.com/go/recaptchaenterprise/v2 v2.14.2 - cloud.google.com/go/recommendationengine v0.8.11 - cloud.google.com/go/recommender v1.12.7 - cloud.google.com/go/redis v1.16.4 - cloud.google.com/go/resourcemanager v1.9.11 - cloud.google.com/go/resourcesettings v1.7.4 - cloud.google.com/go/retail v1.17.4 - cloud.google.com/go/run v1.4.0 - cloud.google.com/go/scheduler v1.10.12 - cloud.google.com/go/secretmanager v1.13.5 - cloud.google.com/go/security v1.17.4 - cloud.google.com/go/securitycenter v1.33.1 + cloud.google.com/go/recaptchaenterprise/v2 v2.16.0 + cloud.google.com/go/recommendationengine v0.9.0 + cloud.google.com/go/recommender v1.13.0 + cloud.google.com/go/redis v1.17.0 + cloud.google.com/go/resourcemanager v1.10.0 + cloud.google.com/go/resourcesettings v1.8.0 + cloud.google.com/go/retail v1.18.0 + cloud.google.com/go/run v1.5.0 + cloud.google.com/go/scheduler v1.11.0 + cloud.google.com/go/secretmanager v1.14.0 + cloud.google.com/go/security v1.18.0 + cloud.google.com/go/securitycenter v1.35.0 cloud.google.com/go/servicecontrol v1.11.1 - cloud.google.com/go/servicedirectory v1.11.11 + cloud.google.com/go/servicedirectory v1.12.0 cloud.google.com/go/servicemanagement v1.8.0 cloud.google.com/go/serviceusage v1.6.0 - cloud.google.com/go/shell v1.7.11 - cloud.google.com/go/spanner v1.65.0 - cloud.google.com/go/speech v1.24.0 + cloud.google.com/go/shell v1.8.0 + cloud.google.com/go/spanner v1.67.0 + cloud.google.com/go/speech v1.25.0 cloud.google.com/go/storageinsights v0.0.0 - cloud.google.com/go/storagetransfer v1.10.10 + cloud.google.com/go/storagetransfer v1.11.0 cloud.google.com/go/support v0.0.0 - cloud.google.com/go/talent v1.6.12 - cloud.google.com/go/texttospeech v1.7.11 - cloud.google.com/go/tpu v1.6.11 - cloud.google.com/go/trace v1.10.11 - cloud.google.com/go/translate v1.10.7 - cloud.google.com/go/video v1.22.0 - cloud.google.com/go/videointelligence v1.11.11 - cloud.google.com/go/vision/v2 v2.8.6 - cloud.google.com/go/vmmigration v1.7.11 - cloud.google.com/go/vmwareengine v1.2.0 - cloud.google.com/go/vpcaccess v1.7.11 - cloud.google.com/go/webrisk v1.9.11 - cloud.google.com/go/websecurityscanner v1.6.11 - cloud.google.com/go/workflows v1.12.10 + cloud.google.com/go/talent v1.7.0 + cloud.google.com/go/texttospeech v1.8.0 + cloud.google.com/go/tpu v1.7.0 + cloud.google.com/go/trace v1.11.0 + cloud.google.com/go/translate v1.12.0 + cloud.google.com/go/video v1.23.0 + cloud.google.com/go/videointelligence v1.12.0 + cloud.google.com/go/vision/v2 v2.9.0 + cloud.google.com/go/vmmigration v1.8.0 + cloud.google.com/go/vmwareengine v1.3.0 + cloud.google.com/go/vpcaccess v1.8.0 + cloud.google.com/go/webrisk v1.10.0 + cloud.google.com/go/websecurityscanner v1.7.0 + cloud.google.com/go/workflows v1.13.0 cloud.google.com/go/workstations v0.0.0-00010101000000-000000000000 github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect - google.golang.org/api v0.193.0 - google.golang.org/genproto v0.0.0-20240814211410-ddb44dafa142 + google.golang.org/api v0.194.0 + google.golang.org/genproto v0.0.0-20240827150818-7e3bb234dfed ) require ( @@ -167,12 +167,11 @@ require ( cloud.google.com/go/streetview v0.0.0-00010101000000-000000000000 cloud.google.com/go/telcoautomation v0.0.0-00010101000000-000000000000 cloud.google.com/go/visionai v0.0.0-00010101000000-000000000000 - google.golang.org/genproto/googleapis/api v0.0.0-20240814211410-ddb44dafa142 google.golang.org/protobuf v1.34.2 ) require ( - cloud.google.com/go/auth v0.9.0 // indirect + cloud.google.com/go/auth v0.9.1 // indirect cloud.google.com/go/auth/oauth2adapt v0.2.4 // indirect cloud.google.com/go/compute/metadata v0.5.0 // indirect github.com/felixge/httpsnoop v1.0.4 // indirect @@ -180,14 +179,14 @@ require ( github.com/go-logr/stdr v1.2.2 // indirect github.com/google/s2a-go v0.1.8 // indirect github.com/google/uuid v1.6.0 // indirect - github.com/googleapis/enterprise-certificate-proxy v0.3.2 // indirect + github.com/googleapis/enterprise-certificate-proxy v0.3.3 // indirect github.com/googleapis/gax-go/v2 v2.13.0 // indirect go.opencensus.io v0.24.0 // indirect - go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.52.0 // indirect - go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.53.0 // indirect - go.opentelemetry.io/otel v1.28.0 // indirect - go.opentelemetry.io/otel/metric v1.28.0 // indirect - go.opentelemetry.io/otel/trace v1.28.0 // indirect + go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.54.0 // indirect + go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.54.0 // indirect + go.opentelemetry.io/otel v1.29.0 // indirect + go.opentelemetry.io/otel/metric v1.29.0 // indirect + go.opentelemetry.io/otel/trace v1.29.0 // indirect golang.org/x/crypto v0.26.0 // indirect golang.org/x/net v0.28.0 // indirect golang.org/x/oauth2 v0.22.0 // indirect @@ -195,7 +194,8 @@ require ( golang.org/x/sys v0.24.0 // indirect golang.org/x/text v0.17.0 // indirect golang.org/x/time v0.6.0 // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20240814211410-ddb44dafa142 // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20240827150818-7e3bb234dfed // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20240827150818-7e3bb234dfed // indirect google.golang.org/grpc v1.65.0 // indirect ) diff --git a/internal/generated/snippets/go.sum b/internal/generated/snippets/go.sum index 6ad71940083c..249535f0e332 100644 --- a/internal/generated/snippets/go.sum +++ b/internal/generated/snippets/go.sum @@ -1,11 +1,12 @@ cel.dev/expr v0.15.0/go.mod h1:TRSuuV7DlVCE/uwv5QbAiW/v8l5O8C4eEPHeu7gf7Sg= +cel.dev/expr v0.16.0/go.mod h1:TRSuuV7DlVCE/uwv5QbAiW/v8l5O8C4eEPHeu7gf7Sg= cloud.google.com/go/auth v0.2.1/go.mod h1:khQRBNrvNoHiHhV1iu2x8fSnlNbCaVHilznW5MAI5GY= cloud.google.com/go/auth v0.3.0/go.mod h1:lBv6NKTWp8E3LPzmO1TbiiRKc4drLOfHsgmlH9ogv5w= cloud.google.com/go/auth v0.5.1/go.mod h1:vbZT8GjzDf3AVqCcQmqeeM32U9HBFc32vVVAbwDsa6s= cloud.google.com/go/auth v0.7.0/go.mod h1:D+WqdrpcjmiCgWrXmLLxOVq1GACoE36chW6KXoEvuIw= cloud.google.com/go/auth v0.7.2/go.mod h1:VEc4p5NNxycWQTMQEDQF0bd6aTMb6VgYDXEwiJJQAbs= -cloud.google.com/go/auth v0.9.0 h1:cYhKl1JUhynmxjXfrk4qdPc6Amw7i+GC9VLflgT0p5M= -cloud.google.com/go/auth v0.9.0/go.mod h1:2HsApZBr9zGZhC9QAXsYVYaWk8kNUt37uny+XVKi7wM= +cloud.google.com/go/auth v0.9.1 h1:+pMtLEV2k0AXKvs/tGZojuj6QaioxfUjOpMsG5Gtx+w= +cloud.google.com/go/auth v0.9.1/go.mod h1:Sw8ocT5mhhXxFklyhT12Eiy0ed6tTrPMCJjSI8KhYLk= cloud.google.com/go/auth/oauth2adapt v0.2.2/go.mod h1:wcYjgpZI9+Yu7LyYBg4pqSiaRkfEK3GQcpb7C/uyF1Q= cloud.google.com/go/auth/oauth2adapt v0.2.4 h1:0GWE/FUsXhf6C+jAkWgYm7X9tK8cuEIfy19DBn6B6bY= cloud.google.com/go/auth/oauth2adapt v0.2.4/go.mod h1:jC/jOpwFP6JBxhB3P5Rr0a9HLMC/Pe3eaL4NmdvqPtc= @@ -71,6 +72,7 @@ github.com/cncf/xds/go v0.0.0-20231109132714-523115ebc101/go.mod h1:eXthEFrGJvWH github.com/cncf/xds/go v0.0.0-20231128003011-0fa0005c9caa/go.mod h1:x/1Gn8zydmfq8dk6e9PdstVsDgu9RuyIIJqAaF//0IM= github.com/cncf/xds/go v0.0.0-20240318125728-8a4994d93e50/go.mod h1:5e1+Vvlzido69INQaVO6d87Qn543Xr6nooe9Kz7oBFM= github.com/cncf/xds/go v0.0.0-20240423153145-555b57ec207b/go.mod h1:W+zGtBO5Y1IgJhy4+A9GOqVhqLpfZi+vwmdNXUehLA8= +github.com/cncf/xds/go v0.0.0-20240822171458-6449f94b4d59/go.mod h1:W+zGtBO5Y1IgJhy4+A9GOqVhqLpfZi+vwmdNXUehLA8= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= @@ -92,6 +94,7 @@ github.com/envoyproxy/go-control-plane v0.11.0/go.mod h1:VnHyVMpzcLvCFt9yUz1UnCw github.com/envoyproxy/go-control-plane v0.11.1-0.20230524094728-9239064ad72f/go.mod h1:sfYdkwUW4BA3PbKjySwjJy+O4Pu0h62rlqCMHNk+K+Q= github.com/envoyproxy/go-control-plane v0.11.1/go.mod h1:uhMcXKCQMEJHiAb0w+YGefQLaTEw+YhGluxZkrTmD0g= github.com/envoyproxy/go-control-plane v0.12.0/go.mod h1:ZBTaoJ23lqITozF0M6G4/IragXCQKCnYbmlmtHvwRG0= +github.com/envoyproxy/go-control-plane v0.13.0/go.mod h1:GRaKG3dwvFoTg4nj7aXdZnvMg4d7nvT/wl9WgVXn3Q8= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/envoyproxy/protoc-gen-validate v0.6.7/go.mod h1:dyJXwwfPK2VSqiB9Klm1J6romD608Ba7Hij42vrOBCo= github.com/envoyproxy/protoc-gen-validate v0.9.1/go.mod h1:OKNgG7TCp5pF4d6XftA0++PMirau2/yoOwVac3AbF2w= @@ -100,6 +103,7 @@ github.com/envoyproxy/protoc-gen-validate v0.10.1/go.mod h1:DRjgyB0I43LtJapqN6Ni github.com/envoyproxy/protoc-gen-validate v1.0.1/go.mod h1:0vj8bNkYbSTNS2PIyH87KZaeN4x9zpL9Qt8fQC7d+vs= github.com/envoyproxy/protoc-gen-validate v1.0.2/go.mod h1:GpiZQP3dDbg4JouG/NNS7QWXpgx6x8QiMKdmN72jogE= github.com/envoyproxy/protoc-gen-validate v1.0.4/go.mod h1:qys6tmnRsYrQqIhm2bvKZH4Blx/1gTIZ2UKVY1M+Yew= +github.com/envoyproxy/protoc-gen-validate v1.1.0/go.mod h1:sXRDRVmzEbkM7CVcM06s9shE/m23dg3wzjl0UWqJ2q4= github.com/ettle/strcase v0.1.1/go.mod h1:hzDLsPC7/lwKyBOywSHEP89nt2pDgdy+No1NBA9o9VY= github.com/fatih/color v1.10.0/go.mod h1:ELkj/draVOlAH/xkhN6mQ50Qd0MPOk5AAr3maGEBuJM= github.com/fatih/color v1.13.0/go.mod h1:kLAiJbzzSOZDVNGyDpeOxJ47H46qBXwg5ILebYFFOfk= @@ -167,6 +171,7 @@ github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6 github.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/google/btree v1.1.2/go.mod h1:qOPhT0dTNdNzV6Z/lhRX0YXUafgPLFUh+gZMl761Gm4= +github.com/google/btree v1.1.3/go.mod h1:qOPhT0dTNdNzV6Z/lhRX0YXUafgPLFUh+gZMl761Gm4= github.com/google/flatbuffers v23.5.26+incompatible/go.mod h1:1AeVuKshWv4vARoZatz6mlQ0JxURH0Kv5+zNeJKJCa8= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= @@ -183,6 +188,7 @@ github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeN github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-pkcs11 v0.2.1-0.20230907215043-c6f79328ddf9/go.mod h1:6eQoGcuNJpa7jnd5pMGdkSaQpNDYvPlXWMcjXXThLlY= +github.com/google/go-pkcs11 v0.3.0/go.mod h1:6eQoGcuNJpa7jnd5pMGdkSaQpNDYvPlXWMcjXXThLlY= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/martian/v3 v3.3.3/go.mod h1:iEPrYcgCF7jA9OtScMFQyAlZZ4YXTKEtJ1E6RWzmBA0= github.com/google/pprof v0.0.0-20221118152302-e6195bd50e26/go.mod h1:dDKJzRmX4S37WGHujM7tX//fmj1uioxKzKxz3lo4HJo= @@ -203,8 +209,9 @@ github.com/googleapis/enterprise-certificate-proxy v0.2.1/go.mod h1:AwSRAtLfXpU5 github.com/googleapis/enterprise-certificate-proxy v0.2.3/go.mod h1:AwSRAtLfXpU5Nm3pW+v7rGDHp09LsPtGY9MduiEsR9k= github.com/googleapis/enterprise-certificate-proxy v0.2.4/go.mod h1:AwSRAtLfXpU5Nm3pW+v7rGDHp09LsPtGY9MduiEsR9k= github.com/googleapis/enterprise-certificate-proxy v0.3.1/go.mod h1:VLSiSSBs/ksPL8kq3OBOQ6WRI2QnaFynd1DCjZ62+V0= -github.com/googleapis/enterprise-certificate-proxy v0.3.2 h1:Vie5ybvEvT75RniqhfFxPRy3Bf7vr3h0cechB90XaQs= github.com/googleapis/enterprise-certificate-proxy v0.3.2/go.mod h1:VLSiSSBs/ksPL8kq3OBOQ6WRI2QnaFynd1DCjZ62+V0= +github.com/googleapis/enterprise-certificate-proxy v0.3.3 h1:QRje2j5GZimBzlbhGA2V2QlGNgL8G6e+wGo/+/2bWI0= +github.com/googleapis/enterprise-certificate-proxy v0.3.3/go.mod h1:YKe7cfqYXjKGpGvmSg28/fFvhNzinZQm8DGnaburhGA= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= github.com/googleapis/gax-go/v2 v2.1.0/go.mod h1:Q3nei7sK6ybPYH7twZdmQpAd1MKb7pfu6SK+H1/DsU0= github.com/googleapis/gax-go/v2 v2.1.1/go.mod h1:hddJymUZASv3XPyGkUpKj8pPO47Rmb0eJc8R6ouapiM= @@ -258,6 +265,7 @@ github.com/lyft/protoc-gen-star v0.6.0/go.mod h1:TGAoBVkt8w7MPG72TrKIu85MIdXwDuz github.com/lyft/protoc-gen-star v0.6.1/go.mod h1:TGAoBVkt8w7MPG72TrKIu85MIdXwDuzJYeZuUPFPNwA= github.com/lyft/protoc-gen-star/v2 v2.0.1/go.mod h1:RcCdONR2ScXaYnQC5tUzxzlpA3WVYF7/opLeUgcQs/o= github.com/lyft/protoc-gen-star/v2 v2.0.3/go.mod h1:amey7yeodaJhXSbf/TlLvWiqQfLOSpEk//mLlc+axEk= +github.com/lyft/protoc-gen-star/v2 v2.0.4-0.20230330145011-496ad1ac90a4/go.mod h1:amey7yeodaJhXSbf/TlLvWiqQfLOSpEk//mLlc+axEk= github.com/mattn/go-colorable v0.1.8/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= github.com/mattn/go-colorable v0.1.9/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= @@ -282,6 +290,7 @@ github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINE github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/sftp v1.10.1/go.mod h1:lYOWFsE0bwd1+KfKJaKeuokY15vzFx25BLbzYYoAxZI= github.com/pkg/sftp v1.13.1/go.mod h1:3HaPG6Dq1ILlpPZRO0HVMrsydcdLt6HRDccSgb87qRg= +github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10/go.mod h1:t/avpk3KcrXxUnYOhZhMXJlSEyie6gQbtLq5NM3loB8= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= @@ -289,6 +298,7 @@ github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6T github.com/prometheus/client_model v0.3.0/go.mod h1:LDGWKZIo7rky3hgvBe+caln+Dr3dPggB5dvjtD7w9+w= github.com/prometheus/client_model v0.4.0/go.mod h1:oMQmHW1/JoDwqLtg57MGgP/Fb1CJEYF2imWWhWtMkYU= github.com/prometheus/client_model v0.5.0/go.mod h1:dTiFglRmd66nLR9Pv9f0mZi7B7fk5Pm3gvsjB5tr+kI= +github.com/prometheus/client_model v0.6.0/go.mod h1:NTQHnmxFpouOD0DpvP4XujX3CdOAGQPoaGhyTchlyt8= github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= @@ -337,37 +347,44 @@ go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0= go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo= go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.47.0/go.mod h1:r9vWsPS/3AQItv3OSlEJ/E4mbrhUbbw18meOjArPtKQ= go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.49.0/go.mod h1:Mjt1i1INqiaoZOMGR1RIUJN+i3ChKoFRqzrRQhlkbs0= -go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.52.0 h1:vS1Ao/R55RNV4O7TA2Qopok8yN+X0LIP6RVWLFkprck= go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.52.0/go.mod h1:BMsdeOxN04K0L5FNUBfjFdvwWGNe/rkmSwH4Aelu/X0= +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.54.0 h1:r6I7RJCN86bpD/FQwedZ0vSixDpwuWREjW9oRMsmqDc= +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.54.0/go.mod h1:B9yO6b04uB80CzjedvewuqDhxJxi11s7/GtiGa8bAjI= go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.47.0/go.mod h1:SK2UL73Zy1quvRPonmOmRDiWk1KBV3LyIeeIxcEApWw= go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0/go.mod h1:p8pYQP+m5XfbZm9fxtSKAbM6oIllS7s2AfxrChvc7iw= -go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.53.0 h1:4K4tsIXefpVJtvA/8srF4V4y0akAoPHkIslgAkjixJA= go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.53.0/go.mod h1:jjdQuTGVsXV4vSs+CJ2qYDeDPf9yIJV23qlIzBm73Vg= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.54.0 h1:TT4fX+nBOA/+LUkobKGW1ydGcn+G3vRw9+g5HwCphpk= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.54.0/go.mod h1:L7UH0GbB0p47T4Rri3uHjbpCFYrVrwc1I25QhNPiGK8= go.opentelemetry.io/otel v1.22.0/go.mod h1:eoV4iAi3Ea8LkAEI9+GFT44O6T/D0GWAVFyZVCC6pMI= go.opentelemetry.io/otel v1.23.0/go.mod h1:YCycw9ZeKhcJFrb34iVSkyT0iczq/zYDtZYFufObyB0= go.opentelemetry.io/otel v1.24.0/go.mod h1:W7b9Ozg4nkF5tWI5zsXkaKKDjdVjpD4oAt9Qi/MArHo= go.opentelemetry.io/otel v1.27.0/go.mod h1:DMpAK8fzYRzs+bi3rS5REupisuqTheUlSZJ1WnZaPAQ= -go.opentelemetry.io/otel v1.28.0 h1:/SqNcYk+idO0CxKEUOtKQClMK/MimZihKYMruSMViUo= go.opentelemetry.io/otel v1.28.0/go.mod h1:q68ijF8Fc8CnMHKyzqL6akLO46ePnjkgfIMIjUIX9z4= +go.opentelemetry.io/otel v1.29.0 h1:PdomN/Al4q/lN6iBJEN3AwPvUiHPMlt93c8bqTG5Llw= +go.opentelemetry.io/otel v1.29.0/go.mod h1:N/WtXPs1CNCUEx+Agz5uouwCba+i+bJGFicT8SR4NP8= go.opentelemetry.io/otel/metric v1.22.0/go.mod h1:evJGjVpZv0mQ5QBRJoBF64yMuOf4xCWdXjK8pzFvliY= go.opentelemetry.io/otel/metric v1.23.0/go.mod h1:MqUW2X2a6Q8RN96E2/nqNoT+z9BSms20Jb7Bbp+HiTo= go.opentelemetry.io/otel/metric v1.24.0/go.mod h1:VYhLe1rFfxuTXLgj4CBiyz+9WYBA8pNGJgDcSFRKBco= go.opentelemetry.io/otel/metric v1.27.0/go.mod h1:mVFgmRlhljgBiuk/MP/oKylr4hs85GZAylncepAX/ak= -go.opentelemetry.io/otel/metric v1.28.0 h1:f0HGvSl1KRAU1DLgLGFjrwVyismPlnuU6JD6bOeuA5Q= go.opentelemetry.io/otel/metric v1.28.0/go.mod h1:Fb1eVBFZmLVTMb6PPohq3TO9IIhUisDsbJoL/+uQW4s= -go.opentelemetry.io/otel/sdk v1.28.0 h1:b9d7hIry8yZsgtbmM0DKyPWMMUMlK9NEKuIG4aBqWyE= +go.opentelemetry.io/otel/metric v1.29.0 h1:vPf/HFWTNkPu1aYeIsc98l4ktOQaL6LeSoeV2g+8YLc= +go.opentelemetry.io/otel/metric v1.29.0/go.mod h1:auu/QWieFVWx+DmQOUMgj0F8LHWdgalxXqvp7BII/W8= go.opentelemetry.io/otel/sdk v1.28.0/go.mod h1:oYj7ClPUA7Iw3m+r7GeEjz0qckQRJK2B8zjcZEfu7Pg= -go.opentelemetry.io/otel/sdk/metric v1.28.0/go.mod h1:cWPjykihLAPvXKi4iZc1dpER3Jdq2Z0YLse3moQUCpg= +go.opentelemetry.io/otel/sdk v1.29.0 h1:vkqKjk7gwhS8VaWb0POZKmIEDimRCMsopNYnriHyryo= +go.opentelemetry.io/otel/sdk v1.29.0/go.mod h1:pM8Dx5WKnvxLCb+8lG1PRNIDxu9g9b9g59Qr7hfAAok= +go.opentelemetry.io/otel/sdk/metric v1.29.0/go.mod h1:6zZLdCl2fkauYoZIOn/soQIDSWFmNSRcICarHfuhNJQ= go.opentelemetry.io/otel/trace v1.22.0/go.mod h1:RbbHXVqKES9QhzZq/fE5UnOSILqRt40a21sPw2He1xo= go.opentelemetry.io/otel/trace v1.23.0/go.mod h1:GSGTbIClEsuZrGIzoEHqsVfxgn5UkggkflQwDScNUsk= go.opentelemetry.io/otel/trace v1.24.0/go.mod h1:HPc3Xr/cOApsBI154IU0OI0HJexz+aw5uPdbs3UCjNU= go.opentelemetry.io/otel/trace v1.27.0/go.mod h1:6RiD1hkAprV4/q+yd2ln1HG9GoPx39SuvvstaLBl+l4= -go.opentelemetry.io/otel/trace v1.28.0 h1:GhQ9cUuQGmNDd5BTCP2dAvv75RdMxEfTmYejp+lkx9g= go.opentelemetry.io/otel/trace v1.28.0/go.mod h1:jPyXzNPg6da9+38HEwElrQiHlVMTnVfM3/yv2OlIHaI= +go.opentelemetry.io/otel/trace v1.29.0 h1:J/8ZNK4XgR7a21DZUAsbF8pZ5Jcw1VhACmnYt39JTi4= +go.opentelemetry.io/otel/trace v1.29.0/go.mod h1:eHl3w0sp3paPkYstJOmAimxhiFXPg+MMTlEh3nsQgWQ= go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI= go.opentelemetry.io/proto/otlp v0.15.0/go.mod h1:H7XAot3MsfNsj7EXtrA2q5xSNQ10UqI405h3+duxN4U= go.opentelemetry.io/proto/otlp v0.19.0/go.mod h1:H7XAot3MsfNsj7EXtrA2q5xSNQ10UqI405h3+duxN4U= go.opentelemetry.io/proto/otlp v1.0.0/go.mod h1:Sy6pihPLfYHkr3NkUbEhGHFhINUSI/v80hjKIs5JXpM= +go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= @@ -736,8 +753,8 @@ google.golang.org/api v0.176.1/go.mod h1:j2MaSDYcvYV1lkZ1+SMW4IeF90SrEyFA+tluDYW google.golang.org/api v0.177.0/go.mod h1:srbhue4MLjkjbkux5p3dw/ocYOSZTaIEvf7bCOnFQDw= google.golang.org/api v0.184.0/go.mod h1:CeDTtUEiYENAf8PPG5VZW2yNp2VM3VWbCeTioAZBTBA= google.golang.org/api v0.188.0/go.mod h1:VR0d+2SIiWOYG3r/jdm7adPW9hI2aRv9ETOSCQ9Beag= -google.golang.org/api v0.193.0 h1:eOGDoJFsLU+HpCBaDJex2fWiYujAw9KbXgpOAMePoUs= -google.golang.org/api v0.193.0/go.mod h1:Po3YMV1XZx+mTku3cfJrlIYR03wiGrCOsdpC67hjZvw= +google.golang.org/api v0.194.0 h1:dztZKG9HgtIpbI35FhfuSNR/zmaMVdxNlntHj1sIS4s= +google.golang.org/api v0.194.0/go.mod h1:AgvUFdojGANh3vI+P7EVnxj3AISHllxGCJSFmggmnd0= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= @@ -811,8 +828,10 @@ google.golang.org/genproto v0.0.0-20240227224415-6ceb2ff114de/go.mod h1:VUhTRKeH google.golang.org/genproto v0.0.0-20240604185151-ef581f913117/go.mod h1:lesfX/+9iA+3OdqeCpoDddJaNxVB1AB6tD7EfqMmprc= google.golang.org/genproto v0.0.0-20240708141625-4ad9e859172b/go.mod h1:FfBgJBJg9GcpPvKIuHSZ/aE1g2ecGL74upMzGZjiGEY= google.golang.org/genproto v0.0.0-20240711142825-46eb208f015d/go.mod h1:FfBgJBJg9GcpPvKIuHSZ/aE1g2ecGL74upMzGZjiGEY= -google.golang.org/genproto v0.0.0-20240814211410-ddb44dafa142 h1:oLiyxGgE+rt22duwci1+TG7bg2/L1LQsXwfjPlmuJA0= google.golang.org/genproto v0.0.0-20240814211410-ddb44dafa142/go.mod h1:G11eXq53iI5Q+kyNOmCvnzBaxEA2Q/Ik5Tj7nqBE8j4= +google.golang.org/genproto v0.0.0-20240822170219-fc7c04adadcd/go.mod h1:JB1IzdOfYpNW7QBoS3aYEw5Zl2Q3OEeNWY/Nb99hSyk= +google.golang.org/genproto v0.0.0-20240827150818-7e3bb234dfed h1:4C4dbrVFtfIp3GXJdMX1Sj25mahfn5DywOo65/2ISQ8= +google.golang.org/genproto v0.0.0-20240827150818-7e3bb234dfed/go.mod h1:ICjniACoWvcDz8c8bOsHVKuuSGDJy1z5M4G0DM3HzTc= google.golang.org/genproto/googleapis/api v0.0.0-20230525234020-1aefcd67740a/go.mod h1:ts19tUU+Z0ZShN1y3aPyq2+O3d5FUNNgT6FtOzmrNn8= google.golang.org/genproto/googleapis/api v0.0.0-20230525234035-dd9d682886f9/go.mod h1:vHYtlOoi6TsQ3Uk2yxR7NI5z8uoV+3pZtR4jmHIkRig= google.golang.org/genproto/googleapis/api v0.0.0-20230526203410-71b5a4ffd15e/go.mod h1:vHYtlOoi6TsQ3Uk2yxR7NI5z8uoV+3pZtR4jmHIkRig= @@ -849,8 +868,11 @@ google.golang.org/genproto/googleapis/api v0.0.0-20240610135401-a8a62080eff3/go. google.golang.org/genproto/googleapis/api v0.0.0-20240701130421-f6361c86f094/go.mod h1:fJ/e3If/Q67Mj99hin0hMhiNyCRmt6BQ2aWIJshUSJw= google.golang.org/genproto/googleapis/api v0.0.0-20240711142825-46eb208f015d/go.mod h1:mw8MG/Qz5wfgYr6VqVCiZcHe/GJEfI+oGGDCohaVgB0= google.golang.org/genproto/googleapis/api v0.0.0-20240730163845-b1a4ccb954bf/go.mod h1:OFMYQFHJ4TM3JRlWDZhJbZfra2uqc3WLBZiaaqP4DtU= -google.golang.org/genproto/googleapis/api v0.0.0-20240814211410-ddb44dafa142 h1:wKguEg1hsxI2/L3hUYrpo1RVi48K+uTyzKqprwLXsb8= google.golang.org/genproto/googleapis/api v0.0.0-20240814211410-ddb44dafa142/go.mod h1:d6be+8HhtEtucleCbxpPW9PA9XwISACu8nvpPqF0BVo= +google.golang.org/genproto/googleapis/api v0.0.0-20240823204242-4ba0660f739c/go.mod h1:fO8wJzT2zbQbAjbIoos1285VfEIYKDDY+Dt+WpTkh6g= +google.golang.org/genproto/googleapis/api v0.0.0-20240826202546-f6391c0de4c7/go.mod h1:OCdP9MfskevB/rbYvHTsXTtKC+3bHWajPdoKgjcYkfo= +google.golang.org/genproto/googleapis/api v0.0.0-20240827150818-7e3bb234dfed h1:3RgNmBoI9MZhsj3QxC+AP/qQhNwpCLOvYDYYsFrhFt0= +google.golang.org/genproto/googleapis/api v0.0.0-20240827150818-7e3bb234dfed/go.mod h1:OCdP9MfskevB/rbYvHTsXTtKC+3bHWajPdoKgjcYkfo= google.golang.org/genproto/googleapis/bytestream v0.0.0-20230530153820-e85fd2cbaebc/go.mod h1:ylj+BE99M198VPbBh6A8d9n3w8fChvyLK3wwBOjXBFA= google.golang.org/genproto/googleapis/bytestream v0.0.0-20231012201019-e917dd12ba7a/go.mod h1:+34luvCflYKiKylNwGJfn9cFBbcL/WrkciMmDmsTQ/A= google.golang.org/genproto/googleapis/bytestream v0.0.0-20240205150955-31a09d347014/go.mod h1:EhZbXt+eY4Yr3YVaEGLdNZF5viWowOJZ8KTPqjYMKzg= @@ -908,8 +930,12 @@ google.golang.org/genproto/googleapis/rpc v0.0.0-20240711142825-46eb208f015d/go. google.golang.org/genproto/googleapis/rpc v0.0.0-20240722135656-d784300faade/go.mod h1:Ue6ibwXGpU+dqIcODieyLOcgj7z8+IcskoNIgZxtrFY= google.golang.org/genproto/googleapis/rpc v0.0.0-20240725223205-93522f1f2a9f/go.mod h1:Ue6ibwXGpU+dqIcODieyLOcgj7z8+IcskoNIgZxtrFY= google.golang.org/genproto/googleapis/rpc v0.0.0-20240730163845-b1a4ccb954bf/go.mod h1:Ue6ibwXGpU+dqIcODieyLOcgj7z8+IcskoNIgZxtrFY= -google.golang.org/genproto/googleapis/rpc v0.0.0-20240814211410-ddb44dafa142 h1:e7S5W7MGGLaSu8j3YjdezkZ+m1/Nm0uRVRMEMGk26Xs= google.golang.org/genproto/googleapis/rpc v0.0.0-20240814211410-ddb44dafa142/go.mod h1:UqMtugtsSgubUsoxbuAoiCXvqvErP7Gf0so0mK9tHxU= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240822170219-fc7c04adadcd/go.mod h1:UqMtugtsSgubUsoxbuAoiCXvqvErP7Gf0so0mK9tHxU= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240823204242-4ba0660f739c/go.mod h1:UqMtugtsSgubUsoxbuAoiCXvqvErP7Gf0so0mK9tHxU= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240826202546-f6391c0de4c7/go.mod h1:UqMtugtsSgubUsoxbuAoiCXvqvErP7Gf0so0mK9tHxU= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240827150818-7e3bb234dfed h1:J6izYgfBXAI3xTKLgxzTmUltdYaLsuBxFCgDHWJ/eXg= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240827150818-7e3bb234dfed/go.mod h1:UqMtugtsSgubUsoxbuAoiCXvqvErP7Gf0so0mK9tHxU= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= diff --git a/vertexai/go.mod b/vertexai/go.mod index f3d9ab55f456..61b225e37248 100644 --- a/vertexai/go.mod +++ b/vertexai/go.mod @@ -6,6 +6,7 @@ require ( cloud.google.com/go v0.115.1 cloud.google.com/go/aiplatform v1.68.0 github.com/google/go-cmp v0.6.0 + golang.org/x/text v0.17.0 google.golang.org/api v0.194.0 google.golang.org/genproto v0.0.0-20240814211410-ddb44dafa142 google.golang.org/protobuf v1.34.2 @@ -35,7 +36,6 @@ require ( golang.org/x/oauth2 v0.22.0 // indirect golang.org/x/sync v0.8.0 // indirect golang.org/x/sys v0.24.0 // indirect - golang.org/x/text v0.17.0 // indirect golang.org/x/time v0.6.0 // indirect google.golang.org/genproto/googleapis/api v0.0.0-20240814211410-ddb44dafa142 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20240814211410-ddb44dafa142 // indirect From e36a397f2beaf7b32aeae6514e046ddff68356af Mon Sep 17 00:00:00 2001 From: happy-qiao Date: Fri, 30 Aug 2024 13:26:26 -0700 Subject: [PATCH 3/8] resolve comments --- vertexai/genai/tokenizer/corpora_test.go | 101 +++++++++++++++-------- 1 file changed, 66 insertions(+), 35 deletions(-) diff --git a/vertexai/genai/tokenizer/corpora_test.go b/vertexai/genai/tokenizer/corpora_test.go index 0a5cb6b41f8a..fee024f3914b 100644 --- a/vertexai/genai/tokenizer/corpora_test.go +++ b/vertexai/genai/tokenizer/corpora_test.go @@ -20,6 +20,7 @@ import ( "fmt" "io" "log" + "os" "net/http" "regexp" "strings" @@ -33,17 +34,25 @@ import ( "golang.org/x/text/transform" ) -// FileInfo holds the name and content of a file in the zip archive -type CorporaInfo struct { +// corporaInfo holds the name and content of a file in the zip archive +type corporaInfo struct { Name string Content []byte } -// fileGenerator is a generator function that returns a channel to iterate over files in the zip archive -func CorporaGenerator(url string) (<-chan CorporaInfo, <-chan error) { - out := make(chan CorporaInfo) +// corporaGenerator is a generator function that returns a channel to iterate over files in the zip archive +func corporaGenerator(url string) (<-chan corporaInfo, <-chan error) { + out := make(chan corporaInfo) errCh := make(chan error, 1) + // This function is run as a goroutine because it performs a network request + // to download the zip file, which can take some time. Running it in a + // separate goroutine prevents the function from blocking the caller + // while the download is in progress. + // + // The goroutine writes the downloaded file information to the `out` channel + // and any errors to the `errCh` channel. This allows the caller to process + // files as they become available and handle errors gracefully. go func() { defer close(out) defer close(errCh) @@ -88,7 +97,7 @@ func CorporaGenerator(url string) (<-chan CorporaInfo, <-chan error) { } fileReader.Close() - out <- CorporaInfo{ + out <- corporaInfo{ Name: file.Name[len("udhr/"):], Content: content, } @@ -101,21 +110,44 @@ func CorporaGenerator(url string) (<-chan CorporaInfo, <-chan error) { return out, errCh } -// UdhrCorpusReader struct to hold encoding patterns and skip set -type UdhrCorpusReader struct { - Encodings []EncodingPattern +// udhrCorpus represents the Universal Declaration of Human Rights (UDHR) corpus. +// This corpus contains translations of the UDHR into many languages, +// stored in a specific directory structure within a zip archive. +// +// The files in the corpus follow a naming convention: +// _