From 2ffe5d5d649af1956edf7724d74902903d5d6ae5 Mon Sep 17 00:00:00 2001 From: Rakhat Zhuman Date: Sat, 1 Apr 2023 07:01:16 +0600 Subject: [PATCH 1/9] Add testcases to check upsert functionality Signed-off-by: Rakhat Zhuman --- CHANGELOG.md | 1 + .../bulk_indexer_integration_test.go | 336 +++++++++++------- 2 files changed, 215 insertions(+), 122 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index bd082d175..fe4700bd9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -20,6 +20,7 @@ Inspired from [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) - Adds `Err()` function to Response for detailed errors ([#246](https://github.com/opensearch-project/opensearch-go/pull/246)) - Adds Point In Time API ([#253](https://github.com/opensearch-project/opensearch-go/pull/253)) - Adds InfoResp type ([#253](https://github.com/opensearch-project/opensearch-go/pull/253)) +- Adds testcases to check upsert functionality ([#207](https://github.com/opensearch-project/opensearch-go/issues/207)) ### Changed diff --git a/opensearchutil/bulk_indexer_integration_test.go b/opensearchutil/bulk_indexer_integration_test.go index 3d8d00fa2..a07677af4 100644 --- a/opensearchutil/bulk_indexer_integration_test.go +++ b/opensearchutil/bulk_indexer_integration_test.go @@ -24,6 +24,7 @@ // specific language governing permissions and limitations // under the License. +//go:build integration // +build integration package opensearchutil_test @@ -34,7 +35,6 @@ import ( "os" "strconv" "strings" - "sync/atomic" "testing" "time" @@ -44,158 +44,250 @@ import ( ) func TestBulkIndexerIntegration(t *testing.T) { - body := `{"body":"Lorem ipsum dolor sit amet, consectetur adipisicing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat."}` + testRecordCount := uint64(10000) testCases := []struct { name string - CompressRequestBodyEnabled bool + compressRequestBodyEnabled bool + tests []struct { + name string + action string + body string + numItems uint64 + numIndexed uint64 + numCreated uint64 + numUpdated uint64 + numFailed uint64 + } }{ { - name: "Without body compression", - CompressRequestBodyEnabled: false, + name: "With body compression", + compressRequestBodyEnabled: true, + tests: []struct { + name string + action string + body string + numItems uint64 + numIndexed uint64 + numCreated uint64 + numUpdated uint64 + numFailed uint64 + }{ + { + name: "Index", + action: "index", + body: `{"title":"bar"}`, + numItems: testRecordCount, + numIndexed: testRecordCount, + numCreated: 0, + numUpdated: 0, + numFailed: 0, + }, + { + name: "Upsert", + action: "update", + body: `{"doc":{"title":"qwe"}, "doc_as_upsert": true}`, + numItems: testRecordCount, + numIndexed: 0, + numCreated: 0, + numUpdated: testRecordCount, + numFailed: 0, + }, + { + name: "Create", + action: "create", + body: `{"title":"bar"}`, + numItems: testRecordCount, + numIndexed: 0, + numCreated: 0, + numUpdated: 0, + numFailed: testRecordCount, + }, + }, }, { - name: "With body compression", - CompressRequestBodyEnabled: true, + name: "Without body compression", + compressRequestBodyEnabled: false, + tests: []struct { + name string + action string + body string + numItems uint64 + numIndexed uint64 + numCreated uint64 + numUpdated uint64 + numFailed uint64 + }{ + { + name: "Index", + action: "index", + body: `{"title":"bar"}`, + numItems: testRecordCount, + numIndexed: testRecordCount, + numCreated: 0, + numUpdated: 0, + numFailed: 0, + }, + { + name: "Upsert", + action: "update", + body: `{"doc":{"title":"qwe"}, "doc_as_upsert": true}`, + numItems: testRecordCount, + numIndexed: 0, + numCreated: 0, + numUpdated: testRecordCount, + numFailed: 0, + }, + { + name: "Create", + action: "create", + body: `{"title":"bar"}`, + numItems: testRecordCount, + numIndexed: 0, + numCreated: 0, + numUpdated: 0, + numFailed: testRecordCount, + }, + }, }, } - for _, tt := range testCases { - t.Run(tt.name, func(t *testing.T) { - t.Run("Default", func(t *testing.T) { - var countSuccessful uint64 - indexName := "test-bulk-integration" - - client, _ := opensearch.NewClient(opensearch.Config{ - CompressRequestBody: tt.CompressRequestBodyEnabled, - Logger: &opensearchtransport.ColorLogger{Output: os.Stdout}, - }) + for _, c := range testCases { + indexName := "test-bulk-integration" - client.Indices.Delete([]string{indexName}, client.Indices.Delete.WithIgnoreUnavailable(true)) - client.Indices.Create( - indexName, - client.Indices.Create.WithBody(strings.NewReader(`{"settings": {"number_of_shards": 1, "number_of_replicas": 0, "refresh_interval":"5s"}}`)), - client.Indices.Create.WithWaitForActiveShards("1")) + client, _ := opensearch.NewClient(opensearch.Config{ + CompressRequestBody: c.compressRequestBodyEnabled, + Logger: &opensearchtransport.ColorLogger{Output: os.Stdout}, + }) - bi, _ := opensearchutil.NewBulkIndexer(opensearchutil.BulkIndexerConfig{ - Index: indexName, - Client: client, - // FlushBytes: 3e+6, - }) + client.Indices.Delete([]string{indexName}, client.Indices.Delete.WithIgnoreUnavailable(true)) + client.Indices.Create( + indexName, + client.Indices.Create.WithBody(strings.NewReader(`{"settings": {"number_of_shards": 1, "number_of_replicas": 0, "refresh_interval":"5s"}}`)), + client.Indices.Create.WithWaitForActiveShards("1")) - numItems := 100000 - start := time.Now().UTC() - - for i := 1; i <= numItems; i++ { - err := bi.Add(context.Background(), opensearchutil.BulkIndexerItem{ - Action: "index", - DocumentID: strconv.Itoa(i), - Body: strings.NewReader(body), - OnSuccess: func(ctx context.Context, item opensearchutil.BulkIndexerItem, res opensearchutil.BulkIndexerResponseItem) { - atomic.AddUint64(&countSuccessful, 1) - }, + for _, tt := range c.tests { + t.Run(tt.name, func(t *testing.T) { + t.Run(c.name, func(t *testing.T) { + bi, _ := opensearchutil.NewBulkIndexer(opensearchutil.BulkIndexerConfig{ + Index: indexName, + Client: client, + ErrorTrace: true, + Human: true, + Pretty: true, + // FlushBytes: 3e+6, }) - if err != nil { - t.Fatalf("Unexpected error: %s", err) - } - } - if err := bi.Close(context.Background()); err != nil { - t.Errorf("Unexpected error: %s", err) - } + start := time.Now().UTC() - stats := bi.Stats() + for i := 1; i <= int(tt.numItems); i++ { + err := bi.Add(context.Background(), opensearchutil.BulkIndexerItem{ + Index: indexName, + Action: tt.action, + DocumentID: strconv.Itoa(i), + Body: strings.NewReader(tt.body), + }) + if err != nil { + t.Fatalf("Unexpected error: %s", err) + } + } - if stats.NumAdded != uint64(numItems) { - t.Errorf("Unexpected NumAdded: want=%d, got=%d", numItems, stats.NumAdded) - } + if err := bi.Close(context.Background()); err != nil { + t.Errorf("Unexpected error: %s", err) + } - if stats.NumIndexed != uint64(numItems) { - t.Errorf("Unexpected NumIndexed: want=%d, got=%d", numItems, stats.NumIndexed) - } + stats := bi.Stats() - if stats.NumFailed != 0 { - t.Errorf("Unexpected NumFailed: want=0, got=%d", stats.NumFailed) - } + if stats.NumAdded != tt.numItems { + t.Errorf("Unexpected NumAdded: want=%d, got=%d", tt.numItems, stats.NumAdded) + } - if countSuccessful != uint64(numItems) { - t.Errorf("Unexpected countSuccessful: want=%d, got=%d", numItems, countSuccessful) - } + if stats.NumIndexed != tt.numIndexed { + t.Errorf("Unexpected NumIndexed: want=%d, got=%d", tt.numItems, stats.NumIndexed) + } - fmt.Printf(" Added %d documents to indexer. Succeeded: %d. Failed: %d. Requests: %d. Duration: %s (%.0f docs/sec)\n", - stats.NumAdded, - stats.NumFlushed, - stats.NumFailed, - stats.NumRequests, - time.Since(start).Truncate(time.Millisecond), - 1000.0/float64(time.Since(start)/time.Millisecond)*float64(stats.NumFlushed)) - }) + if stats.NumUpdated != tt.numUpdated { + t.Errorf("Unexpected NumUpdated: want=%d, got=%d", tt.numUpdated, stats.NumUpdated) + } - t.Run("Multiple indices", func(t *testing.T) { - client, _ := opensearch.NewClient(opensearch.Config{ - CompressRequestBody: tt.CompressRequestBodyEnabled, - Logger: &opensearchtransport.ColorLogger{Output: os.Stdout}, - }) + if stats.NumCreated != tt.numCreated { + t.Errorf("Unexpected NumCreated: want=%d, got=%d", tt.numCreated, stats.NumCreated) + } - bi, _ := opensearchutil.NewBulkIndexer(opensearchutil.BulkIndexerConfig{ - Index: "test-index-a", - Client: client, + if stats.NumFailed != tt.numFailed { + t.Errorf("Unexpected NumFailed: want=0, got=%d", stats.NumFailed) + } + + fmt.Printf(" Added %d documents to indexer. Succeeded: %d. Failed: %d. Requests: %d. Duration: %s (%.0f docs/sec)\n", + stats.NumAdded, + stats.NumFlushed, + stats.NumFailed, + stats.NumRequests, + time.Since(start).Truncate(time.Millisecond), + 1000.0/float64(time.Since(start)/time.Millisecond)*float64(stats.NumFlushed)) }) - // Default index - for i := 1; i <= 10; i++ { - err := bi.Add(context.Background(), opensearchutil.BulkIndexerItem{ - Action: "index", - DocumentID: strconv.Itoa(i), - Body: strings.NewReader(body), + t.Run("Multiple indices", func(t *testing.T) { + bi, _ := opensearchutil.NewBulkIndexer(opensearchutil.BulkIndexerConfig{ + Index: "test-index-a", + Client: client, }) - if err != nil { - t.Fatalf("Unexpected error: %s", err) + + // Default index + for i := 1; i <= 10; i++ { + err := bi.Add(context.Background(), opensearchutil.BulkIndexerItem{ + Action: "index", + DocumentID: strconv.Itoa(i), + Body: strings.NewReader(tt.body), + }) + if err != nil { + t.Fatalf("Unexpected error: %s", err) + } } - } - - // Index 1 - for i := 1; i <= 10; i++ { - err := bi.Add(context.Background(), opensearchutil.BulkIndexerItem{ - Action: "index", - Index: "test-index-b", - Body: strings.NewReader(body), - }) - if err != nil { - t.Fatalf("Unexpected error: %s", err) + + // Index 1 + for i := 1; i <= 10; i++ { + err := bi.Add(context.Background(), opensearchutil.BulkIndexerItem{ + Action: "index", + Index: "test-index-b", + Body: strings.NewReader(tt.body), + }) + if err != nil { + t.Fatalf("Unexpected error: %s", err) + } } - } - - // Index 2 - for i := 1; i <= 10; i++ { - err := bi.Add(context.Background(), opensearchutil.BulkIndexerItem{ - Action: "index", - Index: "test-index-c", - Body: strings.NewReader(body), - }) + + // Index 2 + for i := 1; i <= 10; i++ { + err := bi.Add(context.Background(), opensearchutil.BulkIndexerItem{ + Action: "index", + Index: "test-index-c", + Body: strings.NewReader(tt.body), + }) + if err != nil { + t.Fatalf("Unexpected error: %s", err) + } + } + + if err := bi.Close(context.Background()); err != nil { + t.Errorf("Unexpected error: %s", err) + } + stats := bi.Stats() + + expectedIndexed := 10 + 10 + 10 + if stats.NumIndexed != uint64(expectedIndexed) { + t.Errorf("Unexpected NumIndexed: want=%d, got=%d", expectedIndexed, stats.NumIndexed) + } + + res, err := client.Indices.Exists([]string{"test-index-a", "test-index-b", "test-index-c"}) if err != nil { t.Fatalf("Unexpected error: %s", err) } - } - - if err := bi.Close(context.Background()); err != nil { - t.Errorf("Unexpected error: %s", err) - } - stats := bi.Stats() - - expectedIndexed := 10 + 10 + 10 - if stats.NumIndexed != uint64(expectedIndexed) { - t.Errorf("Unexpected NumIndexed: want=%d, got=%d", expectedIndexed, stats.NumIndexed) - } - - res, err := client.Indices.Exists([]string{"test-index-a", "test-index-b", "test-index-c"}) - if err != nil { - t.Fatalf("Unexpected error: %s", err) - } - if res.StatusCode != 200 { - t.Errorf("Expected indices to exist, but got a [%s] response", res.Status()) - } + if res.StatusCode != 200 { + t.Errorf("Expected indices to exist, but got a [%s] response", res.Status()) + } + }) }) - }) + } } } From b134910096e6c38be4620951102c033ffff51e14 Mon Sep 17 00:00:00 2001 From: Rakhat Zhuman Date: Thu, 6 Apr 2023 19:21:50 +0600 Subject: [PATCH 2/9] Changed type of _source field Signed-off-by: Rakhat Zhuman --- opensearchapi/api.bulk.go | 18 +- opensearchapi/api.delete_by_query.go | 18 +- opensearchapi/api.document_test.go | 327 +++++++++++++++++++++++++++ opensearchapi/api.exists.go | 20 +- opensearchapi/api.exists_source.go | 20 +- opensearchapi/api.explain.go | 20 +- opensearchapi/api.get.go | 17 +- opensearchapi/api.get_source.go | 20 +- opensearchapi/api.mget.go | 18 +- opensearchapi/api.search.go | 18 +- opensearchapi/api.update.go | 20 +- opensearchapi/api.update_by_query.go | 18 +- 12 files changed, 473 insertions(+), 61 deletions(-) create mode 100644 opensearchapi/api.document_test.go diff --git a/opensearchapi/api.bulk.go b/opensearchapi/api.bulk.go index 261deb0d0..378b1e8a2 100644 --- a/opensearchapi/api.bulk.go +++ b/opensearchapi/api.bulk.go @@ -55,7 +55,7 @@ type Bulk func(body io.Reader, o ...func(*BulkRequest)) (*Response, error) // BulkRequest configures the Bulk API request. // type BulkRequest struct { - Index string + Index string Body io.Reader @@ -63,7 +63,7 @@ type BulkRequest struct { Refresh string RequireAlias *bool Routing string - Source []string + Source interface{} SourceExcludes []string SourceIncludes []string Timeout time.Duration @@ -116,8 +116,16 @@ func (r BulkRequest) Do(ctx context.Context, transport Transport) (*Response, er params["routing"] = r.Routing } - if len(r.Source) > 0 { - params["_source"] = strings.Join(r.Source, ",") + if source, ok := r.Source.(bool); ok { + params["_source"] = strconv.FormatBool(source) + } + + if source, ok := r.Source.(string); ok && source != "" { + params["_source"] = source + } + + if sources, ok := r.Source.([]string); ok && len(sources) > 0 { + params["_source"] = strings.Join(sources, ",") } if len(r.SourceExcludes) > 0 { @@ -249,7 +257,7 @@ func (f Bulk) WithRouting(v string) func(*BulkRequest) { // WithSource - true or false to return the _source field or not, or default list of fields to return, can be overridden on each sub-request. // -func (f Bulk) WithSource(v ...string) func(*BulkRequest) { +func (f Bulk) WithSource(v interface{}) func(*BulkRequest) { return func(r *BulkRequest) { r.Source = v } diff --git a/opensearchapi/api.delete_by_query.go b/opensearchapi/api.delete_by_query.go index 0e9509bc1..fa2616165 100644 --- a/opensearchapi/api.delete_by_query.go +++ b/opensearchapi/api.delete_by_query.go @@ -56,7 +56,7 @@ type DeleteByQuery func(index []string, body io.Reader, o ...func(*DeleteByQuery // DeleteByQueryRequest configures the Delete By Query API request. // type DeleteByQueryRequest struct { - Index []string + Index []string Body io.Reader @@ -84,7 +84,7 @@ type DeleteByQueryRequest struct { Size *int Slices interface{} Sort []string - Source []string + Source interface{} SourceExcludes []string SourceIncludes []string Stats []string @@ -219,8 +219,16 @@ func (r DeleteByQueryRequest) Do(ctx context.Context, transport Transport) (*Res params["sort"] = strings.Join(r.Sort, ",") } - if len(r.Source) > 0 { - params["_source"] = strings.Join(r.Source, ",") + if source, ok := r.Source.(bool); ok { + params["_source"] = strconv.FormatBool(source) + } + + if source, ok := r.Source.(string); ok && source != "" { + params["_source"] = source + } + + if sources, ok := r.Source.([]string); ok && len(sources) > 0 { + params["_source"] = strings.Join(sources, ",") } if len(r.SourceExcludes) > 0 { @@ -520,7 +528,7 @@ func (f DeleteByQuery) WithSort(v ...string) func(*DeleteByQueryRequest) { // WithSource - true or false to return the _source field or not, or a list of fields to return. // -func (f DeleteByQuery) WithSource(v ...string) func(*DeleteByQueryRequest) { +func (f DeleteByQuery) WithSource(v interface{}) func(*DeleteByQueryRequest) { return func(r *DeleteByQueryRequest) { r.Source = v } diff --git a/opensearchapi/api.document_test.go b/opensearchapi/api.document_test.go new file mode 100644 index 000000000..7687dfa12 --- /dev/null +++ b/opensearchapi/api.document_test.go @@ -0,0 +1,327 @@ +// SPDX-License-Identifier: Apache-2.0 +// +// The OpenSearch Contributors require contributions made to +// this file be licensed under the Apache-2.0 license or a +// compatible open source license. +// +// Modifications Copyright OpenSearch Contributors. See +// GitHub history for details. + +//go:build integration +// +build integration + +package opensearchapi_test + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "github.com/opensearch-project/opensearch-go/v2" + "github.com/opensearch-project/opensearch-go/v2/opensearchapi" + "github.com/stretchr/testify/require" + "io/ioutil" + "net/http" + "regexp" + "strings" + "testing" + "time" +) + +func TestDocumentRequest_Do(t *testing.T) { + index := fmt.Sprintf("demo-index-%s", time.Now().Format("2006-01-02-15-04-05")) + + tests := []struct { + name string + r DataStreamRequest + want *opensearchapi.Response + wantBody string + wantErr bool + }{ + // Create document + { + name: "TestCreateRequest_Do", + r: opensearchapi.CreateRequest{ + Index: index, + DocumentID: "1", + Body: strings.NewReader(`{ "title": "Moneyball", "director": "Bennett Miller", "year": "2011" }`), + }, + want: &opensearchapi.Response{ + StatusCode: 201, + Header: http.Header{ + "Content-Type": []string{"application/json; charset=UTF-8"}, + "Location": []string{fmt.Sprintf("/%s/_doc/1", index)}, + }, + }, + wantBody: fmt.Sprintf(`{"_index":"%s","_id":"1","_version":1,"result":"created","_shards":{"total":2,"successful":1,"failed":0},"_seq_no":0,"_primary_term":1}`, index), + wantErr: false, + }, + { + name: "TestCreateRequest_Do", + r: opensearchapi.CreateRequest{ + Index: index, + DocumentID: "2", + Body: strings.NewReader(`{ "title": "Tenet", "director": "Christopher Nolan", "year": "2019" }`), + }, + want: &opensearchapi.Response{ + StatusCode: 201, + Header: http.Header{ + "Content-Type": []string{"application/json; charset=UTF-8"}, + "Location": []string{fmt.Sprintf("/%s/_doc/2", index)}, + }, + }, + wantBody: fmt.Sprintf(`{"_index":"%s","_id":"2","_version":1,"result":"created","_shards":{"total":2,"successful":1,"failed":0},"_seq_no":1,"_primary_term":1}`, index), + wantErr: false, + }, + + // Get document + { + name: "TestGetRequest_Do", + r: opensearchapi.GetRequest{ + Index: index, + DocumentID: "2", + Source: true, + }, + want: &opensearchapi.Response{ + StatusCode: 200, + Header: http.Header{ + "Content-Type": []string{"application/json; charset=UTF-8"}, + }, + }, + wantBody: fmt.Sprintf(`{"_id":"2", "_index":"%s", "_primary_term":1, "_seq_no":1, "_source": {"director":"Christopher Nolan", "title":"Tenet", "year":"2019"}, "_version":1, "found":true}`, index), + wantErr: false, + }, + // Get multiple documents + { + name: "TestMultiGetRequest_Do. Source parameter is a bool and slice of strings", + r: opensearchapi.MgetRequest{ + Index: index, + Body: strings.NewReader(`{ "docs": [ { "_id": "1", "_source": true }, { "_id": "2", "_source": [ "title" ] } ] }`), + // seems to does not work + Source: false, + }, + want: &opensearchapi.Response{ + StatusCode: 200, + Header: http.Header{ + "Content-Type": []string{"application/json; charset=UTF-8"}, + }, + }, + wantBody: fmt.Sprintf(`{ "docs": [ { "_id": "1", "_index": "%s", "_primary_term": 1, "_seq_no": 0, "_source": { "director": "Bennett Miller", "title": "Moneyball", "year": "2011" }, "_version": 1, "found": true }, { "_id": "2", "_index": "%s", "_primary_term": 1, "_seq_no": 1, "_source": {"title":"Tenet"}, "_version": 1, "found": true } ] }`, index, index), + wantErr: false, + }, + // Get source document + { + name: "TestGetSourceRequest_Do. Source parameter is a bool and slice of strings", + r: opensearchapi.GetSourceRequest{ + Index: index, + DocumentID: "2", + }, + want: &opensearchapi.Response{ + StatusCode: 200, + Header: http.Header{ + "Content-Type": []string{"application/json; charset=UTF-8"}, + }, + }, + wantBody: `{"director":"Christopher Nolan", "title":"Tenet", "year":"2019"}`, + wantErr: false, + }, + + // Exists document + { + name: "TestExistsRequest_Do", + r: opensearchapi.ExistsRequest{ + Index: index, + DocumentID: "2", + Source: true, + }, + want: &opensearchapi.Response{ + StatusCode: 200, + Header: http.Header{ + "Content-Type": []string{"application/json; charset=UTF-8"}, + "Content-Length": []string{"189"}, + }, + }, + wantBody: ``, + wantErr: false, + }, + + // Explain document + //{ + // name: "TestExplainRequest_Do", + // r: opensearchapi.ExplainRequest{ + // Index: index, + // DocumentID: "2", + // Body: strings.NewReader(`{ "query": { "match": {"year": "2019" } } }`), + // Human: true, + // ErrorTrace: true, + // }, + // want: &opensearchapi.Response{ + // StatusCode: 200, + // Header: http.Header{ + // "Content-Type": []string{"application/json; charset=UTF-8"}, + // }, + // }, + // wantBody: ``, + // wantErr: false, + //}, + + // Search document + { + name: "TestSearchRequest_Do. Source parameter is a slice of strings", + r: opensearchapi.SearchRequest{ + Index: []string{index}, + Body: strings.NewReader(`{ "query": { "match": { "title": "Tenet" } } }`), + //Source: true, + }, + want: &opensearchapi.Response{ + StatusCode: 200, + Header: http.Header{ + "Content-Type": []string{"application/json; charset=UTF-8"}, + }, + }, + wantBody: fmt.Sprintf(`{"_shards": {"failed":0, "skipped":0, "successful":4, "total":4}, "hits":{"hits":[], "max_score": null, "total": {"relation":"eq", "value":0}}, "timed_out":false, "took":0}`), + wantErr: false, + }, + + // Update document + { + name: "TestUpdateRequest_Do", + r: opensearchapi.UpdateRequest{ + Index: index, + DocumentID: "1", + Body: strings.NewReader(`{ "doc": { "title": "Moneyball", "director": "Bennett", "year": "2012" } }`), + Source: nil, + }, + want: &opensearchapi.Response{ + StatusCode: 200, + Header: http.Header{ + "Content-Type": []string{"application/json; charset=UTF-8"}, + }, + }, + wantBody: fmt.Sprintf(`{"_index":"%s","_id":"1","_version":2,"result":"updated","_shards":{"total":2,"successful":1,"failed":0},"_seq_no":2,"_primary_term":1}`, index), + wantErr: false, + }, + { + name: "TestUpdateRequest_Do. Source parameter is bool", + r: opensearchapi.UpdateRequest{ + Index: index, + DocumentID: "1", + Body: strings.NewReader(`{ "doc": { "title": "Moneyball", "director": "Bennett", "year": "2012" } }`), + Source: true, + }, + want: &opensearchapi.Response{ + StatusCode: 200, + Header: http.Header{ + "Content-Type": []string{"application/json; charset=UTF-8"}, + }, + }, + wantBody: fmt.Sprintf(`{"_index":"%s","_id":"1","_version":2,"result":"noop","_shards":{"total":0,"successful":0,"failed":0},"_seq_no":2,"_primary_term":1,"get":{"_seq_no":2,"_primary_term":1,"found":true,"_source":{"title":"Moneyball","director":"Bennett","year":"2012"}}}`, index), + wantErr: false, + }, + { + name: "TestUpdateRequest_Do. Source parameter is a slice of strings", + r: opensearchapi.UpdateRequest{ + Index: index, + DocumentID: "1", + Body: strings.NewReader(`{ "doc": { "title": "Moneyball", "director": "Bennett", "year": "2012" } }`), + Source: []string{"true"}, + }, + want: &opensearchapi.Response{ + StatusCode: 200, + Header: http.Header{ + "Content-Type": []string{"application/json; charset=UTF-8"}, + }, + }, + wantBody: fmt.Sprintf(`{"_index":"%s","_id":"1","_version":2,"result":"noop","_shards":{"total":0,"successful":0,"failed":0},"_seq_no":2,"_primary_term":1,"get":{"_seq_no":2,"_primary_term":1,"found":true,"_source":{"title":"Moneyball","director":"Bennett","year":"2012"}}}`, index), + wantErr: false, + }, + { + name: "TestUpdateRequest_Do. Source Excludes", + r: opensearchapi.UpdateRequest{ + Index: index, + DocumentID: "1", + Body: strings.NewReader(`{ "doc": { "title": "Moneyball", "director": "Bennett", "year": "2012" } }`), + Source: []string{"true"}, + SourceExcludes: []string{"director"}, + }, + want: &opensearchapi.Response{ + StatusCode: 200, + Header: http.Header{ + "Content-Type": []string{"application/json; charset=UTF-8"}, + }, + }, + wantBody: fmt.Sprintf(`{"_index":"%s","_id":"1","_version":2,"result":"noop","_shards":{"total":0,"successful":0,"failed":0},"_seq_no":2,"_primary_term":1,"get":{"_seq_no":2,"_primary_term":1,"found":true,"_source":{"title":"Moneyball","year":"2012"}}}`, index), + wantErr: false, + }, + + // Bulk document + { + name: "TestBulkRequest_Do.", + r: opensearchapi.BulkRequest{ + Index: index, + Body: strings.NewReader(`{ "index": { "_index": "movies", "_id": "tt1979320" } } +{ "title": "Rush", "year": 2013 } +{ "create": { "_index": "movies", "_id": "tt1392214" } } +{ "title": "Prisoners", "year": 2013 } +{ "update": { "_index": "movies", "_id": "tt0816711" } } +{ "doc" : { "title": "World War Z" } } +`), + Source: []string{"true"}, + }, + want: &opensearchapi.Response{ + StatusCode: 200, + Header: http.Header{ + "Content-Type": []string{"application/json; charset=UTF-8"}, + }, + }, + wantErr: false, + }, + } + + client, err := opensearch.NewDefaultClient() + require.NoError(t, err) + + iCreate := opensearchapi.IndicesCreateRequest{ + Index: index, + Pretty: true, + Human: true, + ErrorTrace: true, + Body: strings.NewReader(fmt.Sprintf(`{"settings": {"index": {"number_of_shards": 4}}}`)), + } + + iCreateResponse, err := iCreate.Do(context.Background(), client) + require.NoError(t, err) + require.Equalf(t, false, iCreateResponse.IsError(), + "Error when creating index template: %s", iCreateResponse.String()) + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := tt.r.Do(context.Background(), client) + if (err != nil) != tt.wantErr { + t.Errorf("Do() error = %+v, wantErr %v", err, tt.wantErr) + return + } + + require.Equalf(t, got.IsError(), tt.wantErr, "Do() got = %v, want %v", got.IsError(), tt.wantErr) + require.Equalf(t, got.StatusCode, tt.want.StatusCode, "Do() got = %v, want %v", got.StatusCode, tt.want.StatusCode) + + if tt.wantBody != "" { + require.Equalf(t, got.Header, tt.want.Header, "Do() got = %v, want %v", got.Header, tt.want.Header) + + defer got.Body.Close() + body, err := ioutil.ReadAll(got.Body) + require.NoError(t, err) + + buffer := new(bytes.Buffer) + err = json.Compact(buffer, body) + require.NoError(t, err) + + // ignore took field, since it is dynamic + took := regexp.MustCompile(`"took":\d+`) + actual := took.ReplaceAllString(buffer.String(), `"took":0`) + + require.JSONEqf(t, tt.wantBody, actual, "Do() got = %v, want %v", got.String(), tt.wantBody) + } + }) + } +} diff --git a/opensearchapi/api.exists.go b/opensearchapi/api.exists.go index 7f2eb1b21..6aa046d75 100644 --- a/opensearchapi/api.exists.go +++ b/opensearchapi/api.exists.go @@ -53,14 +53,14 @@ type Exists func(index string, id string, o ...func(*ExistsRequest)) (*Response, // ExistsRequest configures the Exists API request. // type ExistsRequest struct { - Index string - DocumentID string + Index string + DocumentID string Preference string Realtime *bool Refresh *bool Routing string - Source []string + Source interface{} SourceExcludes []string SourceIncludes []string StoredFields []string @@ -113,8 +113,16 @@ func (r ExistsRequest) Do(ctx context.Context, transport Transport) (*Response, params["routing"] = r.Routing } - if len(r.Source) > 0 { - params["_source"] = strings.Join(r.Source, ",") + if source, ok := r.Source.(bool); ok { + params["_source"] = strconv.FormatBool(source) + } + + if source, ok := r.Source.(string); ok && source != "" { + params["_source"] = source + } + + if sources, ok := r.Source.([]string); ok && len(sources) > 0 { + params["_source"] = strings.Join(sources, ",") } if len(r.SourceExcludes) > 0 { @@ -238,7 +246,7 @@ func (f Exists) WithRouting(v string) func(*ExistsRequest) { // WithSource - true or false to return the _source field or not, or a list of fields to return. // -func (f Exists) WithSource(v ...string) func(*ExistsRequest) { +func (f Exists) WithSource(v interface{}) func(*ExistsRequest) { return func(r *ExistsRequest) { r.Source = v } diff --git a/opensearchapi/api.exists_source.go b/opensearchapi/api.exists_source.go index 0ae58dc60..b911dfb56 100644 --- a/opensearchapi/api.exists_source.go +++ b/opensearchapi/api.exists_source.go @@ -53,14 +53,14 @@ type ExistsSource func(index string, id string, o ...func(*ExistsSourceRequest)) // ExistsSourceRequest configures the Exists Source API request. // type ExistsSourceRequest struct { - Index string - DocumentID string + Index string + DocumentID string Preference string Realtime *bool Refresh *bool Routing string - Source []string + Source interface{} SourceExcludes []string SourceIncludes []string Version *int @@ -113,8 +113,16 @@ func (r ExistsSourceRequest) Do(ctx context.Context, transport Transport) (*Resp params["routing"] = r.Routing } - if len(r.Source) > 0 { - params["_source"] = strings.Join(r.Source, ",") + if source, ok := r.Source.(bool); ok { + params["_source"] = strconv.FormatBool(source) + } + + if source, ok := r.Source.(string); ok && source != "" { + params["_source"] = source + } + + if sources, ok := r.Source.([]string); ok && len(sources) > 0 { + params["_source"] = strings.Join(sources, ",") } if len(r.SourceExcludes) > 0 { @@ -234,7 +242,7 @@ func (f ExistsSource) WithRouting(v string) func(*ExistsSourceRequest) { // WithSource - true or false to return the _source field or not, or a list of fields to return. // -func (f ExistsSource) WithSource(v ...string) func(*ExistsSourceRequest) { +func (f ExistsSource) WithSource(v interface{}) func(*ExistsSourceRequest) { return func(r *ExistsSourceRequest) { r.Source = v } diff --git a/opensearchapi/api.explain.go b/opensearchapi/api.explain.go index 0d7440bb1..0682a86f4 100644 --- a/opensearchapi/api.explain.go +++ b/opensearchapi/api.explain.go @@ -54,8 +54,8 @@ type Explain func(index string, id string, o ...func(*ExplainRequest)) (*Respons // ExplainRequest configures the Explain API request. // type ExplainRequest struct { - Index string - DocumentID string + Index string + DocumentID string Body io.Reader @@ -67,7 +67,7 @@ type ExplainRequest struct { Preference string Query string Routing string - Source []string + Source interface{} SourceExcludes []string SourceIncludes []string StoredFields []string @@ -135,8 +135,16 @@ func (r ExplainRequest) Do(ctx context.Context, transport Transport) (*Response, params["routing"] = r.Routing } - if len(r.Source) > 0 { - params["_source"] = strings.Join(r.Source, ",") + if source, ok := r.Source.(bool); ok { + params["_source"] = strconv.FormatBool(source) + } + + if source, ok := r.Source.(string); ok && source != "" { + params["_source"] = source + } + + if sources, ok := r.Source.([]string); ok && len(sources) > 0 { + params["_source"] = strings.Join(sources, ",") } if len(r.SourceExcludes) > 0 { @@ -296,7 +304,7 @@ func (f Explain) WithRouting(v string) func(*ExplainRequest) { // WithSource - true or false to return the _source field or not, or a list of fields to return. // -func (f Explain) WithSource(v ...string) func(*ExplainRequest) { +func (f Explain) WithSource(v interface{}) func(*ExplainRequest) { return func(r *ExplainRequest) { r.Source = v } diff --git a/opensearchapi/api.get.go b/opensearchapi/api.get.go index 4f3ba388c..889176a0b 100644 --- a/opensearchapi/api.get.go +++ b/opensearchapi/api.get.go @@ -53,14 +53,14 @@ type Get func(index string, id string, o ...func(*GetRequest)) (*Response, error // GetRequest configures the Get API request. // type GetRequest struct { - Index string - DocumentID string + Index string + DocumentID string Preference string Realtime *bool Refresh *bool Routing string - Source []string + Source interface{} SourceExcludes []string SourceIncludes []string StoredFields []string @@ -113,8 +113,13 @@ func (r GetRequest) Do(ctx context.Context, transport Transport) (*Response, err params["routing"] = r.Routing } - if len(r.Source) > 0 { - params["_source"] = strings.Join(r.Source, ",") + if source, ok := r.Source.(bool); ok { + params["_source"] = strconv.FormatBool(source) + } else { + sources, ok := r.Source.([]string) + if ok && len(sources) > 0 { + params["_source"] = strings.Join(sources, ",") + } } if len(r.SourceExcludes) > 0 { @@ -238,7 +243,7 @@ func (f Get) WithRouting(v string) func(*GetRequest) { // WithSource - true or false to return the _source field or not, or a list of fields to return. // -func (f Get) WithSource(v ...string) func(*GetRequest) { +func (f Get) WithSource(v interface{}) func(*GetRequest) { return func(r *GetRequest) { r.Source = v } diff --git a/opensearchapi/api.get_source.go b/opensearchapi/api.get_source.go index 42341dbd4..5bebc5fb4 100644 --- a/opensearchapi/api.get_source.go +++ b/opensearchapi/api.get_source.go @@ -53,14 +53,14 @@ type GetSource func(index string, id string, o ...func(*GetSourceRequest)) (*Res // GetSourceRequest configures the Get Source API request. // type GetSourceRequest struct { - Index string - DocumentID string + Index string + DocumentID string Preference string Realtime *bool Refresh *bool Routing string - Source []string + Source interface{} SourceExcludes []string SourceIncludes []string Version *int @@ -113,8 +113,16 @@ func (r GetSourceRequest) Do(ctx context.Context, transport Transport) (*Respons params["routing"] = r.Routing } - if len(r.Source) > 0 { - params["_source"] = strings.Join(r.Source, ",") + if source, ok := r.Source.(bool); ok { + params["_source"] = strconv.FormatBool(source) + } + + if source, ok := r.Source.(string); ok && source != "" { + params["_source"] = source + } + + if sources, ok := r.Source.([]string); ok && len(sources) > 0 { + params["_source"] = strings.Join(sources, ",") } if len(r.SourceExcludes) > 0 { @@ -234,7 +242,7 @@ func (f GetSource) WithRouting(v string) func(*GetSourceRequest) { // WithSource - true or false to return the _source field or not, or a list of fields to return. // -func (f GetSource) WithSource(v ...string) func(*GetSourceRequest) { +func (f GetSource) WithSource(v interface{}) func(*GetSourceRequest) { return func(r *GetSourceRequest) { r.Source = v } diff --git a/opensearchapi/api.mget.go b/opensearchapi/api.mget.go index 6555a6552..f3d7f1ab3 100644 --- a/opensearchapi/api.mget.go +++ b/opensearchapi/api.mget.go @@ -54,7 +54,7 @@ type Mget func(body io.Reader, o ...func(*MgetRequest)) (*Response, error) // MgetRequest configures the Mget API request. // type MgetRequest struct { - Index string + Index string Body io.Reader @@ -62,7 +62,7 @@ type MgetRequest struct { Realtime *bool Refresh *bool Routing string - Source []string + Source interface{} SourceExcludes []string SourceIncludes []string StoredFields []string @@ -114,8 +114,16 @@ func (r MgetRequest) Do(ctx context.Context, transport Transport) (*Response, er params["routing"] = r.Routing } - if len(r.Source) > 0 { - params["_source"] = strings.Join(r.Source, ",") + if source, ok := r.Source.(bool); ok { + params["_source"] = strconv.FormatBool(source) + } + + if source, ok := r.Source.(string); ok && source != "" { + params["_source"] = source + } + + if sources, ok := r.Source.([]string); ok && len(sources) > 0 { + params["_source"] = strings.Join(sources, ",") } if len(r.SourceExcludes) > 0 { @@ -243,7 +251,7 @@ func (f Mget) WithRouting(v string) func(*MgetRequest) { // WithSource - true or false to return the _source field or not, or a list of fields to return. // -func (f Mget) WithSource(v ...string) func(*MgetRequest) { +func (f Mget) WithSource(v interface{}) func(*MgetRequest) { return func(r *MgetRequest) { r.Source = v } diff --git a/opensearchapi/api.search.go b/opensearchapi/api.search.go index 9797e5444..c78e581ca 100644 --- a/opensearchapi/api.search.go +++ b/opensearchapi/api.search.go @@ -56,7 +56,7 @@ type Search func(o ...func(*SearchRequest)) (*Response, error) // SearchRequest configures the Search API request. // type SearchRequest struct { - Index []string + Index []string Body io.Reader @@ -88,7 +88,7 @@ type SearchRequest struct { SeqNoPrimaryTerm *bool Size *int Sort []string - Source []string + Source interface{} SourceExcludes []string SourceIncludes []string Stats []string @@ -247,8 +247,16 @@ func (r SearchRequest) Do(ctx context.Context, transport Transport) (*Response, params["sort"] = strings.Join(r.Sort, ",") } - if len(r.Source) > 0 { - params["_source"] = strings.Join(r.Source, ",") + if source, ok := r.Source.(bool); ok { + params["_source"] = strconv.FormatBool(source) + } + + if source, ok := r.Source.(string); ok && source != "" { + params["_source"] = source + } + + if sources, ok := r.Source.([]string); ok && len(sources) > 0 { + params["_source"] = strings.Join(sources, ",") } if len(r.SourceExcludes) > 0 { @@ -620,7 +628,7 @@ func (f Search) WithSort(v ...string) func(*SearchRequest) { // WithSource - true or false to return the _source field or not, or a list of fields to return. // -func (f Search) WithSource(v ...string) func(*SearchRequest) { +func (f Search) WithSource(v interface{}) func(*SearchRequest) { return func(r *SearchRequest) { r.Source = v } diff --git a/opensearchapi/api.update.go b/opensearchapi/api.update.go index 61442601f..30742011d 100644 --- a/opensearchapi/api.update.go +++ b/opensearchapi/api.update.go @@ -55,8 +55,8 @@ type Update func(index string, id string, body io.Reader, o ...func(*UpdateReque // UpdateRequest configures the Update API request. // type UpdateRequest struct { - Index string - DocumentID string + Index string + DocumentID string Body io.Reader @@ -67,7 +67,7 @@ type UpdateRequest struct { RequireAlias *bool RetryOnConflict *int Routing string - Source []string + Source interface{} SourceExcludes []string SourceIncludes []string Timeout time.Duration @@ -132,8 +132,16 @@ func (r UpdateRequest) Do(ctx context.Context, transport Transport) (*Response, params["routing"] = r.Routing } - if len(r.Source) > 0 { - params["_source"] = strings.Join(r.Source, ",") + if source, ok := r.Source.(bool); ok { + params["_source"] = strconv.FormatBool(source) + } + + if source, ok := r.Source.(string); ok && source != "" { + params["_source"] = source + } + + if sources, ok := r.Source.([]string); ok && len(sources) > 0 { + params["_source"] = strings.Join(sources, ",") } if len(r.SourceExcludes) > 0 { @@ -281,7 +289,7 @@ func (f Update) WithRouting(v string) func(*UpdateRequest) { // WithSource - true or false to return the _source field or not, or a list of fields to return. // -func (f Update) WithSource(v ...string) func(*UpdateRequest) { +func (f Update) WithSource(v interface{}) func(*UpdateRequest) { return func(r *UpdateRequest) { r.Source = v } diff --git a/opensearchapi/api.update_by_query.go b/opensearchapi/api.update_by_query.go index 22d30beb2..bb2d49718 100644 --- a/opensearchapi/api.update_by_query.go +++ b/opensearchapi/api.update_by_query.go @@ -57,7 +57,7 @@ type UpdateByQuery func(index []string, o ...func(*UpdateByQueryRequest)) (*Resp // UpdateByQueryRequest configures the Update By Query API request. // type UpdateByQueryRequest struct { - Index []string + Index []string Body io.Reader @@ -86,7 +86,7 @@ type UpdateByQueryRequest struct { Size *int Slices interface{} Sort []string - Source []string + Source interface{} SourceExcludes []string SourceIncludes []string Stats []string @@ -226,8 +226,16 @@ func (r UpdateByQueryRequest) Do(ctx context.Context, transport Transport) (*Res params["sort"] = strings.Join(r.Sort, ",") } - if len(r.Source) > 0 { - params["_source"] = strings.Join(r.Source, ",") + if source, ok := r.Source.(bool); ok { + params["_source"] = strconv.FormatBool(source) + } + + if source, ok := r.Source.(string); ok && source != "" { + params["_source"] = source + } + + if sources, ok := r.Source.([]string); ok && len(sources) > 0 { + params["_source"] = strings.Join(sources, ",") } if len(r.SourceExcludes) > 0 { @@ -547,7 +555,7 @@ func (f UpdateByQuery) WithSort(v ...string) func(*UpdateByQueryRequest) { // WithSource - true or false to return the _source field or not, or a list of fields to return. // -func (f UpdateByQuery) WithSource(v ...string) func(*UpdateByQueryRequest) { +func (f UpdateByQuery) WithSource(v interface{}) func(*UpdateByQueryRequest) { return func(r *UpdateByQueryRequest) { r.Source = v } From 12ba3c2d50d2afb2ba5e14a6e55518f39d7aff13 Mon Sep 17 00:00:00 2001 From: Rakhat Zhuman Date: Thu, 6 Apr 2023 20:17:01 +0600 Subject: [PATCH 3/9] Fixed changes in CHANGELOG.md Signed-off-by: Rakhat Zhuman --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index b16a9db3f..a6800d635 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -24,6 +24,8 @@ Inspired from [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) - Adds markdown linter ([#261](https://github.com/opensearch-project/opensearch-go/pull/261)) - Adds testcases to check upsert functionality ([#207](https://github.com/opensearch-project/opensearch-go/issues/207)) - Added @Jakob3xD to co-maintainers ([#270](https://github.com/opensearch-project/opensearch-go/pull/270)) +- Adds dynamic type to _source field ([#158](https://github.com/opensearch-project/opensearch-go/issues/158)) +- Adds testcases for Document API ([#280](https://github.com/opensearch-project/opensearch-go/issues/280)) ### Changed From 0e755f03f0f6a10b868869bfd1eb939cc52fc295 Mon Sep 17 00:00:00 2001 From: Rakhat Zhuman Date: Thu, 6 Apr 2023 20:44:33 +0600 Subject: [PATCH 4/9] Fixed markdown linter issues Signed-off-by: Rakhat Zhuman --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a6800d635..17e52d100 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -24,7 +24,7 @@ Inspired from [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) - Adds markdown linter ([#261](https://github.com/opensearch-project/opensearch-go/pull/261)) - Adds testcases to check upsert functionality ([#207](https://github.com/opensearch-project/opensearch-go/issues/207)) - Added @Jakob3xD to co-maintainers ([#270](https://github.com/opensearch-project/opensearch-go/pull/270)) -- Adds dynamic type to _source field ([#158](https://github.com/opensearch-project/opensearch-go/issues/158)) +- Adds dynamic type to \_source field ([#158](https://github.com/opensearch-project/opensearch-go/issues/158)) - Adds testcases for Document API ([#280](https://github.com/opensearch-project/opensearch-go/issues/280)) ### Changed From f331c6c9a4b41b926a72fbfc148a5d483c72e010 Mon Sep 17 00:00:00 2001 From: Rakhat Zhuman Date: Thu, 6 Apr 2023 21:13:32 +0600 Subject: [PATCH 5/9] Fixed version conflicts Signed-off-by: Rakhat Zhuman --- opensearchapi/api.document_test.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/opensearchapi/api.document_test.go b/opensearchapi/api.document_test.go index 7687dfa12..2997ba29e 100644 --- a/opensearchapi/api.document_test.go +++ b/opensearchapi/api.document_test.go @@ -29,7 +29,7 @@ import ( ) func TestDocumentRequest_Do(t *testing.T) { - index := fmt.Sprintf("demo-index-%s", time.Now().Format("2006-01-02-15-04-05")) + index := fmt.Sprintf("index-%s", time.Now().Format("2006-01-02-15-04-05")) tests := []struct { name string @@ -319,6 +319,8 @@ func TestDocumentRequest_Do(t *testing.T) { // ignore took field, since it is dynamic took := regexp.MustCompile(`"took":\d+`) actual := took.ReplaceAllString(buffer.String(), `"took":0`) + // ignore _type field, since it is legacy + actual = strings.ReplaceAll(actual, `"_type":"_doc",`, "") require.JSONEqf(t, tt.wantBody, actual, "Do() got = %v, want %v", got.String(), tt.wantBody) } From 1e6952da4c53fd785cb506c5b91a293c1778e342 Mon Sep 17 00:00:00 2001 From: Rakhat Zhuman Date: Fri, 7 Apr 2023 20:49:03 +0600 Subject: [PATCH 6/9] Changed if constructor Signed-off-by: Rakhat Zhuman --- CHANGELOG.md | 1 - opensearchapi/api.bulk.go | 8 ++------ opensearchapi/api.delete_by_query.go | 8 ++------ opensearchapi/api.document_test.go | 5 +---- opensearchapi/api.exists.go | 8 ++------ opensearchapi/api.exists_source.go | 8 ++------ opensearchapi/api.explain.go | 8 ++------ opensearchapi/api.get.go | 9 ++++----- opensearchapi/api.get_source.go | 8 ++------ opensearchapi/api.mget.go | 8 ++------ opensearchapi/api.search.go | 8 ++------ opensearchapi/api.update.go | 8 ++------ opensearchapi/api.update_by_query.go | 8 ++------ 13 files changed, 25 insertions(+), 70 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 17e52d100..404b64854 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -20,7 +20,6 @@ Inspired from [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) - Adds `Err()` function to Response for detailed errors ([#246](https://github.com/opensearch-project/opensearch-go/pull/246)) - Adds Point In Time API ([#253](https://github.com/opensearch-project/opensearch-go/pull/253)) - Adds InfoResp type ([#253](https://github.com/opensearch-project/opensearch-go/pull/253)) -- Adds testcases to check upsert functionality ([#207](https://github.com/opensearch-project/opensearch-go/issues/207)) - Adds markdown linter ([#261](https://github.com/opensearch-project/opensearch-go/pull/261)) - Adds testcases to check upsert functionality ([#207](https://github.com/opensearch-project/opensearch-go/issues/207)) - Added @Jakob3xD to co-maintainers ([#270](https://github.com/opensearch-project/opensearch-go/pull/270)) diff --git a/opensearchapi/api.bulk.go b/opensearchapi/api.bulk.go index 378b1e8a2..b19db6a21 100644 --- a/opensearchapi/api.bulk.go +++ b/opensearchapi/api.bulk.go @@ -118,13 +118,9 @@ func (r BulkRequest) Do(ctx context.Context, transport Transport) (*Response, er if source, ok := r.Source.(bool); ok { params["_source"] = strconv.FormatBool(source) - } - - if source, ok := r.Source.(string); ok && source != "" { + } else if source, ok := r.Source.(string); ok && source != "" { params["_source"] = source - } - - if sources, ok := r.Source.([]string); ok && len(sources) > 0 { + } else if sources, ok := r.Source.([]string); ok && len(sources) > 0 { params["_source"] = strings.Join(sources, ",") } diff --git a/opensearchapi/api.delete_by_query.go b/opensearchapi/api.delete_by_query.go index fa2616165..f5cb18357 100644 --- a/opensearchapi/api.delete_by_query.go +++ b/opensearchapi/api.delete_by_query.go @@ -221,13 +221,9 @@ func (r DeleteByQueryRequest) Do(ctx context.Context, transport Transport) (*Res if source, ok := r.Source.(bool); ok { params["_source"] = strconv.FormatBool(source) - } - - if source, ok := r.Source.(string); ok && source != "" { + } else if source, ok := r.Source.(string); ok && source != "" { params["_source"] = source - } - - if sources, ok := r.Source.([]string); ok && len(sources) > 0 { + } else if sources, ok := r.Source.([]string); ok && len(sources) > 0 { params["_source"] = strings.Join(sources, ",") } diff --git a/opensearchapi/api.document_test.go b/opensearchapi/api.document_test.go index 2997ba29e..ac4544b0e 100644 --- a/opensearchapi/api.document_test.go +++ b/opensearchapi/api.document_test.go @@ -289,10 +289,8 @@ func TestDocumentRequest_Do(t *testing.T) { Body: strings.NewReader(fmt.Sprintf(`{"settings": {"index": {"number_of_shards": 4}}}`)), } - iCreateResponse, err := iCreate.Do(context.Background(), client) + _, err = iCreate.Do(context.Background(), client) require.NoError(t, err) - require.Equalf(t, false, iCreateResponse.IsError(), - "Error when creating index template: %s", iCreateResponse.String()) for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { @@ -302,7 +300,6 @@ func TestDocumentRequest_Do(t *testing.T) { return } - require.Equalf(t, got.IsError(), tt.wantErr, "Do() got = %v, want %v", got.IsError(), tt.wantErr) require.Equalf(t, got.StatusCode, tt.want.StatusCode, "Do() got = %v, want %v", got.StatusCode, tt.want.StatusCode) if tt.wantBody != "" { diff --git a/opensearchapi/api.exists.go b/opensearchapi/api.exists.go index 6aa046d75..40aec3cc2 100644 --- a/opensearchapi/api.exists.go +++ b/opensearchapi/api.exists.go @@ -115,13 +115,9 @@ func (r ExistsRequest) Do(ctx context.Context, transport Transport) (*Response, if source, ok := r.Source.(bool); ok { params["_source"] = strconv.FormatBool(source) - } - - if source, ok := r.Source.(string); ok && source != "" { + } else if source, ok := r.Source.(string); ok && source != "" { params["_source"] = source - } - - if sources, ok := r.Source.([]string); ok && len(sources) > 0 { + } else if sources, ok := r.Source.([]string); ok && len(sources) > 0 { params["_source"] = strings.Join(sources, ",") } diff --git a/opensearchapi/api.exists_source.go b/opensearchapi/api.exists_source.go index b911dfb56..54ce0c25e 100644 --- a/opensearchapi/api.exists_source.go +++ b/opensearchapi/api.exists_source.go @@ -115,13 +115,9 @@ func (r ExistsSourceRequest) Do(ctx context.Context, transport Transport) (*Resp if source, ok := r.Source.(bool); ok { params["_source"] = strconv.FormatBool(source) - } - - if source, ok := r.Source.(string); ok && source != "" { + } else if source, ok := r.Source.(string); ok && source != "" { params["_source"] = source - } - - if sources, ok := r.Source.([]string); ok && len(sources) > 0 { + } else if sources, ok := r.Source.([]string); ok && len(sources) > 0 { params["_source"] = strings.Join(sources, ",") } diff --git a/opensearchapi/api.explain.go b/opensearchapi/api.explain.go index 0682a86f4..264262868 100644 --- a/opensearchapi/api.explain.go +++ b/opensearchapi/api.explain.go @@ -137,13 +137,9 @@ func (r ExplainRequest) Do(ctx context.Context, transport Transport) (*Response, if source, ok := r.Source.(bool); ok { params["_source"] = strconv.FormatBool(source) - } - - if source, ok := r.Source.(string); ok && source != "" { + } else if source, ok := r.Source.(string); ok && source != "" { params["_source"] = source - } - - if sources, ok := r.Source.([]string); ok && len(sources) > 0 { + } else if sources, ok := r.Source.([]string); ok && len(sources) > 0 { params["_source"] = strings.Join(sources, ",") } diff --git a/opensearchapi/api.get.go b/opensearchapi/api.get.go index 889176a0b..bff13b716 100644 --- a/opensearchapi/api.get.go +++ b/opensearchapi/api.get.go @@ -115,11 +115,10 @@ func (r GetRequest) Do(ctx context.Context, transport Transport) (*Response, err if source, ok := r.Source.(bool); ok { params["_source"] = strconv.FormatBool(source) - } else { - sources, ok := r.Source.([]string) - if ok && len(sources) > 0 { - params["_source"] = strings.Join(sources, ",") - } + } else if source, ok := r.Source.(string); ok && source != "" { + params["_source"] = source + } else if sources, ok := r.Source.([]string); ok && len(sources) > 0 { + params["_source"] = strings.Join(sources, ",") } if len(r.SourceExcludes) > 0 { diff --git a/opensearchapi/api.get_source.go b/opensearchapi/api.get_source.go index 5bebc5fb4..459879d66 100644 --- a/opensearchapi/api.get_source.go +++ b/opensearchapi/api.get_source.go @@ -115,13 +115,9 @@ func (r GetSourceRequest) Do(ctx context.Context, transport Transport) (*Respons if source, ok := r.Source.(bool); ok { params["_source"] = strconv.FormatBool(source) - } - - if source, ok := r.Source.(string); ok && source != "" { + } else if source, ok := r.Source.(string); ok && source != "" { params["_source"] = source - } - - if sources, ok := r.Source.([]string); ok && len(sources) > 0 { + } else if sources, ok := r.Source.([]string); ok && len(sources) > 0 { params["_source"] = strings.Join(sources, ",") } diff --git a/opensearchapi/api.mget.go b/opensearchapi/api.mget.go index f3d7f1ab3..546050f9e 100644 --- a/opensearchapi/api.mget.go +++ b/opensearchapi/api.mget.go @@ -116,13 +116,9 @@ func (r MgetRequest) Do(ctx context.Context, transport Transport) (*Response, er if source, ok := r.Source.(bool); ok { params["_source"] = strconv.FormatBool(source) - } - - if source, ok := r.Source.(string); ok && source != "" { + } else if source, ok := r.Source.(string); ok && source != "" { params["_source"] = source - } - - if sources, ok := r.Source.([]string); ok && len(sources) > 0 { + } else if sources, ok := r.Source.([]string); ok && len(sources) > 0 { params["_source"] = strings.Join(sources, ",") } diff --git a/opensearchapi/api.search.go b/opensearchapi/api.search.go index c78e581ca..ca2c59291 100644 --- a/opensearchapi/api.search.go +++ b/opensearchapi/api.search.go @@ -249,13 +249,9 @@ func (r SearchRequest) Do(ctx context.Context, transport Transport) (*Response, if source, ok := r.Source.(bool); ok { params["_source"] = strconv.FormatBool(source) - } - - if source, ok := r.Source.(string); ok && source != "" { + } else if source, ok := r.Source.(string); ok && source != "" { params["_source"] = source - } - - if sources, ok := r.Source.([]string); ok && len(sources) > 0 { + } else if sources, ok := r.Source.([]string); ok && len(sources) > 0 { params["_source"] = strings.Join(sources, ",") } diff --git a/opensearchapi/api.update.go b/opensearchapi/api.update.go index 30742011d..f17ea760f 100644 --- a/opensearchapi/api.update.go +++ b/opensearchapi/api.update.go @@ -134,13 +134,9 @@ func (r UpdateRequest) Do(ctx context.Context, transport Transport) (*Response, if source, ok := r.Source.(bool); ok { params["_source"] = strconv.FormatBool(source) - } - - if source, ok := r.Source.(string); ok && source != "" { + } else if source, ok := r.Source.(string); ok && source != "" { params["_source"] = source - } - - if sources, ok := r.Source.([]string); ok && len(sources) > 0 { + } else if sources, ok := r.Source.([]string); ok && len(sources) > 0 { params["_source"] = strings.Join(sources, ",") } diff --git a/opensearchapi/api.update_by_query.go b/opensearchapi/api.update_by_query.go index bb2d49718..2b6103878 100644 --- a/opensearchapi/api.update_by_query.go +++ b/opensearchapi/api.update_by_query.go @@ -228,13 +228,9 @@ func (r UpdateByQueryRequest) Do(ctx context.Context, transport Transport) (*Res if source, ok := r.Source.(bool); ok { params["_source"] = strconv.FormatBool(source) - } - - if source, ok := r.Source.(string); ok && source != "" { + } else if source, ok := r.Source.(string); ok && source != "" { params["_source"] = source - } - - if sources, ok := r.Source.([]string); ok && len(sources) > 0 { + } else if sources, ok := r.Source.([]string); ok && len(sources) > 0 { params["_source"] = strings.Join(sources, ",") } From 0165ce7005423122cb4c65a4ffa4e60c70efcf58 Mon Sep 17 00:00:00 2001 From: Rakhat Zhuman Date: Tue, 11 Apr 2023 07:10:05 +0600 Subject: [PATCH 7/9] Remove commented part Signed-off-by: Rakhat Zhuman --- opensearchapi/api.document_test.go | 20 -------------------- 1 file changed, 20 deletions(-) diff --git a/opensearchapi/api.document_test.go b/opensearchapi/api.document_test.go index ac4544b0e..ade5784ac 100644 --- a/opensearchapi/api.document_test.go +++ b/opensearchapi/api.document_test.go @@ -145,26 +145,6 @@ func TestDocumentRequest_Do(t *testing.T) { wantErr: false, }, - // Explain document - //{ - // name: "TestExplainRequest_Do", - // r: opensearchapi.ExplainRequest{ - // Index: index, - // DocumentID: "2", - // Body: strings.NewReader(`{ "query": { "match": {"year": "2019" } } }`), - // Human: true, - // ErrorTrace: true, - // }, - // want: &opensearchapi.Response{ - // StatusCode: 200, - // Header: http.Header{ - // "Content-Type": []string{"application/json; charset=UTF-8"}, - // }, - // }, - // wantBody: ``, - // wantErr: false, - //}, - // Search document { name: "TestSearchRequest_Do. Source parameter is a slice of strings", From 3fb0a98367a860037816cc1d1169161fbede74e8 Mon Sep 17 00:00:00 2001 From: Rakhat Zhuman Date: Tue, 11 Apr 2023 14:08:26 +0600 Subject: [PATCH 8/9] Update CHANGELOG.md Signed-off-by: Rakhat Zhuman --- CHANGELOG.md | 2 +- opensearchapi/api.document_test.go | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 69d909aff..16ca5ed8f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -22,7 +22,7 @@ Inspired from [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) - Adds InfoResp type ([#253](https://github.com/opensearch-project/opensearch-go/pull/253)) - Adds markdown linter ([#261](https://github.com/opensearch-project/opensearch-go/pull/261)) - Adds testcases to check upsert functionality ([#207](https://github.com/opensearch-project/opensearch-go/issues/207)) -- Added @Jakob3xD to co-maintainers ([#270](https://github.com/opensearch-project/opensearch-go/pull/270)) +- Adds @Jakob3xD to co-maintainers ([#270](https://github.com/opensearch-project/opensearch-go/pull/270)) - Adds dynamic type to \_source field ([#158](https://github.com/opensearch-project/opensearch-go/issues/158)) - Adds testcases for Document API ([#280](https://github.com/opensearch-project/opensearch-go/issues/280)) diff --git a/opensearchapi/api.document_test.go b/opensearchapi/api.document_test.go index ade5784ac..c0e039a6a 100644 --- a/opensearchapi/api.document_test.go +++ b/opensearchapi/api.document_test.go @@ -149,9 +149,9 @@ func TestDocumentRequest_Do(t *testing.T) { { name: "TestSearchRequest_Do. Source parameter is a slice of strings", r: opensearchapi.SearchRequest{ - Index: []string{index}, - Body: strings.NewReader(`{ "query": { "match": { "title": "Tenet" } } }`), - //Source: true, + Index: []string{index}, + Body: strings.NewReader(`{ "query": { "match": { "title": "Tenet" } } }`), + Source: true, }, want: &opensearchapi.Response{ StatusCode: 200, From 96ce8223ba92c7f2a9990f1cb375f5d06a90c491 Mon Sep 17 00:00:00 2001 From: Rakhat Zhuman Date: Tue, 11 Apr 2023 21:10:06 +0600 Subject: [PATCH 9/9] Add test for explain method Signed-off-by: Rakhat Zhuman --- opensearchapi/api.document_test.go | 30 +++++++++++++++++++++++++++++- 1 file changed, 29 insertions(+), 1 deletion(-) diff --git a/opensearchapi/api.document_test.go b/opensearchapi/api.document_test.go index c0e039a6a..d9ead2f8f 100644 --- a/opensearchapi/api.document_test.go +++ b/opensearchapi/api.document_test.go @@ -37,6 +37,7 @@ func TestDocumentRequest_Do(t *testing.T) { want *opensearchapi.Response wantBody string wantErr bool + refresh bool }{ // Create document { @@ -145,6 +146,26 @@ func TestDocumentRequest_Do(t *testing.T) { wantErr: false, }, + // Explain document + { + name: "TestExplainRequest_Do", + r: opensearchapi.ExplainRequest{ + Index: index, + DocumentID: "2", + Query: `title: "Tenet"`, + Source: true, + }, + want: &opensearchapi.Response{ + StatusCode: 200, + Header: http.Header{ + "Content-Type": []string{"application/json; charset=UTF-8"}, + }, + }, + wantBody: ``, + wantErr: false, + refresh: true, + }, + // Search document { name: "TestSearchRequest_Do. Source parameter is a slice of strings", @@ -159,8 +180,9 @@ func TestDocumentRequest_Do(t *testing.T) { "Content-Type": []string{"application/json; charset=UTF-8"}, }, }, - wantBody: fmt.Sprintf(`{"_shards": {"failed":0, "skipped":0, "successful":4, "total":4}, "hits":{"hits":[], "max_score": null, "total": {"relation":"eq", "value":0}}, "timed_out":false, "took":0}`), + wantBody: fmt.Sprintf(`{"took":0,"timed_out":false,"_shards":{"total":4,"successful":4,"skipped":0,"failed":0},"hits":{"total":{"value":1,"relation":"eq"},"max_score":0.6931471,"hits":[{"_index":"%s","_id":"2","_score":0.6931471,"_source":{ "title": "Tenet", "director": "Christopher Nolan", "year": "2019" }}]}}`, index), wantErr: false, + refresh: true, }, // Update document @@ -273,6 +295,12 @@ func TestDocumentRequest_Do(t *testing.T) { require.NoError(t, err) for _, tt := range tests { + if tt.refresh { + refresh, err := client.Indices.Refresh() + require.NoError(t, err) + t.Logf("response: %+v", refresh) + } + t.Run(tt.name, func(t *testing.T) { got, err := tt.r.Do(context.Background(), client) if (err != nil) != tt.wantErr {