diff --git a/.codegen/api.go.tmpl b/.codegen/api.go.tmpl index 13d7e80c5..844b99730 100644 --- a/.codegen/api.go.tmpl +++ b/.codegen/api.go.tmpl @@ -9,6 +9,7 @@ import ( "time" "github.com/databricks/databricks-sdk-go/retries" "github.com/databricks/databricks-sdk-go/client" + "github.com/databricks/databricks-sdk-go/listing" "github.com/databricks/databricks-sdk-go/useragent" ) {{range .Services}} @@ -162,70 +163,69 @@ func (a *{{.Service.Name}}API) {{.PascalName}}AndWait(ctx context.Context{{if .R {{.Comment "// " 80}} // // This method is generated by Databricks SDK Code Generator. -func (a *{{.Service.Name}}API) {{.PascalName}}All(ctx context.Context{{if .Request}}, request {{.Request.PascalName}}{{end}}) ([]{{ template "type" .Pagination.Entity }}, error) { - {{if .Pagination.MultiRequest}}var results []{{.Pagination.Entity.PascalName}} - {{ if .Pagination.Limit -}} - var totalCount {{template "type" .Pagination.Limit.Entity}} = 0 - {{ end -}} - ctx = useragent.InContext(ctx, "sdk-feature", "pagination") - {{if .NeedsOffsetDedupe -}} - // deduplicate items that may have been added during iteration - seen := map[{{template "type" .IdentifierField.Entity}}]bool{} - {{end}}{{if eq .Pagination.Increment 1 -}} +func (a *{{.Service.Name}}API) {{.PascalName}}(ctx context.Context{{if .Request}}, request {{.Request.PascalName}}{{end}}) {{ template "paginated-return-type" . }} { + {{ if not .Request }}request := struct{}{}{{end}} + {{if eq .Pagination.Increment 1 -}} request.{{.Pagination.Offset.PascalName}} = 1 // start iterating from the first page - {{end}}for { - response, err := a.impl.{{.PascalName}}(ctx{{if .Request}}, request{{end}}) - if err != nil { - return nil, err - } - if len(response.{{.Pagination.Results.PascalName}}) == 0 { - break - } - for _, v := range response.{{.Pagination.Results.PascalName}} { - {{- if .NeedsOffsetDedupe -}} - id := v{{ template "field-path" .IdFieldPath }} - if seen[id] { - // item was added during iteration - continue - } - seen[id] = true - {{- end}} - results = append(results, v) + {{end}} + getNextPage := func(ctx context.Context, req {{ template "paginated-request-type" . }}) (*{{ .Response.PascalName }}, error) { + ctx = useragent.InContext(ctx, "sdk-feature", "pagination") + return a.impl.{{.PascalName}}(ctx{{if .Request}}, req{{end}}) + } + getItems := func(resp *{{ .Response.PascalName }}) []{{ template "type" .Pagination.Entity }} { + return {{if .Pagination.Results}}resp.{{.Pagination.Results.PascalName}}{{else}}resp{{end}} + } + {{if .Pagination.MultiRequest -}} + getNextReq := func(resp *{{ .Response.PascalName }}) *{{.Request.PascalName}} { + {{if .Pagination.Token -}} + if resp.{{.Pagination.Token.Bind.PascalName}} == "" { + return nil } - {{ if and .Pagination.Token .Pagination.Limit -}} - count := {{template "type" .Pagination.Limit.Entity}}(len(response.{{.Pagination.Results.PascalName}})) - totalCount += count - {{ else if .Pagination.Offset -}} - count := {{template "type" .Pagination.Offset.Entity}}(len(response.{{.Pagination.Results.PascalName}})) - totalCount += count - {{ end -}} - {{if eq .Path "/api/2.0/clusters/events" -}} - if response.NextPage == nil { - break + request.{{.Pagination.Token.PollField.PascalName}} = resp.{{.Pagination.Token.Bind.PascalName}} + {{- else if eq .Path "/api/2.0/clusters/events" -}} + if resp.NextPage == nil { + return nil } - request = *response.NextPage - {{- else if .Pagination.Token -}} - request.{{.Pagination.Token.PollField.PascalName}} = response.{{.Pagination.Token.Bind.PascalName}} - if response.{{.Pagination.Token.Bind.PascalName}} == "" { - break + request = *resp.NextPage + {{ else -}} + if len(getItems(resp)) == 0 { + return nil } - {{- else if eq .Pagination.Increment 1 -}} - request.{{.Pagination.Offset.PascalName}}++ + {{ if eq .Pagination.Increment 1 -}} + request.{{.Pagination.Offset.PascalName}} = resp.{{.Pagination.Offset.PascalName}} + 1 {{- else -}} - request.{{.Pagination.Offset.PascalName}} += {{template "type" .Pagination.Offset.Entity}}(len(response.{{.Pagination.Results.PascalName}})) - {{- end}} - {{ if .Pagination.Limit -}} - limit := request.{{.Pagination.Limit.PascalName}} - if limit > 0 && totalCount >= limit { - break - } - {{- end -}} - } - return results, nil{{else if .Pagination.Results}}response, err := a.impl.{{.PascalName}}(ctx{{if .Request}}, request{{end}}) - if err != nil { - return nil, err + request.{{.Pagination.Offset.PascalName}} = resp.{{.Pagination.Offset.PascalName}} + {{template "type" .Pagination.Offset.Entity}}(len(resp.{{.Pagination.Results.PascalName}})) + {{- end}}{{ end }} + return &request } - return response.{{.Pagination.Results.PascalName}}, nil{{else}}return a.impl.{{.PascalName}}(ctx, request){{end}} + {{- end}} + iterator := listing.NewIterator( + &request, + getNextPage, + getItems, + {{if .Pagination.MultiRequest}}getNextReq{{else}}nil{{end}}) + {{ if .NeedsOffsetDedupe -}} + dedupedIterator := listing.NewDedupeIterator[{{ template "type" .Pagination.Entity }}, {{ template "type" .IdentifierField.Entity }}]( + iterator, + func(item {{ template "type" .Pagination.Entity }}) {{ template "type" .IdentifierField.Entity }} { + return item{{ template "field-path" .IdFieldPath }} + }) + return dedupedIterator + {{- else -}} + return iterator + {{- end }} +} + +{{.Comment "// " 80}} +// +// This method is generated by Databricks SDK Code Generator. +func (a *{{.Service.Name}}API) {{.PascalName}}All(ctx context.Context{{if .Request}}, request {{.Request.PascalName}}{{end}}) ([]{{ template "type" .Pagination.Entity }}, error) { + iterator := a.{{.PascalName}}(ctx{{if .Request}}, request{{end}}) + {{ if .Pagination.Limit -}} + return listing.ToSliceN[{{ template "type" .Pagination.Entity }}, {{ template "type" .Pagination.Limit.Entity }}](ctx, iterator, request.{{.Pagination.Limit.PascalName}}) + {{ else -}} + return listing.ToSlice[{{ template "type" .Pagination.Entity }}](ctx, iterator) + {{- end }} } {{end}}{{if .NamedIdMap}} // {{.NamedIdMap.PascalName}} calls [{{.Service.Name}}API.{{.PascalName}}{{if not .NamedIdMap.Direct}}All{{end -}}] and creates a map of results with [{{.NamedIdMap.Entity.PascalName}}]{{ template "field-path" .NamedIdMap.NamePath }} as key and [{{.NamedIdMap.Entity.PascalName}}]{{ template "field-path" .NamedIdMap.IdPath}} as value. @@ -301,4 +301,14 @@ func (a *{{.Service.Name}}API) {{.Shortcut.PascalName}}AndWait(ctx context.Conte {{- end}} {{- define "field-path" -}} {{- range .}}.{{.PascalName}}{{end}} +{{- end -}} +{{- define "paginated-request-type" -}} + {{if .Request}}{{.Request.PascalName}}{{else}}struct{}{{end}} +{{- end -}} +{{- define "paginated-return-type" -}} + {{ if .NeedsOffsetDedupe -}} + *listing.DeduplicatingIterator[{{ template "type" .Pagination.Entity }}, {{ template "type" .IdentifierField.Entity }}] + {{- else -}} + *listing.PaginatingIterator[{{ template "paginated-request-type" .}}, *{{ .Response.PascalName }}, {{ template "type" .Pagination.Entity }}] + {{- end -}} {{- end -}} \ No newline at end of file diff --git a/listing/listing.go b/listing/listing.go new file mode 100644 index 000000000..aa2ab97ee --- /dev/null +++ b/listing/listing.go @@ -0,0 +1,219 @@ +package listing + +import ( + "context" + "errors" +) + +// Iterator[T] is an iterator over items of type T. It is similar to +// java.util.Iterator. It is not thread-safe. +type Iterator[T any] interface { + // HasNext returns true if there are more items to iterate over. HasNext + // will also return true if the iterator needs to fetch the next page of + // items from the underlying source and the request fails, even if there + // are no more items to iterate over. In this case, Next will return the + // error. + HasNext(context.Context) bool + + // Next returns the next item in the iterator. If there are no more items + // to iterate over, it returns ErrNoMoreItems. If there was an error + // fetching the next page of items, it returns that error. Once Next + // returns ErrNoMoreItems or an error, it will continue to return that + // error. + Next(context.Context) (T, error) +} + +// ToSlice returns all items from the iterator as a slice. If there was an +// error fetching items at any time, it returns that error. +func ToSlice[T any](ctx context.Context, it Iterator[T]) ([]T, error) { + var items []T + for it.HasNext(ctx) { + item, err := it.Next(ctx) + if err != nil { + return nil, err + } + items = append(items, item) + } + return items, nil +} + +// ToSliceN returns the first N items from the iterator as a slice. If there +// was an error fetching items at any time, it returns that error. +func ToSliceN[T any, Limit ~int | ~int64](ctx context.Context, it Iterator[T], n Limit) ([]T, error) { + var items []T + for it.HasNext(ctx) && Limit(len(items)) < n { + item, err := it.Next(ctx) + if err != nil { + return nil, err + } + items = append(items, item) + } + return items, nil +} + +// Use struct{} for Req for iterators that don't need any request structure. +type PaginatingIterator[Req, Resp, T any] struct { + // nextReq is the request to be used to fetch the next page. If nil, then + // there is no next page to fetch. + nextReq *Req + + // getNextPage fetches the next page of items, returning the deserialized + // response and error. + getNextPage func(context.Context, Req) (Resp, error) + + // getItems selects the items being iterated over from the response. + getItems func(Resp) []T + + // getNextReq is used to get the next request to be used in the next page. + // If nil, then there is no next page to fetch. + getNextReq func(Resp) *Req + + // currentPage is the current page of items. + currentPage []T + + // currentPageIdx is the index of the next item from currentPage to return. + currentPageIdx int + + // lastErr is the last error returned by getNextPage. + lastErr error +} + +var ErrNoMoreItems = errors.New("no more items") + +// Returns a new iterator. The iterator will fetch the next page of items +// lazily, when needed. nextReq is the request to be used to fetch the initial +// page. If nil, then no page will be fetched. getNextPage fetches the next +// page of items, returning the deserialized response and error. getItems +// selects the items being iterated over from the response. getNextReq is used +// to get the next request to be used in the next page. If the returned value +// is nil, then there is no next page to fetch. +func NewIterator[Req, Resp, T any]( + nextReq *Req, + getNextPage func(context.Context, Req) (Resp, error), + getItems func(Resp) []T, + getNextReq func(Resp) *Req, +) *PaginatingIterator[Req, Resp, T] { + return &PaginatingIterator[Req, Resp, T]{ + nextReq: nextReq, + getNextPage: getNextPage, + getItems: getItems, + getNextReq: getNextReq, + } +} + +func (i *PaginatingIterator[Req, Resp, T]) loadNextPageIfNeeded(ctx context.Context) error { + if i.currentPageIdx < len(i.currentPage) { + return nil + } + if i.nextReq == nil { + i.currentPage = nil + i.currentPageIdx = 0 + return nil + } + if i.lastErr != nil { + return i.lastErr + } + + // Keep loading pages while we have a next request and the current page is + // empty. + i.currentPage = nil + i.currentPageIdx = 0 + // Endpoints using token-based pagination may return an empty page with a + // next token. We need to keep fetching pages until we get a non-empty + // page or there are no more pages. + for i.nextReq != nil && len(i.currentPage) == 0 { + resp, err := i.getNextPage(ctx, *i.nextReq) + i.lastErr = err + if err != nil { + return err + } + i.nextReq = i.getNextReq(resp) + i.currentPage = i.getItems(resp) + } + return nil +} + +func (i *PaginatingIterator[Req, Resp, T]) Next(ctx context.Context) (T, error) { + var t T + err := i.loadNextPageIfNeeded(ctx) + if err != nil { + return t, err + } + if i.currentPageIdx >= len(i.currentPage) { + return t, ErrNoMoreItems + } + item := i.currentPage[i.currentPageIdx] + i.currentPageIdx++ + return item, nil +} + +func (i *PaginatingIterator[Req, Resp, T]) HasNext(ctx context.Context) bool { + err := i.loadNextPageIfNeeded(ctx) + // As described in the documentation for HasNext, if there was an error + // fetching the next page, we still return true to allow the user to handle + // the error in Next. + if err != nil { + return true + } + return i.currentPageIdx < len(i.currentPage) +} + +type DeduplicatingIterator[T any, Id comparable] struct { + it Iterator[T] + getId func(T) Id + seen map[Id]struct{} + current *T +} + +func NewDedupeIterator[T any, Id comparable](it Iterator[T], getId func(T) Id) *DeduplicatingIterator[T, Id] { + return &DeduplicatingIterator[T, Id]{ + it: it, + getId: getId, + seen: make(map[Id]struct{}), + } +} + +func (i *DeduplicatingIterator[T, Id]) Next(ctx context.Context) (T, error) { + if i.current != nil { + t := *i.current + i.current = nil + return t, nil + } + for { + t, err := i.it.Next(ctx) + if err != nil { + return t, err + } + id := i.getId(t) + if _, ok := i.seen[id]; !ok { + i.seen[id] = struct{}{} + return t, nil + } + } +} + +func (i *DeduplicatingIterator[T, Id]) HasNext(ctx context.Context) bool { + if i.current != nil { + return true + } + // To compute HasNext in DeduplicatingIterator, we need to actually fetch + // the next item from the underlying iterator and compare it to seen items. + // However, the retrieved item cannot be discarded, as it needs to be + // returned by the next call to Next. So we store the item in current and + // return it in the next call to Next, after which current is set to nil. + for { + t, err := i.it.Next(ctx) + if errors.Is(err, ErrNoMoreItems) { + return false + } + if err != nil { + return true + } + id := i.getId(t) + if _, ok := i.seen[id]; !ok { + i.current = &t + i.seen[id] = struct{}{} + return true + } + } +} diff --git a/listing/listing_test.go b/listing/listing_test.go new file mode 100644 index 000000000..9893d5e28 --- /dev/null +++ b/listing/listing_test.go @@ -0,0 +1,217 @@ +package listing_test + +import ( + "context" + "errors" + "testing" + + "github.com/databricks/databricks-sdk-go/listing" + "github.com/stretchr/testify/assert" +) + +type requestResponse struct { + req string + page map[string][]int +} + +func makeIterator(reqsAndPages []requestResponse) listing.Iterator[int] { + reqs := make([]string, len(reqsAndPages)) + pages := make(map[string]map[string][]int) + for i, reqAndPage := range reqsAndPages { + reqs[i] = reqAndPage.req + pages[reqAndPage.req] = reqAndPage.page + } + nextReq := reqs[0] + iterator := listing.NewIterator(&nextReq, + func(ctx context.Context, req string) (map[string][]int, error) { + return pages[req], nil + }, + func(resp map[string][]int) []int { + return resp["page"] + }, + func(resp map[string][]int) *string { + for i, r := range reqs { + if r == nextReq && i+1 < len(reqs) { + nextReq = reqs[i+1] + return &nextReq + } + } + return nil + }, + ) + return iterator +} + +func TestIterator(t *testing.T) { + t.Run("basic iteration", func(t *testing.T) { + rrs := []requestResponse{ + {req: "page1", page: map[string][]int{"page": {1, 2}}}, + {req: "page2", page: map[string][]int{"page": {3, 4}}}, + {req: "page3", page: map[string][]int{"page": {5, 6}}}, + } + iterator := makeIterator(rrs) + + for i := 1; iterator.HasNext(context.Background()); i++ { + item, err := iterator.Next(context.Background()) + assert.NoError(t, err) + assert.Equal(t, i, item) + } + + _, err := iterator.Next(context.Background()) + assert.ErrorIs(t, err, listing.ErrNoMoreItems) + }) + + t.Run("error propagation", func(t *testing.T) { + expectedErr := errors.New("some error") + iterator := listing.NewIterator[struct{}, []int, struct{}](&struct{}{}, + func(ctx context.Context, req struct{}) ([]int, error) { + return nil, expectedErr + }, + nil, + nil, + ) + + _, err := iterator.Next(context.Background()) + assert.ErrorIs(t, err, expectedErr) + }) + + t.Run("error propagation from HasNext", func(t *testing.T) { + expectedErr := errors.New("some error") + iterator := listing.NewIterator[struct{}, []int, struct{}](&struct{}{}, + func(ctx context.Context, req struct{}) ([]int, error) { + return nil, expectedErr + }, + nil, + nil, + ) + + b := iterator.HasNext(context.Background()) + assert.True(t, b) + // Error from HasNext is stored and returned on call to Next + _, err := iterator.Next(context.Background()) + assert.ErrorIs(t, err, expectedErr) + // Error is cached + _, err2 := iterator.Next(context.Background()) + assert.Equal(t, err, err2) + }) + + t.Run("iteration with empty page in the middle", func(t *testing.T) { + rrs := []requestResponse{ + {req: "page1", page: map[string][]int{"page": {1, 2}}}, + {req: "page2", page: map[string][]int{"page": {}}}, + {req: "page3", page: map[string][]int{"page": {3, 4}}}, + } + iterator := makeIterator(rrs) + + for i := 1; i <= 4; i++ { + item, err := iterator.Next(context.Background()) + assert.NoError(t, err) + assert.Equal(t, i, item) + } + + _, err := iterator.Next(context.Background()) + assert.ErrorIs(t, err, listing.ErrNoMoreItems) + }) + + t.Run("iteration with empty page at the end", func(t *testing.T) { + rrs := []requestResponse{ + {req: "page1", page: map[string][]int{"page": {1, 2}}}, + {req: "page2", page: map[string][]int{"page": {3, 4}}}, + {req: "page3", page: map[string][]int{"page": {}}}, + } + iterator := makeIterator(rrs) + + for i := 1; i <= 4; i++ { + item, err := iterator.Next(context.Background()) + assert.NoError(t, err) + assert.Equal(t, i, item) + } + + _, err := iterator.Next(context.Background()) + assert.ErrorIs(t, err, listing.ErrNoMoreItems) + }) + + t.Run("ToSlice returns all items", func(t *testing.T) { + rrs := []requestResponse{ + {req: "page1", page: map[string][]int{"page": {1, 2}}}, + {req: "page2", page: map[string][]int{"page": {3, 4}}}, + {req: "page3", page: map[string][]int{"page": {5, 6}}}, + } + iterator := makeIterator(rrs) + + items, err := listing.ToSlice(context.Background(), iterator) + assert.NoError(t, err) + assert.Equal(t, []int{1, 2, 3, 4, 5, 6}, items) + }) + + t.Run("ToSliceN returns the first N items", func(t *testing.T) { + rrs := []requestResponse{ + {req: "page1", page: map[string][]int{"page": {1, 2}}}, + {req: "page2", page: map[string][]int{"page": {3, 4}}}, + {req: "page3", page: map[string][]int{"page": {5, 6}}}, + } + iterator := makeIterator(rrs) + + items, err := listing.ToSliceN(context.Background(), iterator, 3) + assert.NoError(t, err) + assert.Equal(t, []int{1, 2, 3}, items) + }) +} + +func TestDedupeIterator(t *testing.T) { + t.Run("basic iteration", func(t *testing.T) { + rrs := []requestResponse{ + {req: "page1", page: map[string][]int{"page": {1, 2, 2}}}, + {req: "page2", page: map[string][]int{"page": {3, 4, 4}}}, + {req: "page3", page: map[string][]int{"page": {5, 5, 6}}}, + } + iterator := makeIterator(rrs) + dedupeIterator := listing.NewDedupeIterator[int, int]( + iterator, func(a int) int { return a }) + + for i := 1; i <= 6; i++ { + item, err := dedupeIterator.Next(context.Background()) + assert.NoError(t, err) + assert.Equal(t, i, item) + } + + _, err := dedupeIterator.Next(context.Background()) + assert.ErrorIs(t, err, listing.ErrNoMoreItems) + }) + + t.Run("HasNext caches appropriately", func(t *testing.T) { + rrs := []requestResponse{ + {req: "page1", page: map[string][]int{"page": {1, 2, 2}}}, + {req: "page2", page: map[string][]int{"page": {3, 4, 4}}}, + {req: "page3", page: map[string][]int{"page": {5, 5, 6}}}, + } + iterator := makeIterator(rrs) + dedupeIterator := listing.NewDedupeIterator[int, int]( + iterator, func(a int) int { return a }) + + values := make([]int, 0) + for dedupeIterator.HasNext(context.Background()) { + assert.True(t, dedupeIterator.HasNext(context.Background())) + v, err := dedupeIterator.Next(context.Background()) + values = append(values, v) + assert.NoError(t, err) + } + assert.Equal(t, []int{1, 2, 3, 4, 5, 6}, values) + }) + + t.Run("HasNext returns true when there is an error", func(t *testing.T) { + expectedErr := errors.New("some error") + iterator := listing.NewIterator[struct{}, []int, int](&struct{}{}, + func(ctx context.Context, req struct{}) ([]int, error) { + return nil, expectedErr + }, + nil, + nil, + ) + dedupeIterator := listing.NewDedupeIterator[int, int]( + iterator, func(a int) int { return a }) + assert.True(t, dedupeIterator.HasNext(context.Background())) + _, err := dedupeIterator.Next(context.Background()) + assert.ErrorIs(t, err, expectedErr) + }) +} diff --git a/service/billing/api.go b/service/billing/api.go index 22ad7f146..cab8f4913 100755 --- a/service/billing/api.go +++ b/service/billing/api.go @@ -8,6 +8,7 @@ import ( "fmt" "github.com/databricks/databricks-sdk-go/client" + "github.com/databricks/databricks-sdk-go/listing" "github.com/databricks/databricks-sdk-go/useragent" ) @@ -130,12 +131,34 @@ func (a *BudgetsAPI) GetByBudgetId(ctx context.Context, budgetId string) (*Wrapp // for each day that the budget is configured to include. // // This method is generated by Databricks SDK Code Generator. -func (a *BudgetsAPI) ListAll(ctx context.Context) ([]BudgetWithStatus, error) { - response, err := a.impl.List(ctx) - if err != nil { - return nil, err +func (a *BudgetsAPI) List(ctx context.Context) *listing.PaginatingIterator[struct{}, *BudgetList, BudgetWithStatus] { + request := struct{}{} + + getNextPage := func(ctx context.Context, req struct{}) (*BudgetList, error) { + ctx = useragent.InContext(ctx, "sdk-feature", "pagination") + return a.impl.List(ctx) } - return response.Budgets, nil + getItems := func(resp *BudgetList) []BudgetWithStatus { + return resp.Budgets + } + + iterator := listing.NewIterator( + &request, + getNextPage, + getItems, + nil) + return iterator +} + +// Get all budgets. +// +// Gets all budgets associated with this account, including noncumulative status +// for each day that the budget is configured to include. +// +// This method is generated by Databricks SDK Code Generator. +func (a *BudgetsAPI) ListAll(ctx context.Context) ([]BudgetWithStatus, error) { + iterator := a.List(ctx) + return listing.ToSlice[BudgetWithStatus](ctx, iterator) } // BudgetWithStatusNameToBudgetIdMap calls [BudgetsAPI.ListAll] and creates a map of results with [BudgetWithStatus].Name as key and [BudgetWithStatus].BudgetId as value. @@ -341,12 +364,33 @@ func (a *LogDeliveryAPI) GetByLogDeliveryConfigurationId(ctx context.Context, lo // specified by ID. // // This method is generated by Databricks SDK Code Generator. -func (a *LogDeliveryAPI) ListAll(ctx context.Context, request ListLogDeliveryRequest) ([]LogDeliveryConfiguration, error) { - response, err := a.impl.List(ctx, request) - if err != nil { - return nil, err +func (a *LogDeliveryAPI) List(ctx context.Context, request ListLogDeliveryRequest) *listing.PaginatingIterator[ListLogDeliveryRequest, *WrappedLogDeliveryConfigurations, LogDeliveryConfiguration] { + + getNextPage := func(ctx context.Context, req ListLogDeliveryRequest) (*WrappedLogDeliveryConfigurations, error) { + ctx = useragent.InContext(ctx, "sdk-feature", "pagination") + return a.impl.List(ctx, req) } - return response.LogDeliveryConfigurations, nil + getItems := func(resp *WrappedLogDeliveryConfigurations) []LogDeliveryConfiguration { + return resp.LogDeliveryConfigurations + } + + iterator := listing.NewIterator( + &request, + getNextPage, + getItems, + nil) + return iterator +} + +// Get all log delivery configurations. +// +// Gets all Databricks log delivery configurations associated with an account +// specified by ID. +// +// This method is generated by Databricks SDK Code Generator. +func (a *LogDeliveryAPI) ListAll(ctx context.Context, request ListLogDeliveryRequest) ([]LogDeliveryConfiguration, error) { + iterator := a.List(ctx, request) + return listing.ToSlice[LogDeliveryConfiguration](ctx, iterator) } // LogDeliveryConfigurationConfigNameToConfigIdMap calls [LogDeliveryAPI.ListAll] and creates a map of results with [LogDeliveryConfiguration].ConfigName as key and [LogDeliveryConfiguration].ConfigId as value. diff --git a/service/catalog/api.go b/service/catalog/api.go index c4c2799cb..22e9a98fa 100755 --- a/service/catalog/api.go +++ b/service/catalog/api.go @@ -8,6 +8,7 @@ import ( "fmt" "github.com/databricks/databricks-sdk-go/client" + "github.com/databricks/databricks-sdk-go/listing" "github.com/databricks/databricks-sdk-go/useragent" ) @@ -92,12 +93,33 @@ func (a *AccountMetastoreAssignmentsAPI) GetByWorkspaceId(ctx context.Context, w // metastore. // // This method is generated by Databricks SDK Code Generator. -func (a *AccountMetastoreAssignmentsAPI) ListAll(ctx context.Context, request ListAccountMetastoreAssignmentsRequest) ([]int64, error) { - response, err := a.impl.List(ctx, request) - if err != nil { - return nil, err +func (a *AccountMetastoreAssignmentsAPI) List(ctx context.Context, request ListAccountMetastoreAssignmentsRequest) *listing.PaginatingIterator[ListAccountMetastoreAssignmentsRequest, *ListAccountMetastoreAssignmentsResponse, int64] { + + getNextPage := func(ctx context.Context, req ListAccountMetastoreAssignmentsRequest) (*ListAccountMetastoreAssignmentsResponse, error) { + ctx = useragent.InContext(ctx, "sdk-feature", "pagination") + return a.impl.List(ctx, req) + } + getItems := func(resp *ListAccountMetastoreAssignmentsResponse) []int64 { + return resp.WorkspaceIds } - return response.WorkspaceIds, nil + + iterator := listing.NewIterator( + &request, + getNextPage, + getItems, + nil) + return iterator +} + +// Get all workspaces assigned to a metastore. +// +// Gets a list of all Databricks workspace IDs that have been assigned to given +// metastore. +// +// This method is generated by Databricks SDK Code Generator. +func (a *AccountMetastoreAssignmentsAPI) ListAll(ctx context.Context, request ListAccountMetastoreAssignmentsRequest) ([]int64, error) { + iterator := a.List(ctx, request) + return listing.ToSlice[int64](ctx, iterator) } // Get all workspaces assigned to a metastore. @@ -190,12 +212,33 @@ func (a *AccountMetastoresAPI) GetByMetastoreId(ctx context.Context, metastoreId // Gets all Unity Catalog metastores associated with an account specified by ID. // // This method is generated by Databricks SDK Code Generator. -func (a *AccountMetastoresAPI) ListAll(ctx context.Context) ([]MetastoreInfo, error) { - response, err := a.impl.List(ctx) - if err != nil { - return nil, err +func (a *AccountMetastoresAPI) List(ctx context.Context) *listing.PaginatingIterator[struct{}, *ListMetastoresResponse, MetastoreInfo] { + request := struct{}{} + + getNextPage := func(ctx context.Context, req struct{}) (*ListMetastoresResponse, error) { + ctx = useragent.InContext(ctx, "sdk-feature", "pagination") + return a.impl.List(ctx) } - return response.Metastores, nil + getItems := func(resp *ListMetastoresResponse) []MetastoreInfo { + return resp.Metastores + } + + iterator := listing.NewIterator( + &request, + getNextPage, + getItems, + nil) + return iterator +} + +// Get all metastores associated with an account. +// +// Gets all Unity Catalog metastores associated with an account specified by ID. +// +// This method is generated by Databricks SDK Code Generator. +func (a *AccountMetastoresAPI) ListAll(ctx context.Context) ([]MetastoreInfo, error) { + iterator := a.List(ctx) + return listing.ToSlice[MetastoreInfo](ctx, iterator) } // Update a metastore. @@ -511,12 +554,37 @@ func (a *CatalogsAPI) GetByName(ctx context.Context, name string) (*CatalogInfo, // the array. // // This method is generated by Databricks SDK Code Generator. -func (a *CatalogsAPI) ListAll(ctx context.Context) ([]CatalogInfo, error) { - response, err := a.impl.List(ctx) - if err != nil { - return nil, err +func (a *CatalogsAPI) List(ctx context.Context) *listing.PaginatingIterator[struct{}, *ListCatalogsResponse, CatalogInfo] { + request := struct{}{} + + getNextPage := func(ctx context.Context, req struct{}) (*ListCatalogsResponse, error) { + ctx = useragent.InContext(ctx, "sdk-feature", "pagination") + return a.impl.List(ctx) } - return response.Catalogs, nil + getItems := func(resp *ListCatalogsResponse) []CatalogInfo { + return resp.Catalogs + } + + iterator := listing.NewIterator( + &request, + getNextPage, + getItems, + nil) + return iterator +} + +// List catalogs. +// +// Gets an array of catalogs in the metastore. If the caller is the metastore +// admin, all catalogs will be retrieved. Otherwise, only catalogs owned by the +// caller (or for which the caller has the **USE_CATALOG** privilege) will be +// retrieved. There is no guarantee of a specific ordering of the elements in +// the array. +// +// This method is generated by Databricks SDK Code Generator. +func (a *CatalogsAPI) ListAll(ctx context.Context) ([]CatalogInfo, error) { + iterator := a.List(ctx) + return listing.ToSlice[CatalogInfo](ctx, iterator) } // Update a catalog. @@ -613,12 +681,33 @@ func (a *ConnectionsAPI) GetByNameArg(ctx context.Context, nameArg string) (*Con // List all connections. // // This method is generated by Databricks SDK Code Generator. -func (a *ConnectionsAPI) ListAll(ctx context.Context) ([]ConnectionInfo, error) { - response, err := a.impl.List(ctx) - if err != nil { - return nil, err +func (a *ConnectionsAPI) List(ctx context.Context) *listing.PaginatingIterator[struct{}, *ListConnectionsResponse, ConnectionInfo] { + request := struct{}{} + + getNextPage := func(ctx context.Context, req struct{}) (*ListConnectionsResponse, error) { + ctx = useragent.InContext(ctx, "sdk-feature", "pagination") + return a.impl.List(ctx) + } + getItems := func(resp *ListConnectionsResponse) []ConnectionInfo { + return resp.Connections } - return response.Connections, nil + + iterator := listing.NewIterator( + &request, + getNextPage, + getItems, + nil) + return iterator +} + +// List connections. +// +// List all connections. +// +// This method is generated by Databricks SDK Code Generator. +func (a *ConnectionsAPI) ListAll(ctx context.Context) ([]ConnectionInfo, error) { + iterator := a.List(ctx) + return listing.ToSlice[ConnectionInfo](ctx, iterator) } // ConnectionInfoNameToFullNameMap calls [ConnectionsAPI.ListAll] and creates a map of results with [ConnectionInfo].Name as key and [ConnectionInfo].FullName as value. @@ -776,12 +865,37 @@ func (a *ExternalLocationsAPI) GetByName(ctx context.Context, name string) (*Ext // array. // // This method is generated by Databricks SDK Code Generator. -func (a *ExternalLocationsAPI) ListAll(ctx context.Context) ([]ExternalLocationInfo, error) { - response, err := a.impl.List(ctx) - if err != nil { - return nil, err +func (a *ExternalLocationsAPI) List(ctx context.Context) *listing.PaginatingIterator[struct{}, *ListExternalLocationsResponse, ExternalLocationInfo] { + request := struct{}{} + + getNextPage := func(ctx context.Context, req struct{}) (*ListExternalLocationsResponse, error) { + ctx = useragent.InContext(ctx, "sdk-feature", "pagination") + return a.impl.List(ctx) } - return response.ExternalLocations, nil + getItems := func(resp *ListExternalLocationsResponse) []ExternalLocationInfo { + return resp.ExternalLocations + } + + iterator := listing.NewIterator( + &request, + getNextPage, + getItems, + nil) + return iterator +} + +// List external locations. +// +// Gets an array of external locations (__ExternalLocationInfo__ objects) from +// the metastore. The caller must be a metastore admin, the owner of the +// external location, or a user that has some privilege on the external +// location. There is no guarantee of a specific ordering of the elements in the +// array. +// +// This method is generated by Databricks SDK Code Generator. +func (a *ExternalLocationsAPI) ListAll(ctx context.Context) ([]ExternalLocationInfo, error) { + iterator := a.List(ctx) + return listing.ToSlice[ExternalLocationInfo](ctx, iterator) } // Update an external location. @@ -903,12 +1017,38 @@ func (a *FunctionsAPI) GetByName(ctx context.Context, name string) (*FunctionInf // the array. // // This method is generated by Databricks SDK Code Generator. -func (a *FunctionsAPI) ListAll(ctx context.Context, request ListFunctionsRequest) ([]FunctionInfo, error) { - response, err := a.impl.List(ctx, request) - if err != nil { - return nil, err +func (a *FunctionsAPI) List(ctx context.Context, request ListFunctionsRequest) *listing.PaginatingIterator[ListFunctionsRequest, *ListFunctionsResponse, FunctionInfo] { + + getNextPage := func(ctx context.Context, req ListFunctionsRequest) (*ListFunctionsResponse, error) { + ctx = useragent.InContext(ctx, "sdk-feature", "pagination") + return a.impl.List(ctx, req) + } + getItems := func(resp *ListFunctionsResponse) []FunctionInfo { + return resp.Functions } - return response.Functions, nil + + iterator := listing.NewIterator( + &request, + getNextPage, + getItems, + nil) + return iterator +} + +// List functions. +// +// List functions within the specified parent catalog and schema. If the user is +// a metastore admin, all functions are returned in the output list. Otherwise, +// the user must have the **USE_CATALOG** privilege on the catalog and the +// **USE_SCHEMA** privilege on the schema, and the output list contains only +// functions for which either the user has the **EXECUTE** privilege or the user +// is the owner. There is no guarantee of a specific ordering of the elements in +// the array. +// +// This method is generated by Databricks SDK Code Generator. +func (a *FunctionsAPI) ListAll(ctx context.Context, request ListFunctionsRequest) ([]FunctionInfo, error) { + iterator := a.List(ctx, request) + return listing.ToSlice[FunctionInfo](ctx, iterator) } // FunctionInfoNameToFullNameMap calls [FunctionsAPI.ListAll] and creates a map of results with [FunctionInfo].Name as key and [FunctionInfo].FullName as value. @@ -1139,12 +1279,35 @@ func (a *MetastoresAPI) GetById(ctx context.Context, id string) (*MetastoreInfo, // specific ordering of the elements in the array. // // This method is generated by Databricks SDK Code Generator. -func (a *MetastoresAPI) ListAll(ctx context.Context) ([]MetastoreInfo, error) { - response, err := a.impl.List(ctx) - if err != nil { - return nil, err +func (a *MetastoresAPI) List(ctx context.Context) *listing.PaginatingIterator[struct{}, *ListMetastoresResponse, MetastoreInfo] { + request := struct{}{} + + getNextPage := func(ctx context.Context, req struct{}) (*ListMetastoresResponse, error) { + ctx = useragent.InContext(ctx, "sdk-feature", "pagination") + return a.impl.List(ctx) + } + getItems := func(resp *ListMetastoresResponse) []MetastoreInfo { + return resp.Metastores } - return response.Metastores, nil + + iterator := listing.NewIterator( + &request, + getNextPage, + getItems, + nil) + return iterator +} + +// List metastores. +// +// Gets an array of the available metastores (as __MetastoreInfo__ objects). The +// caller must be an admin to retrieve this info. There is no guarantee of a +// specific ordering of the elements in the array. +// +// This method is generated by Databricks SDK Code Generator. +func (a *MetastoresAPI) ListAll(ctx context.Context) ([]MetastoreInfo, error) { + iterator := a.List(ctx) + return listing.ToSlice[MetastoreInfo](ctx, iterator) } // MetastoreInfoNameToMetastoreIdMap calls [MetastoresAPI.ListAll] and creates a map of results with [MetastoreInfo].Name as key and [MetastoreInfo].MetastoreId as value. @@ -1375,33 +1538,50 @@ func (a *ModelVersionsAPI) GetByAliasByFullNameAndAlias(ctx context.Context, ful // There is no guarantee of a specific ordering of the elements in the response. // // This method is generated by Databricks SDK Code Generator. -func (a *ModelVersionsAPI) ListAll(ctx context.Context, request ListModelVersionsRequest) ([]ModelVersionInfo, error) { - var results []ModelVersionInfo - var totalCount int = 0 - ctx = useragent.InContext(ctx, "sdk-feature", "pagination") - for { - response, err := a.impl.List(ctx, request) - if err != nil { - return nil, err - } - if len(response.ModelVersions) == 0 { - break - } - for _, v := range response.ModelVersions { - results = append(results, v) - } - count := int(len(response.ModelVersions)) - totalCount += count - request.PageToken = response.NextPageToken - if response.NextPageToken == "" { - break - } - limit := request.MaxResults - if limit > 0 && totalCount >= limit { - break +func (a *ModelVersionsAPI) List(ctx context.Context, request ListModelVersionsRequest) *listing.PaginatingIterator[ListModelVersionsRequest, *ListModelVersionsResponse, ModelVersionInfo] { + + getNextPage := func(ctx context.Context, req ListModelVersionsRequest) (*ListModelVersionsResponse, error) { + ctx = useragent.InContext(ctx, "sdk-feature", "pagination") + return a.impl.List(ctx, req) + } + getItems := func(resp *ListModelVersionsResponse) []ModelVersionInfo { + return resp.ModelVersions + } + getNextReq := func(resp *ListModelVersionsResponse) *ListModelVersionsRequest { + if resp.NextPageToken == "" { + return nil } + request.PageToken = resp.NextPageToken + return &request } - return results, nil + iterator := listing.NewIterator( + &request, + getNextPage, + getItems, + getNextReq) + return iterator +} + +// List Model Versions. +// +// List model versions. You can list model versions under a particular schema, +// or list all model versions in the current metastore. +// +// The returned models are filtered based on the privileges of the calling user. +// For example, the metastore admin is able to list all the model versions. A +// regular user needs to be the owner or have the **EXECUTE** privilege on the +// parent registered model to recieve the model versions in the response. For +// the latter case, the caller must also be the owner or have the +// **USE_CATALOG** privilege on the parent catalog and the **USE_SCHEMA** +// privilege on the parent schema. +// +// There is no guarantee of a specific ordering of the elements in the response. +// +// This method is generated by Databricks SDK Code Generator. +func (a *ModelVersionsAPI) ListAll(ctx context.Context, request ListModelVersionsRequest) ([]ModelVersionInfo, error) { + iterator := a.List(ctx, request) + return listing.ToSliceN[ModelVersionInfo, int](ctx, iterator, request.MaxResults) + } // List Model Versions. @@ -1608,33 +1788,50 @@ func (a *RegisteredModelsAPI) GetByFullName(ctx context.Context, fullName string // There is no guarantee of a specific ordering of the elements in the response. // // This method is generated by Databricks SDK Code Generator. -func (a *RegisteredModelsAPI) ListAll(ctx context.Context, request ListRegisteredModelsRequest) ([]RegisteredModelInfo, error) { - var results []RegisteredModelInfo - var totalCount int = 0 - ctx = useragent.InContext(ctx, "sdk-feature", "pagination") - for { - response, err := a.impl.List(ctx, request) - if err != nil { - return nil, err - } - if len(response.RegisteredModels) == 0 { - break - } - for _, v := range response.RegisteredModels { - results = append(results, v) - } - count := int(len(response.RegisteredModels)) - totalCount += count - request.PageToken = response.NextPageToken - if response.NextPageToken == "" { - break - } - limit := request.MaxResults - if limit > 0 && totalCount >= limit { - break +func (a *RegisteredModelsAPI) List(ctx context.Context, request ListRegisteredModelsRequest) *listing.PaginatingIterator[ListRegisteredModelsRequest, *ListRegisteredModelsResponse, RegisteredModelInfo] { + + getNextPage := func(ctx context.Context, req ListRegisteredModelsRequest) (*ListRegisteredModelsResponse, error) { + ctx = useragent.InContext(ctx, "sdk-feature", "pagination") + return a.impl.List(ctx, req) + } + getItems := func(resp *ListRegisteredModelsResponse) []RegisteredModelInfo { + return resp.RegisteredModels + } + getNextReq := func(resp *ListRegisteredModelsResponse) *ListRegisteredModelsRequest { + if resp.NextPageToken == "" { + return nil } + request.PageToken = resp.NextPageToken + return &request } - return results, nil + iterator := listing.NewIterator( + &request, + getNextPage, + getItems, + getNextReq) + return iterator +} + +// List Registered Models. +// +// List registered models. You can list registered models under a particular +// schema, or list all registered models in the current metastore. +// +// The returned models are filtered based on the privileges of the calling user. +// For example, the metastore admin is able to list all the registered models. A +// regular user needs to be the owner or have the **EXECUTE** privilege on the +// registered model to recieve the registered models in the response. For the +// latter case, the caller must also be the owner or have the **USE_CATALOG** +// privilege on the parent catalog and the **USE_SCHEMA** privilege on the +// parent schema. +// +// There is no guarantee of a specific ordering of the elements in the response. +// +// This method is generated by Databricks SDK Code Generator. +func (a *RegisteredModelsAPI) ListAll(ctx context.Context, request ListRegisteredModelsRequest) ([]RegisteredModelInfo, error) { + iterator := a.List(ctx, request) + return listing.ToSliceN[RegisteredModelInfo, int](ctx, iterator, request.MaxResults) + } // RegisteredModelInfoNameToFullNameMap calls [RegisteredModelsAPI.ListAll] and creates a map of results with [RegisteredModelInfo].Name as key and [RegisteredModelInfo].FullName as value. @@ -1804,12 +2001,36 @@ func (a *SchemasAPI) GetByFullName(ctx context.Context, fullName string) (*Schem // There is no guarantee of a specific ordering of the elements in the array. // // This method is generated by Databricks SDK Code Generator. -func (a *SchemasAPI) ListAll(ctx context.Context, request ListSchemasRequest) ([]SchemaInfo, error) { - response, err := a.impl.List(ctx, request) - if err != nil { - return nil, err +func (a *SchemasAPI) List(ctx context.Context, request ListSchemasRequest) *listing.PaginatingIterator[ListSchemasRequest, *ListSchemasResponse, SchemaInfo] { + + getNextPage := func(ctx context.Context, req ListSchemasRequest) (*ListSchemasResponse, error) { + ctx = useragent.InContext(ctx, "sdk-feature", "pagination") + return a.impl.List(ctx, req) + } + getItems := func(resp *ListSchemasResponse) []SchemaInfo { + return resp.Schemas } - return response.Schemas, nil + + iterator := listing.NewIterator( + &request, + getNextPage, + getItems, + nil) + return iterator +} + +// List schemas. +// +// Gets an array of schemas for a catalog in the metastore. If the caller is the +// metastore admin or the owner of the parent catalog, all schemas for the +// catalog will be retrieved. Otherwise, only schemas owned by the caller (or +// for which the caller has the **USE_SCHEMA** privilege) will be retrieved. +// There is no guarantee of a specific ordering of the elements in the array. +// +// This method is generated by Databricks SDK Code Generator. +func (a *SchemasAPI) ListAll(ctx context.Context, request ListSchemasRequest) ([]SchemaInfo, error) { + iterator := a.List(ctx, request) + return listing.ToSlice[SchemaInfo](ctx, iterator) } // SchemaInfoNameToFullNameMap calls [SchemasAPI.ListAll] and creates a map of results with [SchemaInfo].Name as key and [SchemaInfo].FullName as value. @@ -1969,12 +2190,37 @@ func (a *StorageCredentialsAPI) GetByName(ctx context.Context, name string) (*St // of the elements in the array. // // This method is generated by Databricks SDK Code Generator. -func (a *StorageCredentialsAPI) ListAll(ctx context.Context) ([]StorageCredentialInfo, error) { - response, err := a.impl.List(ctx) - if err != nil { - return nil, err +func (a *StorageCredentialsAPI) List(ctx context.Context) *listing.PaginatingIterator[struct{}, *ListStorageCredentialsResponse, StorageCredentialInfo] { + request := struct{}{} + + getNextPage := func(ctx context.Context, req struct{}) (*ListStorageCredentialsResponse, error) { + ctx = useragent.InContext(ctx, "sdk-feature", "pagination") + return a.impl.List(ctx) } - return response.StorageCredentials, nil + getItems := func(resp *ListStorageCredentialsResponse) []StorageCredentialInfo { + return resp.StorageCredentials + } + + iterator := listing.NewIterator( + &request, + getNextPage, + getItems, + nil) + return iterator +} + +// List credentials. +// +// Gets an array of storage credentials (as __StorageCredentialInfo__ objects). +// The array is limited to only those storage credentials the caller has +// permission to access. If the caller is a metastore admin, all storage +// credentials will be retrieved. There is no guarantee of a specific ordering +// of the elements in the array. +// +// This method is generated by Databricks SDK Code Generator. +func (a *StorageCredentialsAPI) ListAll(ctx context.Context) ([]StorageCredentialInfo, error) { + iterator := a.List(ctx) + return listing.ToSlice[StorageCredentialInfo](ctx, iterator) } // StorageCredentialInfoNameToIdMap calls [StorageCredentialsAPI.ListAll] and creates a map of results with [StorageCredentialInfo].Name as key and [StorageCredentialInfo].Id as value. @@ -2089,12 +2335,33 @@ func (a *SystemSchemasAPI) Enable(ctx context.Context, request EnableRequest) er // account admin or a metastore admin. // // This method is generated by Databricks SDK Code Generator. -func (a *SystemSchemasAPI) ListAll(ctx context.Context, request ListSystemSchemasRequest) ([]SystemSchemaInfo, error) { - response, err := a.impl.List(ctx, request) - if err != nil { - return nil, err +func (a *SystemSchemasAPI) List(ctx context.Context, request ListSystemSchemasRequest) *listing.PaginatingIterator[ListSystemSchemasRequest, *ListSystemSchemasResponse, SystemSchemaInfo] { + + getNextPage := func(ctx context.Context, req ListSystemSchemasRequest) (*ListSystemSchemasResponse, error) { + ctx = useragent.InContext(ctx, "sdk-feature", "pagination") + return a.impl.List(ctx, req) } - return response.Schemas, nil + getItems := func(resp *ListSystemSchemasResponse) []SystemSchemaInfo { + return resp.Schemas + } + + iterator := listing.NewIterator( + &request, + getNextPage, + getItems, + nil) + return iterator +} + +// List system schemas. +// +// Gets an array of system schemas for a metastore. The caller must be an +// account admin or a metastore admin. +// +// This method is generated by Databricks SDK Code Generator. +func (a *SystemSchemasAPI) ListAll(ctx context.Context, request ListSystemSchemasRequest) ([]SystemSchemaInfo, error) { + iterator := a.List(ctx, request) + return listing.ToSlice[SystemSchemaInfo](ctx, iterator) } // List system schemas. @@ -2288,33 +2555,44 @@ func (a *TablesAPI) GetByFullName(ctx context.Context, fullName string) (*TableI // guarantee of a specific ordering of the elements in the array. // // This method is generated by Databricks SDK Code Generator. -func (a *TablesAPI) ListAll(ctx context.Context, request ListTablesRequest) ([]TableInfo, error) { - var results []TableInfo - var totalCount int = 0 - ctx = useragent.InContext(ctx, "sdk-feature", "pagination") - for { - response, err := a.impl.List(ctx, request) - if err != nil { - return nil, err - } - if len(response.Tables) == 0 { - break - } - for _, v := range response.Tables { - results = append(results, v) - } - count := int(len(response.Tables)) - totalCount += count - request.PageToken = response.NextPageToken - if response.NextPageToken == "" { - break - } - limit := request.MaxResults - if limit > 0 && totalCount >= limit { - break +func (a *TablesAPI) List(ctx context.Context, request ListTablesRequest) *listing.PaginatingIterator[ListTablesRequest, *ListTablesResponse, TableInfo] { + + getNextPage := func(ctx context.Context, req ListTablesRequest) (*ListTablesResponse, error) { + ctx = useragent.InContext(ctx, "sdk-feature", "pagination") + return a.impl.List(ctx, req) + } + getItems := func(resp *ListTablesResponse) []TableInfo { + return resp.Tables + } + getNextReq := func(resp *ListTablesResponse) *ListTablesRequest { + if resp.NextPageToken == "" { + return nil } + request.PageToken = resp.NextPageToken + return &request } - return results, nil + iterator := listing.NewIterator( + &request, + getNextPage, + getItems, + getNextReq) + return iterator +} + +// List tables. +// +// Gets an array of all tables for the current metastore under the parent +// catalog and schema. The caller must be a metastore admin or an owner of (or +// have the **SELECT** privilege on) the table. For the latter case, the caller +// must also be the owner or have the **USE_CATALOG** privilege on the parent +// catalog and the **USE_SCHEMA** privilege on the parent schema. There is no +// guarantee of a specific ordering of the elements in the array. +// +// This method is generated by Databricks SDK Code Generator. +func (a *TablesAPI) ListAll(ctx context.Context, request ListTablesRequest) ([]TableInfo, error) { + iterator := a.List(ctx, request) + return listing.ToSliceN[TableInfo, int](ctx, iterator, request.MaxResults) + } // TableInfoNameToTableIdMap calls [TablesAPI.ListAll] and creates a map of results with [TableInfo].Name as key and [TableInfo].TableId as value. @@ -2385,33 +2663,49 @@ func (a *TablesAPI) GetByName(ctx context.Context, name string) (*TableInfo, err // There is no guarantee of a specific ordering of the elements in the array. // // This method is generated by Databricks SDK Code Generator. -func (a *TablesAPI) ListSummariesAll(ctx context.Context, request ListSummariesRequest) ([]TableSummary, error) { - var results []TableSummary - var totalCount int = 0 - ctx = useragent.InContext(ctx, "sdk-feature", "pagination") - for { - response, err := a.impl.ListSummaries(ctx, request) - if err != nil { - return nil, err - } - if len(response.Tables) == 0 { - break - } - for _, v := range response.Tables { - results = append(results, v) - } - count := int(len(response.Tables)) - totalCount += count - request.PageToken = response.NextPageToken - if response.NextPageToken == "" { - break - } - limit := request.MaxResults - if limit > 0 && totalCount >= limit { - break +func (a *TablesAPI) ListSummaries(ctx context.Context, request ListSummariesRequest) *listing.PaginatingIterator[ListSummariesRequest, *ListTableSummariesResponse, TableSummary] { + + getNextPage := func(ctx context.Context, req ListSummariesRequest) (*ListTableSummariesResponse, error) { + ctx = useragent.InContext(ctx, "sdk-feature", "pagination") + return a.impl.ListSummaries(ctx, req) + } + getItems := func(resp *ListTableSummariesResponse) []TableSummary { + return resp.Tables + } + getNextReq := func(resp *ListTableSummariesResponse) *ListSummariesRequest { + if resp.NextPageToken == "" { + return nil } + request.PageToken = resp.NextPageToken + return &request } - return results, nil + iterator := listing.NewIterator( + &request, + getNextPage, + getItems, + getNextReq) + return iterator +} + +// List table summaries. +// +// Gets an array of summaries for tables for a schema and catalog within the +// metastore. The table summaries returned are either: +// +// * summaries for all tables (within the current metastore and parent catalog +// and schema), when the user is a metastore admin, or: * summaries for all +// tables and schemas (within the current metastore and parent catalog) for +// which the user has ownership or the **SELECT** privilege on the table and +// ownership or **USE_SCHEMA** privilege on the schema, provided that the user +// also has ownership or the **USE_CATALOG** privilege on the parent catalog. +// +// There is no guarantee of a specific ordering of the elements in the array. +// +// This method is generated by Databricks SDK Code Generator. +func (a *TablesAPI) ListSummariesAll(ctx context.Context, request ListSummariesRequest) ([]TableSummary, error) { + iterator := a.ListSummaries(ctx, request) + return listing.ToSliceN[TableSummary, int](ctx, iterator, request.MaxResults) + } // Update a table owner. @@ -2524,12 +2818,42 @@ func (a *VolumesAPI) DeleteByFullNameArg(ctx context.Context, fullNameArg string // There is no guarantee of a specific ordering of the elements in the array. // // This method is generated by Databricks SDK Code Generator. -func (a *VolumesAPI) ListAll(ctx context.Context, request ListVolumesRequest) ([]VolumeInfo, error) { - response, err := a.impl.List(ctx, request) - if err != nil { - return nil, err +func (a *VolumesAPI) List(ctx context.Context, request ListVolumesRequest) *listing.PaginatingIterator[ListVolumesRequest, *ListVolumesResponseContent, VolumeInfo] { + + getNextPage := func(ctx context.Context, req ListVolumesRequest) (*ListVolumesResponseContent, error) { + ctx = useragent.InContext(ctx, "sdk-feature", "pagination") + return a.impl.List(ctx, req) } - return response.Volumes, nil + getItems := func(resp *ListVolumesResponseContent) []VolumeInfo { + return resp.Volumes + } + + iterator := listing.NewIterator( + &request, + getNextPage, + getItems, + nil) + return iterator +} + +// List Volumes. +// +// Gets an array of all volumes for the current metastore under the parent +// catalog and schema. +// +// The returned volumes are filtered based on the privileges of the calling +// user. For example, the metastore admin is able to list all the volumes. A +// regular user needs to be the owner or have the **READ VOLUME** privilege on +// the volume to recieve the volumes in the response. For the latter case, the +// caller must also be the owner or have the **USE_CATALOG** privilege on the +// parent catalog and the **USE_SCHEMA** privilege on the parent schema. +// +// There is no guarantee of a specific ordering of the elements in the array. +// +// This method is generated by Databricks SDK Code Generator. +func (a *VolumesAPI) ListAll(ctx context.Context, request ListVolumesRequest) ([]VolumeInfo, error) { + iterator := a.List(ctx, request) + return listing.ToSlice[VolumeInfo](ctx, iterator) } // VolumeInfoNameToVolumeIdMap calls [VolumesAPI.ListAll] and creates a map of results with [VolumeInfo].Name as key and [VolumeInfo].VolumeId as value. diff --git a/service/compute/api.go b/service/compute/api.go index 0c814bbbd..3170063e8 100755 --- a/service/compute/api.go +++ b/service/compute/api.go @@ -9,6 +9,7 @@ import ( "time" "github.com/databricks/databricks-sdk-go/client" + "github.com/databricks/databricks-sdk-go/listing" "github.com/databricks/databricks-sdk-go/retries" "github.com/databricks/databricks-sdk-go/useragent" ) @@ -153,12 +154,32 @@ func (a *ClusterPoliciesAPI) GetPermissionsByClusterPolicyId(ctx context.Context // Returns a list of policies accessible by the requesting user. // // This method is generated by Databricks SDK Code Generator. -func (a *ClusterPoliciesAPI) ListAll(ctx context.Context, request ListClusterPoliciesRequest) ([]Policy, error) { - response, err := a.impl.List(ctx, request) - if err != nil { - return nil, err +func (a *ClusterPoliciesAPI) List(ctx context.Context, request ListClusterPoliciesRequest) *listing.PaginatingIterator[ListClusterPoliciesRequest, *ListPoliciesResponse, Policy] { + + getNextPage := func(ctx context.Context, req ListClusterPoliciesRequest) (*ListPoliciesResponse, error) { + ctx = useragent.InContext(ctx, "sdk-feature", "pagination") + return a.impl.List(ctx, req) + } + getItems := func(resp *ListPoliciesResponse) []Policy { + return resp.Policies } - return response.Policies, nil + + iterator := listing.NewIterator( + &request, + getNextPage, + getItems, + nil) + return iterator +} + +// List cluster policies. +// +// Returns a list of policies accessible by the requesting user. +// +// This method is generated by Databricks SDK Code Generator. +func (a *ClusterPoliciesAPI) ListAll(ctx context.Context, request ListClusterPoliciesRequest) ([]Policy, error) { + iterator := a.List(ctx, request) + return listing.ToSlice[Policy](ctx, iterator) } // PolicyNameToPolicyIdMap calls [ClusterPoliciesAPI.ListAll] and creates a map of results with [Policy].Name as key and [Policy].PolicyId as value. @@ -583,33 +604,42 @@ func (a *ClustersAPI) EditAndWait(ctx context.Context, editCluster EditCluster, // nparameters necessary to request the next page of events. // // This method is generated by Databricks SDK Code Generator. -func (a *ClustersAPI) EventsAll(ctx context.Context, request GetEvents) ([]ClusterEvent, error) { - var results []ClusterEvent - var totalCount int64 = 0 - ctx = useragent.InContext(ctx, "sdk-feature", "pagination") - for { - response, err := a.impl.Events(ctx, request) - if err != nil { - return nil, err - } - if len(response.Events) == 0 { - break - } - for _, v := range response.Events { - results = append(results, v) - } - count := int64(len(response.Events)) - totalCount += count - if response.NextPage == nil { - break - } - request = *response.NextPage - limit := request.Limit - if limit > 0 && totalCount >= limit { - break +func (a *ClustersAPI) Events(ctx context.Context, request GetEvents) *listing.PaginatingIterator[GetEvents, *GetEventsResponse, ClusterEvent] { + + getNextPage := func(ctx context.Context, req GetEvents) (*GetEventsResponse, error) { + ctx = useragent.InContext(ctx, "sdk-feature", "pagination") + return a.impl.Events(ctx, req) + } + getItems := func(resp *GetEventsResponse) []ClusterEvent { + return resp.Events + } + getNextReq := func(resp *GetEventsResponse) *GetEvents { + if resp.NextPage == nil { + return nil } + request = *resp.NextPage + + return &request } - return results, nil + iterator := listing.NewIterator( + &request, + getNextPage, + getItems, + getNextReq) + return iterator +} + +// List cluster activity events. +// +// Retrieves a list of events about the activity of a cluster. This API is +// paginated. If there are more events to read, the response includes all the +// nparameters necessary to request the next page of events. +// +// This method is generated by Databricks SDK Code Generator. +func (a *ClustersAPI) EventsAll(ctx context.Context, request GetEvents) ([]ClusterEvent, error) { + iterator := a.Events(ctx, request) + return listing.ToSliceN[ClusterEvent, int64](ctx, iterator, request.Limit) + } // Get cluster info. @@ -677,12 +707,40 @@ func (a *ClustersAPI) GetPermissionsByClusterId(ctx context.Context, clusterId s // terminated job clusters. // // This method is generated by Databricks SDK Code Generator. -func (a *ClustersAPI) ListAll(ctx context.Context, request ListClustersRequest) ([]ClusterDetails, error) { - response, err := a.impl.List(ctx, request) - if err != nil { - return nil, err +func (a *ClustersAPI) List(ctx context.Context, request ListClustersRequest) *listing.PaginatingIterator[ListClustersRequest, *ListClustersResponse, ClusterDetails] { + + getNextPage := func(ctx context.Context, req ListClustersRequest) (*ListClustersResponse, error) { + ctx = useragent.InContext(ctx, "sdk-feature", "pagination") + return a.impl.List(ctx, req) + } + getItems := func(resp *ListClustersResponse) []ClusterDetails { + return resp.Clusters } - return response.Clusters, nil + + iterator := listing.NewIterator( + &request, + getNextPage, + getItems, + nil) + return iterator +} + +// List all clusters. +// +// Return information about all pinned clusters, active clusters, up to 200 of +// the most recently terminated all-purpose clusters in the past 30 days, and up +// to 30 of the most recently terminated job clusters in the past 30 days. +// +// For example, if there is 1 pinned cluster, 4 active clusters, 45 terminated +// all-purpose clusters in the past 30 days, and 50 terminated job clusters in +// the past 30 days, then this API returns the 1 pinned cluster, 4 active +// clusters, all 45 terminated all-purpose clusters, and the 30 most recently +// terminated job clusters. +// +// This method is generated by Databricks SDK Code Generator. +func (a *ClustersAPI) ListAll(ctx context.Context, request ListClustersRequest) ([]ClusterDetails, error) { + iterator := a.List(ctx, request) + return listing.ToSlice[ClusterDetails](ctx, iterator) } // ClusterDetailsClusterNameToClusterIdMap calls [ClustersAPI.ListAll] and creates a map of results with [ClusterDetails].ClusterName as key and [ClusterDetails].ClusterId as value. @@ -1491,12 +1549,36 @@ func (a *GlobalInitScriptsAPI) GetByScriptId(ctx context.Context, scriptId strin // script](#operation/get-script) operation. // // This method is generated by Databricks SDK Code Generator. -func (a *GlobalInitScriptsAPI) ListAll(ctx context.Context) ([]GlobalInitScriptDetails, error) { - response, err := a.impl.List(ctx) - if err != nil { - return nil, err +func (a *GlobalInitScriptsAPI) List(ctx context.Context) *listing.PaginatingIterator[struct{}, *ListGlobalInitScriptsResponse, GlobalInitScriptDetails] { + request := struct{}{} + + getNextPage := func(ctx context.Context, req struct{}) (*ListGlobalInitScriptsResponse, error) { + ctx = useragent.InContext(ctx, "sdk-feature", "pagination") + return a.impl.List(ctx) + } + getItems := func(resp *ListGlobalInitScriptsResponse) []GlobalInitScriptDetails { + return resp.Scripts } - return response.Scripts, nil + + iterator := listing.NewIterator( + &request, + getNextPage, + getItems, + nil) + return iterator +} + +// Get init scripts. +// +// Get a list of all global init scripts for this workspace. This returns all +// properties for each script but **not** the script contents. To retrieve the +// contents of a script, use the [get a global init +// script](#operation/get-script) operation. +// +// This method is generated by Databricks SDK Code Generator. +func (a *GlobalInitScriptsAPI) ListAll(ctx context.Context) ([]GlobalInitScriptDetails, error) { + iterator := a.List(ctx) + return listing.ToSlice[GlobalInitScriptDetails](ctx, iterator) } // GlobalInitScriptDetailsNameToScriptIdMap calls [GlobalInitScriptsAPI.ListAll] and creates a map of results with [GlobalInitScriptDetails].Name as key and [GlobalInitScriptDetails].ScriptId as value. @@ -1691,12 +1773,33 @@ func (a *InstancePoolsAPI) GetPermissionsByInstancePoolId(ctx context.Context, i // Gets a list of instance pools with their statistics. // // This method is generated by Databricks SDK Code Generator. -func (a *InstancePoolsAPI) ListAll(ctx context.Context) ([]InstancePoolAndStats, error) { - response, err := a.impl.List(ctx) - if err != nil { - return nil, err +func (a *InstancePoolsAPI) List(ctx context.Context) *listing.PaginatingIterator[struct{}, *ListInstancePools, InstancePoolAndStats] { + request := struct{}{} + + getNextPage := func(ctx context.Context, req struct{}) (*ListInstancePools, error) { + ctx = useragent.InContext(ctx, "sdk-feature", "pagination") + return a.impl.List(ctx) } - return response.InstancePools, nil + getItems := func(resp *ListInstancePools) []InstancePoolAndStats { + return resp.InstancePools + } + + iterator := listing.NewIterator( + &request, + getNextPage, + getItems, + nil) + return iterator +} + +// List instance pool info. +// +// Gets a list of instance pools with their statistics. +// +// This method is generated by Databricks SDK Code Generator. +func (a *InstancePoolsAPI) ListAll(ctx context.Context) ([]InstancePoolAndStats, error) { + iterator := a.List(ctx) + return listing.ToSlice[InstancePoolAndStats](ctx, iterator) } // InstancePoolAndStatsInstancePoolNameToInstancePoolIdMap calls [InstancePoolsAPI.ListAll] and creates a map of results with [InstancePoolAndStats].InstancePoolName as key and [InstancePoolAndStats].InstancePoolId as value. @@ -1836,12 +1939,35 @@ func (a *InstanceProfilesAPI) Edit(ctx context.Context, request InstanceProfile) // This API is available to all users. // // This method is generated by Databricks SDK Code Generator. -func (a *InstanceProfilesAPI) ListAll(ctx context.Context) ([]InstanceProfile, error) { - response, err := a.impl.List(ctx) - if err != nil { - return nil, err +func (a *InstanceProfilesAPI) List(ctx context.Context) *listing.PaginatingIterator[struct{}, *ListInstanceProfilesResponse, InstanceProfile] { + request := struct{}{} + + getNextPage := func(ctx context.Context, req struct{}) (*ListInstanceProfilesResponse, error) { + ctx = useragent.InContext(ctx, "sdk-feature", "pagination") + return a.impl.List(ctx) } - return response.InstanceProfiles, nil + getItems := func(resp *ListInstanceProfilesResponse) []InstanceProfile { + return resp.InstanceProfiles + } + + iterator := listing.NewIterator( + &request, + getNextPage, + getItems, + nil) + return iterator +} + +// List available instance profiles. +// +// List the instance profiles that the calling user can use to launch a cluster. +// +// This API is available to all users. +// +// This method is generated by Databricks SDK Code Generator. +func (a *InstanceProfilesAPI) ListAll(ctx context.Context) ([]InstanceProfile, error) { + iterator := a.List(ctx) + return listing.ToSlice[InstanceProfile](ctx, iterator) } // Remove the instance profile. @@ -1941,12 +2067,46 @@ func (a *LibrariesAPI) AllClusterStatuses(ctx context.Context) (*ListAllClusterL // guarantee. // // This method is generated by Databricks SDK Code Generator. -func (a *LibrariesAPI) ClusterStatusAll(ctx context.Context, request ClusterStatusRequest) ([]LibraryFullStatus, error) { - response, err := a.impl.ClusterStatus(ctx, request) - if err != nil { - return nil, err +func (a *LibrariesAPI) ClusterStatus(ctx context.Context, request ClusterStatusRequest) *listing.PaginatingIterator[ClusterStatusRequest, *ClusterLibraryStatuses, LibraryFullStatus] { + + getNextPage := func(ctx context.Context, req ClusterStatusRequest) (*ClusterLibraryStatuses, error) { + ctx = useragent.InContext(ctx, "sdk-feature", "pagination") + return a.impl.ClusterStatus(ctx, req) + } + getItems := func(resp *ClusterLibraryStatuses) []LibraryFullStatus { + return resp.LibraryStatuses } - return response.LibraryStatuses, nil + + iterator := listing.NewIterator( + &request, + getNextPage, + getItems, + nil) + return iterator +} + +// Get status. +// +// Get the status of libraries on a cluster. A status will be available for all +// libraries installed on this cluster via the API or the libraries UI as well +// as libraries set to be installed on all clusters via the libraries UI. The +// order of returned libraries will be as follows. +// +// 1. Libraries set to be installed on this cluster will be returned first. +// Within this group, the final order will be order in which the libraries were +// added to the cluster. +// +// 2. Libraries set to be installed on all clusters are returned next. Within +// this group there is no order guarantee. +// +// 3. Libraries that were previously requested on this cluster or on all +// clusters, but now marked for removal. Within this group there is no order +// guarantee. +// +// This method is generated by Databricks SDK Code Generator. +func (a *LibrariesAPI) ClusterStatusAll(ctx context.Context, request ClusterStatusRequest) ([]LibraryFullStatus, error) { + iterator := a.ClusterStatus(ctx, request) + return listing.ToSlice[LibraryFullStatus](ctx, iterator) } // Get status. @@ -2049,31 +2209,37 @@ func (a *PolicyFamiliesAPI) GetByPolicyFamilyId(ctx context.Context, policyFamil // Retrieve a list of policy families. This API is paginated. // // This method is generated by Databricks SDK Code Generator. -func (a *PolicyFamiliesAPI) ListAll(ctx context.Context, request ListPolicyFamiliesRequest) ([]PolicyFamily, error) { - var results []PolicyFamily - var totalCount int64 = 0 - ctx = useragent.InContext(ctx, "sdk-feature", "pagination") - for { - response, err := a.impl.List(ctx, request) - if err != nil { - return nil, err - } - if len(response.PolicyFamilies) == 0 { - break - } - for _, v := range response.PolicyFamilies { - results = append(results, v) - } - count := int64(len(response.PolicyFamilies)) - totalCount += count - request.PageToken = response.NextPageToken - if response.NextPageToken == "" { - break - } - limit := request.MaxResults - if limit > 0 && totalCount >= limit { - break +func (a *PolicyFamiliesAPI) List(ctx context.Context, request ListPolicyFamiliesRequest) *listing.PaginatingIterator[ListPolicyFamiliesRequest, *ListPolicyFamiliesResponse, PolicyFamily] { + + getNextPage := func(ctx context.Context, req ListPolicyFamiliesRequest) (*ListPolicyFamiliesResponse, error) { + ctx = useragent.InContext(ctx, "sdk-feature", "pagination") + return a.impl.List(ctx, req) + } + getItems := func(resp *ListPolicyFamiliesResponse) []PolicyFamily { + return resp.PolicyFamilies + } + getNextReq := func(resp *ListPolicyFamiliesResponse) *ListPolicyFamiliesRequest { + if resp.NextPageToken == "" { + return nil } + request.PageToken = resp.NextPageToken + return &request } - return results, nil + iterator := listing.NewIterator( + &request, + getNextPage, + getItems, + getNextReq) + return iterator +} + +// List policy families. +// +// Retrieve a list of policy families. This API is paginated. +// +// This method is generated by Databricks SDK Code Generator. +func (a *PolicyFamiliesAPI) ListAll(ctx context.Context, request ListPolicyFamiliesRequest) ([]PolicyFamily, error) { + iterator := a.List(ctx, request) + return listing.ToSliceN[PolicyFamily, int64](ctx, iterator, request.MaxResults) + } diff --git a/service/files/api.go b/service/files/api.go index dd8eb2c3b..6f4353001 100755 --- a/service/files/api.go +++ b/service/files/api.go @@ -7,6 +7,8 @@ import ( "context" "github.com/databricks/databricks-sdk-go/client" + "github.com/databricks/databricks-sdk-go/listing" + "github.com/databricks/databricks-sdk-go/useragent" ) func NewDbfs(client *client.DatabricksClient) *DbfsAPI { @@ -140,12 +142,42 @@ func (a *DbfsAPI) GetStatusByPath(ctx context.Context, path string) (*FileInfo, // which provides the same functionality without timing out. // // This method is generated by Databricks SDK Code Generator. -func (a *DbfsAPI) ListAll(ctx context.Context, request ListDbfsRequest) ([]FileInfo, error) { - response, err := a.impl.List(ctx, request) - if err != nil { - return nil, err +func (a *DbfsAPI) List(ctx context.Context, request ListDbfsRequest) *listing.PaginatingIterator[ListDbfsRequest, *ListStatusResponse, FileInfo] { + + getNextPage := func(ctx context.Context, req ListDbfsRequest) (*ListStatusResponse, error) { + ctx = useragent.InContext(ctx, "sdk-feature", "pagination") + return a.impl.List(ctx, req) + } + getItems := func(resp *ListStatusResponse) []FileInfo { + return resp.Files } - return response.Files, nil + + iterator := listing.NewIterator( + &request, + getNextPage, + getItems, + nil) + return iterator +} + +// List directory contents or file details. +// +// List the contents of a directory, or details of the file. If the file or +// directory does not exist, this call throws an exception with +// `RESOURCE_DOES_NOT_EXIST`. +// +// When calling list on a large directory, the list operation will time out +// after approximately 60 seconds. We strongly recommend using list only on +// directories containing less than 10K files and discourage using the DBFS REST +// API for operations that list more than 10K files. Instead, we recommend that +// you perform such operations in the context of a cluster, using the [File +// system utility (dbutils.fs)](/dev-tools/databricks-utils.html#dbutils-fs), +// which provides the same functionality without timing out. +// +// This method is generated by Databricks SDK Code Generator. +func (a *DbfsAPI) ListAll(ctx context.Context, request ListDbfsRequest) ([]FileInfo, error) { + iterator := a.List(ctx, request) + return listing.ToSlice[FileInfo](ctx, iterator) } // List directory contents or file details. diff --git a/service/iam/api.go b/service/iam/api.go index 45b59de49..5c6b1d027 100755 --- a/service/iam/api.go +++ b/service/iam/api.go @@ -8,6 +8,7 @@ import ( "fmt" "github.com/databricks/databricks-sdk-go/client" + "github.com/databricks/databricks-sdk-go/listing" "github.com/databricks/databricks-sdk-go/useragent" ) @@ -202,12 +203,32 @@ func (a *AccountGroupsAPI) GetById(ctx context.Context, id string) (*Group, erro // Gets all details of the groups associated with the Databricks account. // // This method is generated by Databricks SDK Code Generator. -func (a *AccountGroupsAPI) ListAll(ctx context.Context, request ListAccountGroupsRequest) ([]Group, error) { - response, err := a.impl.List(ctx, request) - if err != nil { - return nil, err +func (a *AccountGroupsAPI) List(ctx context.Context, request ListAccountGroupsRequest) *listing.PaginatingIterator[ListAccountGroupsRequest, *ListGroupsResponse, Group] { + + getNextPage := func(ctx context.Context, req ListAccountGroupsRequest) (*ListGroupsResponse, error) { + ctx = useragent.InContext(ctx, "sdk-feature", "pagination") + return a.impl.List(ctx, req) } - return response.Resources, nil + getItems := func(resp *ListGroupsResponse) []Group { + return resp.Resources + } + + iterator := listing.NewIterator( + &request, + getNextPage, + getItems, + nil) + return iterator +} + +// List group details. +// +// Gets all details of the groups associated with the Databricks account. +// +// This method is generated by Databricks SDK Code Generator. +func (a *AccountGroupsAPI) ListAll(ctx context.Context, request ListAccountGroupsRequest) ([]Group, error) { + iterator := a.List(ctx, request) + return listing.ToSlice[Group](ctx, iterator) } // GroupDisplayNameToIdMap calls [AccountGroupsAPI.ListAll] and creates a map of results with [Group].DisplayName as key and [Group].Id as value. @@ -355,12 +376,32 @@ func (a *AccountServicePrincipalsAPI) GetById(ctx context.Context, id string) (* // Gets the set of service principals associated with a Databricks account. // // This method is generated by Databricks SDK Code Generator. -func (a *AccountServicePrincipalsAPI) ListAll(ctx context.Context, request ListAccountServicePrincipalsRequest) ([]ServicePrincipal, error) { - response, err := a.impl.List(ctx, request) - if err != nil { - return nil, err +func (a *AccountServicePrincipalsAPI) List(ctx context.Context, request ListAccountServicePrincipalsRequest) *listing.PaginatingIterator[ListAccountServicePrincipalsRequest, *ListServicePrincipalResponse, ServicePrincipal] { + + getNextPage := func(ctx context.Context, req ListAccountServicePrincipalsRequest) (*ListServicePrincipalResponse, error) { + ctx = useragent.InContext(ctx, "sdk-feature", "pagination") + return a.impl.List(ctx, req) + } + getItems := func(resp *ListServicePrincipalResponse) []ServicePrincipal { + return resp.Resources } - return response.Resources, nil + + iterator := listing.NewIterator( + &request, + getNextPage, + getItems, + nil) + return iterator +} + +// List service principals. +// +// Gets the set of service principals associated with a Databricks account. +// +// This method is generated by Databricks SDK Code Generator. +func (a *AccountServicePrincipalsAPI) ListAll(ctx context.Context, request ListAccountServicePrincipalsRequest) ([]ServicePrincipal, error) { + iterator := a.List(ctx, request) + return listing.ToSlice[ServicePrincipal](ctx, iterator) } // ServicePrincipalDisplayNameToIdMap calls [AccountServicePrincipalsAPI.ListAll] and creates a map of results with [ServicePrincipal].DisplayName as key and [ServicePrincipal].Id as value. @@ -517,12 +558,32 @@ func (a *AccountUsersAPI) GetById(ctx context.Context, id string) (*User, error) // Gets details for all the users associated with a Databricks account. // // This method is generated by Databricks SDK Code Generator. -func (a *AccountUsersAPI) ListAll(ctx context.Context, request ListAccountUsersRequest) ([]User, error) { - response, err := a.impl.List(ctx, request) - if err != nil { - return nil, err +func (a *AccountUsersAPI) List(ctx context.Context, request ListAccountUsersRequest) *listing.PaginatingIterator[ListAccountUsersRequest, *ListUsersResponse, User] { + + getNextPage := func(ctx context.Context, req ListAccountUsersRequest) (*ListUsersResponse, error) { + ctx = useragent.InContext(ctx, "sdk-feature", "pagination") + return a.impl.List(ctx, req) } - return response.Resources, nil + getItems := func(resp *ListUsersResponse) []User { + return resp.Resources + } + + iterator := listing.NewIterator( + &request, + getNextPage, + getItems, + nil) + return iterator +} + +// List users. +// +// Gets details for all the users associated with a Databricks account. +// +// This method is generated by Databricks SDK Code Generator. +func (a *AccountUsersAPI) ListAll(ctx context.Context, request ListAccountUsersRequest) ([]User, error) { + iterator := a.List(ctx, request) + return listing.ToSlice[User](ctx, iterator) } // UserUserNameToIdMap calls [AccountUsersAPI.ListAll] and creates a map of results with [User].UserName as key and [User].Id as value. @@ -706,12 +767,32 @@ func (a *GroupsAPI) GetById(ctx context.Context, id string) (*Group, error) { // Gets all details of the groups associated with the Databricks workspace. // // This method is generated by Databricks SDK Code Generator. -func (a *GroupsAPI) ListAll(ctx context.Context, request ListGroupsRequest) ([]Group, error) { - response, err := a.impl.List(ctx, request) - if err != nil { - return nil, err +func (a *GroupsAPI) List(ctx context.Context, request ListGroupsRequest) *listing.PaginatingIterator[ListGroupsRequest, *ListGroupsResponse, Group] { + + getNextPage := func(ctx context.Context, req ListGroupsRequest) (*ListGroupsResponse, error) { + ctx = useragent.InContext(ctx, "sdk-feature", "pagination") + return a.impl.List(ctx, req) + } + getItems := func(resp *ListGroupsResponse) []Group { + return resp.Resources } - return response.Resources, nil + + iterator := listing.NewIterator( + &request, + getNextPage, + getItems, + nil) + return iterator +} + +// List group details. +// +// Gets all details of the groups associated with the Databricks workspace. +// +// This method is generated by Databricks SDK Code Generator. +func (a *GroupsAPI) ListAll(ctx context.Context, request ListGroupsRequest) ([]Group, error) { + iterator := a.List(ctx, request) + return listing.ToSlice[Group](ctx, iterator) } // GroupDisplayNameToIdMap calls [GroupsAPI.ListAll] and creates a map of results with [Group].DisplayName as key and [Group].Id as value. @@ -984,12 +1065,32 @@ func (a *ServicePrincipalsAPI) GetById(ctx context.Context, id string) (*Service // Gets the set of service principals associated with a Databricks workspace. // // This method is generated by Databricks SDK Code Generator. -func (a *ServicePrincipalsAPI) ListAll(ctx context.Context, request ListServicePrincipalsRequest) ([]ServicePrincipal, error) { - response, err := a.impl.List(ctx, request) - if err != nil { - return nil, err +func (a *ServicePrincipalsAPI) List(ctx context.Context, request ListServicePrincipalsRequest) *listing.PaginatingIterator[ListServicePrincipalsRequest, *ListServicePrincipalResponse, ServicePrincipal] { + + getNextPage := func(ctx context.Context, req ListServicePrincipalsRequest) (*ListServicePrincipalResponse, error) { + ctx = useragent.InContext(ctx, "sdk-feature", "pagination") + return a.impl.List(ctx, req) } - return response.Resources, nil + getItems := func(resp *ListServicePrincipalResponse) []ServicePrincipal { + return resp.Resources + } + + iterator := listing.NewIterator( + &request, + getNextPage, + getItems, + nil) + return iterator +} + +// List service principals. +// +// Gets the set of service principals associated with a Databricks workspace. +// +// This method is generated by Databricks SDK Code Generator. +func (a *ServicePrincipalsAPI) ListAll(ctx context.Context, request ListServicePrincipalsRequest) ([]ServicePrincipal, error) { + iterator := a.List(ctx, request) + return listing.ToSlice[ServicePrincipal](ctx, iterator) } // ServicePrincipalDisplayNameToIdMap calls [ServicePrincipalsAPI.ListAll] and creates a map of results with [ServicePrincipal].DisplayName as key and [ServicePrincipal].Id as value. @@ -1161,12 +1262,32 @@ func (a *UsersAPI) GetPermissions(ctx context.Context) (*PasswordPermissions, er // Gets details for all the users associated with a Databricks workspace. // // This method is generated by Databricks SDK Code Generator. -func (a *UsersAPI) ListAll(ctx context.Context, request ListUsersRequest) ([]User, error) { - response, err := a.impl.List(ctx, request) - if err != nil { - return nil, err +func (a *UsersAPI) List(ctx context.Context, request ListUsersRequest) *listing.PaginatingIterator[ListUsersRequest, *ListUsersResponse, User] { + + getNextPage := func(ctx context.Context, req ListUsersRequest) (*ListUsersResponse, error) { + ctx = useragent.InContext(ctx, "sdk-feature", "pagination") + return a.impl.List(ctx, req) + } + getItems := func(resp *ListUsersResponse) []User { + return resp.Resources } - return response.Resources, nil + + iterator := listing.NewIterator( + &request, + getNextPage, + getItems, + nil) + return iterator +} + +// List users. +// +// Gets details for all the users associated with a Databricks workspace. +// +// This method is generated by Databricks SDK Code Generator. +func (a *UsersAPI) ListAll(ctx context.Context, request ListUsersRequest) ([]User, error) { + iterator := a.List(ctx, request) + return listing.ToSlice[User](ctx, iterator) } // UserUserNameToIdMap calls [UsersAPI.ListAll] and creates a map of results with [User].UserName as key and [User].Id as value. @@ -1324,12 +1445,33 @@ func (a *WorkspaceAssignmentAPI) GetByWorkspaceId(ctx context.Context, workspace // Databricks workspace. // // This method is generated by Databricks SDK Code Generator. -func (a *WorkspaceAssignmentAPI) ListAll(ctx context.Context, request ListWorkspaceAssignmentRequest) ([]PermissionAssignment, error) { - response, err := a.impl.List(ctx, request) - if err != nil { - return nil, err +func (a *WorkspaceAssignmentAPI) List(ctx context.Context, request ListWorkspaceAssignmentRequest) *listing.PaginatingIterator[ListWorkspaceAssignmentRequest, *PermissionAssignments, PermissionAssignment] { + + getNextPage := func(ctx context.Context, req ListWorkspaceAssignmentRequest) (*PermissionAssignments, error) { + ctx = useragent.InContext(ctx, "sdk-feature", "pagination") + return a.impl.List(ctx, req) + } + getItems := func(resp *PermissionAssignments) []PermissionAssignment { + return resp.PermissionAssignments } - return response.PermissionAssignments, nil + + iterator := listing.NewIterator( + &request, + getNextPage, + getItems, + nil) + return iterator +} + +// Get permission assignments. +// +// Get the permission assignments for the specified Databricks account and +// Databricks workspace. +// +// This method is generated by Databricks SDK Code Generator. +func (a *WorkspaceAssignmentAPI) ListAll(ctx context.Context, request ListWorkspaceAssignmentRequest) ([]PermissionAssignment, error) { + iterator := a.List(ctx, request) + return listing.ToSlice[PermissionAssignment](ctx, iterator) } // Get permission assignments. diff --git a/service/iam/model.go b/service/iam/model.go index 4d1a0f3d0..f08919356 100755 --- a/service/iam/model.go +++ b/service/iam/model.go @@ -229,7 +229,7 @@ type Group struct { Groups []ComplexValue `json:"groups,omitempty"` // Databricks group ID - Id string `json:"id,omitempty"` + Id string `json:"id,omitempty" url:"-"` Members []ComplexValue `json:"members,omitempty"` // Container for the group identifier. Workspace local versus account. @@ -1070,7 +1070,7 @@ type User struct { Groups []ComplexValue `json:"groups,omitempty"` // Databricks user ID. - Id string `json:"id,omitempty"` + Id string `json:"id,omitempty" url:"-"` Name *Name `json:"name,omitempty"` diff --git a/service/jobs/api.go b/service/jobs/api.go index 6bbe4c593..3fc54c127 100755 --- a/service/jobs/api.go +++ b/service/jobs/api.go @@ -9,6 +9,7 @@ import ( "time" "github.com/databricks/databricks-sdk-go/client" + "github.com/databricks/databricks-sdk-go/listing" "github.com/databricks/databricks-sdk-go/retries" "github.com/databricks/databricks-sdk-go/useragent" ) @@ -327,33 +328,39 @@ func (a *JobsAPI) GetRunOutputByRunId(ctx context.Context, runId int64) (*RunOut // Retrieves a list of jobs. // // This method is generated by Databricks SDK Code Generator. -func (a *JobsAPI) ListAll(ctx context.Context, request ListJobsRequest) ([]BaseJob, error) { - var results []BaseJob - var totalCount int = 0 - ctx = useragent.InContext(ctx, "sdk-feature", "pagination") - for { - response, err := a.impl.List(ctx, request) - if err != nil { - return nil, err - } - if len(response.Jobs) == 0 { - break - } - for _, v := range response.Jobs { - results = append(results, v) - } - count := int(len(response.Jobs)) - totalCount += count - request.PageToken = response.NextPageToken - if response.NextPageToken == "" { - break - } - limit := request.Limit - if limit > 0 && totalCount >= limit { - break +func (a *JobsAPI) List(ctx context.Context, request ListJobsRequest) *listing.PaginatingIterator[ListJobsRequest, *ListJobsResponse, BaseJob] { + + getNextPage := func(ctx context.Context, req ListJobsRequest) (*ListJobsResponse, error) { + ctx = useragent.InContext(ctx, "sdk-feature", "pagination") + return a.impl.List(ctx, req) + } + getItems := func(resp *ListJobsResponse) []BaseJob { + return resp.Jobs + } + getNextReq := func(resp *ListJobsResponse) *ListJobsRequest { + if resp.NextPageToken == "" { + return nil } + request.PageToken = resp.NextPageToken + return &request } - return results, nil + iterator := listing.NewIterator( + &request, + getNextPage, + getItems, + getNextReq) + return iterator +} + +// List jobs. +// +// Retrieves a list of jobs. +// +// This method is generated by Databricks SDK Code Generator. +func (a *JobsAPI) ListAll(ctx context.Context, request ListJobsRequest) ([]BaseJob, error) { + iterator := a.List(ctx, request) + return listing.ToSliceN[BaseJob, int](ctx, iterator, request.Limit) + } // BaseJobSettingsNameToJobIdMap calls [JobsAPI.ListAll] and creates a map of results with [BaseJob].Settings.Name as key and [BaseJob].JobId as value. @@ -414,33 +421,39 @@ func (a *JobsAPI) GetBySettingsName(ctx context.Context, name string) (*BaseJob, // List runs in descending order by start time. // // This method is generated by Databricks SDK Code Generator. -func (a *JobsAPI) ListRunsAll(ctx context.Context, request ListRunsRequest) ([]BaseRun, error) { - var results []BaseRun - var totalCount int = 0 - ctx = useragent.InContext(ctx, "sdk-feature", "pagination") - for { - response, err := a.impl.ListRuns(ctx, request) - if err != nil { - return nil, err - } - if len(response.Runs) == 0 { - break - } - for _, v := range response.Runs { - results = append(results, v) - } - count := int(len(response.Runs)) - totalCount += count - request.PageToken = response.NextPageToken - if response.NextPageToken == "" { - break - } - limit := request.Limit - if limit > 0 && totalCount >= limit { - break +func (a *JobsAPI) ListRuns(ctx context.Context, request ListRunsRequest) *listing.PaginatingIterator[ListRunsRequest, *ListRunsResponse, BaseRun] { + + getNextPage := func(ctx context.Context, req ListRunsRequest) (*ListRunsResponse, error) { + ctx = useragent.InContext(ctx, "sdk-feature", "pagination") + return a.impl.ListRuns(ctx, req) + } + getItems := func(resp *ListRunsResponse) []BaseRun { + return resp.Runs + } + getNextReq := func(resp *ListRunsResponse) *ListRunsRequest { + if resp.NextPageToken == "" { + return nil } + request.PageToken = resp.NextPageToken + return &request } - return results, nil + iterator := listing.NewIterator( + &request, + getNextPage, + getItems, + getNextReq) + return iterator +} + +// List job runs. +// +// List runs in descending order by start time. +// +// This method is generated by Databricks SDK Code Generator. +func (a *JobsAPI) ListRunsAll(ctx context.Context, request ListRunsRequest) ([]BaseRun, error) { + iterator := a.ListRuns(ctx, request) + return listing.ToSliceN[BaseRun, int](ctx, iterator, request.Limit) + } // Repair a job run. diff --git a/service/jobs/jobs_usage_test.go b/service/jobs/jobs_usage_test.go index 2130b26b6..d05292d5b 100755 --- a/service/jobs/jobs_usage_test.go +++ b/service/jobs/jobs_usage_test.go @@ -382,6 +382,64 @@ func ExampleJobsAPI_ListAll_jobsApiFullIntegration() { } +func ExampleJobsAPI_ListRuns_jobsApiFullIntegration() { + ctx := context.Background() + w, err := databricks.NewWorkspaceClient() + if err != nil { + panic(err) + } + + notebookPath := func() string { + me, err := w.CurrentUser.Me(ctx) + if err != nil { + panic(err) + } + return filepath.Join("/Users", me.UserName, fmt.Sprintf("sdk-%x", time.Now().UnixNano())) + }() + + clusterId := func() string { + clusterId := os.Getenv("DATABRICKS_CLUSTER_ID") + err := w.Clusters.EnsureClusterIsRunning(ctx, clusterId) + if err != nil { + panic(err) + } + return clusterId + }() + + createdJob, err := w.Jobs.Create(ctx, jobs.CreateJob{ + Name: fmt.Sprintf("sdk-%x", time.Now().UnixNano()), + Tasks: []jobs.Task{jobs.Task{ + Description: "test", + ExistingClusterId: clusterId, + NotebookTask: &jobs.NotebookTask{ + NotebookPath: notebookPath, + }, + TaskKey: "test", + TimeoutSeconds: 0, + }}, + }) + if err != nil { + panic(err) + } + logger.Infof(ctx, "found %v", createdJob) + + runList, err := w.Jobs.ListRunsAll(ctx, jobs.ListRunsRequest{ + JobId: createdJob.JobId, + }) + if err != nil { + panic(err) + } + logger.Infof(ctx, "found %v", runList) + + // cleanup + + err = w.Jobs.DeleteByJobId(ctx, createdJob.JobId) + if err != nil { + panic(err) + } + +} + func ExampleJobsAPI_RepairRun_jobsApiFullIntegration() { ctx := context.Background() w, err := databricks.NewWorkspaceClient() diff --git a/service/ml/api.go b/service/ml/api.go index eea99d654..a01153788 100755 --- a/service/ml/api.go +++ b/service/ml/api.go @@ -7,6 +7,7 @@ import ( "context" "github.com/databricks/databricks-sdk-go/client" + "github.com/databricks/databricks-sdk-go/listing" "github.com/databricks/databricks-sdk-go/useragent" ) @@ -126,33 +127,39 @@ func (a *ExperimentsAPI) GetExperiment(ctx context.Context, request GetExperimen // Gets a list of all values for the specified metric for a given run. // // This method is generated by Databricks SDK Code Generator. -func (a *ExperimentsAPI) GetHistoryAll(ctx context.Context, request GetHistoryRequest) ([]Metric, error) { - var results []Metric - var totalCount int = 0 - ctx = useragent.InContext(ctx, "sdk-feature", "pagination") - for { - response, err := a.impl.GetHistory(ctx, request) - if err != nil { - return nil, err - } - if len(response.Metrics) == 0 { - break - } - for _, v := range response.Metrics { - results = append(results, v) - } - count := int(len(response.Metrics)) - totalCount += count - request.PageToken = response.NextPageToken - if response.NextPageToken == "" { - break - } - limit := request.MaxResults - if limit > 0 && totalCount >= limit { - break +func (a *ExperimentsAPI) GetHistory(ctx context.Context, request GetHistoryRequest) *listing.PaginatingIterator[GetHistoryRequest, *GetMetricHistoryResponse, Metric] { + + getNextPage := func(ctx context.Context, req GetHistoryRequest) (*GetMetricHistoryResponse, error) { + ctx = useragent.InContext(ctx, "sdk-feature", "pagination") + return a.impl.GetHistory(ctx, req) + } + getItems := func(resp *GetMetricHistoryResponse) []Metric { + return resp.Metrics + } + getNextReq := func(resp *GetMetricHistoryResponse) *GetHistoryRequest { + if resp.NextPageToken == "" { + return nil } + request.PageToken = resp.NextPageToken + return &request } - return results, nil + iterator := listing.NewIterator( + &request, + getNextPage, + getItems, + getNextReq) + return iterator +} + +// Get history of a given metric within a run. +// +// Gets a list of all values for the specified metric for a given run. +// +// This method is generated by Databricks SDK Code Generator. +func (a *ExperimentsAPI) GetHistoryAll(ctx context.Context, request GetHistoryRequest) ([]Metric, error) { + iterator := a.GetHistory(ctx, request) + return listing.ToSliceN[Metric, int](ctx, iterator, request.MaxResults) + } // Get experiment permission levels. @@ -207,26 +214,39 @@ func (a *ExperimentsAPI) GetRun(ctx context.Context, request GetRunRequest) (*Ge // specified, the response contains only artifacts with the specified prefix.", // // This method is generated by Databricks SDK Code Generator. -func (a *ExperimentsAPI) ListArtifactsAll(ctx context.Context, request ListArtifactsRequest) ([]FileInfo, error) { - var results []FileInfo - ctx = useragent.InContext(ctx, "sdk-feature", "pagination") - for { - response, err := a.impl.ListArtifacts(ctx, request) - if err != nil { - return nil, err - } - if len(response.Files) == 0 { - break - } - for _, v := range response.Files { - results = append(results, v) - } - request.PageToken = response.NextPageToken - if response.NextPageToken == "" { - break +func (a *ExperimentsAPI) ListArtifacts(ctx context.Context, request ListArtifactsRequest) *listing.PaginatingIterator[ListArtifactsRequest, *ListArtifactsResponse, FileInfo] { + + getNextPage := func(ctx context.Context, req ListArtifactsRequest) (*ListArtifactsResponse, error) { + ctx = useragent.InContext(ctx, "sdk-feature", "pagination") + return a.impl.ListArtifacts(ctx, req) + } + getItems := func(resp *ListArtifactsResponse) []FileInfo { + return resp.Files + } + getNextReq := func(resp *ListArtifactsResponse) *ListArtifactsRequest { + if resp.NextPageToken == "" { + return nil } + request.PageToken = resp.NextPageToken + return &request } - return results, nil + iterator := listing.NewIterator( + &request, + getNextPage, + getItems, + getNextReq) + return iterator +} + +// Get all artifacts. +// +// List artifacts for a run. Takes an optional `artifact_path` prefix. If it is +// specified, the response contains only artifacts with the specified prefix.", +// +// This method is generated by Databricks SDK Code Generator. +func (a *ExperimentsAPI) ListArtifactsAll(ctx context.Context, request ListArtifactsRequest) ([]FileInfo, error) { + iterator := a.ListArtifacts(ctx, request) + return listing.ToSlice[FileInfo](ctx, iterator) } // List experiments. @@ -234,33 +254,39 @@ func (a *ExperimentsAPI) ListArtifactsAll(ctx context.Context, request ListArtif // Gets a list of all experiments. // // This method is generated by Databricks SDK Code Generator. -func (a *ExperimentsAPI) ListExperimentsAll(ctx context.Context, request ListExperimentsRequest) ([]Experiment, error) { - var results []Experiment - var totalCount int = 0 - ctx = useragent.InContext(ctx, "sdk-feature", "pagination") - for { - response, err := a.impl.ListExperiments(ctx, request) - if err != nil { - return nil, err - } - if len(response.Experiments) == 0 { - break - } - for _, v := range response.Experiments { - results = append(results, v) - } - count := int(len(response.Experiments)) - totalCount += count - request.PageToken = response.NextPageToken - if response.NextPageToken == "" { - break - } - limit := request.MaxResults - if limit > 0 && totalCount >= limit { - break +func (a *ExperimentsAPI) ListExperiments(ctx context.Context, request ListExperimentsRequest) *listing.PaginatingIterator[ListExperimentsRequest, *ListExperimentsResponse, Experiment] { + + getNextPage := func(ctx context.Context, req ListExperimentsRequest) (*ListExperimentsResponse, error) { + ctx = useragent.InContext(ctx, "sdk-feature", "pagination") + return a.impl.ListExperiments(ctx, req) + } + getItems := func(resp *ListExperimentsResponse) []Experiment { + return resp.Experiments + } + getNextReq := func(resp *ListExperimentsResponse) *ListExperimentsRequest { + if resp.NextPageToken == "" { + return nil } + request.PageToken = resp.NextPageToken + return &request } - return results, nil + iterator := listing.NewIterator( + &request, + getNextPage, + getItems, + getNextReq) + return iterator +} + +// List experiments. +// +// Gets a list of all experiments. +// +// This method is generated by Databricks SDK Code Generator. +func (a *ExperimentsAPI) ListExperimentsAll(ctx context.Context, request ListExperimentsRequest) ([]Experiment, error) { + iterator := a.ListExperiments(ctx, request) + return listing.ToSliceN[Experiment, int](ctx, iterator, request.MaxResults) + } // Log a batch. @@ -373,26 +399,38 @@ func (a *ExperimentsAPI) RestoreRuns(ctx context.Context, request RestoreRuns) ( // Searches for experiments that satisfy specified search criteria. // // This method is generated by Databricks SDK Code Generator. -func (a *ExperimentsAPI) SearchExperimentsAll(ctx context.Context, request SearchExperiments) ([]Experiment, error) { - var results []Experiment - ctx = useragent.InContext(ctx, "sdk-feature", "pagination") - for { - response, err := a.impl.SearchExperiments(ctx, request) - if err != nil { - return nil, err - } - if len(response.Experiments) == 0 { - break - } - for _, v := range response.Experiments { - results = append(results, v) - } - request.PageToken = response.NextPageToken - if response.NextPageToken == "" { - break +func (a *ExperimentsAPI) SearchExperiments(ctx context.Context, request SearchExperiments) *listing.PaginatingIterator[SearchExperiments, *SearchExperimentsResponse, Experiment] { + + getNextPage := func(ctx context.Context, req SearchExperiments) (*SearchExperimentsResponse, error) { + ctx = useragent.InContext(ctx, "sdk-feature", "pagination") + return a.impl.SearchExperiments(ctx, req) + } + getItems := func(resp *SearchExperimentsResponse) []Experiment { + return resp.Experiments + } + getNextReq := func(resp *SearchExperimentsResponse) *SearchExperiments { + if resp.NextPageToken == "" { + return nil } + request.PageToken = resp.NextPageToken + return &request } - return results, nil + iterator := listing.NewIterator( + &request, + getNextPage, + getItems, + getNextReq) + return iterator +} + +// Search experiments. +// +// Searches for experiments that satisfy specified search criteria. +// +// This method is generated by Databricks SDK Code Generator. +func (a *ExperimentsAPI) SearchExperimentsAll(ctx context.Context, request SearchExperiments) ([]Experiment, error) { + iterator := a.SearchExperiments(ctx, request) + return listing.ToSlice[Experiment](ctx, iterator) } // Search for runs. @@ -402,26 +440,40 @@ func (a *ExperimentsAPI) SearchExperimentsAll(ctx context.Context, request Searc // Search expressions can use `mlflowMetric` and `mlflowParam` keys.", // // This method is generated by Databricks SDK Code Generator. -func (a *ExperimentsAPI) SearchRunsAll(ctx context.Context, request SearchRuns) ([]Run, error) { - var results []Run - ctx = useragent.InContext(ctx, "sdk-feature", "pagination") - for { - response, err := a.impl.SearchRuns(ctx, request) - if err != nil { - return nil, err - } - if len(response.Runs) == 0 { - break - } - for _, v := range response.Runs { - results = append(results, v) - } - request.PageToken = response.NextPageToken - if response.NextPageToken == "" { - break +func (a *ExperimentsAPI) SearchRuns(ctx context.Context, request SearchRuns) *listing.PaginatingIterator[SearchRuns, *SearchRunsResponse, Run] { + + getNextPage := func(ctx context.Context, req SearchRuns) (*SearchRunsResponse, error) { + ctx = useragent.InContext(ctx, "sdk-feature", "pagination") + return a.impl.SearchRuns(ctx, req) + } + getItems := func(resp *SearchRunsResponse) []Run { + return resp.Runs + } + getNextReq := func(resp *SearchRunsResponse) *SearchRuns { + if resp.NextPageToken == "" { + return nil } + request.PageToken = resp.NextPageToken + return &request } - return results, nil + iterator := listing.NewIterator( + &request, + getNextPage, + getItems, + getNextReq) + return iterator +} + +// Search for runs. +// +// Searches for runs that satisfy expressions. +// +// Search expressions can use `mlflowMetric` and `mlflowParam` keys.", +// +// This method is generated by Databricks SDK Code Generator. +func (a *ExperimentsAPI) SearchRunsAll(ctx context.Context, request SearchRuns) ([]Run, error) { + iterator := a.SearchRuns(ctx, request) + return listing.ToSlice[Run](ctx, iterator) } // Set a tag. @@ -603,12 +655,32 @@ func (a *ModelRegistryAPI) DeleteWebhook(ctx context.Context, request DeleteWebh // Gets the latest version of a registered model. // // This method is generated by Databricks SDK Code Generator. -func (a *ModelRegistryAPI) GetLatestVersionsAll(ctx context.Context, request GetLatestVersionsRequest) ([]ModelVersion, error) { - response, err := a.impl.GetLatestVersions(ctx, request) - if err != nil { - return nil, err +func (a *ModelRegistryAPI) GetLatestVersions(ctx context.Context, request GetLatestVersionsRequest) *listing.PaginatingIterator[GetLatestVersionsRequest, *GetLatestVersionsResponse, ModelVersion] { + + getNextPage := func(ctx context.Context, req GetLatestVersionsRequest) (*GetLatestVersionsResponse, error) { + ctx = useragent.InContext(ctx, "sdk-feature", "pagination") + return a.impl.GetLatestVersions(ctx, req) + } + getItems := func(resp *GetLatestVersionsResponse) []ModelVersion { + return resp.ModelVersions } - return response.ModelVersions, nil + + iterator := listing.NewIterator( + &request, + getNextPage, + getItems, + nil) + return iterator +} + +// Get the latest version. +// +// Gets the latest version of a registered model. +// +// This method is generated by Databricks SDK Code Generator. +func (a *ModelRegistryAPI) GetLatestVersionsAll(ctx context.Context, request GetLatestVersionsRequest) ([]ModelVersion, error) { + iterator := a.GetLatestVersions(ctx, request) + return listing.ToSlice[ModelVersion](ctx, iterator) } // Get model. @@ -676,33 +748,63 @@ func (a *ModelRegistryAPI) GetPermissionsByRegisteredModelId(ctx context.Context // __max_results__. // // This method is generated by Databricks SDK Code Generator. -func (a *ModelRegistryAPI) ListModelsAll(ctx context.Context, request ListModelsRequest) ([]Model, error) { - var results []Model - var totalCount int = 0 - ctx = useragent.InContext(ctx, "sdk-feature", "pagination") - for { - response, err := a.impl.ListModels(ctx, request) - if err != nil { - return nil, err - } - if len(response.RegisteredModels) == 0 { - break - } - for _, v := range response.RegisteredModels { - results = append(results, v) - } - count := int(len(response.RegisteredModels)) - totalCount += count - request.PageToken = response.NextPageToken - if response.NextPageToken == "" { - break - } - limit := request.MaxResults - if limit > 0 && totalCount >= limit { - break +func (a *ModelRegistryAPI) ListModels(ctx context.Context, request ListModelsRequest) *listing.PaginatingIterator[ListModelsRequest, *ListModelsResponse, Model] { + + getNextPage := func(ctx context.Context, req ListModelsRequest) (*ListModelsResponse, error) { + ctx = useragent.InContext(ctx, "sdk-feature", "pagination") + return a.impl.ListModels(ctx, req) + } + getItems := func(resp *ListModelsResponse) []Model { + return resp.RegisteredModels + } + getNextReq := func(resp *ListModelsResponse) *ListModelsRequest { + if resp.NextPageToken == "" { + return nil } + request.PageToken = resp.NextPageToken + return &request + } + iterator := listing.NewIterator( + &request, + getNextPage, + getItems, + getNextReq) + return iterator +} + +// List models. +// +// Lists all available registered models, up to the limit specified in +// __max_results__. +// +// This method is generated by Databricks SDK Code Generator. +func (a *ModelRegistryAPI) ListModelsAll(ctx context.Context, request ListModelsRequest) ([]Model, error) { + iterator := a.ListModels(ctx, request) + return listing.ToSliceN[Model, int](ctx, iterator, request.MaxResults) + +} + +// List transition requests. +// +// Gets a list of all open stage transition requests for the model version. +// +// This method is generated by Databricks SDK Code Generator. +func (a *ModelRegistryAPI) ListTransitionRequests(ctx context.Context, request ListTransitionRequestsRequest) *listing.PaginatingIterator[ListTransitionRequestsRequest, *ListTransitionRequestsResponse, Activity] { + + getNextPage := func(ctx context.Context, req ListTransitionRequestsRequest) (*ListTransitionRequestsResponse, error) { + ctx = useragent.InContext(ctx, "sdk-feature", "pagination") + return a.impl.ListTransitionRequests(ctx, req) } - return results, nil + getItems := func(resp *ListTransitionRequestsResponse) []Activity { + return resp.Requests + } + + iterator := listing.NewIterator( + &request, + getNextPage, + getItems, + nil) + return iterator } // List transition requests. @@ -711,11 +813,8 @@ func (a *ModelRegistryAPI) ListModelsAll(ctx context.Context, request ListModels // // This method is generated by Databricks SDK Code Generator. func (a *ModelRegistryAPI) ListTransitionRequestsAll(ctx context.Context, request ListTransitionRequestsRequest) ([]Activity, error) { - response, err := a.impl.ListTransitionRequests(ctx, request) - if err != nil { - return nil, err - } - return response.Requests, nil + iterator := a.ListTransitionRequests(ctx, request) + return listing.ToSlice[Activity](ctx, iterator) } // List registry webhooks. @@ -725,26 +824,40 @@ func (a *ModelRegistryAPI) ListTransitionRequestsAll(ctx context.Context, reques // Lists all registry webhooks. // // This method is generated by Databricks SDK Code Generator. -func (a *ModelRegistryAPI) ListWebhooksAll(ctx context.Context, request ListWebhooksRequest) ([]RegistryWebhook, error) { - var results []RegistryWebhook - ctx = useragent.InContext(ctx, "sdk-feature", "pagination") - for { - response, err := a.impl.ListWebhooks(ctx, request) - if err != nil { - return nil, err - } - if len(response.Webhooks) == 0 { - break - } - for _, v := range response.Webhooks { - results = append(results, v) - } - request.PageToken = response.NextPageToken - if response.NextPageToken == "" { - break +func (a *ModelRegistryAPI) ListWebhooks(ctx context.Context, request ListWebhooksRequest) *listing.PaginatingIterator[ListWebhooksRequest, *ListRegistryWebhooks, RegistryWebhook] { + + getNextPage := func(ctx context.Context, req ListWebhooksRequest) (*ListRegistryWebhooks, error) { + ctx = useragent.InContext(ctx, "sdk-feature", "pagination") + return a.impl.ListWebhooks(ctx, req) + } + getItems := func(resp *ListRegistryWebhooks) []RegistryWebhook { + return resp.Webhooks + } + getNextReq := func(resp *ListRegistryWebhooks) *ListWebhooksRequest { + if resp.NextPageToken == "" { + return nil } + request.PageToken = resp.NextPageToken + return &request } - return results, nil + iterator := listing.NewIterator( + &request, + getNextPage, + getItems, + getNextReq) + return iterator +} + +// List registry webhooks. +// +// **NOTE:** This endpoint is in Public Preview. +// +// Lists all registry webhooks. +// +// This method is generated by Databricks SDK Code Generator. +func (a *ModelRegistryAPI) ListWebhooksAll(ctx context.Context, request ListWebhooksRequest) ([]RegistryWebhook, error) { + iterator := a.ListWebhooks(ctx, request) + return listing.ToSlice[RegistryWebhook](ctx, iterator) } // Reject a transition request. @@ -766,33 +879,39 @@ func (a *ModelRegistryAPI) RenameModel(ctx context.Context, request RenameModelR // Searches for specific model versions based on the supplied __filter__. // // This method is generated by Databricks SDK Code Generator. -func (a *ModelRegistryAPI) SearchModelVersionsAll(ctx context.Context, request SearchModelVersionsRequest) ([]ModelVersion, error) { - var results []ModelVersion - var totalCount int = 0 - ctx = useragent.InContext(ctx, "sdk-feature", "pagination") - for { - response, err := a.impl.SearchModelVersions(ctx, request) - if err != nil { - return nil, err - } - if len(response.ModelVersions) == 0 { - break - } - for _, v := range response.ModelVersions { - results = append(results, v) - } - count := int(len(response.ModelVersions)) - totalCount += count - request.PageToken = response.NextPageToken - if response.NextPageToken == "" { - break - } - limit := request.MaxResults - if limit > 0 && totalCount >= limit { - break +func (a *ModelRegistryAPI) SearchModelVersions(ctx context.Context, request SearchModelVersionsRequest) *listing.PaginatingIterator[SearchModelVersionsRequest, *SearchModelVersionsResponse, ModelVersion] { + + getNextPage := func(ctx context.Context, req SearchModelVersionsRequest) (*SearchModelVersionsResponse, error) { + ctx = useragent.InContext(ctx, "sdk-feature", "pagination") + return a.impl.SearchModelVersions(ctx, req) + } + getItems := func(resp *SearchModelVersionsResponse) []ModelVersion { + return resp.ModelVersions + } + getNextReq := func(resp *SearchModelVersionsResponse) *SearchModelVersionsRequest { + if resp.NextPageToken == "" { + return nil } + request.PageToken = resp.NextPageToken + return &request } - return results, nil + iterator := listing.NewIterator( + &request, + getNextPage, + getItems, + getNextReq) + return iterator +} + +// Searches model versions. +// +// Searches for specific model versions based on the supplied __filter__. +// +// This method is generated by Databricks SDK Code Generator. +func (a *ModelRegistryAPI) SearchModelVersionsAll(ctx context.Context, request SearchModelVersionsRequest) ([]ModelVersion, error) { + iterator := a.SearchModelVersions(ctx, request) + return listing.ToSliceN[ModelVersion, int](ctx, iterator, request.MaxResults) + } // Search models. @@ -800,33 +919,39 @@ func (a *ModelRegistryAPI) SearchModelVersionsAll(ctx context.Context, request S // Search for registered models based on the specified __filter__. // // This method is generated by Databricks SDK Code Generator. -func (a *ModelRegistryAPI) SearchModelsAll(ctx context.Context, request SearchModelsRequest) ([]Model, error) { - var results []Model - var totalCount int = 0 - ctx = useragent.InContext(ctx, "sdk-feature", "pagination") - for { - response, err := a.impl.SearchModels(ctx, request) - if err != nil { - return nil, err - } - if len(response.RegisteredModels) == 0 { - break - } - for _, v := range response.RegisteredModels { - results = append(results, v) - } - count := int(len(response.RegisteredModels)) - totalCount += count - request.PageToken = response.NextPageToken - if response.NextPageToken == "" { - break - } - limit := request.MaxResults - if limit > 0 && totalCount >= limit { - break +func (a *ModelRegistryAPI) SearchModels(ctx context.Context, request SearchModelsRequest) *listing.PaginatingIterator[SearchModelsRequest, *SearchModelsResponse, Model] { + + getNextPage := func(ctx context.Context, req SearchModelsRequest) (*SearchModelsResponse, error) { + ctx = useragent.InContext(ctx, "sdk-feature", "pagination") + return a.impl.SearchModels(ctx, req) + } + getItems := func(resp *SearchModelsResponse) []Model { + return resp.RegisteredModels + } + getNextReq := func(resp *SearchModelsResponse) *SearchModelsRequest { + if resp.NextPageToken == "" { + return nil } + request.PageToken = resp.NextPageToken + return &request } - return results, nil + iterator := listing.NewIterator( + &request, + getNextPage, + getItems, + getNextReq) + return iterator +} + +// Search models. +// +// Search for registered models based on the specified __filter__. +// +// This method is generated by Databricks SDK Code Generator. +func (a *ModelRegistryAPI) SearchModelsAll(ctx context.Context, request SearchModelsRequest) ([]Model, error) { + iterator := a.SearchModels(ctx, request) + return listing.ToSliceN[Model, int](ctx, iterator, request.MaxResults) + } // Set a tag. diff --git a/service/ml/model_registry_usage_test.go b/service/ml/model_registry_usage_test.go index 72de73538..b5ccf9028 100755 --- a/service/ml/model_registry_usage_test.go +++ b/service/ml/model_registry_usage_test.go @@ -75,7 +75,7 @@ func ExampleModelRegistryAPI_CreateModel_models() { } -func ExampleModelRegistryAPI_CreateModel_modelVersionComments() { +func ExampleModelRegistryAPI_CreateModel_modelVersions() { ctx := context.Background() w, err := databricks.NewWorkspaceClient() if err != nil { @@ -92,7 +92,7 @@ func ExampleModelRegistryAPI_CreateModel_modelVersionComments() { } -func ExampleModelRegistryAPI_CreateModel_modelVersions() { +func ExampleModelRegistryAPI_CreateModel_modelVersionComments() { ctx := context.Background() w, err := databricks.NewWorkspaceClient() if err != nil { @@ -109,7 +109,7 @@ func ExampleModelRegistryAPI_CreateModel_modelVersions() { } -func ExampleModelRegistryAPI_CreateModelVersion_modelVersions() { +func ExampleModelRegistryAPI_CreateModelVersion_modelVersionComments() { ctx := context.Background() w, err := databricks.NewWorkspaceClient() if err != nil { @@ -124,18 +124,18 @@ func ExampleModelRegistryAPI_CreateModelVersion_modelVersions() { } logger.Infof(ctx, "found %v", model) - created, err := w.ModelRegistry.CreateModelVersion(ctx, ml.CreateModelVersionRequest{ + mv, err := w.ModelRegistry.CreateModelVersion(ctx, ml.CreateModelVersionRequest{ Name: model.RegisteredModel.Name, Source: "dbfs:/tmp", }) if err != nil { panic(err) } - logger.Infof(ctx, "found %v", created) + logger.Infof(ctx, "found %v", mv) } -func ExampleModelRegistryAPI_CreateModelVersion_modelVersionComments() { +func ExampleModelRegistryAPI_CreateModelVersion_modelVersions() { ctx := context.Background() w, err := databricks.NewWorkspaceClient() if err != nil { @@ -150,14 +150,14 @@ func ExampleModelRegistryAPI_CreateModelVersion_modelVersionComments() { } logger.Infof(ctx, "found %v", model) - mv, err := w.ModelRegistry.CreateModelVersion(ctx, ml.CreateModelVersionRequest{ + created, err := w.ModelRegistry.CreateModelVersion(ctx, ml.CreateModelVersionRequest{ Name: model.RegisteredModel.Name, Source: "dbfs:/tmp", }) if err != nil { panic(err) } - logger.Infof(ctx, "found %v", mv) + logger.Infof(ctx, "found %v", created) } diff --git a/service/oauth2/api.go b/service/oauth2/api.go index 3d6e0c382..62cc1431b 100755 --- a/service/oauth2/api.go +++ b/service/oauth2/api.go @@ -7,6 +7,7 @@ import ( "context" "github.com/databricks/databricks-sdk-go/client" + "github.com/databricks/databricks-sdk-go/listing" "github.com/databricks/databricks-sdk-go/useragent" ) @@ -89,12 +90,34 @@ func (a *CustomAppIntegrationAPI) GetByIntegrationId(ctx context.Context, integr // account // // This method is generated by Databricks SDK Code Generator. -func (a *CustomAppIntegrationAPI) ListAll(ctx context.Context) ([]GetCustomAppIntegrationOutput, error) { - response, err := a.impl.List(ctx) - if err != nil { - return nil, err +func (a *CustomAppIntegrationAPI) List(ctx context.Context) *listing.PaginatingIterator[struct{}, *GetCustomAppIntegrationsOutput, GetCustomAppIntegrationOutput] { + request := struct{}{} + + getNextPage := func(ctx context.Context, req struct{}) (*GetCustomAppIntegrationsOutput, error) { + ctx = useragent.InContext(ctx, "sdk-feature", "pagination") + return a.impl.List(ctx) } - return response.Apps, nil + getItems := func(resp *GetCustomAppIntegrationsOutput) []GetCustomAppIntegrationOutput { + return resp.Apps + } + + iterator := listing.NewIterator( + &request, + getNextPage, + getItems, + nil) + return iterator +} + +// Get custom oauth app integrations. +// +// Get the list of custom oauth app integrations for the specified Databricks +// account +// +// This method is generated by Databricks SDK Code Generator. +func (a *CustomAppIntegrationAPI) ListAll(ctx context.Context) ([]GetCustomAppIntegrationOutput, error) { + iterator := a.List(ctx) + return listing.ToSlice[GetCustomAppIntegrationOutput](ctx, iterator) } // Updates Custom OAuth App Integration. @@ -196,33 +219,39 @@ func (a *OAuthPublishedAppsAPI) Impl() OAuthPublishedAppsService { // Get all the available published OAuth apps in Databricks. // // This method is generated by Databricks SDK Code Generator. -func (a *OAuthPublishedAppsAPI) ListAll(ctx context.Context, request ListOAuthPublishedAppsRequest) ([]PublishedAppOutput, error) { - var results []PublishedAppOutput - var totalCount int64 = 0 - ctx = useragent.InContext(ctx, "sdk-feature", "pagination") - for { - response, err := a.impl.List(ctx, request) - if err != nil { - return nil, err - } - if len(response.Apps) == 0 { - break - } - for _, v := range response.Apps { - results = append(results, v) - } - count := int64(len(response.Apps)) - totalCount += count - request.PageToken = response.NextPageToken - if response.NextPageToken == "" { - break - } - limit := request.PageSize - if limit > 0 && totalCount >= limit { - break +func (a *OAuthPublishedAppsAPI) List(ctx context.Context, request ListOAuthPublishedAppsRequest) *listing.PaginatingIterator[ListOAuthPublishedAppsRequest, *GetPublishedAppsOutput, PublishedAppOutput] { + + getNextPage := func(ctx context.Context, req ListOAuthPublishedAppsRequest) (*GetPublishedAppsOutput, error) { + ctx = useragent.InContext(ctx, "sdk-feature", "pagination") + return a.impl.List(ctx, req) + } + getItems := func(resp *GetPublishedAppsOutput) []PublishedAppOutput { + return resp.Apps + } + getNextReq := func(resp *GetPublishedAppsOutput) *ListOAuthPublishedAppsRequest { + if resp.NextPageToken == "" { + return nil } + request.PageToken = resp.NextPageToken + return &request } - return results, nil + iterator := listing.NewIterator( + &request, + getNextPage, + getItems, + getNextReq) + return iterator +} + +// Get all the published OAuth apps. +// +// Get all the available published OAuth apps in Databricks. +// +// This method is generated by Databricks SDK Code Generator. +func (a *OAuthPublishedAppsAPI) ListAll(ctx context.Context, request ListOAuthPublishedAppsRequest) ([]PublishedAppOutput, error) { + iterator := a.List(ctx, request) + return listing.ToSliceN[PublishedAppOutput, int64](ctx, iterator, request.PageSize) + } func NewPublishedAppIntegration(client *client.DatabricksClient) *PublishedAppIntegrationAPI { @@ -304,12 +333,34 @@ func (a *PublishedAppIntegrationAPI) GetByIntegrationId(ctx context.Context, int // account // // This method is generated by Databricks SDK Code Generator. -func (a *PublishedAppIntegrationAPI) ListAll(ctx context.Context) ([]GetPublishedAppIntegrationOutput, error) { - response, err := a.impl.List(ctx) - if err != nil { - return nil, err +func (a *PublishedAppIntegrationAPI) List(ctx context.Context) *listing.PaginatingIterator[struct{}, *GetPublishedAppIntegrationsOutput, GetPublishedAppIntegrationOutput] { + request := struct{}{} + + getNextPage := func(ctx context.Context, req struct{}) (*GetPublishedAppIntegrationsOutput, error) { + ctx = useragent.InContext(ctx, "sdk-feature", "pagination") + return a.impl.List(ctx) + } + getItems := func(resp *GetPublishedAppIntegrationsOutput) []GetPublishedAppIntegrationOutput { + return resp.Apps } - return response.Apps, nil + + iterator := listing.NewIterator( + &request, + getNextPage, + getItems, + nil) + return iterator +} + +// Get published oauth app integrations. +// +// Get the list of published oauth app integrations for the specified Databricks +// account +// +// This method is generated by Databricks SDK Code Generator. +func (a *PublishedAppIntegrationAPI) ListAll(ctx context.Context) ([]GetPublishedAppIntegrationOutput, error) { + iterator := a.List(ctx) + return listing.ToSlice[GetPublishedAppIntegrationOutput](ctx, iterator) } // Updates Published OAuth App Integration. @@ -390,12 +441,34 @@ func (a *ServicePrincipalSecretsAPI) DeleteByServicePrincipalIdAndSecretId(ctx c // the secret values. // // This method is generated by Databricks SDK Code Generator. -func (a *ServicePrincipalSecretsAPI) ListAll(ctx context.Context, request ListServicePrincipalSecretsRequest) ([]SecretInfo, error) { - response, err := a.impl.List(ctx, request) - if err != nil { - return nil, err +func (a *ServicePrincipalSecretsAPI) List(ctx context.Context, request ListServicePrincipalSecretsRequest) *listing.PaginatingIterator[ListServicePrincipalSecretsRequest, *ListServicePrincipalSecretsResponse, SecretInfo] { + + getNextPage := func(ctx context.Context, req ListServicePrincipalSecretsRequest) (*ListServicePrincipalSecretsResponse, error) { + ctx = useragent.InContext(ctx, "sdk-feature", "pagination") + return a.impl.List(ctx, req) } - return response.Secrets, nil + getItems := func(resp *ListServicePrincipalSecretsResponse) []SecretInfo { + return resp.Secrets + } + + iterator := listing.NewIterator( + &request, + getNextPage, + getItems, + nil) + return iterator +} + +// List service principal secrets. +// +// List all secrets associated with the given service principal. This operation +// only returns information about the secrets themselves and does not include +// the secret values. +// +// This method is generated by Databricks SDK Code Generator. +func (a *ServicePrincipalSecretsAPI) ListAll(ctx context.Context, request ListServicePrincipalSecretsRequest) ([]SecretInfo, error) { + iterator := a.List(ctx, request) + return listing.ToSlice[SecretInfo](ctx, iterator) } // List service principal secrets. diff --git a/service/pipelines/api.go b/service/pipelines/api.go index 463af5059..b3f445388 100755 --- a/service/pipelines/api.go +++ b/service/pipelines/api.go @@ -9,6 +9,7 @@ import ( "time" "github.com/databricks/databricks-sdk-go/client" + "github.com/databricks/databricks-sdk-go/listing" "github.com/databricks/databricks-sdk-go/retries" "github.com/databricks/databricks-sdk-go/useragent" ) @@ -253,33 +254,39 @@ func (a *PipelinesAPI) GetUpdateByPipelineIdAndUpdateId(ctx context.Context, pip // Retrieves events for a pipeline. // // This method is generated by Databricks SDK Code Generator. -func (a *PipelinesAPI) ListPipelineEventsAll(ctx context.Context, request ListPipelineEventsRequest) ([]PipelineEvent, error) { - var results []PipelineEvent - var totalCount int = 0 - ctx = useragent.InContext(ctx, "sdk-feature", "pagination") - for { - response, err := a.impl.ListPipelineEvents(ctx, request) - if err != nil { - return nil, err - } - if len(response.Events) == 0 { - break - } - for _, v := range response.Events { - results = append(results, v) - } - count := int(len(response.Events)) - totalCount += count - request.PageToken = response.NextPageToken - if response.NextPageToken == "" { - break - } - limit := request.MaxResults - if limit > 0 && totalCount >= limit { - break +func (a *PipelinesAPI) ListPipelineEvents(ctx context.Context, request ListPipelineEventsRequest) *listing.PaginatingIterator[ListPipelineEventsRequest, *ListPipelineEventsResponse, PipelineEvent] { + + getNextPage := func(ctx context.Context, req ListPipelineEventsRequest) (*ListPipelineEventsResponse, error) { + ctx = useragent.InContext(ctx, "sdk-feature", "pagination") + return a.impl.ListPipelineEvents(ctx, req) + } + getItems := func(resp *ListPipelineEventsResponse) []PipelineEvent { + return resp.Events + } + getNextReq := func(resp *ListPipelineEventsResponse) *ListPipelineEventsRequest { + if resp.NextPageToken == "" { + return nil } + request.PageToken = resp.NextPageToken + return &request } - return results, nil + iterator := listing.NewIterator( + &request, + getNextPage, + getItems, + getNextReq) + return iterator +} + +// List pipeline events. +// +// Retrieves events for a pipeline. +// +// This method is generated by Databricks SDK Code Generator. +func (a *PipelinesAPI) ListPipelineEventsAll(ctx context.Context, request ListPipelineEventsRequest) ([]PipelineEvent, error) { + iterator := a.ListPipelineEvents(ctx, request) + return listing.ToSliceN[PipelineEvent, int](ctx, iterator, request.MaxResults) + } // List pipeline events. @@ -296,33 +303,39 @@ func (a *PipelinesAPI) ListPipelineEventsByPipelineId(ctx context.Context, pipel // Lists pipelines defined in the Delta Live Tables system. // // This method is generated by Databricks SDK Code Generator. -func (a *PipelinesAPI) ListPipelinesAll(ctx context.Context, request ListPipelinesRequest) ([]PipelineStateInfo, error) { - var results []PipelineStateInfo - var totalCount int = 0 - ctx = useragent.InContext(ctx, "sdk-feature", "pagination") - for { - response, err := a.impl.ListPipelines(ctx, request) - if err != nil { - return nil, err - } - if len(response.Statuses) == 0 { - break - } - for _, v := range response.Statuses { - results = append(results, v) - } - count := int(len(response.Statuses)) - totalCount += count - request.PageToken = response.NextPageToken - if response.NextPageToken == "" { - break - } - limit := request.MaxResults - if limit > 0 && totalCount >= limit { - break +func (a *PipelinesAPI) ListPipelines(ctx context.Context, request ListPipelinesRequest) *listing.PaginatingIterator[ListPipelinesRequest, *ListPipelinesResponse, PipelineStateInfo] { + + getNextPage := func(ctx context.Context, req ListPipelinesRequest) (*ListPipelinesResponse, error) { + ctx = useragent.InContext(ctx, "sdk-feature", "pagination") + return a.impl.ListPipelines(ctx, req) + } + getItems := func(resp *ListPipelinesResponse) []PipelineStateInfo { + return resp.Statuses + } + getNextReq := func(resp *ListPipelinesResponse) *ListPipelinesRequest { + if resp.NextPageToken == "" { + return nil } + request.PageToken = resp.NextPageToken + return &request } - return results, nil + iterator := listing.NewIterator( + &request, + getNextPage, + getItems, + getNextReq) + return iterator +} + +// List pipelines. +// +// Lists pipelines defined in the Delta Live Tables system. +// +// This method is generated by Databricks SDK Code Generator. +func (a *PipelinesAPI) ListPipelinesAll(ctx context.Context, request ListPipelinesRequest) ([]PipelineStateInfo, error) { + iterator := a.ListPipelines(ctx, request) + return listing.ToSliceN[PipelineStateInfo, int](ctx, iterator, request.MaxResults) + } // PipelineStateInfoNameToPipelineIdMap calls [PipelinesAPI.ListPipelinesAll] and creates a map of results with [PipelineStateInfo].Name as key and [PipelineStateInfo].PipelineId as value. diff --git a/service/serving/api.go b/service/serving/api.go index 067017fbf..7c62111e4 100755 --- a/service/serving/api.go +++ b/service/serving/api.go @@ -9,6 +9,7 @@ import ( "time" "github.com/databricks/databricks-sdk-go/client" + "github.com/databricks/databricks-sdk-go/listing" "github.com/databricks/databricks-sdk-go/retries" "github.com/databricks/databricks-sdk-go/useragent" ) @@ -252,12 +253,31 @@ func (a *ServingEndpointsAPI) GetPermissionsByServingEndpointId(ctx context.Cont // Retrieve all serving endpoints. // // This method is generated by Databricks SDK Code Generator. -func (a *ServingEndpointsAPI) ListAll(ctx context.Context) ([]ServingEndpoint, error) { - response, err := a.impl.List(ctx) - if err != nil { - return nil, err +func (a *ServingEndpointsAPI) List(ctx context.Context) *listing.PaginatingIterator[struct{}, *ListEndpointsResponse, ServingEndpoint] { + request := struct{}{} + + getNextPage := func(ctx context.Context, req struct{}) (*ListEndpointsResponse, error) { + ctx = useragent.InContext(ctx, "sdk-feature", "pagination") + return a.impl.List(ctx) + } + getItems := func(resp *ListEndpointsResponse) []ServingEndpoint { + return resp.Endpoints } - return response.Endpoints, nil + + iterator := listing.NewIterator( + &request, + getNextPage, + getItems, + nil) + return iterator +} + +// Retrieve all serving endpoints. +// +// This method is generated by Databricks SDK Code Generator. +func (a *ServingEndpointsAPI) ListAll(ctx context.Context) ([]ServingEndpoint, error) { + iterator := a.List(ctx) + return listing.ToSlice[ServingEndpoint](ctx, iterator) } // Retrieve the most recent log lines associated with a given serving endpoint's diff --git a/service/settings/api.go b/service/settings/api.go index e4129f4cb..2987ce7b8 100755 --- a/service/settings/api.go +++ b/service/settings/api.go @@ -8,6 +8,7 @@ import ( "fmt" "github.com/databricks/databricks-sdk-go/client" + "github.com/databricks/databricks-sdk-go/listing" "github.com/databricks/databricks-sdk-go/useragent" ) @@ -116,12 +117,33 @@ func (a *AccountIpAccessListsAPI) GetByIpAccessListId(ctx context.Context, ipAcc // Gets all IP access lists for the specified account. // // This method is generated by Databricks SDK Code Generator. -func (a *AccountIpAccessListsAPI) ListAll(ctx context.Context) ([]IpAccessListInfo, error) { - response, err := a.impl.List(ctx) - if err != nil { - return nil, err +func (a *AccountIpAccessListsAPI) List(ctx context.Context) *listing.PaginatingIterator[struct{}, *GetIpAccessListsResponse, IpAccessListInfo] { + request := struct{}{} + + getNextPage := func(ctx context.Context, req struct{}) (*GetIpAccessListsResponse, error) { + ctx = useragent.InContext(ctx, "sdk-feature", "pagination") + return a.impl.List(ctx) } - return response.IpAccessLists, nil + getItems := func(resp *GetIpAccessListsResponse) []IpAccessListInfo { + return resp.IpAccessLists + } + + iterator := listing.NewIterator( + &request, + getNextPage, + getItems, + nil) + return iterator +} + +// Get access lists. +// +// Gets all IP access lists for the specified account. +// +// This method is generated by Databricks SDK Code Generator. +func (a *AccountIpAccessListsAPI) ListAll(ctx context.Context) ([]IpAccessListInfo, error) { + iterator := a.List(ctx) + return listing.ToSlice[IpAccessListInfo](ctx, iterator) } // IpAccessListInfoLabelToListIdMap calls [AccountIpAccessListsAPI.ListAll] and creates a map of results with [IpAccessListInfo].Label as key and [IpAccessListInfo].ListId as value. @@ -412,12 +434,33 @@ func (a *IpAccessListsAPI) GetByIpAccessListId(ctx context.Context, ipAccessList // Gets all IP access lists for the specified workspace. // // This method is generated by Databricks SDK Code Generator. -func (a *IpAccessListsAPI) ListAll(ctx context.Context) ([]IpAccessListInfo, error) { - response, err := a.impl.List(ctx) - if err != nil { - return nil, err +func (a *IpAccessListsAPI) List(ctx context.Context) *listing.PaginatingIterator[struct{}, *ListIpAccessListResponse, IpAccessListInfo] { + request := struct{}{} + + getNextPage := func(ctx context.Context, req struct{}) (*ListIpAccessListResponse, error) { + ctx = useragent.InContext(ctx, "sdk-feature", "pagination") + return a.impl.List(ctx) } - return response.IpAccessLists, nil + getItems := func(resp *ListIpAccessListResponse) []IpAccessListInfo { + return resp.IpAccessLists + } + + iterator := listing.NewIterator( + &request, + getNextPage, + getItems, + nil) + return iterator +} + +// Get access lists. +// +// Gets all IP access lists for the specified workspace. +// +// This method is generated by Databricks SDK Code Generator. +func (a *IpAccessListsAPI) ListAll(ctx context.Context) ([]IpAccessListInfo, error) { + iterator := a.List(ctx) + return listing.ToSlice[IpAccessListInfo](ctx, iterator) } // IpAccessListInfoLabelToListIdMap calls [IpAccessListsAPI.ListAll] and creates a map of results with [IpAccessListInfo].Label as key and [IpAccessListInfo].ListId as value. @@ -654,12 +697,32 @@ func (a *TokenManagementAPI) GetPermissions(ctx context.Context) (*TokenPermissi // Lists all tokens associated with the specified workspace or user. // // This method is generated by Databricks SDK Code Generator. -func (a *TokenManagementAPI) ListAll(ctx context.Context, request ListTokenManagementRequest) ([]TokenInfo, error) { - response, err := a.impl.List(ctx, request) - if err != nil { - return nil, err +func (a *TokenManagementAPI) List(ctx context.Context, request ListTokenManagementRequest) *listing.PaginatingIterator[ListTokenManagementRequest, *ListTokensResponse, TokenInfo] { + + getNextPage := func(ctx context.Context, req ListTokenManagementRequest) (*ListTokensResponse, error) { + ctx = useragent.InContext(ctx, "sdk-feature", "pagination") + return a.impl.List(ctx, req) } - return response.TokenInfos, nil + getItems := func(resp *ListTokensResponse) []TokenInfo { + return resp.TokenInfos + } + + iterator := listing.NewIterator( + &request, + getNextPage, + getItems, + nil) + return iterator +} + +// List all tokens. +// +// Lists all tokens associated with the specified workspace or user. +// +// This method is generated by Databricks SDK Code Generator. +func (a *TokenManagementAPI) ListAll(ctx context.Context, request ListTokenManagementRequest) ([]TokenInfo, error) { + iterator := a.List(ctx, request) + return listing.ToSlice[TokenInfo](ctx, iterator) } // TokenInfoCommentToTokenIdMap calls [TokenManagementAPI.ListAll] and creates a map of results with [TokenInfo].Comment as key and [TokenInfo].TokenId as value. @@ -796,12 +859,33 @@ func (a *TokensAPI) DeleteByTokenId(ctx context.Context, tokenId string) error { // Lists all the valid tokens for a user-workspace pair. // // This method is generated by Databricks SDK Code Generator. -func (a *TokensAPI) ListAll(ctx context.Context) ([]TokenInfo, error) { - response, err := a.impl.List(ctx) - if err != nil { - return nil, err +func (a *TokensAPI) List(ctx context.Context) *listing.PaginatingIterator[struct{}, *ListTokensResponse, TokenInfo] { + request := struct{}{} + + getNextPage := func(ctx context.Context, req struct{}) (*ListTokensResponse, error) { + ctx = useragent.InContext(ctx, "sdk-feature", "pagination") + return a.impl.List(ctx) } - return response.TokenInfos, nil + getItems := func(resp *ListTokensResponse) []TokenInfo { + return resp.TokenInfos + } + + iterator := listing.NewIterator( + &request, + getNextPage, + getItems, + nil) + return iterator +} + +// List tokens. +// +// Lists all the valid tokens for a user-workspace pair. +// +// This method is generated by Databricks SDK Code Generator. +func (a *TokensAPI) ListAll(ctx context.Context) ([]TokenInfo, error) { + iterator := a.List(ctx) + return listing.ToSlice[TokenInfo](ctx, iterator) } // TokenInfoCommentToTokenIdMap calls [TokensAPI.ListAll] and creates a map of results with [TokenInfo].Comment as key and [TokenInfo].TokenId as value. diff --git a/service/sharing/api.go b/service/sharing/api.go index de305dc5c..3cf721818 100755 --- a/service/sharing/api.go +++ b/service/sharing/api.go @@ -8,6 +8,7 @@ import ( "fmt" "github.com/databricks/databricks-sdk-go/client" + "github.com/databricks/databricks-sdk-go/listing" "github.com/databricks/databricks-sdk-go/service/catalog" "github.com/databricks/databricks-sdk-go/useragent" ) @@ -95,33 +96,41 @@ func (a *CleanRoomsAPI) GetByNameArg(ctx context.Context, nameArg string) (*Clea // a specific ordering of the elements in the array. // // This method is generated by Databricks SDK Code Generator. -func (a *CleanRoomsAPI) ListAll(ctx context.Context, request ListCleanRoomsRequest) ([]CleanRoomInfo, error) { - var results []CleanRoomInfo - var totalCount int = 0 - ctx = useragent.InContext(ctx, "sdk-feature", "pagination") - for { - response, err := a.impl.List(ctx, request) - if err != nil { - return nil, err - } - if len(response.CleanRooms) == 0 { - break - } - for _, v := range response.CleanRooms { - results = append(results, v) - } - count := int(len(response.CleanRooms)) - totalCount += count - request.PageToken = response.NextPageToken - if response.NextPageToken == "" { - break - } - limit := request.MaxResults - if limit > 0 && totalCount >= limit { - break +func (a *CleanRoomsAPI) List(ctx context.Context, request ListCleanRoomsRequest) *listing.PaginatingIterator[ListCleanRoomsRequest, *ListCleanRoomsResponse, CleanRoomInfo] { + + getNextPage := func(ctx context.Context, req ListCleanRoomsRequest) (*ListCleanRoomsResponse, error) { + ctx = useragent.InContext(ctx, "sdk-feature", "pagination") + return a.impl.List(ctx, req) + } + getItems := func(resp *ListCleanRoomsResponse) []CleanRoomInfo { + return resp.CleanRooms + } + getNextReq := func(resp *ListCleanRoomsResponse) *ListCleanRoomsRequest { + if resp.NextPageToken == "" { + return nil } + request.PageToken = resp.NextPageToken + return &request } - return results, nil + iterator := listing.NewIterator( + &request, + getNextPage, + getItems, + getNextReq) + return iterator +} + +// List clean rooms. +// +// Gets an array of data object clean rooms from the metastore. The caller must +// be a metastore admin or the owner of the clean room. There is no guarantee of +// a specific ordering of the elements in the array. +// +// This method is generated by Databricks SDK Code Generator. +func (a *CleanRoomsAPI) ListAll(ctx context.Context, request ListCleanRoomsRequest) ([]CleanRoomInfo, error) { + iterator := a.List(ctx, request) + return listing.ToSliceN[CleanRoomInfo, int](ctx, iterator, request.MaxResults) + } // Update a clean room. @@ -228,12 +237,35 @@ func (a *ProvidersAPI) GetByName(ctx context.Context, name string) (*ProviderInf // specific ordering of the elements in the array. // // This method is generated by Databricks SDK Code Generator. -func (a *ProvidersAPI) ListAll(ctx context.Context, request ListProvidersRequest) ([]ProviderInfo, error) { - response, err := a.impl.List(ctx, request) - if err != nil { - return nil, err +func (a *ProvidersAPI) List(ctx context.Context, request ListProvidersRequest) *listing.PaginatingIterator[ListProvidersRequest, *ListProvidersResponse, ProviderInfo] { + + getNextPage := func(ctx context.Context, req ListProvidersRequest) (*ListProvidersResponse, error) { + ctx = useragent.InContext(ctx, "sdk-feature", "pagination") + return a.impl.List(ctx, req) + } + getItems := func(resp *ListProvidersResponse) []ProviderInfo { + return resp.Providers } - return response.Providers, nil + + iterator := listing.NewIterator( + &request, + getNextPage, + getItems, + nil) + return iterator +} + +// List providers. +// +// Gets an array of available authentication providers. The caller must either +// be a metastore admin or the owner of the providers. Providers not owned by +// the caller are not included in the response. There is no guarantee of a +// specific ordering of the elements in the array. +// +// This method is generated by Databricks SDK Code Generator. +func (a *ProvidersAPI) ListAll(ctx context.Context, request ListProvidersRequest) ([]ProviderInfo, error) { + iterator := a.List(ctx, request) + return listing.ToSlice[ProviderInfo](ctx, iterator) } // ProviderInfoNameToMetastoreIdMap calls [ProvidersAPI.ListAll] and creates a map of results with [ProviderInfo].Name as key and [ProviderInfo].MetastoreId as value. @@ -268,12 +300,34 @@ func (a *ProvidersAPI) ProviderInfoNameToMetastoreIdMap(ctx context.Context, req // * the caller is a metastore admin, or * the caller is the owner. // // This method is generated by Databricks SDK Code Generator. -func (a *ProvidersAPI) ListSharesAll(ctx context.Context, request ListSharesRequest) ([]ProviderShare, error) { - response, err := a.impl.ListShares(ctx, request) - if err != nil { - return nil, err +func (a *ProvidersAPI) ListShares(ctx context.Context, request ListSharesRequest) *listing.PaginatingIterator[ListSharesRequest, *ListProviderSharesResponse, ProviderShare] { + + getNextPage := func(ctx context.Context, req ListSharesRequest) (*ListProviderSharesResponse, error) { + ctx = useragent.InContext(ctx, "sdk-feature", "pagination") + return a.impl.ListShares(ctx, req) + } + getItems := func(resp *ListProviderSharesResponse) []ProviderShare { + return resp.Shares } - return response.Shares, nil + + iterator := listing.NewIterator( + &request, + getNextPage, + getItems, + nil) + return iterator +} + +// List shares by Provider. +// +// Gets an array of a specified provider's shares within the metastore where: +// +// * the caller is a metastore admin, or * the caller is the owner. +// +// This method is generated by Databricks SDK Code Generator. +func (a *ProvidersAPI) ListSharesAll(ctx context.Context, request ListSharesRequest) ([]ProviderShare, error) { + iterator := a.ListShares(ctx, request) + return listing.ToSlice[ProviderShare](ctx, iterator) } // List shares by Provider. @@ -465,12 +519,35 @@ func (a *RecipientsAPI) GetByName(ctx context.Context, name string) (*RecipientI // guarantee of a specific ordering of the elements in the array. // // This method is generated by Databricks SDK Code Generator. -func (a *RecipientsAPI) ListAll(ctx context.Context, request ListRecipientsRequest) ([]RecipientInfo, error) { - response, err := a.impl.List(ctx, request) - if err != nil { - return nil, err +func (a *RecipientsAPI) List(ctx context.Context, request ListRecipientsRequest) *listing.PaginatingIterator[ListRecipientsRequest, *ListRecipientsResponse, RecipientInfo] { + + getNextPage := func(ctx context.Context, req ListRecipientsRequest) (*ListRecipientsResponse, error) { + ctx = useragent.InContext(ctx, "sdk-feature", "pagination") + return a.impl.List(ctx, req) + } + getItems := func(resp *ListRecipientsResponse) []RecipientInfo { + return resp.Recipients } - return response.Recipients, nil + + iterator := listing.NewIterator( + &request, + getNextPage, + getItems, + nil) + return iterator +} + +// List share recipients. +// +// Gets an array of all share recipients within the current metastore where: +// +// * the caller is a metastore admin, or * the caller is the owner. There is no +// guarantee of a specific ordering of the elements in the array. +// +// This method is generated by Databricks SDK Code Generator. +func (a *RecipientsAPI) ListAll(ctx context.Context, request ListRecipientsRequest) ([]RecipientInfo, error) { + iterator := a.List(ctx, request) + return listing.ToSlice[RecipientInfo](ctx, iterator) } // RecipientInfoNameToMetastoreIdMap calls [RecipientsAPI.ListAll] and creates a map of results with [RecipientInfo].Name as key and [RecipientInfo].MetastoreId as value. @@ -617,12 +694,35 @@ func (a *SharesAPI) GetByName(ctx context.Context, name string) (*ShareInfo, err // specific ordering of the elements in the array. // // This method is generated by Databricks SDK Code Generator. -func (a *SharesAPI) ListAll(ctx context.Context) ([]ShareInfo, error) { - response, err := a.impl.List(ctx) - if err != nil { - return nil, err +func (a *SharesAPI) List(ctx context.Context) *listing.PaginatingIterator[struct{}, *ListSharesResponse, ShareInfo] { + request := struct{}{} + + getNextPage := func(ctx context.Context, req struct{}) (*ListSharesResponse, error) { + ctx = useragent.InContext(ctx, "sdk-feature", "pagination") + return a.impl.List(ctx) } - return response.Shares, nil + getItems := func(resp *ListSharesResponse) []ShareInfo { + return resp.Shares + } + + iterator := listing.NewIterator( + &request, + getNextPage, + getItems, + nil) + return iterator +} + +// List shares. +// +// Gets an array of data object shares from the metastore. The caller must be a +// metastore admin or the owner of the share. There is no guarantee of a +// specific ordering of the elements in the array. +// +// This method is generated by Databricks SDK Code Generator. +func (a *SharesAPI) ListAll(ctx context.Context) ([]ShareInfo, error) { + iterator := a.List(ctx) + return listing.ToSlice[ShareInfo](ctx, iterator) } // Get permissions. diff --git a/service/sql/api.go b/service/sql/api.go index 5075caae7..c461b658a 100755 --- a/service/sql/api.go +++ b/service/sql/api.go @@ -9,6 +9,7 @@ import ( "time" "github.com/databricks/databricks-sdk-go/client" + "github.com/databricks/databricks-sdk-go/listing" "github.com/databricks/databricks-sdk-go/retries" "github.com/databricks/databricks-sdk-go/useragent" ) @@ -285,39 +286,46 @@ func (a *DashboardsAPI) GetByDashboardId(ctx context.Context, dashboardId string // Fetch a paginated list of dashboard objects. // // This method is generated by Databricks SDK Code Generator. -func (a *DashboardsAPI) ListAll(ctx context.Context, request ListDashboardsRequest) ([]Dashboard, error) { - var results []Dashboard - var totalCount int = 0 - ctx = useragent.InContext(ctx, "sdk-feature", "pagination") - // deduplicate items that may have been added during iteration - seen := map[string]bool{} +func (a *DashboardsAPI) List(ctx context.Context, request ListDashboardsRequest) *listing.DeduplicatingIterator[Dashboard, string] { + request.Page = 1 // start iterating from the first page - for { - response, err := a.impl.List(ctx, request) - if err != nil { - return nil, err - } - if len(response.Results) == 0 { - break - } - for _, v := range response.Results { - id := v.Id - if seen[id] { - // item was added during iteration - continue - } - seen[id] = true - results = append(results, v) - } - count := int(len(response.Results)) - totalCount += count - request.Page++ - limit := request.PageSize - if limit > 0 && totalCount >= limit { - break + + getNextPage := func(ctx context.Context, req ListDashboardsRequest) (*ListResponse, error) { + ctx = useragent.InContext(ctx, "sdk-feature", "pagination") + return a.impl.List(ctx, req) + } + getItems := func(resp *ListResponse) []Dashboard { + return resp.Results + } + getNextReq := func(resp *ListResponse) *ListDashboardsRequest { + if len(getItems(resp)) == 0 { + return nil } + request.Page = resp.Page + 1 + return &request } - return results, nil + iterator := listing.NewIterator( + &request, + getNextPage, + getItems, + getNextReq) + dedupedIterator := listing.NewDedupeIterator[Dashboard, string]( + iterator, + func(item Dashboard) string { + return item.Id + }) + return dedupedIterator +} + +// Get dashboard objects. +// +// Fetch a paginated list of dashboard objects. +// +// This method is generated by Databricks SDK Code Generator. +func (a *DashboardsAPI) ListAll(ctx context.Context, request ListDashboardsRequest) ([]Dashboard, error) { + iterator := a.List(ctx, request) + return listing.ToSliceN[Dashboard, int](ctx, iterator, request.PageSize) + } // DashboardNameToIdMap calls [DashboardsAPI.ListAll] and creates a map of results with [Dashboard].Name as key and [Dashboard].Id as value. @@ -641,39 +649,47 @@ func (a *QueriesAPI) GetByQueryId(ctx context.Context, queryId string) (*Query, // term. // // This method is generated by Databricks SDK Code Generator. -func (a *QueriesAPI) ListAll(ctx context.Context, request ListQueriesRequest) ([]Query, error) { - var results []Query - var totalCount int = 0 - ctx = useragent.InContext(ctx, "sdk-feature", "pagination") - // deduplicate items that may have been added during iteration - seen := map[string]bool{} +func (a *QueriesAPI) List(ctx context.Context, request ListQueriesRequest) *listing.DeduplicatingIterator[Query, string] { + request.Page = 1 // start iterating from the first page - for { - response, err := a.impl.List(ctx, request) - if err != nil { - return nil, err - } - if len(response.Results) == 0 { - break - } - for _, v := range response.Results { - id := v.Id - if seen[id] { - // item was added during iteration - continue - } - seen[id] = true - results = append(results, v) - } - count := int(len(response.Results)) - totalCount += count - request.Page++ - limit := request.PageSize - if limit > 0 && totalCount >= limit { - break + + getNextPage := func(ctx context.Context, req ListQueriesRequest) (*QueryList, error) { + ctx = useragent.InContext(ctx, "sdk-feature", "pagination") + return a.impl.List(ctx, req) + } + getItems := func(resp *QueryList) []Query { + return resp.Results + } + getNextReq := func(resp *QueryList) *ListQueriesRequest { + if len(getItems(resp)) == 0 { + return nil } + request.Page = resp.Page + 1 + return &request } - return results, nil + iterator := listing.NewIterator( + &request, + getNextPage, + getItems, + getNextReq) + dedupedIterator := listing.NewDedupeIterator[Query, string]( + iterator, + func(item Query) string { + return item.Id + }) + return dedupedIterator +} + +// Get a list of queries. +// +// Gets a list of queries. Optionally, this list can be filtered by a search +// term. +// +// This method is generated by Databricks SDK Code Generator. +func (a *QueriesAPI) ListAll(ctx context.Context, request ListQueriesRequest) ([]Query, error) { + iterator := a.List(ctx, request) + return listing.ToSliceN[Query, int](ctx, iterator, request.PageSize) + } // QueryNameToIdMap calls [QueriesAPI.ListAll] and creates a map of results with [Query].Name as key and [Query].Id as value. @@ -780,33 +796,41 @@ func (a *QueryHistoryAPI) Impl() QueryHistoryService { // You can filter by user ID, warehouse ID, status, and time range. // // This method is generated by Databricks SDK Code Generator. -func (a *QueryHistoryAPI) ListAll(ctx context.Context, request ListQueryHistoryRequest) ([]QueryInfo, error) { - var results []QueryInfo - var totalCount int = 0 - ctx = useragent.InContext(ctx, "sdk-feature", "pagination") - for { - response, err := a.impl.List(ctx, request) - if err != nil { - return nil, err - } - if len(response.Res) == 0 { - break - } - for _, v := range response.Res { - results = append(results, v) - } - count := int(len(response.Res)) - totalCount += count - request.PageToken = response.NextPageToken - if response.NextPageToken == "" { - break - } - limit := request.MaxResults - if limit > 0 && totalCount >= limit { - break +func (a *QueryHistoryAPI) List(ctx context.Context, request ListQueryHistoryRequest) *listing.PaginatingIterator[ListQueryHistoryRequest, *ListQueriesResponse, QueryInfo] { + + getNextPage := func(ctx context.Context, req ListQueryHistoryRequest) (*ListQueriesResponse, error) { + ctx = useragent.InContext(ctx, "sdk-feature", "pagination") + return a.impl.List(ctx, req) + } + getItems := func(resp *ListQueriesResponse) []QueryInfo { + return resp.Res + } + getNextReq := func(resp *ListQueriesResponse) *ListQueryHistoryRequest { + if resp.NextPageToken == "" { + return nil } + request.PageToken = resp.NextPageToken + return &request } - return results, nil + iterator := listing.NewIterator( + &request, + getNextPage, + getItems, + getNextReq) + return iterator +} + +// List Queries. +// +// List the history of queries through SQL warehouses. +// +// You can filter by user ID, warehouse ID, status, and time range. +// +// This method is generated by Databricks SDK Code Generator. +func (a *QueryHistoryAPI) ListAll(ctx context.Context, request ListQueryHistoryRequest) ([]QueryInfo, error) { + iterator := a.List(ctx, request) + return listing.ToSliceN[QueryInfo, int](ctx, iterator, request.MaxResults) + } func NewQueryVisualizations(client *client.DatabricksClient) *QueryVisualizationsAPI { @@ -1375,12 +1399,32 @@ func (a *WarehousesAPI) GetWorkspaceWarehouseConfig(ctx context.Context) (*GetWo // Lists all SQL warehouses that a user has manager permissions on. // // This method is generated by Databricks SDK Code Generator. -func (a *WarehousesAPI) ListAll(ctx context.Context, request ListWarehousesRequest) ([]EndpointInfo, error) { - response, err := a.impl.List(ctx, request) - if err != nil { - return nil, err +func (a *WarehousesAPI) List(ctx context.Context, request ListWarehousesRequest) *listing.PaginatingIterator[ListWarehousesRequest, *ListWarehousesResponse, EndpointInfo] { + + getNextPage := func(ctx context.Context, req ListWarehousesRequest) (*ListWarehousesResponse, error) { + ctx = useragent.InContext(ctx, "sdk-feature", "pagination") + return a.impl.List(ctx, req) + } + getItems := func(resp *ListWarehousesResponse) []EndpointInfo { + return resp.Warehouses } - return response.Warehouses, nil + + iterator := listing.NewIterator( + &request, + getNextPage, + getItems, + nil) + return iterator +} + +// List warehouses. +// +// Lists all SQL warehouses that a user has manager permissions on. +// +// This method is generated by Databricks SDK Code Generator. +func (a *WarehousesAPI) ListAll(ctx context.Context, request ListWarehousesRequest) ([]EndpointInfo, error) { + iterator := a.List(ctx, request) + return listing.ToSlice[EndpointInfo](ctx, iterator) } // EndpointInfoNameToIdMap calls [WarehousesAPI.ListAll] and creates a map of results with [EndpointInfo].Name as key and [EndpointInfo].Id as value. diff --git a/service/workspace/api.go b/service/workspace/api.go index 2c9d57db3..cd70b2948 100755 --- a/service/workspace/api.go +++ b/service/workspace/api.go @@ -8,6 +8,7 @@ import ( "fmt" "github.com/databricks/databricks-sdk-go/client" + "github.com/databricks/databricks-sdk-go/listing" "github.com/databricks/databricks-sdk-go/useragent" ) @@ -91,12 +92,34 @@ func (a *GitCredentialsAPI) GetByCredentialId(ctx context.Context, credentialId // supported. // // This method is generated by Databricks SDK Code Generator. -func (a *GitCredentialsAPI) ListAll(ctx context.Context) ([]CredentialInfo, error) { - response, err := a.impl.List(ctx) - if err != nil { - return nil, err +func (a *GitCredentialsAPI) List(ctx context.Context) *listing.PaginatingIterator[struct{}, *GetCredentialsResponse, CredentialInfo] { + request := struct{}{} + + getNextPage := func(ctx context.Context, req struct{}) (*GetCredentialsResponse, error) { + ctx = useragent.InContext(ctx, "sdk-feature", "pagination") + return a.impl.List(ctx) } - return response.Credentials, nil + getItems := func(resp *GetCredentialsResponse) []CredentialInfo { + return resp.Credentials + } + + iterator := listing.NewIterator( + &request, + getNextPage, + getItems, + nil) + return iterator +} + +// Get Git credentials. +// +// Lists the calling user's Git credentials. One credential per user is +// supported. +// +// This method is generated by Databricks SDK Code Generator. +func (a *GitCredentialsAPI) ListAll(ctx context.Context) ([]CredentialInfo, error) { + iterator := a.List(ctx) + return listing.ToSlice[CredentialInfo](ctx, iterator) } // CredentialInfoGitProviderToCredentialIdMap calls [GitCredentialsAPI.ListAll] and creates a map of results with [CredentialInfo].GitProvider as key and [CredentialInfo].CredentialId as value. @@ -276,26 +299,39 @@ func (a *ReposAPI) GetPermissionsByRepoId(ctx context.Context, repoId string) (* // paginated with each page containing twenty repos. // // This method is generated by Databricks SDK Code Generator. -func (a *ReposAPI) ListAll(ctx context.Context, request ListReposRequest) ([]RepoInfo, error) { - var results []RepoInfo - ctx = useragent.InContext(ctx, "sdk-feature", "pagination") - for { - response, err := a.impl.List(ctx, request) - if err != nil { - return nil, err - } - if len(response.Repos) == 0 { - break - } - for _, v := range response.Repos { - results = append(results, v) - } - request.NextPageToken = response.NextPageToken - if response.NextPageToken == "" { - break +func (a *ReposAPI) List(ctx context.Context, request ListReposRequest) *listing.PaginatingIterator[ListReposRequest, *ListReposResponse, RepoInfo] { + + getNextPage := func(ctx context.Context, req ListReposRequest) (*ListReposResponse, error) { + ctx = useragent.InContext(ctx, "sdk-feature", "pagination") + return a.impl.List(ctx, req) + } + getItems := func(resp *ListReposResponse) []RepoInfo { + return resp.Repos + } + getNextReq := func(resp *ListReposResponse) *ListReposRequest { + if resp.NextPageToken == "" { + return nil } + request.NextPageToken = resp.NextPageToken + return &request } - return results, nil + iterator := listing.NewIterator( + &request, + getNextPage, + getItems, + getNextReq) + return iterator +} + +// Get repos. +// +// Returns repos that the calling user has Manage permissions on. Results are +// paginated with each page containing twenty repos. +// +// This method is generated by Databricks SDK Code Generator. +func (a *ReposAPI) ListAll(ctx context.Context, request ListReposRequest) ([]RepoInfo, error) { + iterator := a.List(ctx, request) + return listing.ToSlice[RepoInfo](ctx, iterator) } // RepoInfoPathToIdMap calls [ReposAPI.ListAll] and creates a map of results with [RepoInfo].Path as key and [RepoInfo].Id as value. @@ -510,12 +546,37 @@ func (a *SecretsAPI) GetSecret(ctx context.Context, request GetSecretRequest) (* // call. // // This method is generated by Databricks SDK Code Generator. -func (a *SecretsAPI) ListAclsAll(ctx context.Context, request ListAclsRequest) ([]AclItem, error) { - response, err := a.impl.ListAcls(ctx, request) - if err != nil { - return nil, err +func (a *SecretsAPI) ListAcls(ctx context.Context, request ListAclsRequest) *listing.PaginatingIterator[ListAclsRequest, *ListAclsResponse, AclItem] { + + getNextPage := func(ctx context.Context, req ListAclsRequest) (*ListAclsResponse, error) { + ctx = useragent.InContext(ctx, "sdk-feature", "pagination") + return a.impl.ListAcls(ctx, req) + } + getItems := func(resp *ListAclsResponse) []AclItem { + return resp.Items } - return response.Items, nil + + iterator := listing.NewIterator( + &request, + getNextPage, + getItems, + nil) + return iterator +} + +// Lists ACLs. +// +// List the ACLs for a given secret scope. Users must have the `MANAGE` +// permission to invoke this API. +// +// Throws `RESOURCE_DOES_NOT_EXIST` if no such secret scope exists. Throws +// `PERMISSION_DENIED` if the user does not have permission to make this API +// call. +// +// This method is generated by Databricks SDK Code Generator. +func (a *SecretsAPI) ListAclsAll(ctx context.Context, request ListAclsRequest) ([]AclItem, error) { + iterator := a.ListAcls(ctx, request) + return listing.ToSlice[AclItem](ctx, iterator) } // Lists ACLs. @@ -532,6 +593,33 @@ func (a *SecretsAPI) ListAclsByScope(ctx context.Context, scope string) (*ListAc }) } +// List all scopes. +// +// Lists all secret scopes available in the workspace. +// +// Throws `PERMISSION_DENIED` if the user does not have permission to make this +// API call. +// +// This method is generated by Databricks SDK Code Generator. +func (a *SecretsAPI) ListScopes(ctx context.Context) *listing.PaginatingIterator[struct{}, *ListScopesResponse, SecretScope] { + request := struct{}{} + + getNextPage := func(ctx context.Context, req struct{}) (*ListScopesResponse, error) { + ctx = useragent.InContext(ctx, "sdk-feature", "pagination") + return a.impl.ListScopes(ctx) + } + getItems := func(resp *ListScopesResponse) []SecretScope { + return resp.Scopes + } + + iterator := listing.NewIterator( + &request, + getNextPage, + getItems, + nil) + return iterator +} + // List all scopes. // // Lists all secret scopes available in the workspace. @@ -541,11 +629,38 @@ func (a *SecretsAPI) ListAclsByScope(ctx context.Context, scope string) (*ListAc // // This method is generated by Databricks SDK Code Generator. func (a *SecretsAPI) ListScopesAll(ctx context.Context) ([]SecretScope, error) { - response, err := a.impl.ListScopes(ctx) - if err != nil { - return nil, err + iterator := a.ListScopes(ctx) + return listing.ToSlice[SecretScope](ctx, iterator) +} + +// List secret keys. +// +// Lists the secret keys that are stored at this scope. This is a metadata-only +// operation; secret data cannot be retrieved using this API. Users need the +// READ permission to make this call. +// +// The lastUpdatedTimestamp returned is in milliseconds since epoch. Throws +// `RESOURCE_DOES_NOT_EXIST` if no such secret scope exists. Throws +// `PERMISSION_DENIED` if the user does not have permission to make this API +// call. +// +// This method is generated by Databricks SDK Code Generator. +func (a *SecretsAPI) ListSecrets(ctx context.Context, request ListSecretsRequest) *listing.PaginatingIterator[ListSecretsRequest, *ListSecretsResponse, SecretMetadata] { + + getNextPage := func(ctx context.Context, req ListSecretsRequest) (*ListSecretsResponse, error) { + ctx = useragent.InContext(ctx, "sdk-feature", "pagination") + return a.impl.ListSecrets(ctx, req) } - return response.Scopes, nil + getItems := func(resp *ListSecretsResponse) []SecretMetadata { + return resp.Secrets + } + + iterator := listing.NewIterator( + &request, + getNextPage, + getItems, + nil) + return iterator } // List secret keys. @@ -561,11 +676,8 @@ func (a *SecretsAPI) ListScopesAll(ctx context.Context) ([]SecretScope, error) { // // This method is generated by Databricks SDK Code Generator. func (a *SecretsAPI) ListSecretsAll(ctx context.Context, request ListSecretsRequest) ([]SecretMetadata, error) { - response, err := a.impl.ListSecrets(ctx, request) - if err != nil { - return nil, err - } - return response.Secrets, nil + iterator := a.ListSecrets(ctx, request) + return listing.ToSlice[SecretMetadata](ctx, iterator) } // List secret keys. @@ -773,12 +885,34 @@ func (a *WorkspaceAPI) Import(ctx context.Context, request Import) error { // `RESOURCE_DOES_NOT_EXIST`. // // This method is generated by Databricks SDK Code Generator. -func (a *WorkspaceAPI) ListAll(ctx context.Context, request ListWorkspaceRequest) ([]ObjectInfo, error) { - response, err := a.impl.List(ctx, request) - if err != nil { - return nil, err +func (a *WorkspaceAPI) List(ctx context.Context, request ListWorkspaceRequest) *listing.PaginatingIterator[ListWorkspaceRequest, *ListResponse, ObjectInfo] { + + getNextPage := func(ctx context.Context, req ListWorkspaceRequest) (*ListResponse, error) { + ctx = useragent.InContext(ctx, "sdk-feature", "pagination") + return a.impl.List(ctx, req) } - return response.Objects, nil + getItems := func(resp *ListResponse) []ObjectInfo { + return resp.Objects + } + + iterator := listing.NewIterator( + &request, + getNextPage, + getItems, + nil) + return iterator +} + +// List contents. +// +// Lists the contents of a directory, or the object if it is not a directory. If +// the input path does not exist, this call returns an error +// `RESOURCE_DOES_NOT_EXIST`. +// +// This method is generated by Databricks SDK Code Generator. +func (a *WorkspaceAPI) ListAll(ctx context.Context, request ListWorkspaceRequest) ([]ObjectInfo, error) { + iterator := a.List(ctx, request) + return listing.ToSlice[ObjectInfo](ctx, iterator) } // ObjectInfoPathToObjectIdMap calls [WorkspaceAPI.ListAll] and creates a map of results with [ObjectInfo].Path as key and [ObjectInfo].ObjectId as value.