Skip to content

Commit

Permalink
Support pagination in non-policy data sources
Browse files Browse the repository at this point in the history 
These data sources were limited to 1K objects, and recommendation is
to use policy counterparts. However some customers that can not yet
move to policy require support for MP data sources at scale.
The support was not added to fabric data sources that are not
expected to reach triple digit scale.
  • Loading branch information
@annakhm
annakhm committed Aug 18, 2021
1 parent a223a8f commit ff3d912
Show file tree
Hide file tree
Showing 9 changed files with 229 additions and 108 deletions.
39 changes: 25 additions & 14 deletions nsxt/data_source_nsxt_firewall_section.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -60,25 +60,36 @@ func dataSourceNsxtFirewallSectionRead(d *schema.ResourceData, m interface{}) er
}
obj = objGet
} else if objName != "" {
// Get by full name
// TODO use 2nd parameter localVarOptionals for paging
objList, _, err := nsxClient.ServicesApi.ListSections(nsxClient.Context, nil)
if err != nil {
return fmt.Errorf("Error while reading Firewall sections: %v", err)
}
// go over the list to find the correct one
found := false
for _, objInList := range objList.Results {
if objInList.DisplayName == objName {
if found {
return fmt.Errorf("Found multiple Firewall sections with name '%s'", objName)
// Get by full name
lister := func(info *paginationInfo) error {
objList, _, err := nsxClient.ServicesApi.ListSections(nsxClient.Context, info.LocalVarOptionals)
if err != nil {
return fmt.Errorf("Error while reading Firewall sections: %v", err)
}

info.PageCount = int64(len(objList.Results))
info.TotalCount = objList.ResultCount
info.Cursor = objList.Cursor

// go over the list to find the correct one
for _, objInList := range objList.Results {
if objInList.DisplayName == objName {
if found {
return fmt.Errorf("Found multiple Firewall sections with name '%s'", objName)
}
obj = objInList
found = true
}
obj = objInList
found = true
}
return nil
}
total, err := handlePagination(lister)
if err != nil {
return err
}
if !found {
return fmt.Errorf("Firewall section with name '%s' was not found among %d sections", objName, len(objList.Results))
return fmt.Errorf("Firewall section with name '%s' was not found among %d sections", objName, total)
}
} else {
return fmt.Errorf("Error obtaining Firewall section ID or name during read")
Expand Down
37 changes: 25 additions & 12 deletions nsxt/data_source_nsxt_ip_pool.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -61,23 +61,36 @@ func dataSourceNsxtIPPoolRead(d *schema.ResourceData, m interface{}) error {
obj = objGet
} else if objName != "" {
// Get by full name
objList, _, err := nsxClient.PoolManagementApi.ListIpPools(nsxClient.Context, nil)
if err != nil {
return fmt.Errorf("Error while reading IP pool: %v", err)
}
// go over the list to find the correct one
found := false
for _, objInList := range objList.Results {
if objInList.DisplayName == objName {
if found {
return fmt.Errorf("Found multiple IP pool with name '%s'", objName)
lister := func(info *paginationInfo) error {
objList, _, err := nsxClient.PoolManagementApi.ListIpPools(nsxClient.Context, info.LocalVarOptionals)
if err != nil {
return fmt.Errorf("Error while reading IP pool: %v", err)
}

info.PageCount = int64(len(objList.Results))
info.TotalCount = objList.ResultCount
info.Cursor = objList.Cursor

// go over the list to find the correct one
for _, objInList := range objList.Results {
if objInList.DisplayName == objName {
if found {
return fmt.Errorf("Found multiple IP pool with name '%s'", objName)
}
obj = objInList
found = true
}
obj = objInList
found = true
}
return nil
}

total, err := handlePagination(lister)
if err != nil {
return err
}
if !found {
return fmt.Errorf("IP pool '%s' was not found out of %d objects", objName, len(objList.Results))
return fmt.Errorf("IP pool '%s' was not found out of %d objects", objName, total)
}
} else {
return fmt.Errorf("Error obtaining IP pool ID or name during read")
Expand Down
40 changes: 26 additions & 14 deletions nsxt/data_source_nsxt_logical_tier0_router.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -80,24 +80,36 @@ func dataSourceNsxtLogicalTier0RouterRead(d *schema.ResourceData, m interface{})
return fmt.Errorf("Error obtaining logical tier0 router ID or name during read")
} else {
// Get by full name/prefix
// TODO use 2nd parameter localVarOptionals for paging
objList, _, err := nsxClient.LogicalRoutingAndServicesApi.ListLogicalRouters(nsxClient.Context, nil)
if err != nil {
return fmt.Errorf("Error while reading logical tier0 routers: %v", err)
}
// go over the list to find the correct one (prefer a perfect match. If not - prefix match)
var perfectMatch []manager.LogicalRouter
var prefixMatch []manager.LogicalRouter
for _, objInList := range objList.Results {
if objInList.RouterType == "TIER0" {
if strings.HasPrefix(objInList.DisplayName, objName) {
prefixMatch = append(prefixMatch, objInList)
}
if objInList.DisplayName == objName {
perfectMatch = append(perfectMatch, objInList)
lister := func(info *paginationInfo) error {
objList, _, err := nsxClient.LogicalRoutingAndServicesApi.ListLogicalRouters(nsxClient.Context, info.LocalVarOptionals)
if err != nil {
return fmt.Errorf("Error while reading logical tier0 routers: %v", err)
}

info.PageCount = int64(len(objList.Results))
info.TotalCount = objList.ResultCount
info.Cursor = objList.Cursor
// go over the list to find the correct one (prefer a perfect match. If not - prefix match)
for _, objInList := range objList.Results {
if objInList.RouterType == "TIER0" {
if strings.HasPrefix(objInList.DisplayName, objName) {
prefixMatch = append(prefixMatch, objInList)
}
if objInList.DisplayName == objName {
perfectMatch = append(perfectMatch, objInList)
}
}
}
return nil
}

total, err := handlePagination(lister)
if err != nil {
return err
}

if len(perfectMatch) > 0 {
if len(perfectMatch) > 1 {
return fmt.Errorf("Found multiple logical tier0 routers with name '%s'", objName)
Expand All @@ -109,7 +121,7 @@ func dataSourceNsxtLogicalTier0RouterRead(d *schema.ResourceData, m interface{})
}
obj = prefixMatch[0]
} else {
return fmt.Errorf("Logical tier0 router with name '%s' was not found", objName)
return fmt.Errorf("Logical tier0 router with name '%s' was not found among %d objects", objName, total)
}
}

Expand Down
41 changes: 27 additions & 14 deletions nsxt/data_source_nsxt_logical_tier1_router.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -74,24 +74,37 @@ func dataSourceNsxtLogicalTier1RouterRead(d *schema.ResourceData, m interface{})
return fmt.Errorf("Error obtaining logical tier1 router ID or name during read")
} else {
// Get by full name/prefix
// TODO use 2nd parameter localVarOptionals for paging
objList, _, err := nsxClient.LogicalRoutingAndServicesApi.ListLogicalRouters(nsxClient.Context, nil)
if err != nil {
return fmt.Errorf("Error while reading logical tier1 routers: %v", err)
}
// go over the list to find the correct one (prefer a perfect match. If not - prefix match)
var perfectMatch []manager.LogicalRouter
var prefixMatch []manager.LogicalRouter
for _, objInList := range objList.Results {
if objInList.RouterType == "TIER1" {
if strings.HasPrefix(objInList.DisplayName, objName) {
prefixMatch = append(prefixMatch, objInList)
}
if objInList.DisplayName == objName {
perfectMatch = append(perfectMatch, objInList)
lister := func(info *paginationInfo) error {
objList, _, err := nsxClient.LogicalRoutingAndServicesApi.ListLogicalRouters(nsxClient.Context, info.LocalVarOptionals)
if err != nil {
return fmt.Errorf("Error while reading logical tier1 routers: %v", err)
}

info.PageCount = int64(len(objList.Results))
info.TotalCount = objList.ResultCount
info.Cursor = objList.Cursor

// go over the list to find the correct one (prefer a perfect match. If not - prefix match)
for _, objInList := range objList.Results {
if objInList.RouterType == "TIER1" {
if strings.HasPrefix(objInList.DisplayName, objName) {
prefixMatch = append(prefixMatch, objInList)
}
if objInList.DisplayName == objName {
perfectMatch = append(perfectMatch, objInList)
}
}
}
return nil
}

total, err := handlePagination(lister)
if err != nil {
return err
}

if len(perfectMatch) > 0 {
if len(perfectMatch) > 1 {
return fmt.Errorf("Found multiple logical tier1 routers with name '%s'", objName)
Expand All @@ -103,7 +116,7 @@ func dataSourceNsxtLogicalTier1RouterRead(d *schema.ResourceData, m interface{})
}
obj = prefixMatch[0]
} else {
return fmt.Errorf("Logical tier1 router with name '%s' was not found", objName)
return fmt.Errorf("Logical tier1 router with name '%s' was not found among %d objects", objName, total)
}
}

Expand Down
38 changes: 25 additions & 13 deletions nsxt/data_source_nsxt_mac_pool.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -62,24 +62,36 @@ func dataSourceNsxtMacPoolRead(d *schema.ResourceData, m interface{}) error {
obj = objGet
} else if objName != "" {
// Get by full name
// TODO use 2nd parameter localVarOptionals for paging
objList, _, err := nsxClient.PoolManagementApi.ListMacPools(nsxClient.Context, nil)
if err != nil {
return fmt.Errorf("Error while reading Mac pool: %v", err)
}
// go over the list to find the correct one
found := false
for _, objInList := range objList.Results {
if objInList.DisplayName == objName {
if found {
return fmt.Errorf("Found multiple Mac pool with name '%s'", objName)
lister := func(info *paginationInfo) error {
objList, _, err := nsxClient.PoolManagementApi.ListMacPools(nsxClient.Context, info.LocalVarOptionals)
if err != nil {
return fmt.Errorf("Error while reading Mac pool: %v", err)
}

info.PageCount = int64(len(objList.Results))
info.TotalCount = objList.ResultCount
info.Cursor = objList.Cursor

// go over the list to find the correct one
for _, objInList := range objList.Results {
if objInList.DisplayName == objName {
if found {
return fmt.Errorf("Found multiple Mac pool with name '%s'", objName)
}
obj = objInList
found = true
}
obj = objInList
found = true
}
return nil
}

total, err := handlePagination(lister)
if err != nil {
return err
}
if !found {
return fmt.Errorf("Mac pool with name '%s' was not found among %d pools", objName, len(objList.Results))
return fmt.Errorf("Mac pool with name '%s' was not found among %d pools", objName, total)
}
} else {
return fmt.Errorf("Error obtaining Mac pool ID or name during read")
Expand Down
25 changes: 12 additions & 13 deletions nsxt/data_source_nsxt_ns_group.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -63,22 +63,16 @@ func dataSourceNsxtNsGroupRead(d *schema.ResourceData, m interface{}) error {
obj = objGet
} else if objName != "" {
// Get by full name
// Handle paging here as it is limited to 50 per page in the api, even if configured otherwise
localVarOptionals := make(map[string]interface{})
localVarOptionals["pageSize"] = int64(50)
found := false
total := 0
count := 0
for !found && (total == 0 || count < total) {
objList, _, err := nsxClient.GroupingObjectsApi.ListNSGroups(nsxClient.Context, localVarOptionals)
lister := func(info *paginationInfo) error {
objList, _, err := nsxClient.GroupingObjectsApi.ListNSGroups(nsxClient.Context, info.LocalVarOptionals)
if err != nil {
return fmt.Errorf("Error while reading NS groups: %v", err)
}
if total == 0 && objList.ResultCount > 0 {
// first response
total = int(objList.ResultCount)
}
count += len(objList.Results)
info.PageCount = int64(len(objList.Results))
info.TotalCount = objList.ResultCount
info.Cursor = objList.Cursor

// go over the list to find the correct one
for _, objInList := range objList.Results {
if objInList.DisplayName == objName {
Expand All @@ -89,7 +83,12 @@ func dataSourceNsxtNsGroupRead(d *schema.ResourceData, m interface{}) error {
found = true
}
}
localVarOptionals["cursor"] = objList.Cursor
return nil
}

total, err := handlePagination(lister)
if err != nil {
return err
}
if !found {
return fmt.Errorf("NS group with name '%s' was not found among %d groups", objName, total)
Expand Down
38 changes: 25 additions & 13 deletions nsxt/data_source_nsxt_ns_service.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -61,24 +61,36 @@ func dataSourceNsxtNsServiceRead(d *schema.ResourceData, m interface{}) error {
obj = objGet
} else if objName != "" {
// Get by full name
// TODO use 2nd parameter localVarOptionals for paging
objList, _, err := nsxClient.GroupingObjectsApi.ListNSServices(nsxClient.Context, nil)
if err != nil {
return fmt.Errorf("Error while reading NS services: %v", err)
}
// go over the list to find the correct one
found := false
for _, objInList := range objList.Results {
if objInList.DisplayName == objName {
if found {
return fmt.Errorf("Found multiple NS services with name '%s'", objName)
lister := func(info *paginationInfo) error {
objList, _, err := nsxClient.GroupingObjectsApi.ListNSServices(nsxClient.Context, info.LocalVarOptionals)
if err != nil {
return fmt.Errorf("Error while reading NS services: %v", err)
}
info.PageCount = int64(len(objList.Results))
info.TotalCount = objList.ResultCount
info.Cursor = objList.Cursor

// go over the list to find the correct one
for _, objInList := range objList.Results {
if objInList.DisplayName == objName {
if found {
return fmt.Errorf("Found multiple NS services with name '%s'", objName)
}
obj = objInList
found = true
}
obj = objInList
found = true
}
return nil
}

total, err := handlePagination(lister)
if err != nil {
return err
}

if !found {
return fmt.Errorf("NS service with name '%s' was not found among %d services", objName, len(objList.Results))
return fmt.Errorf("NS service with name '%s' was not found among %d services", objName, total)
}
} else {
return fmt.Errorf("Error obtaining NS service ID or name during read")
Expand Down
Loading

0 comments on commit ff3d912

Please sign in to comment.