Skip to content

Commit

Permalink
fix: fetch children (#189)
Browse files Browse the repository at this point in the history 
* fix: fetch children

* fix: test

* fix: resolve commenys

* refactor code

* fix: test

* fix: test

* coverage test

* fix: test

* fix: test

* fix: test

* coverage test

* remove a comment

---------

Co-authored-by: Lifosmin Simon <lifosmin.simon@gojek.com>
  • Loading branch information
@lifosmin
lifosmin and Lifosmin Simon authored Nov 28, 2024
1 parent 691ddd6 commit 6d09d2b
Show file tree
Hide file tree
Showing 3 changed files with 178 additions and 45 deletions.
2 changes: 1 addition & 1 deletion core/provider/common.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -54,7 +54,7 @@ func normalizeDetails(details map[string]interface{}) (map[string]interface{}, e
return normalized, nil
}

func compareResources(existingResource, newResource domain.Resource) (bool, string) {
func compareResource(existingResource, newResource domain.Resource) (bool, string) {
opts := cmp.Options{
cmpopts.IgnoreFields(domain.Resource{}, "ID", "CreatedAt", "UpdatedAt", "ParentID", "Children"),
cmpopts.EquateEmpty(),
Expand Down
96 changes: 52 additions & 44 deletions core/provider/service.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -575,27 +575,30 @@ func (s *Service) fetchNewResources(ctx context.Context, p *domain.Provider) ([]
return nil, 0, fmt.Errorf("%w: %v", ErrInvalidProviderType, p.Type)
}

existingGuardianResources, err := s.resourceService.Find(ctx, domain.ListResourcesFilter{
existingResources, err := s.resourceService.Find(ctx, domain.ListResourcesFilter{
ProviderType: p.Type,
ProviderURN: p.URN,
})
if err != nil {
return nil, 0, err
}
mapExistingResources := make(map[string]*domain.Resource, len(existingResources))
for _, existing := range existingResources {
mapExistingResources[existing.GlobalURN] = existing
}

newResourcesWithChildren, err := c.GetResources(ctx, p.Config)
if err != nil {
return nil, 0, fmt.Errorf("error fetching resources for %v: %w", p.ID, err)
}
resourceTypeFilterMap := make(map[string]string)
for _, rc := range p.Config.Resources {
if len(rc.Filter) > 0 {
resourceTypeFilterMap[rc.Type] = rc.Filter
}
}

newProviderResources, err := c.GetResources(ctx, p.Config)
if err != nil {
return nil, 0, fmt.Errorf("error fetching resources for %v: %w", p.ID, err)
}
filteredResources := make([]*domain.Resource, 0)
for _, r := range newProviderResources {
for _, r := range newResourcesWithChildren {
if filterExpression, ok := resourceTypeFilterMap[r.Type]; ok {
v, err := evaluator.Expression(filterExpression).EvaluateWithStruct(r)
if err != nil {
Expand All @@ -608,51 +611,56 @@ func (s *Service) fetchNewResources(ctx context.Context, p *domain.Provider) ([]
filteredResources = append(filteredResources, r)
}
}
flattenedProviderResources := flattenResources(filteredResources)

existingProviderResources := map[string]bool{}
updatedResources := []*domain.Resource{}
for _, newResource := range flattenedProviderResources {
found := false
for _, existingResource := range existingGuardianResources {
if existingResource.Type == newResource.Type && existingResource.URN == newResource.URN {
found = true
if existingDetails := existingResource.Details; existingDetails != nil {
if newResource.Details != nil {
for key, value := range existingDetails {
if _, ok := newResource.Details[key]; !ok {
newResource.Details[key] = value
}
}
} else {
newResource.Details = existingDetails
}
if isUpdated, diff := compareResources(*existingResource, *newResource); isUpdated {
s.logger.Debug(ctx, "diff", "resources", diff)
updatedResources = append(updatedResources, newResource)
s.logger.Info(ctx, "resources is updated", "resource", newResource.URN)

newAndUpdatedResources := s.compareResources(ctx, mapExistingResources, filteredResources)
if len(newAndUpdatedResources) == 0 {
return []*domain.Resource{}, 0, nil
}
for _, deletedResource := range mapExistingResources {
deletedResource.IsDeleted = true
newAndUpdatedResources = append(newAndUpdatedResources, deletedResource)
s.logger.Info(ctx, "resource deleted", "resource", deletedResource.GlobalURN)
}

return newAndUpdatedResources, len(newResourcesWithChildren), nil
}

func (s *Service) compareResources(ctx context.Context, existingResources map[string]*domain.Resource, newResources []*domain.Resource) []*domain.Resource {
var res []*domain.Resource
for _, new := range newResources {
new.Children = s.compareResources(ctx, existingResources, new.Children)

existing, exist := existingResources[new.GlobalURN]
if !exist {
// new resource
res = append(res, new)
continue
}
delete(existingResources, new.GlobalURN)
if existingDetails := existing.Details; existingDetails != nil {
if new.Details != nil {
for key, value := range existingDetails {
if _, ok := new.Details[key]; !ok {
new.Details[key] = value
}
}
existingProviderResources[existingResource.ID] = true
break
} else {
new.Details = existingDetails
}
}
if !found {
updatedResources = append(updatedResources, newResource)
s.logger.Info(ctx, "new resource added", "resource", newResource.Name)
if len(new.Children) == 0 {
isUpdated, diff := compareResource(*existing, *new)
if !isUpdated {
continue
}
s.logger.Debug(ctx, "diff", "resources", diff)
s.logger.Info(ctx, "resources is updated", "resource", new.URN)
}
}

// mark IsDeleted of guardian resources that no longer exist in provider
for _, r := range existingGuardianResources {
if _, ok := existingProviderResources[r.ID]; !ok {
r.IsDeleted = true
updatedResources = append(updatedResources, r)
s.logger.Info(ctx, "resource deleted", "resource", r.Name)
}
res = append(res, new)
}

return updatedResources, len(newProviderResources), nil
return res
}

func (s *Service) validateAppealParam(a *domain.Appeal) error {
Expand Down
125 changes: 125 additions & 0 deletions core/provider/service_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -381,6 +381,7 @@ func (s *ServiceTestSuite) TestFetchResources() {
ProviderURN: mockProvider,
Type: "test-resource-type",
URN: "test-resource-urn-2",
GlobalURN: "test-1",
},
{
ID: "1",
Expand All @@ -398,6 +399,7 @@ func (s *ServiceTestSuite) TestFetchResources() {
"x": "y",
},
},
GlobalURN: "test-2",
},
}
newResources := []*domain.Resource{
Expand All @@ -415,13 +417,15 @@ func (s *ServiceTestSuite) TestFetchResources() {
"x": "y",
},
},
GlobalURN: "test-2",
},
{
ID: "12ß",
ProviderType: mockProviderType,
ProviderURN: mockProvider,
Type: "test-resource-type",
URN: "test-resource-urn-2",
GlobalURN: "test-1",
},
}

Expand All @@ -434,6 +438,122 @@ func (s *ServiceTestSuite) TestFetchResources() {
s.Nil(actualError)
})

s.Run("should upsert children on success", func() {
existingResources := []*domain.Resource{
{
ID: "12ß",
ProviderType: mockProviderType,
ProviderURN: mockProvider,
Type: "test-resource-type",
URN: "test-resource-urn-2",
GlobalURN: "test-1",
},
{
ID: "1",
ProviderType: mockProviderType,
ProviderURN: mockProvider,
Type: "test-resource-type",
URN: "test-resource-urn-1",
Details: map[string]interface{}{
"owner": "test-owner",
resource.ReservedDetailsKeyMetadata: map[string]interface{}{
"labels": map[string]string{
"foo": "bar",
"baz": "qux",
},
"x": "y",
},
},
GlobalURN: "test-2",
Children: []*domain.Resource{
{
ID: "12ß",
ProviderType: mockProviderType,
ProviderURN: mockProvider,
Type: "test-resource-type",
URN: "test-resource-urn-2",
GlobalURN: "test-1",
},
},
},
}
newResources := []*domain.Resource{
{
ProviderType: mockProviderType,
ProviderURN: mockProvider,
Type: "test-resource-type",
URN: "test-resource-urn-1",
Details: map[string]interface{}{
resource.ReservedDetailsKeyMetadata: map[string]interface{}{
"labels": map[string]string{
"foo": "bar",
"baz": "qux",
},
"x": "y",
},
},
GlobalURN: "test-2",
Children: []*domain.Resource{
{
ProviderType: mockProviderType,
ProviderURN: mockProvider,
Type: "test-resource-type",
URN: "test-resource-urn-2",
GlobalURN: "test-1",
},
{
ProviderType: mockProviderType,
ProviderURN: mockProvider,
Type: "test-resource-type",
URN: "test-resource-urn-3",
GlobalURN: "test-3",
},
},
},
}
expectedResources := []*domain.Resource{
{
ProviderType: mockProviderType,
ProviderURN: mockProvider,
Type: "test-resource-type",
URN: "test-resource-urn-1",
Details: map[string]interface{}{
"owner": "test-owner",
resource.ReservedDetailsKeyMetadata: map[string]interface{}{
"labels": map[string]string{
"foo": "bar",
"baz": "qux",
},
"x": "y",
},
},
GlobalURN: "test-2",
Children: []*domain.Resource{
{
ID: "2",
ProviderType: mockProviderType,
ProviderURN: mockProvider,
Type: "test-resource-type",
URN: "test-resource-urn-3",
GlobalURN: "test-3",
},
},
},
}

expectedProvider := providers[0]
s.mockProviderRepository.EXPECT().Find(mockCtx).Return([]*domain.Provider{expectedProvider}, nil).Once()
s.mockProvider.EXPECT().GetResources(mockCtx, expectedProvider.Config).Return(newResources, nil).Once()
s.mockResourceService.EXPECT().BulkUpsert(mock.Anything, mock.AnythingOfType("[]*domain.Resource")).
Run(func(_a0 context.Context, resources []*domain.Resource) {
s.Empty(cmp.Diff(expectedResources, resources, cmpopts.IgnoreFields(domain.Resource{}, "ID", "CreatedAt", "UpdatedAt")))
}).Return(nil).Once()
s.mockResourceService.EXPECT().Find(mock.Anything, mock.Anything).Return(existingResources, nil).Once()
actualError := s.service.FetchResources(context.Background())

s.Nil(actualError)
})

s.Run("should upsert all resources on success", func() {
existingResources := []*domain.Resource{
{
Expand All @@ -452,6 +572,7 @@ func (s *ServiceTestSuite) TestFetchResources() {
"x": "y",
},
},
GlobalURN: "test-1",
},
}
newResources := []*domain.Resource{
Expand All @@ -467,12 +588,14 @@ func (s *ServiceTestSuite) TestFetchResources() {
},
},
},
GlobalURN: "test-1",
},
{
ProviderType: mockProviderType,
ProviderURN: mockProvider,
Type: "test-resource-type",
URN: "test-resource-urn-2",
GlobalURN: "test-2",
},
}
expectedResources := []*domain.Resource{
Expand All @@ -489,12 +612,14 @@ func (s *ServiceTestSuite) TestFetchResources() {
},
},
},
GlobalURN: "test-1",
},
{
ProviderType: mockProviderType,
ProviderURN: mockProvider,
Type: "test-resource-type",
URN: "test-resource-urn-2",
GlobalURN: "test-2",
},
}

Expand Down

0 comments on commit 6d09d2b

Please sign in to comment.