From 57bb3f0768eab5f14a1ead78fe377efc722397bd Mon Sep 17 00:00:00 2001 From: Samuel Lucidi Date: Wed, 8 Nov 2023 13:20:04 -0500 Subject: [PATCH] :warning: Add "owner" and "contributors" columns to application import CSV format. (#551) Adds two fields to the import CSV immediately before tags. One is for the Owner and should be empty or contain a single value in the form 'John Doe '. The second is for contributors, and should be empty or a list of one or more values in the previous format, separated by commas. As this would be a comma-delimited field within a CSV, the whole value should be surrounded by quotes. This breaks compatibility with any existing import CSVs since it adds columns in the middle (necessary since the number of tags is unbounded and as such the tag list must come last). Fixes https://github.com/konveyor/tackle2-hub/issues/538 @jortel Do you think we can get away with adding this to migration 11, or do you think I should create a new one? --------- Signed-off-by: Sam Lucidi --- api/import.go | 12 ++- importer/manager.go | 84 +++++++++++++++++++ migration/v11/model/application.go | 2 + test/api/importcsv/api_test.go | 20 ++++- test/api/importcsv/samples.go | 22 +++++ .../importcsv/template_application_import.csv | 8 +- 6 files changed, 138 insertions(+), 10 deletions(-) diff --git a/api/import.go b/api/import.go index 8641470ac..3b8fe19a0 100644 --- a/api/import.go +++ b/api/import.go @@ -18,6 +18,10 @@ const ( RecordTypeDependency = "2" ) +const ( + ExpectedFieldCount = 17 +) + // // Import Statuses const ( @@ -270,8 +274,8 @@ func (h ImportHandler) UploadCSV(ctx *gin.Context) { var imp model.Import switch row[0] { case RecordTypeApplication: - // Check row format - length, expecting 15 fields + tags - if len(row) < 15 { + // Check row format - length, expecting 17 fields + tags + if len(row) < ExpectedFieldCount { h.Respond(ctx, http.StatusBadRequest, gin.H{"errorMessage": "Invalid Application Import CSV format."}) return } @@ -396,10 +400,12 @@ func (h ImportHandler) applicationFromRow(fileName string, row []string) (app mo RepositoryURL: row[12], RepositoryBranch: row[13], RepositoryPath: row[14], + Owner: row[15], + Contributors: row[16], } // Tags - for i := 15; i < len(row); i++ { + for i := ExpectedFieldCount; i < len(row); i++ { if i%2 == 0 { tag := model.ImportTag{ Name: row[i], diff --git a/importer/manager.go b/importer/manager.go index 96bc78732..575e71528 100644 --- a/importer/manager.go +++ b/importer/manager.go @@ -277,6 +277,54 @@ func (m *Manager) createApplication(imp *model.Import) (ok bool) { } } + if imp.Owner != "" { + name, email, parsed := parseStakeholder(imp.Owner) + if !parsed { + imp.ErrorMessage = fmt.Sprintf("Could not parse Owner '%s'.", imp.Owner) + return + } + owner, found := m.findStakeholder(email) + if !found { + if imp.ImportSummary.CreateEntities { + var err error + owner, err = m.createStakeholder(name, email) + if err != nil { + imp.ErrorMessage = fmt.Sprintf("Owner '%s' could not be created.", imp.Owner) + return + } + } else { + imp.ErrorMessage = fmt.Sprintf("Owner '%s' could not be found.", imp.Owner) + return + } + } + app.OwnerID = &owner.ID + } + if imp.Contributors != "" { + fields := strings.Split(imp.Contributors, ",") + for _, f := range fields { + name, email, parsed := parseStakeholder(f) + if !parsed { + imp.ErrorMessage = fmt.Sprintf("Could not parse Contributor '%s'.", f) + return + } + contributor, found := m.findStakeholder(email) + if !found { + if imp.ImportSummary.CreateEntities { + var err error + contributor, err = m.createStakeholder(name, email) + if err != nil { + imp.ErrorMessage = fmt.Sprintf("Contributor '%s' could not be created.", imp.Owner) + return + } + } else { + imp.ErrorMessage = fmt.Sprintf("Contributor '%s' could not be found.", imp.Owner) + return + } + } + app.Contributors = append(app.Contributors, contributor) + } + } + result := m.DB.Create(app) if result.Error != nil { imp.ErrorMessage = result.Error.Error() @@ -295,6 +343,25 @@ func (m *Manager) createApplication(imp *model.Import) (ok bool) { return } +func (m *Manager) createStakeholder(name string, email string) (stakeholder model.Stakeholder, err error) { + stakeholder.Name = name + stakeholder.Email = email + err = m.DB.Create(&stakeholder).Error + if err != nil { + err = liberr.Wrap(err) + } + return +} + +func (m *Manager) findStakeholder(email string) (stakeholder model.Stakeholder, found bool) { + result := m.DB.First(&stakeholder, "email = ?", email) + if result.Error != nil { + return + } + found = true + return +} + // // normalizedName transforms given name to be comparable as same with similar names // Example: normalizedName(" F oo-123 bar! ") returns "foo123bar!" @@ -304,3 +371,20 @@ func normalizedName(name string) (normName string) { normName = invalidSymbols.ReplaceAllString(normName, "") return } + +// +// parseStakeholder attempts to parse a stakeholder's name and an email address +// out of a string like `John Smith `. The pattern is very +// simple and treats anything before the first bracket as the name, +// and anything within the brackets as the email. +func parseStakeholder(s string) (name string, email string, parsed bool) { + pattern := regexp.MustCompile("(.+)\\s<(.+@.+)>") + matches := pattern.FindStringSubmatch(strings.TrimSpace(s)) + if len(matches) != 3 { + return + } + parsed = true + name = matches[1] + email = strings.ToLower(matches[2]) + return +} diff --git a/migration/v11/model/application.go b/migration/v11/model/application.go index 64e1e52a2..f6c1586c2 100644 --- a/migration/v11/model/application.go +++ b/migration/v11/model/application.go @@ -258,6 +258,8 @@ type Import struct { RepositoryURL string RepositoryBranch string RepositoryPath string + Owner string + Contributors string } func (r *Import) AsMap() (m map[string]interface{}) { diff --git a/test/api/importcsv/api_test.go b/test/api/importcsv/api_test.go index 3e7776ec1..8e7596fc8 100644 --- a/test/api/importcsv/api_test.go +++ b/test/api/importcsv/api_test.go @@ -1,13 +1,13 @@ package importcsv import ( + "github.com/konveyor/tackle2-hub/api" + "github.com/konveyor/tackle2-hub/binding" + "github.com/konveyor/tackle2-hub/test/assert" "io/ioutil" "os" "testing" "time" - "github.com/konveyor/tackle2-hub/api" - "github.com/konveyor/tackle2-hub/binding" - "github.com/konveyor/tackle2-hub/test/assert" ) func TestImportCSV(t *testing.T) { @@ -68,6 +68,20 @@ func TestImportCSV(t *testing.T) { if r.ExpectedApplications[i].BusinessService.Name != gotApp.BusinessService.Name { t.Errorf("Mismatch in name of the BusinessService of imported Application: Expected %s, Actual %s", r.ExpectedApplications[i].BusinessService.Name, gotApp.BusinessService.Name) } + if gotApp.Owner == nil || r.ExpectedApplications[i].Owner == nil { + if gotApp.Owner != r.ExpectedApplications[i].Owner { + t.Errorf("Mismatch in value of Owner on imported Application: Expected %v, Actual %v", r.ExpectedApplications[i].Owner, gotApp.BusinessService) + } + } else if r.ExpectedApplications[i].Owner.Name != gotApp.Owner.Name { + t.Errorf("Mismatch in name of the Owner of imported Application: Expected %s, Actual %s", r.ExpectedApplications[i].Owner.Name, gotApp.BusinessService.Name) + } + if len(gotApp.Contributors) != len(r.ExpectedApplications[i].Contributors) { + t.Errorf("Mismatch in number of Contributors: Expected %d, Actual %d", len(r.ExpectedApplications[i].Contributors), len(gotApp.Contributors)) + } else { + for j, contributor := range gotApp.Contributors { + if contributor.Name != r.ExpectedApplications[i].Contributors[j].Name {} + } + } } } diff --git a/test/api/importcsv/samples.go b/test/api/importcsv/samples.go index 56ee10628..aa0423fc9 100644 --- a/test/api/importcsv/samples.go +++ b/test/api/importcsv/samples.go @@ -48,6 +48,9 @@ var ( BusinessService: &api.Ref{ Name: "Retail", }, + Owner: &api.Ref{ + Name: "John Doe", + }, }, { Name: "Inventory", @@ -82,6 +85,14 @@ var ( BusinessService: &api.Ref{ Name: "Retail", }, + Contributors: []api.Ref{ + { + Name: "John Doe", + }, + { + Name: "Jane Smith", + }, + }, }, { Name: "Gateway", @@ -112,6 +123,17 @@ var ( BusinessService: &api.Ref{ Name: "Retail", }, + Owner: &api.Ref{ + Name: "John Doe", + }, + Contributors: []api.Ref{ + { + Name: "John Doe", + }, + { + Name: "Jane Smith", + }, + }, }, }, ExpectedDependencies: []api.Dependency{ diff --git a/test/api/importcsv/template_application_import.csv b/test/api/importcsv/template_application_import.csv index 80f64fdab..0466d30cd 100644 --- a/test/api/importcsv/template_application_import.csv +++ b/test/api/importcsv/template_application_import.csv @@ -1,6 +1,6 @@ -Record Type 1,Application Name,Description,Comments,Business Service,Dependency,Dependency Direction,Binary Group,Binary Artifact,Binary Version,Binary Packaging,Repository Type,Repository URL,Repository Branch,Repository Path,Tag Category 1,Tag 1,Tag Category 2,Tag 2,Tag Category 3,Tag 3,Tag Category 4,Tag 4,Tag Category 5,Tag 5,Tag Category 6,Tag 6,Tag Category 7,Tag 7,Tag Category 8,Tag 8,Tag Category 9,Tag 9,Tag Category 10,Tag 10,Tag Category 11,Tag 11,Tag Category 12,Tag 12,Tag Category 13,Tag 13,Tag Category 14,Tag 14,Tag Category 15,Tag 15,Tag Category 16,Tag 16,Tag Category 17,Tag 17,Tag Category 18,Tag 18,Tag Category 19,Tag 19,Tag Category 20,Tag 20 -1,Customers,Legacy Customers management service,,Retail,,,corp.acme.demo,customers-tomcat,0.0.1-SNAPSHOT,war,git,https://git-acme.local/customers.git,,,Operating System,RHEL 8,Database,Oracle,Language,Java,Runtime,Tomcat,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, -1,Inventory,Inventory service,,Retail,,,corp.acme.demo,inventory,0.1.1-SNAPSHOT,war,git,https://git-acme.local/inventory.git,,,Operating System,RHEL 8,Database,Postgresql,Language,Java,Runtime,Quarkus,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, -1,Gateway,API Gateway,,Retail,,,corp.acme.demo,gateway,0.1.1-SNAPSHOT,war,git,https://git-acme.local/gateway.git,,,Operating System,RHEL 8,,,Language,Java,Runtime,Spring Boot,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +Record Type 1,Application Name,Description,Comments,Business Service,Dependency,Dependency Direction,Binary Group,Binary Artifact,Binary Version,Binary Packaging,Repository Type,Repository URL,Repository Branch,Repository Path,Owner,Contributors,Tag Category 1,Tag 1,Tag Category 2,Tag 2,Tag Category 3,Tag 3,Tag Category 4,Tag 4,Tag Category 5,Tag 5,Tag Category 6,Tag 6,Tag Category 7,Tag 7,Tag Category 8,Tag 8,Tag Category 9,Tag 9,Tag Category 10,Tag 10,Tag Category 11,Tag 11,Tag Category 12,Tag 12,Tag Category 13,Tag 13,Tag Category 14,Tag 14,Tag Category 15,Tag 15,Tag Category 16,Tag 16,Tag Category 17,Tag 17,Tag Category 18,Tag 18,Tag Category 19,Tag 19,Tag Category 20,Tag 20 +1,Customers,Legacy Customers management service,,Retail,,,corp.acme.demo,customers-tomcat,0.0.1-SNAPSHOT,war,git,https://git-acme.local/customers.git,,,John Doe ,,Operating System,RHEL 8,Database,Oracle,Language,Java,Runtime,Tomcat,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +1,Inventory,Inventory service,,Retail,,,corp.acme.demo,inventory,0.1.1-SNAPSHOT,war,git,https://git-acme.local/inventory.git,,,,"John Doe , Jane Smith ",Operating System,RHEL 8,Database,Postgresql,Language,Java,Runtime,Quarkus,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +1,Gateway,API Gateway,,Retail,,,corp.acme.demo,gateway,0.1.1-SNAPSHOT,war,git,https://git-acme.local/gateway.git,,,John Doe ,"John Doe , Jane Smith ",Operating System,RHEL 8,,,Language,Java,Runtime,Spring Boot,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, 2,Gateway,,,,Inventory,southbound 2,Gateway,,,,Customers,southbound