From 74da1fb07d9f7eccf497dce55a991af9a3992f29 Mon Sep 17 00:00:00 2001 From: Austin Canada Date: Tue, 5 Nov 2024 16:17:03 -0500 Subject: [PATCH 01/11] BCDA-8405: Add CSV lambda changes --- bcda/lambda/cclf/main.go | 43 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 43 insertions(+) diff --git a/bcda/lambda/cclf/main.go b/bcda/lambda/cclf/main.go index 43bfa4bdc..6b992ee3e 100644 --- a/bcda/lambda/cclf/main.go +++ b/bcda/lambda/cclf/main.go @@ -60,14 +60,57 @@ func cclfImportHandler(ctx context.Context, sqsEvent events.SQSEvent) (string, e // importing the one file that was sent in the trigger. filepath := fmt.Sprintf("%s/%s", e.S3.Bucket.Name, e.S3.Object.Key) logger.Infof("Reading %s event for file %s", e.EventName, filepath) + parser := cclf.CSVParser{Filepath: e.S3.Object.Key} + if parser.CheckIfAttributionCSVFile(e.S3.Object.Key) { + return handleCSVImport(s3AssumeRoleArn, filepath) + } else { return handleCclfImport(s3AssumeRoleArn, filepath) } } + } logger.Info("No ObjectCreated events found, skipping safely.") return "", nil } +func handleCSVImport(s3AssumeRoleArn, s3ImportPath string) (string, error) { + env := conf.GetEnv("ENV") + appName := conf.GetEnv("APP_NAME") + logger := configureLogger(env, appName) + logger = logger.WithFields(logrus.Fields{"import_filename": s3ImportPath}) + + importer := cclf.CSVImporter{ + Logger: logger, + FileProcessor: &cclf.S3FileProcessor{ + Handler: optout.S3FileHandler{ + Logger: logger, + Endpoint: os.Getenv("LOCAL_STACK_ENDPOINT"), + AssumeRoleArn: s3AssumeRoleArn, + }, + }, + } + + success, failure, skipped, err := importer.ImportCSV(s3AssumeRoleArn, s3ImportPath) + + if err != nil { + logger.Error("error returned from ImportCSV: ", err) + return "", err + } + + if failure > 0 || skipped > 0 { + result := fmt.Sprintf("Successfully imported %v files. Failed to import %v files. Skipped %v files. See logs for more details.", success, failure, skipped) + logger.Error(result) + + err = errors.New("Files skipped or failed import. See logs for more details.") + return result, err + + } + + result := fmt.Sprintf("Completed CSV import. Successfully imported %v files. Failed to import %v files. Skipped %v files. See logs for more details.", success, failure, skipped) + logger.Info(result) + return result, nil +} + func loadBfdS3Params() (string, error) { env := conf.GetEnv("ENV") From 1fefe6a708512527934a9b8dd7f47df7417b78bb Mon Sep 17 00:00:00 2001 From: Austin Canada Date: Tue, 5 Nov 2024 16:18:22 -0500 Subject: [PATCH 02/11] BCDA-8405: Add parser tests --- bcda/cclf/parser_test.go | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/bcda/cclf/parser_test.go b/bcda/cclf/parser_test.go index 83b06c4b8..2cb2935d5 100644 --- a/bcda/cclf/parser_test.go +++ b/bcda/cclf/parser_test.go @@ -293,3 +293,26 @@ func TestGetCCLFMetadata(t *testing.T) { }) } } + +func TestCheckIfAttributionCSVFile(t *testing.T) { + tests := []struct { + name string + path string + testIsCSV bool + }{ + {name: "Is an Attribution CSV File path", path: "P.PCPB.M2014.D00302.T2420001", testIsCSV: true}, + {name: "Is not an Attribution CSV File path", path: "T.PCPB.M2014.D00302.T2420001", testIsCSV: false}, + {name: "Is an Attribution CCLF File path", path: "P.PCPB.M2014.D00302.T2420001", testIsCSV: false}, + } + + for _, test := range tests { + t.Run(test.name, func(sub *testing.T) { + isCSV := CheckIfAttributionCSVFile(test.path) + if test.testIsCSV { + assert.True(sub, isCSV) + } else { + assert.False(sub, isCSV) + } + }) + } +} From c2d33e0e83fe527e0c354fcac426a5b52ee49bf2 Mon Sep 17 00:00:00 2001 From: Austin Canada Date: Tue, 5 Nov 2024 16:20:24 -0500 Subject: [PATCH 03/11] BCDA-8405: Changes to cclf lambda --- bcda/lambda/cclf/main.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bcda/lambda/cclf/main.go b/bcda/lambda/cclf/main.go index 6b992ee3e..075da22fc 100644 --- a/bcda/lambda/cclf/main.go +++ b/bcda/lambda/cclf/main.go @@ -64,10 +64,10 @@ func cclfImportHandler(ctx context.Context, sqsEvent events.SQSEvent) (string, e if parser.CheckIfAttributionCSVFile(e.S3.Object.Key) { return handleCSVImport(s3AssumeRoleArn, filepath) } else { - return handleCclfImport(s3AssumeRoleArn, filepath) + return handleCclfImport(s3AssumeRoleArn, filepath) + } } } - } logger.Info("No ObjectCreated events found, skipping safely.") return "", nil From 0ae73883f59a370edf3af25792583c7d77948a56 Mon Sep 17 00:00:00 2001 From: Austin Canada Date: Tue, 5 Nov 2024 16:20:38 -0500 Subject: [PATCH 04/11] BCDA-8405: Update parser import --- bcda/lambda/cclf/main.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/bcda/lambda/cclf/main.go b/bcda/lambda/cclf/main.go index 075da22fc..08c453868 100644 --- a/bcda/lambda/cclf/main.go +++ b/bcda/lambda/cclf/main.go @@ -60,8 +60,7 @@ func cclfImportHandler(ctx context.Context, sqsEvent events.SQSEvent) (string, e // importing the one file that was sent in the trigger. filepath := fmt.Sprintf("%s/%s", e.S3.Bucket.Name, e.S3.Object.Key) logger.Infof("Reading %s event for file %s", e.EventName, filepath) - parser := cclf.CSVParser{Filepath: e.S3.Object.Key} - if parser.CheckIfAttributionCSVFile(e.S3.Object.Key) { + if cclf.CheckIfAttributionCSVFile(e.S3.Object.Key) { return handleCSVImport(s3AssumeRoleArn, filepath) } else { return handleCclfImport(s3AssumeRoleArn, filepath) From 9ad084424a3e38cf5a53bbaff3c97a8755db8f74 Mon Sep 17 00:00:00 2001 From: Austin Canada Date: Tue, 5 Nov 2024 16:21:41 -0500 Subject: [PATCH 05/11] BCDA-8405: Change cclf calls to attribution --- bcda/lambda/cclf/main.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bcda/lambda/cclf/main.go b/bcda/lambda/cclf/main.go index 08c453868..042b647dd 100644 --- a/bcda/lambda/cclf/main.go +++ b/bcda/lambda/cclf/main.go @@ -30,11 +30,11 @@ func main() { fmt.Println(res) } } else { - lambda.Start(cclfImportHandler) + lambda.Start(attributionImportHandler) } } -func cclfImportHandler(ctx context.Context, sqsEvent events.SQSEvent) (string, error) { +func attributionImportHandler(ctx context.Context, sqsEvent events.SQSEvent) (string, error) { env := conf.GetEnv("ENV") appName := conf.GetEnv("APP_NAME") logger := configureLogger(env, appName) From 4cc088910ab8bd1cd5d68d0fe91e448995615d11 Mon Sep 17 00:00:00 2001 From: Austin Canada Date: Tue, 5 Nov 2024 16:22:47 -0500 Subject: [PATCH 06/11] BCDA-8405: Change cclf lambda tests to attribution --- bcda/lambda/cclf/main_test.go | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/bcda/lambda/cclf/main_test.go b/bcda/lambda/cclf/main_test.go index a2605b930..36161ee8f 100644 --- a/bcda/lambda/cclf/main_test.go +++ b/bcda/lambda/cclf/main_test.go @@ -14,15 +14,15 @@ import ( "github.com/stretchr/testify/suite" ) -type CclfImportMainSuite struct { +type AttributionImportMainSuite struct { suite.Suite } -func TestCclfImportMainSuite(t *testing.T) { - suite.Run(t, new(CclfImportMainSuite)) +func TestAttributionImportMainSuite(t *testing.T) { + suite.Run(t, new(AttributionImportMainSuite)) } -func (s *CclfImportMainSuite) TestImportCCLFDirectory() { +func (s *AttributionImportMainSuite) TestImportCCLFDirectory() { targetACO := "A0001" assert := assert.New(s.T()) @@ -60,7 +60,7 @@ func (s *CclfImportMainSuite) TestImportCCLFDirectory() { path, cleanup := testUtils.CopyToS3(s.T(), tc.path) defer cleanup() - res, err := cclfImportHandler(context.Background(), testUtils.GetSQSEvent(s.T(), path, tc.filename)) + res, err := attributionImportHandler(context.Background(), testUtils.GetSQSEvent(s.T(), path, tc.filename)) if tc.err == nil { assert.Nil(err) @@ -75,8 +75,8 @@ func (s *CclfImportMainSuite) TestImportCCLFDirectory() { } } -func (s *CclfImportMainSuite) TestHandlerMissingS3AssumeRoleArn() { +func (s *AttributionImportMainSuite) TestHandlerMissingS3AssumeRoleArn() { assert := assert.New(s.T()) - _, err := cclfImportHandler(context.Background(), testUtils.GetSQSEvent(s.T(), "doesn't-matter", "fake_filename")) + _, err := attributionImportHandler(context.Background(), testUtils.GetSQSEvent(s.T(), "doesn't-matter", "fake_filename")) assert.Contains(err.Error(), "Error retrieving parameter /cclf-import/bcda/local/bfd-bucket-role-arn from parameter store: ParameterNotFound: Parameter /cclf-import/bcda/local/bfd-bucket-role-arn not found.") } From 19832f3c3a3e2b1e0c115152a5c6a4a3379ce5c1 Mon Sep 17 00:00:00 2001 From: Austin Canada Date: Tue, 5 Nov 2024 16:23:37 -0500 Subject: [PATCH 07/11] BCDA-8405: Add additional parser tests --- bcda/cclf/parser_test.go | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/bcda/cclf/parser_test.go b/bcda/cclf/parser_test.go index 2cb2935d5..77970c23b 100644 --- a/bcda/cclf/parser_test.go +++ b/bcda/cclf/parser_test.go @@ -301,8 +301,12 @@ func TestCheckIfAttributionCSVFile(t *testing.T) { testIsCSV bool }{ {name: "Is an Attribution CSV File path", path: "P.PCPB.M2014.D00302.T2420001", testIsCSV: true}, - {name: "Is not an Attribution CSV File path", path: "T.PCPB.M2014.D00302.T2420001", testIsCSV: false}, - {name: "Is an Attribution CCLF File path", path: "P.PCPB.M2014.D00302.T2420001", testIsCSV: false}, + {name: "Is not an Attribution CSV File path (incorrect first)", path: "T.PCPB.M2014.D00302.T2420001", testIsCSV: false}, + {name: "Is not an Attribution CSV File path (incorrect second)", path: "P.BFD.N2014.D00302.T2420001", testIsCSV: false}, + {name: "Is not an Attribution CSV File path (incorrect third)", path: "P.PCPB.M2014.D00302.T2420001", testIsCSV: false}, + {name: "Is not an Attribution CSV File path (incorrect fourth)", path: "P.PCPB.M2014.D003022.T2420001", testIsCSV: false}, + {name: "Is not an Attribution CSV File path (incorrect fifth)", path: "P.PCPB.M2014.D00302.T24200011", testIsCSV: false}, + {name: "Is not an Attribution CSV File path (CCLF file)", path: "T.BCD.A0001.ZCY18.D181121.T1000000", testIsCSV: false}, } for _, test := range tests { From 5c15a406a4a3bc1e4f2ecabb809f75193e5b52d3 Mon Sep 17 00:00:00 2001 From: Austin Canada Date: Thu, 7 Nov 2024 12:16:48 -0500 Subject: [PATCH 08/11] BCDA-8405: Update parser tests --- bcda/cclf/parser_test.go | 50 ++++++++++++++++++++++++++-------------- 1 file changed, 33 insertions(+), 17 deletions(-) diff --git a/bcda/cclf/parser_test.go b/bcda/cclf/parser_test.go index 77970c23b..eb70102da 100644 --- a/bcda/cclf/parser_test.go +++ b/bcda/cclf/parser_test.go @@ -91,7 +91,8 @@ func TestGetCCLFMetadata(t *testing.T) { {"Invalid date (no 13th month)", sspID, "T.BCD.A0001.ZC0Y18.D181320.T0001000", "failed to parse date", cclfFileMetadata{}}, {"CCLF file too old", sspID, gen(sspProd, startUTC.Add(-365*24*time.Hour)), "out of range", cclfFileMetadata{}}, {"CCLF file too new", sspID, gen(sspProd, startUTC.Add(365*24*time.Hour)), "out of range", cclfFileMetadata{}}, - {"Production SSP file", sspID, sspProdFile, "", + { + "Production SSP file", sspID, sspProdFile, "", cclfFileMetadata{ env: "production", name: sspProdFile, @@ -102,7 +103,8 @@ func TestGetCCLFMetadata(t *testing.T) { fileType: models.FileTypeDefault, }, }, - {"Test SSP file", sspID, sspTestFile, "", + { + "Test SSP file", sspID, sspTestFile, "", cclfFileMetadata{ env: "test", name: sspTestFile, @@ -125,7 +127,8 @@ func TestGetCCLFMetadata(t *testing.T) { fileType: models.FileTypeRunout, }, }, - {"Production CEC file", cecID, cecProdFile, "", + { + "Production CEC file", cecID, cecProdFile, "", cclfFileMetadata{ env: "production", name: cecProdFile, @@ -136,7 +139,8 @@ func TestGetCCLFMetadata(t *testing.T) { fileType: models.FileTypeDefault, }, }, - {"Test CEC file", cecID, cecTestFile, "", + { + "Test CEC file", cecID, cecTestFile, "", cclfFileMetadata{ env: "test", name: cecTestFile, @@ -147,7 +151,8 @@ func TestGetCCLFMetadata(t *testing.T) { fileType: models.FileTypeDefault, }, }, - {"Production NGACO file", ngacoID, ngacoProdFile, "", + { + "Production NGACO file", ngacoID, ngacoProdFile, "", cclfFileMetadata{ env: "production", name: ngacoProdFile, @@ -158,7 +163,8 @@ func TestGetCCLFMetadata(t *testing.T) { fileType: models.FileTypeDefault, }, }, - {"Test NGACO file", ngacoID, ngacoTestFile, "", + { + "Test NGACO file", ngacoID, ngacoTestFile, "", cclfFileMetadata{ env: "test", name: ngacoTestFile, @@ -169,7 +175,8 @@ func TestGetCCLFMetadata(t *testing.T) { fileType: models.FileTypeDefault, }, }, - {"Production CKCC file", ckccID, ckccProdFile, "", + { + "Production CKCC file", ckccID, ckccProdFile, "", cclfFileMetadata{ env: "production", name: ckccProdFile, @@ -180,7 +187,8 @@ func TestGetCCLFMetadata(t *testing.T) { fileType: models.FileTypeDefault, }, }, - {"Test CKCC file", ckccID, ckccTestFile, "", + { + "Test CKCC file", ckccID, ckccTestFile, "", cclfFileMetadata{ env: "test", name: ckccTestFile, @@ -191,7 +199,8 @@ func TestGetCCLFMetadata(t *testing.T) { fileType: models.FileTypeDefault, }, }, - {"Production KCF file", kcfID, kcfProdFile, "", + { + "Production KCF file", kcfID, kcfProdFile, "", cclfFileMetadata{ env: "production", name: kcfProdFile, @@ -202,7 +211,8 @@ func TestGetCCLFMetadata(t *testing.T) { fileType: models.FileTypeDefault, }, }, - {"Test KCF file", kcfID, kcfTestFile, "", + { + "Test KCF file", kcfID, kcfTestFile, "", cclfFileMetadata{ env: "test", name: kcfTestFile, @@ -213,7 +223,8 @@ func TestGetCCLFMetadata(t *testing.T) { fileType: models.FileTypeDefault, }, }, - {"Production DC file", dcID, dcProdFile, "", + { + "Production DC file", dcID, dcProdFile, "", cclfFileMetadata{ env: "production", name: dcProdFile, @@ -224,7 +235,8 @@ func TestGetCCLFMetadata(t *testing.T) { fileType: models.FileTypeDefault, }, }, - {"Test DC file", dcID, dcTestFile, "", + { + "Test DC file", dcID, dcTestFile, "", cclfFileMetadata{ env: "test", name: dcTestFile, @@ -235,7 +247,8 @@ func TestGetCCLFMetadata(t *testing.T) { fileType: models.FileTypeDefault, }, }, - {"Production TEST file", testID, testProdFile, "", + { + "Production TEST file", testID, testProdFile, "", cclfFileMetadata{ env: "production", name: testProdFile, @@ -246,7 +259,8 @@ func TestGetCCLFMetadata(t *testing.T) { fileType: models.FileTypeDefault, }, }, - {"Test TEST file", testID, testTestFile, "", + { + "Test TEST file", testID, testTestFile, "", cclfFileMetadata{ env: "test", name: testTestFile, @@ -257,7 +271,8 @@ func TestGetCCLFMetadata(t *testing.T) { fileType: models.FileTypeDefault, }, }, - {"Production SBX file", sbxID, sbxProdFile, "", + { + "Production SBX file", sbxID, sbxProdFile, "", cclfFileMetadata{ env: "production", name: sbxProdFile, @@ -268,7 +283,8 @@ func TestGetCCLFMetadata(t *testing.T) { fileType: models.FileTypeDefault, }, }, - {"Test SBX file", sbxID, sbxTestFile, "", + { + "Test SBX file", sbxID, sbxTestFile, "", cclfFileMetadata{ env: "test", name: sbxTestFile, @@ -301,7 +317,7 @@ func TestCheckIfAttributionCSVFile(t *testing.T) { testIsCSV bool }{ {name: "Is an Attribution CSV File path", path: "P.PCPB.M2014.D00302.T2420001", testIsCSV: true}, - {name: "Is not an Attribution CSV File path (incorrect first)", path: "T.PCPB.M2014.D00302.T2420001", testIsCSV: false}, + {name: "Is not an Attribution CSV File path (incorrect first)", path: "M.PCPB.M2014.D00302.T2420001", testIsCSV: false}, {name: "Is not an Attribution CSV File path (incorrect second)", path: "P.BFD.N2014.D00302.T2420001", testIsCSV: false}, {name: "Is not an Attribution CSV File path (incorrect third)", path: "P.PCPB.M2014.D00302.T2420001", testIsCSV: false}, {name: "Is not an Attribution CSV File path (incorrect fourth)", path: "P.PCPB.M2014.D003022.T2420001", testIsCSV: false}, From 12b17ed46d7fe3d3b781dc81423ea2784810060c Mon Sep 17 00:00:00 2001 From: Austin Canada Date: Thu, 7 Nov 2024 12:24:43 -0500 Subject: [PATCH 09/11] BCDA-8405: Add CSV parser function --- bcda/cclf/parser.go | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/bcda/cclf/parser.go b/bcda/cclf/parser.go index 326b2e563..412560a22 100644 --- a/bcda/cclf/parser.go +++ b/bcda/cclf/parser.go @@ -26,6 +26,14 @@ func getCMSID(name string) (string, error) { return parts[1], nil } +func CheckIfAttributionCSVFile(filePath string) bool { + pattern := `P\.PCPB\.M\d{4}\.D\d{6}\.T\d{7}` + filenameRegexp := regexp.MustCompile(pattern) + found := filenameRegexp.Match([]byte(filePath)) + return found +} + + func getCCLFFileMetadata(cmsID, fileName string) (cclfFileMetadata, error) { var metadata cclfFileMetadata const ( From 6cea8719b61eb2ddbbcf29b61f025367ee569778 Mon Sep 17 00:00:00 2001 From: Austin Canada Date: Thu, 7 Nov 2024 12:59:56 -0500 Subject: [PATCH 10/11] BCDA-8405: Add opt-out file to tests --- bcda/cclf/parser_test.go | 1 + 1 file changed, 1 insertion(+) diff --git a/bcda/cclf/parser_test.go b/bcda/cclf/parser_test.go index eb70102da..5789a5942 100644 --- a/bcda/cclf/parser_test.go +++ b/bcda/cclf/parser_test.go @@ -323,6 +323,7 @@ func TestCheckIfAttributionCSVFile(t *testing.T) { {name: "Is not an Attribution CSV File path (incorrect fourth)", path: "P.PCPB.M2014.D003022.T2420001", testIsCSV: false}, {name: "Is not an Attribution CSV File path (incorrect fifth)", path: "P.PCPB.M2014.D00302.T24200011", testIsCSV: false}, {name: "Is not an Attribution CSV File path (CCLF file)", path: "T.BCD.A0001.ZCY18.D181121.T1000000", testIsCSV: false}, + {name: "Is not an Attribution CSV File path (opt-out file)", path: "T#EFT.ON.ACO.NGD1800.DPRF.D181120.T1000009", testIsCSV: false}, } for _, test := range tests { From 55d2c8a81916f01a97f8646b4c9ffeca7bc211d1 Mon Sep 17 00:00:00 2001 From: Austin Canada Date: Thu, 7 Nov 2024 17:06:03 -0500 Subject: [PATCH 11/11] BCDA-8405: Add parser updates and tests --- bcda/cclf/parser.go | 3 +-- bcda/cclf/parser_test.go | 1 + 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/bcda/cclf/parser.go b/bcda/cclf/parser.go index 412560a22..b48609847 100644 --- a/bcda/cclf/parser.go +++ b/bcda/cclf/parser.go @@ -27,13 +27,12 @@ func getCMSID(name string) (string, error) { } func CheckIfAttributionCSVFile(filePath string) bool { - pattern := `P\.PCPB\.M\d{4}\.D\d{6}\.T\d{7}` + pattern := `(P|T)\.PCPB\.M\d{4}\.D\d{6}\.T\d{7}` filenameRegexp := regexp.MustCompile(pattern) found := filenameRegexp.Match([]byte(filePath)) return found } - func getCCLFFileMetadata(cmsID, fileName string) (cclfFileMetadata, error) { var metadata cclfFileMetadata const ( diff --git a/bcda/cclf/parser_test.go b/bcda/cclf/parser_test.go index 5789a5942..29f2da1e9 100644 --- a/bcda/cclf/parser_test.go +++ b/bcda/cclf/parser_test.go @@ -88,6 +88,7 @@ func TestGetCCLFMetadata(t *testing.T) { }{ {"Non CCLF0 or CCLF8 file", sspID, "P.A0001.ACO.ZC9Y18.D190108.T2355000", "invalid filename", cclfFileMetadata{}}, {"Unsupported CCLF file type", "Z9999", "P.Z0001.ACO.ZC8Y18.D190108.T2355000", "invalid filename", cclfFileMetadata{}}, + {"Unsupported CSV file type", sspID, "P.PCPB.M2014.D00302.T2420001", "invalid filename", cclfFileMetadata{}}, {"Invalid date (no 13th month)", sspID, "T.BCD.A0001.ZC0Y18.D181320.T0001000", "failed to parse date", cclfFileMetadata{}}, {"CCLF file too old", sspID, gen(sspProd, startUTC.Add(-365*24*time.Hour)), "out of range", cclfFileMetadata{}}, {"CCLF file too new", sspID, gen(sspProd, startUTC.Add(365*24*time.Hour)), "out of range", cclfFileMetadata{}},