Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft: CCLF lambda changes #1008

Merged
merged 11 commits into from
Nov 8, 2024
Merged

Draft: CCLF lambda changes #1008

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
11 commits
Select commit Hold shift + click to select a range
74da1fb
BCDA-8405: Add CSV lambda changes
austincanada Nov 5, 2024
1fefe6a
BCDA-8405: Add parser tests
austincanada Nov 5, 2024
c2d33e0
BCDA-8405: Changes to cclf lambda
austincanada Nov 5, 2024
0ae7388
BCDA-8405: Update parser import
austincanada Nov 5, 2024
9ad0844
BCDA-8405: Change cclf calls to attribution
austincanada Nov 5, 2024
4cc0889
BCDA-8405: Change cclf lambda tests to attribution
austincanada Nov 5, 2024
19832f3
BCDA-8405: Add additional parser tests
austincanada Nov 5, 2024
5c15a40
BCDA-8405: Update parser tests
austincanada Nov 7, 2024
12b17ed
BCDA-8405: Add CSV parser function
austincanada Nov 7, 2024
6cea871
BCDA-8405: Add opt-out file to tests
austincanada Nov 7, 2024
55d2c8a
BCDA-8405: Add parser updates and tests
austincanada Nov 7, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 7 additions & 0 deletions bcda/cclf/parser.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,13 @@ func getCMSID(name string) (string, error) {
return parts[1], nil
}

func CheckIfAttributionCSVFile(filePath string) bool {
pattern := `(P|T)\.PCPB\.M\d{4}\.D\d{6}\.T\d{7}`
filenameRegexp := regexp.MustCompile(pattern)
found := filenameRegexp.Match([]byte(filePath))
return found
}

func getCCLFFileMetadata(cmsID, fileName string) (cclfFileMetadata, error) {
var metadata cclfFileMetadata
const (
Expand Down
77 changes: 61 additions & 16 deletions bcda/cclf/parser_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -88,10 +88,12 @@ func TestGetCCLFMetadata(t *testing.T) {
}{
{"Non CCLF0 or CCLF8 file", sspID, "P.A0001.ACO.ZC9Y18.D190108.T2355000", "invalid filename", cclfFileMetadata{}},
{"Unsupported CCLF file type", "Z9999", "P.Z0001.ACO.ZC8Y18.D190108.T2355000", "invalid filename", cclfFileMetadata{}},
{"Unsupported CSV file type", sspID, "P.PCPB.M2014.D00302.T2420001", "invalid filename", cclfFileMetadata{}},
{"Invalid date (no 13th month)", sspID, "T.BCD.A0001.ZC0Y18.D181320.T0001000", "failed to parse date", cclfFileMetadata{}},
{"CCLF file too old", sspID, gen(sspProd, startUTC.Add(-365*24*time.Hour)), "out of range", cclfFileMetadata{}},
{"CCLF file too new", sspID, gen(sspProd, startUTC.Add(365*24*time.Hour)), "out of range", cclfFileMetadata{}},
{"Production SSP file", sspID, sspProdFile, "",
{
"Production SSP file", sspID, sspProdFile, "",
cclfFileMetadata{
env: "production",
name: sspProdFile,
Expand All @@ -102,7 +104,8 @@ func TestGetCCLFMetadata(t *testing.T) {
fileType: models.FileTypeDefault,
},
},
{"Test SSP file", sspID, sspTestFile, "",
{
"Test SSP file", sspID, sspTestFile, "",
cclfFileMetadata{
env: "test",
name: sspTestFile,
Expand All @@ -125,7 +128,8 @@ func TestGetCCLFMetadata(t *testing.T) {
fileType: models.FileTypeRunout,
},
},
{"Production CEC file", cecID, cecProdFile, "",
{
"Production CEC file", cecID, cecProdFile, "",
cclfFileMetadata{
env: "production",
name: cecProdFile,
Expand All @@ -136,7 +140,8 @@ func TestGetCCLFMetadata(t *testing.T) {
fileType: models.FileTypeDefault,
},
},
{"Test CEC file", cecID, cecTestFile, "",
{
"Test CEC file", cecID, cecTestFile, "",
cclfFileMetadata{
env: "test",
name: cecTestFile,
Expand All @@ -147,7 +152,8 @@ func TestGetCCLFMetadata(t *testing.T) {
fileType: models.FileTypeDefault,
},
},
{"Production NGACO file", ngacoID, ngacoProdFile, "",
{
"Production NGACO file", ngacoID, ngacoProdFile, "",
cclfFileMetadata{
env: "production",
name: ngacoProdFile,
Expand All @@ -158,7 +164,8 @@ func TestGetCCLFMetadata(t *testing.T) {
fileType: models.FileTypeDefault,
},
},
{"Test NGACO file", ngacoID, ngacoTestFile, "",
{
"Test NGACO file", ngacoID, ngacoTestFile, "",
cclfFileMetadata{
env: "test",
name: ngacoTestFile,
Expand All @@ -169,7 +176,8 @@ func TestGetCCLFMetadata(t *testing.T) {
fileType: models.FileTypeDefault,
},
},
{"Production CKCC file", ckccID, ckccProdFile, "",
{
"Production CKCC file", ckccID, ckccProdFile, "",
cclfFileMetadata{
env: "production",
name: ckccProdFile,
Expand All @@ -180,7 +188,8 @@ func TestGetCCLFMetadata(t *testing.T) {
fileType: models.FileTypeDefault,
},
},
{"Test CKCC file", ckccID, ckccTestFile, "",
{
"Test CKCC file", ckccID, ckccTestFile, "",
cclfFileMetadata{
env: "test",
name: ckccTestFile,
Expand All @@ -191,7 +200,8 @@ func TestGetCCLFMetadata(t *testing.T) {
fileType: models.FileTypeDefault,
},
},
{"Production KCF file", kcfID, kcfProdFile, "",
{
"Production KCF file", kcfID, kcfProdFile, "",
cclfFileMetadata{
env: "production",
name: kcfProdFile,
Expand All @@ -202,7 +212,8 @@ func TestGetCCLFMetadata(t *testing.T) {
fileType: models.FileTypeDefault,
},
},
{"Test KCF file", kcfID, kcfTestFile, "",
{
"Test KCF file", kcfID, kcfTestFile, "",
cclfFileMetadata{
env: "test",
name: kcfTestFile,
Expand All @@ -213,7 +224,8 @@ func TestGetCCLFMetadata(t *testing.T) {
fileType: models.FileTypeDefault,
},
},
{"Production DC file", dcID, dcProdFile, "",
{
"Production DC file", dcID, dcProdFile, "",
cclfFileMetadata{
env: "production",
name: dcProdFile,
Expand All @@ -224,7 +236,8 @@ func TestGetCCLFMetadata(t *testing.T) {
fileType: models.FileTypeDefault,
},
},
{"Test DC file", dcID, dcTestFile, "",
{
"Test DC file", dcID, dcTestFile, "",
cclfFileMetadata{
env: "test",
name: dcTestFile,
Expand All @@ -235,7 +248,8 @@ func TestGetCCLFMetadata(t *testing.T) {
fileType: models.FileTypeDefault,
},
},
{"Production TEST file", testID, testProdFile, "",
{
"Production TEST file", testID, testProdFile, "",
cclfFileMetadata{
env: "production",
name: testProdFile,
Expand All @@ -246,7 +260,8 @@ func TestGetCCLFMetadata(t *testing.T) {
fileType: models.FileTypeDefault,
},
},
{"Test TEST file", testID, testTestFile, "",
{
"Test TEST file", testID, testTestFile, "",
cclfFileMetadata{
env: "test",
name: testTestFile,
Expand All @@ -257,7 +272,8 @@ func TestGetCCLFMetadata(t *testing.T) {
fileType: models.FileTypeDefault,
},
},
{"Production SBX file", sbxID, sbxProdFile, "",
{
"Production SBX file", sbxID, sbxProdFile, "",
cclfFileMetadata{
env: "production",
name: sbxProdFile,
Expand All @@ -268,7 +284,8 @@ func TestGetCCLFMetadata(t *testing.T) {
fileType: models.FileTypeDefault,
},
},
{"Test SBX file", sbxID, sbxTestFile, "",
{
"Test SBX file", sbxID, sbxTestFile, "",
carlpartridge marked this conversation as resolved.
Show resolved Hide resolved
cclfFileMetadata{
env: "test",
name: sbxTestFile,
Expand All @@ -293,3 +310,31 @@ func TestGetCCLFMetadata(t *testing.T) {
})
}
}

func TestCheckIfAttributionCSVFile(t *testing.T) {
tests := []struct {
name string
path string
testIsCSV bool
}{
{name: "Is an Attribution CSV File path", path: "P.PCPB.M2014.D00302.T2420001", testIsCSV: true},
{name: "Is not an Attribution CSV File path (incorrect first)", path: "M.PCPB.M2014.D00302.T2420001", testIsCSV: false},
{name: "Is not an Attribution CSV File path (incorrect second)", path: "P.BFD.N2014.D00302.T2420001", testIsCSV: false},
{name: "Is not an Attribution CSV File path (incorrect third)", path: "P.PCPB.M2014.D00302.T2420001", testIsCSV: false},
{name: "Is not an Attribution CSV File path (incorrect fourth)", path: "P.PCPB.M2014.D003022.T2420001", testIsCSV: false},
{name: "Is not an Attribution CSV File path (incorrect fifth)", path: "P.PCPB.M2014.D00302.T24200011", testIsCSV: false},
{name: "Is not an Attribution CSV File path (CCLF file)", path: "T.BCD.A0001.ZCY18.D181121.T1000000", testIsCSV: false},
carlpartridge marked this conversation as resolved.
Show resolved Hide resolved
{name: "Is not an Attribution CSV File path (opt-out file)", path: "T#EFT.ON.ACO.NGD1800.DPRF.D181120.T1000009", testIsCSV: false},
}

for _, test := range tests {
t.Run(test.name, func(sub *testing.T) {
isCSV := CheckIfAttributionCSVFile(test.path)
if test.testIsCSV {
assert.True(sub, isCSV)
} else {
assert.False(sub, isCSV)
}
})
}
}
48 changes: 45 additions & 3 deletions bcda/lambda/cclf/main.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -30,11 +30,11 @@ func main() {
fmt.Println(res)
}
} else {
lambda.Start(cclfImportHandler)
lambda.Start(attributionImportHandler)
}
}

func cclfImportHandler(ctx context.Context, sqsEvent events.SQSEvent) (string, error) {
func attributionImportHandler(ctx context.Context, sqsEvent events.SQSEvent) (string, error) {
env := conf.GetEnv("ENV")
appName := conf.GetEnv("APP_NAME")
logger := configureLogger(env, appName)
Expand All @@ -60,14 +60,56 @@ func cclfImportHandler(ctx context.Context, sqsEvent events.SQSEvent) (string, e
// importing the one file that was sent in the trigger.
filepath := fmt.Sprintf("%s/%s", e.S3.Bucket.Name, e.S3.Object.Key)
logger.Infof("Reading %s event for file %s", e.EventName, filepath)
return handleCclfImport(s3AssumeRoleArn, filepath)
if cclf.CheckIfAttributionCSVFile(e.S3.Object.Key) {
carlpartridge marked this conversation as resolved.
Show resolved Hide resolved
return handleCSVImport(s3AssumeRoleArn, filepath)
} else {
return handleCclfImport(s3AssumeRoleArn, filepath)
}
}
}

logger.Info("No ObjectCreated events found, skipping safely.")
return "", nil
}

func handleCSVImport(s3AssumeRoleArn, s3ImportPath string) (string, error) {
env := conf.GetEnv("ENV")
appName := conf.GetEnv("APP_NAME")
logger := configureLogger(env, appName)
logger = logger.WithFields(logrus.Fields{"import_filename": s3ImportPath})

importer := cclf.CSVImporter{
Logger: logger,
FileProcessor: &cclf.S3FileProcessor{
Handler: optout.S3FileHandler{
Logger: logger,
Endpoint: os.Getenv("LOCAL_STACK_ENDPOINT"),
AssumeRoleArn: s3AssumeRoleArn,
},
},
}

success, failure, skipped, err := importer.ImportCSV(s3AssumeRoleArn, s3ImportPath)
carlpartridge marked this conversation as resolved.
Show resolved Hide resolved

if err != nil {
logger.Error("error returned from ImportCSV: ", err)
return "", err
}

if failure > 0 || skipped > 0 {
result := fmt.Sprintf("Successfully imported %v files. Failed to import %v files. Skipped %v files. See logs for more details.", success, failure, skipped)
logger.Error(result)

err = errors.New("Files skipped or failed import. See logs for more details.")
return result, err

}

result := fmt.Sprintf("Completed CSV import. Successfully imported %v files. Failed to import %v files. Skipped %v files. See logs for more details.", success, failure, skipped)
logger.Info(result)
return result, nil
}

func loadBfdS3Params() (string, error) {
env := conf.GetEnv("ENV")

Expand Down
14 changes: 7 additions & 7 deletions bcda/lambda/cclf/main_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,15 +14,15 @@ import (
"github.com/stretchr/testify/suite"
)

type CclfImportMainSuite struct {
type AttributionImportMainSuite struct {
carlpartridge marked this conversation as resolved.
Show resolved Hide resolved
suite.Suite
}

func TestCclfImportMainSuite(t *testing.T) {
suite.Run(t, new(CclfImportMainSuite))
func TestAttributionImportMainSuite(t *testing.T) {
suite.Run(t, new(AttributionImportMainSuite))
}

func (s *CclfImportMainSuite) TestImportCCLFDirectory() {
func (s *AttributionImportMainSuite) TestImportCCLFDirectory() {
targetACO := "A0001"
assert := assert.New(s.T())

Expand Down Expand Up @@ -60,7 +60,7 @@ func (s *CclfImportMainSuite) TestImportCCLFDirectory() {
path, cleanup := testUtils.CopyToS3(s.T(), tc.path)
defer cleanup()

res, err := cclfImportHandler(context.Background(), testUtils.GetSQSEvent(s.T(), path, tc.filename))
res, err := attributionImportHandler(context.Background(), testUtils.GetSQSEvent(s.T(), path, tc.filename))

if tc.err == nil {
assert.Nil(err)
Expand All @@ -75,8 +75,8 @@ func (s *CclfImportMainSuite) TestImportCCLFDirectory() {
}
}

func (s *CclfImportMainSuite) TestHandlerMissingS3AssumeRoleArn() {
func (s *AttributionImportMainSuite) TestHandlerMissingS3AssumeRoleArn() {
assert := assert.New(s.T())
_, err := cclfImportHandler(context.Background(), testUtils.GetSQSEvent(s.T(), "doesn't-matter", "fake_filename"))
_, err := attributionImportHandler(context.Background(), testUtils.GetSQSEvent(s.T(), "doesn't-matter", "fake_filename"))
assert.Contains(err.Error(), "Error retrieving parameter /cclf-import/bcda/local/bfd-bucket-role-arn from parameter store: ParameterNotFound: Parameter /cclf-import/bcda/local/bfd-bucket-role-arn not found.")
}
Loading