From cb4be2a370aeae67662f6b4f25925456b3566c11 Mon Sep 17 00:00:00 2001 From: Lauren Krugen Date: Fri, 6 Dec 2024 10:29:05 -0700 Subject: [PATCH 01/17] add csv attribution file to integration test --- .../cclf-import-test-integration.yml | 40 +++++++++++++++++-- 1 file changed, 36 insertions(+), 4 deletions(-) diff --git a/.github/workflows/cclf-import-test-integration.yml b/.github/workflows/cclf-import-test-integration.yml index 64e4b50ad..a73768b41 100644 --- a/.github/workflows/cclf-import-test-integration.yml +++ b/.github/workflows/cclf-import-test-integration.yml @@ -33,7 +33,8 @@ jobs: run: working-directory: bcda outputs: - filename: ${{ steps.createfile.outputs.FILENAME }} + cclffilename: ${{ steps.createfile.outputs.cclffilename }} + csvfilename: ${{ steps.createfile.outputs.csvfilename }} steps: - uses: actions/checkout@v4 - uses: aws-actions/configure-aws-credentials@v4 @@ -53,9 +54,15 @@ jobs: year=$(date +'%y') date=$(date +'%y%m%d') time=$(date +'%H%M%S') + fname=T.BCD.A0001.ZCY24.D${date}.T${time}1 cclf8_fname=T.BCD.A0001.ZC8Y24.D${date}.T${time}1 - echo "FILENAME=$cclf8_fname" >> "$GITHUB_OUTPUT" + echo "CCLFFILENAME=$cclf8_fname" >> "$GITHUB_OUTPUT" + + csvname=T.PCPB.M${year}11.D${date}.T${time}1 + echo "CSVFILENAME=$csvname" >> "$GITHUB_OUTPUT" + + mv ../shared_files/cclf/archives/csv/P.PCPB.M2411.D181120.T1000000 ${csvname} unzip ../shared_files/cclf/archives/valid/T.BCD.A0001.ZCY18.D181120.T1000000 @@ -70,6 +77,9 @@ jobs: aws s3 cp --no-progress $fname \ s3://bfd-test-eft/bfdeft01/bcda/in/test/$fname + aws s3 cp --no-progress ${csvname} \ + s3://bfd-test-eft/bfdeft01/bcda/in/test/${csvname} + verify: needs: trigger runs-on: self-hosted @@ -93,7 +103,8 @@ jobs: CONNECTION_INFO=/bcda/test/api/DATABASE_URL - name: Verify CCLF file was ingested env: - FILENAME: ${{needs.trigger.outputs.filename}} + CCLFFILENAME: ${{needs.trigger.outputs.cclffilename}} + CSVFILENAME: ${{needs.trigger.outputs.csvfilename}} PGSSLMODE: require # CAUTION: if changing the script below, validate that sensitive information is not printed in the workflow run: | @@ -101,7 +112,7 @@ jobs: CONNECTION_URL=$(echo $CONNECTION_INFO 2>&1 | sed -E "s/@.*\/bcda/\@$HOST\/bcda/" 2>&1) # Verify that we have a record of the CCLF file in the database - CCLF_FILE=`psql -t "$CONNECTION_URL" -c "SELECT id FROM cclf_files WHERE name = '$FILENAME' LIMIT 1" 2>&1` + CCLF_FILE=`psql -t "$CONNECTION_URL" -c "SELECT id FROM cclf_files WHERE name = '$CCLFFILENAME' LIMIT 1" 2>&1` if [[ $? -ne 0 || -z $CCLF_FILE ]]; then echo "cclf_file query returned zero results or command failed" exit 1 @@ -118,3 +129,24 @@ jobs: exit 1 fi fi + + # Verify that we have a record of the CCLF file in the database + CSV_FILE=`psql -t "$CONNECTION_URL" -c "SELECT id FROM cclf_files WHERE name = '$CSVFILENAME' LIMIT 1" 2>&1` + if [[ $? -ne 0 || -z $CSV_FILE ]]; then + echo "cclf_file query returned zero results or command failed" + exit 1 + else + + # Verify that the correct number of benes were imported into the database. + CSV_BENES=`psql -t "$CONNECTION_URL" -c "SELECT count(mbi) FROM cclf_beneficiaries WHERE file_id = $CSV_FILE" 2>&1` + if [[ $? -ne 0 || -z $CSV_BENES ]]; then + echo "CCLF beneficiaries query returned zero results or command failed" + exit 1 + fi + if [[ $(echo $CSV_BENES | xargs) != "5" ]]; then + echo "expected 6 beneficiaries imported from file, received $CSV_BENES". + exit 1 + fi + fi + + From 0910c4266044da7735403cb1d02483c85e2bd47f Mon Sep 17 00:00:00 2001 From: Lauren Krugen Date: Fri, 6 Dec 2024 10:58:24 -0700 Subject: [PATCH 02/17] name updates --- .github/workflows/cclf-import-test-integration.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/cclf-import-test-integration.yml b/.github/workflows/cclf-import-test-integration.yml index a73768b41..bc9842153 100644 --- a/.github/workflows/cclf-import-test-integration.yml +++ b/.github/workflows/cclf-import-test-integration.yml @@ -130,21 +130,21 @@ jobs: fi fi - # Verify that we have a record of the CCLF file in the database + # Verify that we have a record of the CSV file in the database CSV_FILE=`psql -t "$CONNECTION_URL" -c "SELECT id FROM cclf_files WHERE name = '$CSVFILENAME' LIMIT 1" 2>&1` if [[ $? -ne 0 || -z $CSV_FILE ]]; then - echo "cclf_file query returned zero results or command failed" + echo "csv_file query returned zero results or command failed" exit 1 else # Verify that the correct number of benes were imported into the database. CSV_BENES=`psql -t "$CONNECTION_URL" -c "SELECT count(mbi) FROM cclf_beneficiaries WHERE file_id = $CSV_FILE" 2>&1` if [[ $? -ne 0 || -z $CSV_BENES ]]; then - echo "CCLF beneficiaries query returned zero results or command failed" + echo "CSV beneficiaries query returned zero results or command failed" exit 1 fi if [[ $(echo $CSV_BENES | xargs) != "5" ]]; then - echo "expected 6 beneficiaries imported from file, received $CSV_BENES". + echo "expected 5 beneficiaries imported from file, received $CSV_BENES". exit 1 fi fi From 850eabd87c81fa039e5d37dc9ba4679913886aa9 Mon Sep 17 00:00:00 2001 From: Lauren Krugen Date: Fri, 6 Dec 2024 11:07:13 -0700 Subject: [PATCH 03/17] logging update --- bcda/cclf/parser.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bcda/cclf/parser.go b/bcda/cclf/parser.go index f51be7aa1..000b45378 100644 --- a/bcda/cclf/parser.go +++ b/bcda/cclf/parser.go @@ -72,7 +72,7 @@ func GetCSVMetadata(path string) (csvFileMetadata, error) { } if metadata == (csvFileMetadata{}) { - err := fmt.Errorf("invalid filename for attribution file.") + err := fmt.Errorf("invalid filename for attribution file: %s", path) return metadata, err } From c24c1e1cc08cc7db926873d6aac7ed148809bad2 Mon Sep 17 00:00:00 2001 From: Lauren Krugen Date: Fri, 6 Dec 2024 11:14:58 -0700 Subject: [PATCH 04/17] more logging for troubleshooting --- bcda/cclf/parser.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bcda/cclf/parser.go b/bcda/cclf/parser.go index 000b45378..e8be47368 100644 --- a/bcda/cclf/parser.go +++ b/bcda/cclf/parser.go @@ -64,7 +64,7 @@ func GetCSVMetadata(path string) (csvFileMetadata, error) { if len(parts) == v.AttributionFile.MetadataMatches { metadata, err = validateCSVMetadata(parts) if err != nil { - return csvFileMetadata{}, nil + return csvFileMetadata{}, err } metadata.acoID = v.Model break @@ -72,7 +72,7 @@ func GetCSVMetadata(path string) (csvFileMetadata, error) { } if metadata == (csvFileMetadata{}) { - err := fmt.Errorf("invalid filename for attribution file: %s", path) + err := fmt.Errorf("invalid filename for attribution file: %s", err) return metadata, err } From 61134b01c09066c616d9f346240da0138281f20f Mon Sep 17 00:00:00 2001 From: Lauren Krugen Date: Fri, 6 Dec 2024 11:24:13 -0700 Subject: [PATCH 05/17] more logging for troubleshooting again --- bcda/cclf/parser.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/bcda/cclf/parser.go b/bcda/cclf/parser.go index e8be47368..25049d6fb 100644 --- a/bcda/cclf/parser.go +++ b/bcda/cclf/parser.go @@ -63,6 +63,7 @@ func GetCSVMetadata(path string) (csvFileMetadata, error) { parts := filenameRegexp.FindStringSubmatch(path) if len(parts) == v.AttributionFile.MetadataMatches { metadata, err = validateCSVMetadata(parts) + log.API.Error("regex for aco (%s): %s", v.Model, v.AttributionFile.NamePattern) if err != nil { return csvFileMetadata{}, err } @@ -72,7 +73,7 @@ func GetCSVMetadata(path string) (csvFileMetadata, error) { } if metadata == (csvFileMetadata{}) { - err := fmt.Errorf("invalid filename for attribution file: %s", err) + err := fmt.Errorf("invalid filename for attribution file: %s.", err.Error()) return metadata, err } From 4b1736e18dcdbfc64f307933d6266f3210f72119 Mon Sep 17 00:00:00 2001 From: Lauren Krugen Date: Fri, 6 Dec 2024 11:30:51 -0700 Subject: [PATCH 06/17] troubleshoot api config --- bcda/cclf/parser.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/bcda/cclf/parser.go b/bcda/cclf/parser.go index 25049d6fb..2c1b11a1c 100644 --- a/bcda/cclf/parser.go +++ b/bcda/cclf/parser.go @@ -57,6 +57,9 @@ func GetCSVMetadata(path string) (csvFileMetadata, error) { if err != nil { return csvFileMetadata{}, err } + if acos == nil { + log.API.Error("no aco configs present!") + } for _, v := range acos { filenameRegexp := regexp.MustCompile(v.AttributionFile.NamePattern) From 685ff3e6ff7f416b86fd7be55d6708da31fd79b9 Mon Sep 17 00:00:00 2001 From: Lauren Krugen Date: Fri, 6 Dec 2024 11:41:51 -0700 Subject: [PATCH 07/17] troubleshoot api config not loading --- bcda/cclf/parser.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/bcda/cclf/parser.go b/bcda/cclf/parser.go index 2c1b11a1c..0760c61b7 100644 --- a/bcda/cclf/parser.go +++ b/bcda/cclf/parser.go @@ -41,8 +41,12 @@ type CSVParser struct { func getACOConfigs() ([]service.ACOConfig, error) { configs, err := service.LoadConfig() if err != nil { + log.API.Error("error when fetching configs: %s", err) return []service.ACOConfig{}, err } + if configs == nil { + log.API.Error("no configs loaded: %s", err) + } return configs.ACOConfigs, err } @@ -55,6 +59,7 @@ func GetCSVMetadata(path string) (csvFileMetadata, error) { acos, err := getACOConfigs() if err != nil { + return csvFileMetadata{}, err } if acos == nil { From 988df19fae344a9ac798eb336d0249629bd9a9d4 Mon Sep 17 00:00:00 2001 From: Lauren Krugen Date: Fri, 6 Dec 2024 11:50:13 -0700 Subject: [PATCH 08/17] more troubleshooting --- bcda/cclf/parser.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/bcda/cclf/parser.go b/bcda/cclf/parser.go index 0760c61b7..76ba0aca5 100644 --- a/bcda/cclf/parser.go +++ b/bcda/cclf/parser.go @@ -41,12 +41,13 @@ type CSVParser struct { func getACOConfigs() ([]service.ACOConfig, error) { configs, err := service.LoadConfig() if err != nil { - log.API.Error("error when fetching configs: %s", err) + log.API.Errorf("error when fetching configs: %s", err) return []service.ACOConfig{}, err } if configs == nil { - log.API.Error("no configs loaded: %s", err) + log.API.Errorf("no configs loaded: %s", err) } + log.API.Infof("configs loaded: %s", configs) return configs.ACOConfigs, err } @@ -81,7 +82,6 @@ func GetCSVMetadata(path string) (csvFileMetadata, error) { } if metadata == (csvFileMetadata{}) { - err := fmt.Errorf("invalid filename for attribution file: %s.", err.Error()) return metadata, err } From 67fe52df5a755f8f50e72e699de163ab3e8c1b74 Mon Sep 17 00:00:00 2001 From: Lauren Krugen Date: Tue, 10 Dec 2024 09:11:39 -0700 Subject: [PATCH 09/17] updating importer dependency --- bcda/cclf/parser_test.go | 7 +++++++ bcda/lambda/cclf/main.go | 4 +++- .../cclf/archives/csv/T.PCPB.M2411.D241206.T1732561 | 6 ++++++ 3 files changed, 16 insertions(+), 1 deletion(-) create mode 100644 shared_files/cclf/archives/csv/T.PCPB.M2411.D241206.T1732561 diff --git a/bcda/cclf/parser_test.go b/bcda/cclf/parser_test.go index 2d318a039..f8525c08b 100644 --- a/bcda/cclf/parser_test.go +++ b/bcda/cclf/parser_test.go @@ -109,6 +109,13 @@ func TestValidateCCLFFileName(t *testing.T) { fileType: models.FileTypeDefault, }, }, + {"valid csv test filename", "T.PCPB.M2411." + fileDateTime, nil, csvFileMetadata{ + env: "test", + timestamp: validTime, + perfYear: 24, + fileType: models.FileTypeDefault, + }, + }, {"invalid csv - file date too old", "P.PCPB.M2411.D201101.T0000001", errors.New("out of range"), csvFileMetadata{}}, {"invalid csv - file date in the future", "P.PCPB.M2411." + futureTime.Format(dateFormat), errors.New("out of range"), csvFileMetadata{}}, } diff --git a/bcda/lambda/cclf/main.go b/bcda/lambda/cclf/main.go index 00e244a48..034589b36 100644 --- a/bcda/lambda/cclf/main.go +++ b/bcda/lambda/cclf/main.go @@ -14,6 +14,7 @@ import ( bcdaaws "github.com/CMSgov/bcda-app/bcda/aws" "github.com/CMSgov/bcda-app/bcda/cclf" + "github.com/CMSgov/bcda-app/bcda/database" "github.com/CMSgov/bcda-app/optout" "github.com/CMSgov/bcda-app/conf" @@ -79,7 +80,8 @@ func handleCSVImport(s3AssumeRoleArn, s3ImportPath string) (string, error) { logger = logger.WithFields(logrus.Fields{"import_filename": s3ImportPath}) importer := cclf.CSVImporter{ - Logger: logger, + Logger: logger, + Database: database.Connection, FileProcessor: &cclf.S3FileProcessor{ Handler: optout.S3FileHandler{ Logger: logger, diff --git a/shared_files/cclf/archives/csv/T.PCPB.M2411.D241206.T1732561 b/shared_files/cclf/archives/csv/T.PCPB.M2411.D241206.T1732561 new file mode 100644 index 000000000..0d9c9847c --- /dev/null +++ b/shared_files/cclf/archives/csv/T.PCPB.M2411.D241206.T1732561 @@ -0,0 +1,6 @@ +MBIs +MBI000001 +MBI000002 +MBI000003 +MBI000004 +MBI000005 From ba1e02f431645d5427a85098917e85003ccbb382 Mon Sep 17 00:00:00 2001 From: Lauren Krugen Date: Tue, 10 Dec 2024 11:37:00 -0700 Subject: [PATCH 10/17] debugging --- bcda/cclf/parser.go | 7 +++---- bcda/lambda/cclf/main.go | 29 +++++++++++++++++++++++++++++ 2 files changed, 32 insertions(+), 4 deletions(-) diff --git a/bcda/cclf/parser.go b/bcda/cclf/parser.go index 76ba0aca5..6c923a73b 100644 --- a/bcda/cclf/parser.go +++ b/bcda/cclf/parser.go @@ -60,11 +60,10 @@ func GetCSVMetadata(path string) (csvFileMetadata, error) { acos, err := getACOConfigs() if err != nil { - - return csvFileMetadata{}, err + return csvFileMetadata{}, errors.New("Failed to load ACO configs") } if acos == nil { - log.API.Error("no aco configs present!") + return csvFileMetadata{}, errors.New("No ACO configs found.") } for _, v := range acos { @@ -82,7 +81,7 @@ func GetCSVMetadata(path string) (csvFileMetadata, error) { } if metadata == (csvFileMetadata{}) { - return metadata, err + return metadata, errors.New("Invalid filename for csv attribution file") } metadata.name = path diff --git a/bcda/lambda/cclf/main.go b/bcda/lambda/cclf/main.go index 034589b36..6f138b032 100644 --- a/bcda/lambda/cclf/main.go +++ b/bcda/lambda/cclf/main.go @@ -56,6 +56,10 @@ func attributionImportHandler(ctx context.Context, sqsEvent events.SQSEvent) (st if err != nil { return "", err } + // err = loadBCDAParams() + // if err != nil { + // return "", err + // } // Send the entire filepath into the CCLF Importer so we are only // importing the one file that was sent in the trigger. @@ -91,6 +95,13 @@ func handleCSVImport(s3AssumeRoleArn, s3ImportPath string) (string, error) { }, } + logger.Info("bcda lambda api config: ", os.Getenv("BCDA_API_CONFIG_PATH")) + wd, _ := os.Getwd() + logger.Infof("Working directory: %s", wd) + dirs, _ := os.ReadDir(wd) + for _, v := range dirs { + logger.Infof(v.Name()) + } err := importer.ImportCSV(s3ImportPath) if err != nil { @@ -119,6 +130,24 @@ func loadBfdS3Params() (string, error) { return param, nil } +// func loadBCDAParams() error { +// // loadlambdconfigs and update the path +// os.Getwd() +// env := conf.GetEnv("ENV") +// bcdaSession, err := bcdaaws.NewSession("", os.Getenv("LOCAL_STACK_ENDPOINT")) +// if err != nil { +// return err +// } + +// param, err := bcdaaws.GetParameter(bcdaSession, fmt.Sprintf("/", env)) +// if err != nil { +// return err +// } +// os.Setenv("BCDA_API_CONFIG_PATH", param) + +// return nil +// } + func handleCclfImport(s3AssumeRoleArn, s3ImportPath string) (string, error) { env := conf.GetEnv("ENV") appName := conf.GetEnv("APP_NAME") From 343d681d46706981242bf8a2e3f86802b066ff1c Mon Sep 17 00:00:00 2001 From: Lauren Krugen Date: Wed, 11 Dec 2024 11:15:37 -0700 Subject: [PATCH 11/17] copy config files to lambda zip --- .github/workflows/cclf-import-test-deploy.yml | 1 + bcda/lambda/cclf/main.go | 44 +++++++++---------- conf/config.go | 8 ++++ 3 files changed, 30 insertions(+), 23 deletions(-) diff --git a/.github/workflows/cclf-import-test-deploy.yml b/.github/workflows/cclf-import-test-deploy.yml index 7ca347677..23b523787 100644 --- a/.github/workflows/cclf-import-test-deploy.yml +++ b/.github/workflows/cclf-import-test-deploy.yml @@ -21,6 +21,7 @@ jobs: env: CGO_ENABLED: 0 run: | + cp -r ../conf/configs ./lambda/cclf go build -o bin/bootstrap ./lambda/cclf/main.go zip -j function.zip bin/bootstrap - uses: aws-actions/configure-aws-credentials@v4 diff --git a/bcda/lambda/cclf/main.go b/bcda/lambda/cclf/main.go index 6f138b032..919712e4c 100644 --- a/bcda/lambda/cclf/main.go +++ b/bcda/lambda/cclf/main.go @@ -56,10 +56,10 @@ func attributionImportHandler(ctx context.Context, sqsEvent events.SQSEvent) (st if err != nil { return "", err } - // err = loadBCDAParams() - // if err != nil { - // return "", err - // } + err = loadBCDAParams() + if err != nil { + return "", err + } // Send the entire filepath into the CCLF Importer so we are only // importing the one file that was sent in the trigger. @@ -98,9 +98,17 @@ func handleCSVImport(s3AssumeRoleArn, s3ImportPath string) (string, error) { logger.Info("bcda lambda api config: ", os.Getenv("BCDA_API_CONFIG_PATH")) wd, _ := os.Getwd() logger.Infof("Working directory: %s", wd) - dirs, _ := os.ReadDir(wd) + dirs, _ := os.ReadDir("./") + for _, v := range dirs { + logger.Infof("dir: %s, file: %s", (v.Name())) + } + dirs, _ = os.ReadDir("../") for _, v := range dirs { - logger.Infof(v.Name()) + logger.Infof("dir: ../, file: %s", (v.Name())) + } + dirs, _ = os.ReadDir("../../") + for _, v := range dirs { + logger.Infof("dir: ../.., file: %s", (v.Name())) } err := importer.ImportCSV(s3ImportPath) @@ -130,23 +138,13 @@ func loadBfdS3Params() (string, error) { return param, nil } -// func loadBCDAParams() error { -// // loadlambdconfigs and update the path -// os.Getwd() -// env := conf.GetEnv("ENV") -// bcdaSession, err := bcdaaws.NewSession("", os.Getenv("LOCAL_STACK_ENDPOINT")) -// if err != nil { -// return err -// } - -// param, err := bcdaaws.GetParameter(bcdaSession, fmt.Sprintf("/", env)) -// if err != nil { -// return err -// } -// os.Setenv("BCDA_API_CONFIG_PATH", param) - -// return nil -// } +func loadBCDAParams() error { + // loadlambdconfigs and update the path + env := conf.GetEnv("ENV") + conf.LoadLambdaEnvVars(env) + + return nil +} func handleCclfImport(s3AssumeRoleArn, s3ImportPath string) (string, error) { env := conf.GetEnv("ENV") diff --git a/conf/config.go b/conf/config.go index bfdb0f252..0d9384536 100644 --- a/conf/config.go +++ b/conf/config.go @@ -94,6 +94,14 @@ func init() { } +func LoadLambdaEnvVars(env string) { + envPath := fmt.Sprintf("./configs/%s.yml", env) + var configPaths []string + configPaths = append(configPaths, envPath) + envVars, state = loadConfigs(configPaths...) + +} + // This is the private helper function that sets up viper. This function is // called by the init() function only once during initialization of the package. func loadConfigs(locations ...string) (config, configStatus) { From 16c3049e51c913acd1ad3c9109f37f3c455adfea Mon Sep 17 00:00:00 2001 From: Lauren Krugen Date: Wed, 11 Dec 2024 11:31:43 -0700 Subject: [PATCH 12/17] copy config files to lambda zip --- .github/workflows/cclf-import-test-deploy.yml | 2 +- bcda/lambda/cclf/main.go | 12 +----------- 2 files changed, 2 insertions(+), 12 deletions(-) diff --git a/.github/workflows/cclf-import-test-deploy.yml b/.github/workflows/cclf-import-test-deploy.yml index 23b523787..b3c534e61 100644 --- a/.github/workflows/cclf-import-test-deploy.yml +++ b/.github/workflows/cclf-import-test-deploy.yml @@ -23,7 +23,7 @@ jobs: run: | cp -r ../conf/configs ./lambda/cclf go build -o bin/bootstrap ./lambda/cclf/main.go - zip -j function.zip bin/bootstrap + zip -j function.zip bin/bootstrap ./lambda/cclf/configs - uses: aws-actions/configure-aws-credentials@v4 with: aws-region: ${{ vars.AWS_REGION }} diff --git a/bcda/lambda/cclf/main.go b/bcda/lambda/cclf/main.go index 919712e4c..45447165d 100644 --- a/bcda/lambda/cclf/main.go +++ b/bcda/lambda/cclf/main.go @@ -94,21 +94,12 @@ func handleCSVImport(s3AssumeRoleArn, s3ImportPath string) (string, error) { }, }, } - logger.Info("bcda lambda api config: ", os.Getenv("BCDA_API_CONFIG_PATH")) wd, _ := os.Getwd() logger.Infof("Working directory: %s", wd) dirs, _ := os.ReadDir("./") for _, v := range dirs { - logger.Infof("dir: %s, file: %s", (v.Name())) - } - dirs, _ = os.ReadDir("../") - for _, v := range dirs { - logger.Infof("dir: ../, file: %s", (v.Name())) - } - dirs, _ = os.ReadDir("../../") - for _, v := range dirs { - logger.Infof("dir: ../.., file: %s", (v.Name())) + logger.Infof("dir: %s, file: %s", wd, (v.Name())) } err := importer.ImportCSV(s3ImportPath) @@ -139,7 +130,6 @@ func loadBfdS3Params() (string, error) { } func loadBCDAParams() error { - // loadlambdconfigs and update the path env := conf.GetEnv("ENV") conf.LoadLambdaEnvVars(env) From 3f397909ee144748379161179e6d30415554637a Mon Sep 17 00:00:00 2001 From: Lauren Krugen Date: Wed, 11 Dec 2024 11:59:05 -0700 Subject: [PATCH 13/17] test zip function --- .github/workflows/cclf-import-test-deploy.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/cclf-import-test-deploy.yml b/.github/workflows/cclf-import-test-deploy.yml index b3c534e61..d737f4d01 100644 --- a/.github/workflows/cclf-import-test-deploy.yml +++ b/.github/workflows/cclf-import-test-deploy.yml @@ -21,9 +21,8 @@ jobs: env: CGO_ENABLED: 0 run: | - cp -r ../conf/configs ./lambda/cclf go build -o bin/bootstrap ./lambda/cclf/main.go - zip -j function.zip bin/bootstrap ./lambda/cclf/configs + zip -j function.zip bin/bootstrap ../conf/configs/test.yml - uses: aws-actions/configure-aws-credentials@v4 with: aws-region: ${{ vars.AWS_REGION }} From 4f66dba7b6ddd8ebba8170ab146620078b73cffb Mon Sep 17 00:00:00 2001 From: Lauren Krugen Date: Wed, 11 Dec 2024 13:04:42 -0700 Subject: [PATCH 14/17] update path --- conf/config.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conf/config.go b/conf/config.go index 0d9384536..fd55a38f1 100644 --- a/conf/config.go +++ b/conf/config.go @@ -95,7 +95,7 @@ func init() { } func LoadLambdaEnvVars(env string) { - envPath := fmt.Sprintf("./configs/%s.yml", env) + envPath := fmt.Sprintf("./%s.yml", env) var configPaths []string configPaths = append(configPaths, envPath) envVars, state = loadConfigs(configPaths...) From ae4354632a719d7affe3290c63f51f53f1d22b66 Mon Sep 17 00:00:00 2001 From: Lauren Krugen Date: Wed, 11 Dec 2024 13:45:47 -0700 Subject: [PATCH 15/17] add config files to function root --- .github/workflows/cclf-import-test-deploy.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/cclf-import-test-deploy.yml b/.github/workflows/cclf-import-test-deploy.yml index d737f4d01..4e3be6a55 100644 --- a/.github/workflows/cclf-import-test-deploy.yml +++ b/.github/workflows/cclf-import-test-deploy.yml @@ -22,7 +22,7 @@ jobs: CGO_ENABLED: 0 run: | go build -o bin/bootstrap ./lambda/cclf/main.go - zip -j function.zip bin/bootstrap ../conf/configs/test.yml + zip -j function.zip bin/bootstrap ../conf/configs/dev.yml ../conf/configs/test.yml ../conf/configs/prod.yml - uses: aws-actions/configure-aws-credentials@v4 with: aws-region: ${{ vars.AWS_REGION }} From c138ec38b27863ea8dd92e70df4f5df65faeab1f Mon Sep 17 00:00:00 2001 From: Lauren Krugen Date: Wed, 11 Dec 2024 14:16:02 -0700 Subject: [PATCH 16/17] clean up debugging statements --- bcda/cclf/parser.go | 6 ------ bcda/lambda/cclf/main.go | 8 -------- 2 files changed, 14 deletions(-) diff --git a/bcda/cclf/parser.go b/bcda/cclf/parser.go index 6c923a73b..2fdeb602a 100644 --- a/bcda/cclf/parser.go +++ b/bcda/cclf/parser.go @@ -41,13 +41,8 @@ type CSVParser struct { func getACOConfigs() ([]service.ACOConfig, error) { configs, err := service.LoadConfig() if err != nil { - log.API.Errorf("error when fetching configs: %s", err) return []service.ACOConfig{}, err } - if configs == nil { - log.API.Errorf("no configs loaded: %s", err) - } - log.API.Infof("configs loaded: %s", configs) return configs.ACOConfigs, err } @@ -71,7 +66,6 @@ func GetCSVMetadata(path string) (csvFileMetadata, error) { parts := filenameRegexp.FindStringSubmatch(path) if len(parts) == v.AttributionFile.MetadataMatches { metadata, err = validateCSVMetadata(parts) - log.API.Error("regex for aco (%s): %s", v.Model, v.AttributionFile.NamePattern) if err != nil { return csvFileMetadata{}, err } diff --git a/bcda/lambda/cclf/main.go b/bcda/lambda/cclf/main.go index 45447165d..c9616ea2b 100644 --- a/bcda/lambda/cclf/main.go +++ b/bcda/lambda/cclf/main.go @@ -94,13 +94,6 @@ func handleCSVImport(s3AssumeRoleArn, s3ImportPath string) (string, error) { }, }, } - logger.Info("bcda lambda api config: ", os.Getenv("BCDA_API_CONFIG_PATH")) - wd, _ := os.Getwd() - logger.Infof("Working directory: %s", wd) - dirs, _ := os.ReadDir("./") - for _, v := range dirs { - logger.Infof("dir: %s, file: %s", wd, (v.Name())) - } err := importer.ImportCSV(s3ImportPath) if err != nil { @@ -132,7 +125,6 @@ func loadBfdS3Params() (string, error) { func loadBCDAParams() error { env := conf.GetEnv("ENV") conf.LoadLambdaEnvVars(env) - return nil } From c7d3fa49ab6ebfcd12d9dee56b1fe3cdf77f5622 Mon Sep 17 00:00:00 2001 From: Lauren Krugen Date: Thu, 12 Dec 2024 11:36:28 -0700 Subject: [PATCH 17/17] test fixes --- bcda/cclf/csv_test.go | 2 +- bcda/cclf/local_fileprocessor_test.go | 42 ++++++++++++++++++++++----- bcda/cclf/parser_test.go | 8 ++--- 3 files changed, 40 insertions(+), 12 deletions(-) diff --git a/bcda/cclf/csv_test.go b/bcda/cclf/csv_test.go index 855a8fe90..d6269b588 100644 --- a/bcda/cclf/csv_test.go +++ b/bcda/cclf/csv_test.go @@ -109,7 +109,7 @@ func (s *CSVTestSuite) TestImportCSV_Integration() { }{ {"Import CSV attribution success", filepath.Join(s.basePath, "cclf/archives/csv/P.PCPB.M2411.D181120.T1000000"), 0, []string{"MBI000001", "MBI000002", "MBI000003", "MBI000004", "MBI000005"}, nil}, {"Import CSV attribution that already exists", filepath.Join(s.basePath, "cclf/archives/csv/P.PCPB.M2411.D181121.T1000000"), 0, []string{}, errors.New("already exists")}, - {"Import CSV attribution invalid name", filepath.Join(s.basePath, "cclf/archives/csv/P.PC.M2411.D181120.T1000000"), 0, []string{}, errors.New("invalid filename")}, + {"Import CSV attribution invalid name", filepath.Join(s.basePath, "cclf/archives/csv/P.PC.M2411.D181120.T1000000"), 0, []string{}, errors.New("Invalid filename")}, {"Import Opt Out failure", filepath.Join(s.basePath, "cclf/archives/csv/T#EFT.ON.ACO.NGD1800.DPRF.D181120.T1000010"), 0, []string{}, errors.New("File is type: opt-out. Skipping attribution import.")}, } diff --git a/bcda/cclf/local_fileprocessor_test.go b/bcda/cclf/local_fileprocessor_test.go index a14dc9553..56ff240c2 100644 --- a/bcda/cclf/local_fileprocessor_test.go +++ b/bcda/cclf/local_fileprocessor_test.go @@ -38,10 +38,18 @@ type LocalFileProcessorTestSuite struct { } func (s *LocalFileProcessorTestSuite) SetupTest() { + setupTestHelper(s) +} + +func setupTestHelper(s *LocalFileProcessorTestSuite) { s.basePath, s.cleanup = testUtils.CopyToTemporaryDirectory(s.T(), "../../shared_files/") } func (s *LocalFileProcessorTestSuite) SetupSuite() { + setupSuiteHelper(s) +} + +func setupSuiteHelper(s *LocalFileProcessorTestSuite) { s.cclfRefDate = conf.GetEnv("CCLF_REF_DATE") conf.SetEnv(s.T(), "CCLF_REF_DATE", "181201") // Needed to allow our static CCLF files to continue to be processed dir, err := os.MkdirTemp("", "*") @@ -73,10 +81,18 @@ func (s *LocalFileProcessorTestSuite) SetupSuite() { } func (s *LocalFileProcessorTestSuite) TearDownTest() { + tearDownTestHelper(s) +} + +func tearDownTestHelper(s *LocalFileProcessorTestSuite) { s.cleanup() } func (s *LocalFileProcessorTestSuite) TearDownSuite() { + tearDownSuiteHelper(s) +} + +func tearDownSuiteHelper(s *LocalFileProcessorTestSuite) { conf.SetEnv(s.T(), "CCLF_REF_DATE", s.cclfRefDate) os.RemoveAll(s.pendingDeletionDir) } @@ -401,22 +417,30 @@ func (s *LocalFileProcessorTestSuite) TestCleanUpCSV() { tests := []struct { name string - filepath string + filename string deliverytime time.Time imported bool delFiles int baseFiles int }{ - {"Not imported and expired", filepath.Join(s.basePath, "cclf/archives/csv/P.PCPB.M2411.D181120.T1000000"), expiredTime, false, 1, 1}, - {"Not imported and not expired", filepath.Join(s.basePath, "cclf/archives/csv/P.PCPB.M2411.D181120.T1000000"), time.Now(), false, 1, 1}, - {"Successfully imported", filepath.Join(s.basePath, "cclf/archives/csv/P.PCPB.M2411.D181120.T1000000"), time.Now(), true, 1, 1}, + {"Not imported and expired", "cclf/archives/csv/P.PCPB.M2411.D181120.T1000000", expiredTime, false, 1, 2}, + {"Not imported and not expired", "cclf/archives/csv/P.PCPB.M2411.D181120.T1000000", time.Now(), false, 0, 3}, + {"Successfully imported", "cclf/archives/csv/P.PCPB.M2411.D181120.T1000000", time.Now(), true, 1, 2}, } + runcount := 0 + for _, test := range tests { + s.T().Run(test.name, func(tt *testing.T) { + if runcount != 0 { + setupSuiteHelper(s) + setupTestHelper(s) + } + file.metadata.deliveryDate = test.deliverytime file.imported = test.imported - file.filepath = test.filepath + file.filepath = filepath.Join(s.basePath, test.filename) err := s.csvProcessor.CleanUpCSV(file) assert.Nil(s.T(), err) delDir, err := os.ReadDir(conf.GetEnv("PENDING_DELETION_DIR")) @@ -432,10 +456,14 @@ func (s *LocalFileProcessorTestSuite) TestCleanUpCSV() { s.FailNow("failed to read directory: %s", conf.GetEnv("PENDING_DELETION_DIR"), err) } assert.Len(s.T(), baseDir, test.baseFiles) - if test.baseFiles == 2 { - assert.Equal(s.T(), file.metadata.name, baseDir[0].Name()) + runcount++ + tearDownTestHelper(s) + if runcount < 3 { + tearDownSuiteHelper(s) } + }) + } } diff --git a/bcda/cclf/parser_test.go b/bcda/cclf/parser_test.go index f8525c08b..a713810d6 100644 --- a/bcda/cclf/parser_test.go +++ b/bcda/cclf/parser_test.go @@ -66,10 +66,10 @@ func TestGetCSVMetadata(t *testing.T) { fileType: models.FileTypeDefault, }, }, - {"invalid csv filename", "P.PPB.M2411." + fileDateTime, "invalid filename", csvFileMetadata{}}, - {"invalid csv filename - extra digit", "P.PCPB.M24112." + fileDateTime, "invalid filename", csvFileMetadata{}}, - {"invalid csv filename - env", "A.PCPB.M24112." + fileDateTime, "invalid filename", csvFileMetadata{}}, - {"invalid csv filename - dupe match", "P.PCPBPCPB.M2411." + fileDateTime, "invalid filename", csvFileMetadata{}}, + {"invalid csv filename", "P.PPB.M2411." + fileDateTime, "Invalid filename", csvFileMetadata{}}, + {"invalid csv filename - extra digit", "P.PCPB.M24112." + fileDateTime, "Invalid filename", csvFileMetadata{}}, + {"invalid csv filename - env", "A.PCPB.M24112." + fileDateTime, "Invalid filename", csvFileMetadata{}}, + {"invalid csv filename - dupe match", "P.PCPBPCPB.M2411." + fileDateTime, "Invalid filename", csvFileMetadata{}}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) {