PRMP-642: Create Bulk ODS Update Lambda

lambda/bulk-ods-update/bulk_ods_update.py

@@ -5,7 +5,6 @@
 import csv
 
 import boto3
-from pynamodb.exceptions import DoesNotExist
 
 from utils.enums.trud import OdsDownloadType, TrudItem
 from utils.models.ods_models import PracticeOds, IcbOds
@@ -20,6 +19,8 @@
     ICB_QUARTERLY_FILE_PATH,
     ICB_MONTHLY_FILE_NAME,
     ICB_QUARTERLY_FILE_NAME,
+    GP_WEEKLY_FILE_NAME,
+    GP_WEEKLY_ZIP_FILE_PATH,
 )
 
 logger = logging.getLogger()
@@ -50,8 +51,8 @@ def determine_ods_manifest_download_type() -> OdsDownloadType:
     logger.info("Determining download type")
     today = date.today()
 
-    last_day_of_month = calendar.monthrange(today.year, today.month)[1]
-    last_date_of_month = date(today.year, today.month, last_day_of_month)
+    total_days_in_month = calendar.monthrange(today.year, today.month)[1]
+    last_date_of_month = date(today.year, today.month, total_days_in_month)
 
     last_sunday_of_month = last_date_of_month
 
@@ -81,28 +82,32 @@ def extract_and_process_ods_gp_data(trud_service: TrudApiService):
         gp_ods_releases[0].get("archiveFileUrl")
     )
 
-    eppracur_csv_path = os.path.join(TEMP_DIR, "epraccur.csv")
+    eppracur_csv_path = os.path.join(TEMP_DIR, GP_WEEKLY_FILE_NAME)
 
     epraccur_zip_file = trud_service.unzipping_files(
-        download_file_bytes, "Data/epraccur.zip", TEMP_DIR, True
+        download_file_bytes, GP_WEEKLY_ZIP_FILE_PATH, TEMP_DIR, True
     )
-    trud_service.unzipping_files(epraccur_zip_file,"epraccur.csv", TEMP_DIR)
-
-    logger.info(os.listdir(TEMP_DIR))
+    trud_service.unzipping_files(epraccur_zip_file, GP_WEEKLY_FILE_NAME, TEMP_DIR)
 
     gp_ods_data = trud_csv_to_dict(eppracur_csv_path, GP_FILE_HEADERS)
     gp_ods_data_amended_data = get_amended_records(gp_ods_data)
 
     if gp_ods_data_amended_data:
+        logger.info(
+            f"Found {len(gp_ods_data_amended_data)} amended GP data records to update"
+        )
         compare_and_overwrite(OdsDownloadType.GP, gp_ods_data_amended_data)
         return
 
     logger.info("No amended GP data found")
 
+
 def extract_and_process_ods_icb_data(trud_service: TrudApiService):
     logger.info("Extracting and processing ODS ICB data")
 
-    icb_ods_releases = trud_service.get_release_list(TrudItem.ORG_REF_DATA_MONTHLY, True)
+    icb_ods_releases = trud_service.get_release_list(
+        TrudItem.ORG_REF_DATA_MONTHLY, True
+    )
 
     is_quarterly_release = icb_ods_releases[0].get("name").endswith(".0.0")
     download_file = trud_service.get_download_file(
@@ -127,6 +132,9 @@ def extract_and_process_ods_icb_data(trud_service: TrudApiService):
             icb_ods_data_amended_data = get_amended_records(icb_ods_data)
 
     if icb_ods_data_amended_data:
+        logger.info(
+            f"Found {len(icb_ods_data_amended_data)} amended ICB data records to update"
+        )
         compare_and_overwrite(OdsDownloadType.ICB, icb_ods_data_amended_data)
         return
 
@@ -152,29 +160,29 @@ def trud_csv_to_dict(file_path: str, headers: list[str]) -> list[dict]:
 
 
 def compare_and_overwrite(download_type: OdsDownloadType, data: list[dict]):
-    logger.info(data)
     if download_type == OdsDownloadType.GP:
         logger.info("Comparing GP Practice data")
         for amended_record in data:
             try:
                 practice = PracticeOds(amended_record.get("PracticeOdsCode"))
                 practice.update(
                     actions=[
-                        PracticeOds.practice_name.set(amended_record.get("PracticeName")),
-                        PracticeOds.icb_ods_code.set(amended_record.get("IcbOdsCode"))
+                        PracticeOds.practice_name.set(
+                            amended_record.get("PracticeName")
+                        ),
+                        PracticeOds.icb_ods_code.set(amended_record.get("IcbOdsCode")),
                     ]
                 )
-            except DoesNotExist as e:
-                logger.info(f"Failed to retrieve record by Practice ODS code: {str(e)}")
+            except Exception as e:
+                logger.info(
+                    f"Failed to create/update record by Practice ODS code: {str(e)}"
+                )
 
     if download_type == OdsDownloadType.ICB:
+        logger.info("Comparing ICB data")
         for amended_record in data:
             try:
                 icb = IcbOds(amended_record.get("IcbOdsCode"))
-                icb.update(
-                    actions=[
-                        IcbOds.icb_name.set(amended_record.get("IcbName"))
-                    ]
-                )
-            except DoesNotExist as e:
-                logger.info(f"Failed to retrieve record by ICB ODS code: {str(e)}")
+                icb.update(actions=[IcbOds.icb_name.set(amended_record.get("IcbName"))])
+            except Exception as e:
+                logger.info(f"Failed to create/update record by ICB ODS code: {str(e)}")

lambda/mi-enrichment-requirements.txt

@@ -0,0 +1 @@
+pynamodb==6.0.1

stacks/gp-registrations-mi/scripts/get_latest_ods_csv.py

@@ -10,6 +10,8 @@
     ICB_QUARTERLY_FILE_NAME,
     ICB_FILE_HEADERS,
     GP_FILE_HEADERS,
+    GP_WEEKLY_FILE_NAME,
+    GP_WEEKLY_ZIP_FILE_PATH,
 )
 
 
@@ -48,9 +50,9 @@ def get_gp_latest_ods_csv(service):
         release_list_response[0].get("archiveFileUrl")
     )
     epraccur_zip_file = service.unzipping_files(
-        download_file, "Data/epraccur.zip", byte=True
+        download_file, GP_WEEKLY_ZIP_FILE_PATH, byte=True
     )
-    epraccur_csv_file = service.unzipping_files(epraccur_zip_file, "epraccur.csv")
+    epraccur_csv_file = service.unzipping_files(epraccur_zip_file, GP_WEEKLY_FILE_NAME)
     create_modify_csv(
         epraccur_csv_file,
         "initial_full_gps_ods.csv",

stacks/gp-registrations-mi/terraform/event-enrichment-lambda.tf

@@ -17,7 +17,7 @@ resource "aws_lambda_function" "event_enrichment_lambda" {
       ApplicationRole = "AwsLambdaFunction"
     }
   )
-  layers = [aws_lambda_layer_version.event_enrichment_lambda.arn]
+  layers = [aws_lambda_layer_version.mi_enrichment_lambda_layer.arn]
   environment {
     variables = {
       SPLUNK_CLOUD_EVENT_UPLOADER_SQS_QUEUE_URL = aws_sqs_queue.incoming_mi_events_for_splunk_cloud_event_uploader.url,
@@ -54,11 +54,4 @@ resource "aws_cloudwatch_log_group" "event_enrichment_lambda" {
     }
   )
   retention_in_days = 60
-}
-
-resource "aws_lambda_layer_version" "event_enrichment_lambda" {
-  filename                 = var.event_enrichment_lambda_layer_zip
-  layer_name               = "${var.environment}-${var.event_enrichment_lambda_name}_layer"
-  compatible_runtimes      = ["python3.12"]
-  compatible_architectures = ["x86_64"]
 }

stacks/gp-registrations-mi/terraform/lambda-bulk-ods-update.tf

@@ -6,7 +6,7 @@ resource "aws_lambda_function" "ods_bulk_update" {
   source_code_hash = filebase64sha256(var.bulk_ods_update_lambda_zip)
   runtime          = "python3.12"
   timeout          = 300
-  layers           = [aws_lambda_layer_version.ods_bulk_update_lambda.arn]
+  layers           = [aws_lambda_layer_version.mi_enrichment_lambda_layer.arn]
   environment {
     variables = {
       TRUD_API_KEY_PARAM_NAME      = data.aws_ssm_parameter.trud_api_key.name,
@@ -73,9 +73,3 @@ resource "aws_lambda_permission" "bulk_upload_metadata_schedule_permission" {
   ]
 }
 
-resource "aws_lambda_layer_version" "ods_bulk_update_lambda" {
-  filename                 = var.event_enrichment_lambda_layer_zip
-  layer_name               = "${var.environment}-${var.ods_bulk_update_lambda_name}_layer"
-  compatible_runtimes      = ["python3.12"]
-  compatible_architectures = ["x86_64"]
-}

stacks/gp-registrations-mi/terraform/lambda_layer.tf

@@ -0,0 +1,6 @@
+resource "aws_lambda_layer_version" "mi_enrichment_lambda_layer" {
+  filename                 = var.mi_enrichment_lambda_layer_zip
+  layer_name               = "${var.environment}_mi_enrichment_layer"
+  compatible_runtimes      = ["python3.12"]
+  compatible_architectures = ["x86_64"]
+}

stacks/gp-registrations-mi/terraform/variables.tf

@@ -84,6 +84,12 @@ variable "splunk_cloud_api_token_param_name" {
   description = "SSM param containing splunk cloud api token to send MI events to"
 }
 
+variable "mi_enrichment_lambda_layer_zip" {
+  type        = string
+  description = "Path to zipfile containing relevant packages for MI lambdas"
+  default     = "../../../lambda/build/layers/mi-enrichment.zip"
+}
+
 variable "splunk_cloud_event_uploader_lambda_zip" {
   type        = string
   description = "Path to zipfile containing lambda code for uploading events to splunk cloud"
@@ -96,22 +102,10 @@ variable "event_enrichment_lambda_zip" {
   default     = "../../../lambda/build/event-enrichment.zip"
 }
 
-variable "event_enrichment_lambda_layer_zip" {
-  type        = string
-  description = "Path to zipfile containing lambda layer code for enriching MI events"
-  default     = "../../../lambda/build/layer/event-enrichment.zip"
-}
-
 variable "bulk_ods_update_lambda_zip" {
   type        = string
   description = "Path to zipfile containing lambda code for ODS update"
-  default     = "placeholder_lambda_payload.zip"
-}
-
-variable "bulk_ods_update_lambda_layer_zip" {
-  type        = string
-  description = "Path to zipfile containing lambda layer code for ODS update"
-  default     = "placeholder_lambda_payload.zip"
+  default     = "../../../lambda/build/bulk-ods-update.zip"
 }
 
 variable "s3_event_uploader_lambda_zip" {

tasks_github_actions.sh

@@ -12,11 +12,6 @@ function build_lambda {
     rm -rf $build_dir
     mkdir -p $build_dir
 
-    requirements_file=lambda/$lambda_name/requirements.txt
-    if test -f "$requirements_file"; then
-        build_lambda_layer $lambda_name
-    fi
-
     if test "$lambda_services"; then
         cp -r ./$lambda_services $build_dir
     fi
@@ -28,13 +23,13 @@ function build_lambda {
 }
 
 function build_lambda_layer {
-    lambda_name=$1
-    build_dir=lambda/build/layer/$lambda_name
+    layer_name=$1
+    build_dir=lambda/build/layers/$layer_name
 
     rm -rf $build_dir/python
     mkdir -p $build_dir/python
 
-    requirements_file=lambda/$lambda_name/requirements.txt
+    requirements_file=lambda/$layer_name-requirements.txt
     if test -f "$requirements_file"; then
         python3 -m venv create_layer
         source create_layer/bin/activate
@@ -43,13 +38,14 @@ function build_lambda_layer {
 
     cp -r create_layer/lib $build_dir/python
     pushd $build_dir
-    zip -r -X ../$lambda_name.zip .
+    zip -r -X ../$layer_name.zip .
     popd
 }
 
 echo "--- ${task} ---"
 case "${task}" in
 build-lambdas)
+  build_lambda_layer mi-enrichment
   build_lambda bulk-ods-update utils
   build_lambda error-alarm-alert
   build_lambda splunk-cloud-event-uploader

utils/trud_files.py

@@ -1,3 +1,6 @@
+GP_WEEKLY_ZIP_FILE_PATH = "Data/epraccur.zip"
+GP_WEEKLY_FILE_NAME = "epraccur.csv"
+
 ICB_MONTHLY_FILE_PATH = "eamendam.zip"
 ICB_MONTHLY_FILE_NAME = "eccgam.csv"