From 0deccb11ab6352d40f02b41f721fb7a71e706184 Mon Sep 17 00:00:00 2001 From: abbas-khan10 <127417949+abbas-khan10@users.noreply.github.com> Date: Fri, 16 Aug 2024 15:05:45 +0100 Subject: [PATCH] PRMP-642: Create Bulk ODS Update Lambda (#12) * Add bulk ods update lambda --- .gitignore | 1 + lambda/bulk-ods-update/bulk_ods_update.py | 188 ++++++++++++++++++ lambda/mi-enrichment-requirements.txt | 1 + .../scripts/get_latest_ods_csv.py | 85 ++------ .../terraform/event-enrichment-lambda.tf | 9 +- .../terraform/lambda-bulk-ods-update.tf | 9 +- .../terraform/lambda_layer.tf | 6 + .../terraform/variables.tf | 20 +- tasks_github_actions.sh | 17 +- utils/__init__.py | 0 utils/enums/__init__.py | 0 utils/enums/trud.py | 12 ++ utils/models/__init__.py | 0 utils/models/ods_models.py | 25 +++ utils/services/__init__.py | 0 utils/services/ssm_service.py | 7 + utils/services/trud_api_service.py | 67 +++++++ utils/trud_files.py | 68 +++++++ 18 files changed, 406 insertions(+), 109 deletions(-) create mode 100644 lambda/bulk-ods-update/bulk_ods_update.py create mode 100644 lambda/mi-enrichment-requirements.txt create mode 100644 stacks/gp-registrations-mi/terraform/lambda_layer.tf create mode 100644 utils/__init__.py create mode 100644 utils/enums/__init__.py create mode 100644 utils/enums/trud.py create mode 100644 utils/models/__init__.py create mode 100644 utils/models/ods_models.py create mode 100644 utils/services/__init__.py create mode 100644 utils/services/ssm_service.py create mode 100644 utils/services/trud_api_service.py create mode 100644 utils/trud_files.py diff --git a/.gitignore b/.gitignore index 831bdd7..76fa86b 100644 --- a/.gitignore +++ b/.gitignore @@ -2,6 +2,7 @@ *.tfplan .idea +__pycache__/ */*/__pycache__ */*/_trial_temp diff --git a/lambda/bulk-ods-update/bulk_ods_update.py b/lambda/bulk-ods-update/bulk_ods_update.py new file mode 100644 index 0000000..0fab56b --- /dev/null +++ b/lambda/bulk-ods-update/bulk_ods_update.py @@ -0,0 +1,188 @@ +import os +import tempfile +from datetime import date, timedelta +import calendar +import csv + +import boto3 + +from utils.enums.trud import OdsDownloadType, TrudItem +from utils.models.ods_models import PracticeOds, IcbOds +from utils.services.trud_api_service import TrudApiService + +import logging + +from utils.trud_files import ( + GP_FILE_HEADERS, + ICB_FILE_HEADERS, + ICB_MONTHLY_FILE_PATH, + ICB_QUARTERLY_FILE_PATH, + ICB_MONTHLY_FILE_NAME, + ICB_QUARTERLY_FILE_NAME, + GP_WEEKLY_FILE_NAME, + GP_WEEKLY_ZIP_FILE_PATH, +) + +logger = logging.getLogger() +logger.setLevel(logging.INFO) + +TEMP_DIR = tempfile.mkdtemp(dir="/tmp") + + +def lambda_handler(event, context): + download_type = determine_ods_manifest_download_type() + ssm = boto3.client("ssm") + trud_api_key_param = os.environ.get("TRUD_API_KEY_PARAM_NAME") + trud_api_key = ssm.get_parameter(trud_api_key_param) if trud_api_key_param else "" + trud_service = TrudApiService( + api_key=trud_api_key, + api_url=os.environ.get("TRUD_FHIR_API_URL_PARAM_NAME"), + ) + + extract_and_process_ods_gp_data(trud_service) + + if download_type == OdsDownloadType.BOTH: + extract_and_process_ods_icb_data(trud_service) + + return {"statusCode": 200} + + +def determine_ods_manifest_download_type() -> OdsDownloadType: + logger.info("Determining download type") + today = date.today() + + total_days_in_month = calendar.monthrange(today.year, today.month)[1] + last_date_of_month = date(today.year, today.month, total_days_in_month) + + last_sunday_of_month = last_date_of_month + + while last_sunday_of_month.weekday() != 6: + last_sunday_of_month -= timedelta(days=1) + + is_icb_download_date = today == last_sunday_of_month + + if is_icb_download_date: + logger.info("Download type set to: GP and ICB") + return OdsDownloadType.BOTH + + logger.info("Download type set to: GP") + return OdsDownloadType.GP + + +def extract_and_process_ods_gp_data(trud_service: TrudApiService): + logger.info("Extracting and processing ODS GP data") + + gp_ods_releases = trud_service.get_release_list( + TrudItem.NHS_ODS_WEEKLY, is_latest=True + ) + + logger.info(gp_ods_releases) + + download_file_bytes = trud_service.get_download_file( + gp_ods_releases[0].get("archiveFileUrl") + ) + + eppracur_csv_path = os.path.join(TEMP_DIR, GP_WEEKLY_FILE_NAME) + + epraccur_zip_file = trud_service.unzipping_files( + download_file_bytes, GP_WEEKLY_ZIP_FILE_PATH, TEMP_DIR, True + ) + trud_service.unzipping_files(epraccur_zip_file, GP_WEEKLY_FILE_NAME, TEMP_DIR) + + gp_ods_data = trud_csv_to_dict(eppracur_csv_path, GP_FILE_HEADERS) + gp_ods_data_amended_data = get_amended_records(gp_ods_data) + + if gp_ods_data_amended_data: + logger.info( + f"Found {len(gp_ods_data_amended_data)} amended GP data records to update" + ) + compare_and_overwrite(OdsDownloadType.GP, gp_ods_data_amended_data) + return + + logger.info("No amended GP data found") + + +def extract_and_process_ods_icb_data(trud_service: TrudApiService): + logger.info("Extracting and processing ODS ICB data") + + icb_ods_releases = trud_service.get_release_list( + TrudItem.ORG_REF_DATA_MONTHLY, True + ) + + is_quarterly_release = icb_ods_releases[0].get("name").endswith(".0.0") + download_file = trud_service.get_download_file( + icb_ods_releases[0].get("archiveFileUrl") + ) + + icb_zip_file_path = ( + ICB_MONTHLY_FILE_PATH if not is_quarterly_release else ICB_QUARTERLY_FILE_PATH + ) + icb_csv_file_name = ( + ICB_MONTHLY_FILE_NAME if not is_quarterly_release else ICB_QUARTERLY_FILE_NAME + ) + + icb_ods_data_amended_data = [] + if icb_zip_file := trud_service.unzipping_files( + download_file, icb_zip_file_path, TEMP_DIR, True + ): + if icb_csv_file := trud_service.unzipping_files( + icb_zip_file, icb_csv_file_name, TEMP_DIR + ): + icb_ods_data = trud_csv_to_dict(icb_csv_file, ICB_FILE_HEADERS) + icb_ods_data_amended_data = get_amended_records(icb_ods_data) + + if icb_ods_data_amended_data: + logger.info( + f"Found {len(icb_ods_data_amended_data)} amended ICB data records to update" + ) + compare_and_overwrite(OdsDownloadType.ICB, icb_ods_data_amended_data) + return + + logger.info("No amended ICB data found") + + +def get_amended_records(data: list[dict]) -> list[dict]: + return [ + amended_data + for amended_data in data + if amended_data.get("AmendedRecordIndicator") == "1" + ] + + +def trud_csv_to_dict(file_path: str, headers: list[str]) -> list[dict]: + with open(file_path, mode="r") as csv_file: + csv_reader = csv.DictReader(csv_file) + csv_reader.fieldnames = headers + data_list = [] + for row in csv_reader: + data_list.append(dict(row)) + return data_list + + +def compare_and_overwrite(download_type: OdsDownloadType, data: list[dict]): + if download_type == OdsDownloadType.GP: + logger.info("Comparing GP Practice data") + for amended_record in data: + try: + practice = PracticeOds(amended_record.get("PracticeOdsCode")) + practice.update( + actions=[ + PracticeOds.practice_name.set( + amended_record.get("PracticeName") + ), + PracticeOds.icb_ods_code.set(amended_record.get("IcbOdsCode")), + ] + ) + except Exception as e: + logger.info( + f"Failed to create/update record by Practice ODS code: {str(e)}" + ) + + if download_type == OdsDownloadType.ICB: + logger.info("Comparing ICB data") + for amended_record in data: + try: + icb = IcbOds(amended_record.get("IcbOdsCode")) + icb.update(actions=[IcbOds.icb_name.set(amended_record.get("IcbName"))]) + except Exception as e: + logger.info(f"Failed to create/update record by ICB ODS code: {str(e)}") diff --git a/lambda/mi-enrichment-requirements.txt b/lambda/mi-enrichment-requirements.txt new file mode 100644 index 0000000..bb868cc --- /dev/null +++ b/lambda/mi-enrichment-requirements.txt @@ -0,0 +1 @@ +pynamodb==6.0.1 \ No newline at end of file diff --git a/stacks/gp-registrations-mi/scripts/get_latest_ods_csv.py b/stacks/gp-registrations-mi/scripts/get_latest_ods_csv.py index cb74624..fca41e1 100644 --- a/stacks/gp-registrations-mi/scripts/get_latest_ods_csv.py +++ b/stacks/gp-registrations-mi/scripts/get_latest_ods_csv.py @@ -1,73 +1,18 @@ import csv import sys -from services.trud_api_service import TrudApiService, TrudItem - -GP_FILE_HEADERS = [ - "PracticeOdsCode", - "PracticeName", - "NG", - "HLHG", - "AD1", - "AD2", - "AD3", - "AD4", - "AD5", - "PostCode", - "OD", - "CD", - "Null1", - "Null2", - "IcbOdsCode", - "JPD", - "LPD", - "CTN", - "Null3", - "Null4", - "Null5", - "AM", - "Null6", - "GOR", - "Null7", - "Null8", - "Null9", -] - -ICB_FILE_HEADERS = [ - "IcbOdsCode", - "IcbName", - "NG", - "HLHG", - "AD1", - "AD2", - "AD3", - "AD4", - "AD5", - "PostCode", - "OD", - "CD", - "Null1", - "OSTC", - "Null2", - "Null3", - "Null4", - "Null5", - "Null6", - "Null7", - "Null8", - "AM", - "Null9", - "Null10", - "Null11", - "Null12", - "Null13", -] - -ICB_MONTHLY_FILE_PATH = "eamendam.zip" -ICB_QUARTERLY_FILE_PATH = "ocsissue/data/eccg.zip" - -ICB_MONTHLY_FILE_NAME = "eccgam.csv" -ICB_QUARTERLY_FILE_NAME = "eccg.csv" +from utils.enums.trud import TrudItem +from utils.services.trud_api_service import TrudApiService +from utils.trud_files import ( + ICB_MONTHLY_FILE_PATH, + ICB_QUARTERLY_FILE_PATH, + ICB_MONTHLY_FILE_NAME, + ICB_QUARTERLY_FILE_NAME, + ICB_FILE_HEADERS, + GP_FILE_HEADERS, + GP_WEEKLY_FILE_NAME, + GP_WEEKLY_ZIP_FILE_PATH, +) def create_modify_csv( @@ -105,9 +50,9 @@ def get_gp_latest_ods_csv(service): release_list_response[0].get("archiveFileUrl") ) epraccur_zip_file = service.unzipping_files( - download_file, "Data/epraccur.zip", True + download_file, GP_WEEKLY_ZIP_FILE_PATH, byte=True ) - epraccur_csv_file = service.unzipping_files(epraccur_zip_file, "epraccur.csv") + epraccur_csv_file = service.unzipping_files(epraccur_zip_file, GP_WEEKLY_FILE_NAME) create_modify_csv( epraccur_csv_file, "initial_full_gps_ods.csv", @@ -144,7 +89,7 @@ def get_icb_latest_ods_csv(service): ) if epraccur_zip_file := service.unzipping_files( - download_file, zip_file_path, True + download_file, zip_file_path, byte=True ): if epraccur_csv_file := service.unzipping_files( epraccur_zip_file, csv_file_name diff --git a/stacks/gp-registrations-mi/terraform/event-enrichment-lambda.tf b/stacks/gp-registrations-mi/terraform/event-enrichment-lambda.tf index dbf5db1..ae9eb92 100644 --- a/stacks/gp-registrations-mi/terraform/event-enrichment-lambda.tf +++ b/stacks/gp-registrations-mi/terraform/event-enrichment-lambda.tf @@ -17,7 +17,7 @@ resource "aws_lambda_function" "event_enrichment_lambda" { ApplicationRole = "AwsLambdaFunction" } ) - layers = [aws_lambda_layer_version.event_enrichment_lambda.arn] + layers = [aws_lambda_layer_version.mi_enrichment_lambda_layer.arn] environment { variables = { SPLUNK_CLOUD_EVENT_UPLOADER_SQS_QUEUE_URL = aws_sqs_queue.incoming_mi_events_for_splunk_cloud_event_uploader.url, @@ -54,11 +54,4 @@ resource "aws_cloudwatch_log_group" "event_enrichment_lambda" { } ) retention_in_days = 60 -} - -resource "aws_lambda_layer_version" "event_enrichment_lambda" { - filename = var.event_enrichment_lambda_layer_zip - layer_name = "${var.environment}-${var.event_enrichment_lambda_name}_layer" - compatible_runtimes = ["python3.12"] - compatible_architectures = ["x86_64"] } \ No newline at end of file diff --git a/stacks/gp-registrations-mi/terraform/lambda-bulk-ods-update.tf b/stacks/gp-registrations-mi/terraform/lambda-bulk-ods-update.tf index 058fc74..70911e4 100644 --- a/stacks/gp-registrations-mi/terraform/lambda-bulk-ods-update.tf +++ b/stacks/gp-registrations-mi/terraform/lambda-bulk-ods-update.tf @@ -6,7 +6,7 @@ resource "aws_lambda_function" "ods_bulk_update" { source_code_hash = filebase64sha256(var.bulk_ods_update_lambda_zip) runtime = "python3.12" timeout = 300 - layers = [aws_lambda_layer_version.ods_bulk_update_lambda.arn] + layers = [aws_lambda_layer_version.mi_enrichment_lambda_layer.arn] environment { variables = { TRUD_API_KEY_PARAM_NAME = data.aws_ssm_parameter.trud_api_key.name, @@ -72,10 +72,3 @@ resource "aws_lambda_permission" "bulk_upload_metadata_schedule_permission" { aws_lambda_function.ods_bulk_update, ] } - -resource "aws_lambda_layer_version" "ods_bulk_update_lambda" { - filename = var.event_enrichment_lambda_layer_zip - layer_name = "${var.environment}-${var.ods_bulk_update_lambda_name}_layer" - compatible_runtimes = ["python3.12"] - compatible_architectures = ["x86_64"] -} \ No newline at end of file diff --git a/stacks/gp-registrations-mi/terraform/lambda_layer.tf b/stacks/gp-registrations-mi/terraform/lambda_layer.tf new file mode 100644 index 0000000..063338a --- /dev/null +++ b/stacks/gp-registrations-mi/terraform/lambda_layer.tf @@ -0,0 +1,6 @@ +resource "aws_lambda_layer_version" "mi_enrichment_lambda_layer" { + filename = var.mi_enrichment_lambda_layer_zip + layer_name = "${var.environment}_mi_enrichment_layer" + compatible_runtimes = ["python3.12"] + compatible_architectures = ["x86_64"] +} \ No newline at end of file diff --git a/stacks/gp-registrations-mi/terraform/variables.tf b/stacks/gp-registrations-mi/terraform/variables.tf index c0fb46d..005f1bb 100644 --- a/stacks/gp-registrations-mi/terraform/variables.tf +++ b/stacks/gp-registrations-mi/terraform/variables.tf @@ -84,6 +84,12 @@ variable "splunk_cloud_api_token_param_name" { description = "SSM param containing splunk cloud api token to send MI events to" } +variable "mi_enrichment_lambda_layer_zip" { + type = string + description = "Path to zipfile containing relevant packages for MI lambdas" + default = "../../../lambda/build/layers/mi-enrichment.zip" +} + variable "splunk_cloud_event_uploader_lambda_zip" { type = string description = "Path to zipfile containing lambda code for uploading events to splunk cloud" @@ -96,22 +102,10 @@ variable "event_enrichment_lambda_zip" { default = "../../../lambda/build/event-enrichment.zip" } -variable "event_enrichment_lambda_layer_zip" { - type = string - description = "Path to zipfile containing lambda layer code for enriching MI events" - default = "../../../lambda/build/layer/event-enrichment.zip" -} - variable "bulk_ods_update_lambda_zip" { type = string description = "Path to zipfile containing lambda code for ODS update" - default = "placeholder_lambda_payload.zip" -} - -variable "bulk_ods_update_lambda_layer_zip" { - type = string - description = "Path to zipfile containing lambda layer code for ODS update" - default = "placeholder_lambda_payload.zip" + default = "../../../lambda/build/bulk-ods-update.zip" } variable "s3_event_uploader_lambda_zip" { diff --git a/tasks_github_actions.sh b/tasks_github_actions.sh index 7325930..6079094 100755 --- a/tasks_github_actions.sh +++ b/tasks_github_actions.sh @@ -12,11 +12,6 @@ function build_lambda { rm -rf $build_dir mkdir -p $build_dir - requirements_file=lambda/$lambda_name/requirements.txt - if test -f "$requirements_file"; then - build_lambda_layer $lambda_name - fi - if test "$lambda_services"; then cp -r ./$lambda_services $build_dir fi @@ -28,13 +23,13 @@ function build_lambda { } function build_lambda_layer { - lambda_name=$1 - build_dir=lambda/build/layer/$lambda_name + layer_name=$1 + build_dir=lambda/build/layers/$layer_name rm -rf $build_dir/python mkdir -p $build_dir/python - requirements_file=lambda/$lambda_name/requirements.txt + requirements_file=lambda/$layer_name-requirements.txt if test -f "$requirements_file"; then python3 -m venv create_layer source create_layer/bin/activate @@ -43,16 +38,18 @@ function build_lambda_layer { cp -r create_layer/lib $build_dir/python pushd $build_dir - zip -r -X ../$lambda_name.zip . + zip -r -X ../$layer_name.zip . popd } echo "--- ${task} ---" case "${task}" in build-lambdas) + build_lambda_layer mi-enrichment + build_lambda bulk-ods-update utils build_lambda error-alarm-alert build_lambda splunk-cloud-event-uploader - build_lambda event-enrichment services + build_lambda event-enrichment utils build_lambda s3-event-uploader ;; *) diff --git a/utils/__init__.py b/utils/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/utils/enums/__init__.py b/utils/enums/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/utils/enums/trud.py b/utils/enums/trud.py new file mode 100644 index 0000000..2cceb3b --- /dev/null +++ b/utils/enums/trud.py @@ -0,0 +1,12 @@ +from enum import StrEnum + + +class TrudItem(StrEnum): + NHS_ODS_WEEKLY = "58" + ORG_REF_DATA_MONTHLY = "242" + + +class OdsDownloadType(StrEnum): + GP = "ODS_GP" + ICB = "ODS_ICB" + BOTH = "ODS_GP_ICB" diff --git a/utils/models/__init__.py b/utils/models/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/utils/models/ods_models.py b/utils/models/ods_models.py new file mode 100644 index 0000000..4001feb --- /dev/null +++ b/utils/models/ods_models.py @@ -0,0 +1,25 @@ +import os + +from pynamodb.attributes import UnicodeAttribute, UTCDateTimeAttribute +from pynamodb.models import Model + + +class PracticeOds(Model): + class Meta: + table_name = os.getenv("GP_ODS_DYNAMO_TABLE_NAME") + + practice_ods_code = UnicodeAttribute(hash_key=True, attr_name="PracticeOdsCode") + practice_name = UnicodeAttribute(attr_name="PracticeName") + icb_ods_code = UnicodeAttribute(null=True, attr_name="IcbOdsCode") + supplier_name = UnicodeAttribute(null=True, attr_name="SupplierName") + supplier_last_updated = UTCDateTimeAttribute( + null=True, attr_name="SupplierLastUpdated" + ) + + +class IcbOds(Model): + class Meta: + table_name = os.getenv("ICB_ODS_DYNAMO_TABLE_NAME") + + icb_ods_code = UnicodeAttribute(hash_key=True, attr_name="IcbOdsCode") + icb_name = UnicodeAttribute(attr_name="IcbName") diff --git a/utils/services/__init__.py b/utils/services/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/utils/services/ssm_service.py b/utils/services/ssm_service.py new file mode 100644 index 0000000..b0a1fe3 --- /dev/null +++ b/utils/services/ssm_service.py @@ -0,0 +1,7 @@ +class SsmSecretManager: + def __init__(self, ssm): + self._ssm = ssm + + def get_secret(self, name): + response = self._ssm.get_parameter(Name=name, WithDecryption=True) + return response["Parameter"]["Value"] \ No newline at end of file diff --git a/utils/services/trud_api_service.py b/utils/services/trud_api_service.py new file mode 100644 index 0000000..67dad0e --- /dev/null +++ b/utils/services/trud_api_service.py @@ -0,0 +1,67 @@ +import os +from io import BytesIO +from zipfile import ZipFile +import urllib3 +from urllib3.util.retry import Retry + +import logging + +from utils.enums.trud import TrudItem + +logger = logging.getLogger() +logger.setLevel(logging.INFO) + + +class TrudApiService: + def __init__(self, api_key: str, api_url: str): + self.api_key = api_key + self.api_url = api_url + + retry_strategy = Retry( + total=3, backoff_factor=1, status_forcelist=[400, 404, 500, 502, 503, 504] + ) + + self.http = urllib3.PoolManager(retries=retry_strategy) + + def get_release_list(self, item_number: TrudItem, is_latest=False): + latest = "?latest" if is_latest else "" + url_endpoint = ( + self.api_url + self.api_key + "/items/" + item_number + "/releases" + latest + ) + + try: + trud_response = self.http.request("GET", url_endpoint) + response = trud_response.json().get("releases", []) + trud_response.release_conn() + + return response + except Exception as e: + logger.info(f"An unexpected error occurred: {e}") + raise e + + def get_download_url_by_release( + self, releases_list, break_at_quarterly_release=True + ): + download_url_by_release = {} + for release in releases_list: + download_url_by_release[release["name"]] = release.get("archiveFileUrl") + if break_at_quarterly_release and release["name"].endswith(".0.0"): + break + return download_url_by_release + + def get_download_file(self, download_url): + try: + download_response = self.http.request("GET", download_url) + logger.info(download_response) + return download_response.data + except Exception as e: + logger.info(f"An unexpected error occurred: {e}") + raise e + + def unzipping_files(self, zip_file, path=None, path_to_extract=None, byte: bool = False): + myzip = ZipFile(BytesIO(zip_file) if byte else zip_file) + if path_to_extract is None: + path_to_extract = os.getcwd() + if path in myzip.namelist(): + return myzip.extract(path, path_to_extract) + return None diff --git a/utils/trud_files.py b/utils/trud_files.py new file mode 100644 index 0000000..26b5527 --- /dev/null +++ b/utils/trud_files.py @@ -0,0 +1,68 @@ +GP_WEEKLY_ZIP_FILE_PATH = "Data/epraccur.zip" +GP_WEEKLY_FILE_NAME = "epraccur.csv" + +ICB_MONTHLY_FILE_PATH = "eamendam.zip" +ICB_MONTHLY_FILE_NAME = "eccgam.csv" + +ICB_QUARTERLY_FILE_PATH = "ocsissue/data/eccg.zip" +ICB_QUARTERLY_FILE_NAME = "eccg.csv" + +GP_FILE_HEADERS = [ + "PracticeOdsCode", + "PracticeName", + "NationalGrouping", + "HighLevelHealthGeography", + "AddressLine1", + "AddressLine2", + "AddressLine3", + "AddressLine4", + "AddressLine5", + "Postcode", + "OpenDate", + "CloseDate", + "StatusCode", + "OrganisationSubTypeCode", + "IcbOdsCode", + "JoinParentDate", + "LeftParentDate", + "ContactTelephoneNumber", + "Null", + "Null2", + "Null3", + "AmendedRecordIndicator", + "Null4", + "ProviderPurchaser", + "Null5", + "PracticeType", + "Null6", +] + +ICB_FILE_HEADERS = [ + "IcbOdsCode", + "IcbName", + "NationalGrouping", + "HighLevelHealthGeography", + "AddressLine1", + "AddressLine2", + "AddressLine3", + "AddressLine4", + "AddressLine5", + "Postcode", + "OpenDate", + "CloseDate", + "Null1", + "OrganisationSubTypeCode", + "Null2", + "Null3", + "Null4", + "Null5", + "Null6", + "Null7", + "Null8", + "AmendedRecordIndicator", + "Null9", + "Null10", + "Null11", + "Null12", + "Null13", +]