Ma/primarykeyconstraint

sqlmeshrunner/sqlmesh_runner.py

@@ -0,0 +1,43 @@
+# Databricks notebook source
+# DBTITLE 1,Import Libraries
+import sqlmesh
+import os
+
+# COMMAND ----------
+
+# DBTITLE 1,Get Secrets
+os.environ['DATABRICKS_STATE_DB_USER'] = dbutils.secrets.get(scope = "AzureKeyVaultSecrets", key = "UsernameTo-SqlMeshStateDatabase")
+os.environ['DATABRICKS_STATE_DB_PASSWORD'] = dbutils.secrets.get(scope = "AzureKeyVaultSecrets", key = "PasswordTo-SqlMeshStateDatabase")
+os.environ['DATABRICKS_ACCESS_TOKEN'] = dbutils.secrets.get(scope = "AzureKeyVaultSecrets", key = "TokenTo-DatabricksAPI")
+
+# COMMAND ----------
+
+os.environ['ENABLED_GATEWAYS']="databricks"
+os.environ['DEFAULT_GATEWAY']='databricks'
+os.environ['STATE_SCHEMA']="stroma"
+os.environ['DATABRICKS_SERVER_HOSTNAME']="adb-1868816176859647.7.azuredatabricks.net"
+os.environ['DATABRICKS_HTTP_PATH']="/sql/1.0/warehouses/0aec44b2e70e201d" # X-small
+os.environ['DATABRICKS_CATALOG']="rde_dev"
+os.environ['DATABRICKS_CATALOG_SOURCE']="dbw-lsc-uks-datadev-dlz-01"
+os.environ['DATABRICKS_SCHEMA_BASE']="lth_omop"
+os.environ['DATABRICKS_SCHEMA_VOCAB']="lth_omop"
+os.environ['DATABRICKS_CONCURRENT_TASKS']='8'
+os.environ['DATABRICKS_STATE_DB_HOST']="lscsdeprod-psql.postgres.database.azure.com"
+os.environ['DATABRICKS_STATE_DB_PORT']='5432'
+os.environ['DATABRICKS_STATE_DB_DATABASE']="sqlmesh_stroma"
+
+
+print('Environment variables set.')
+
+# COMMAND ----------
+
+# MAGIC %context ../stroma/
+
+# COMMAND ----------
+
+# Uncomment next line to run a migration when SQLMesh upgrade results in a request for migration.
+#  %migrate
+
+# COMMAND ----------
+
+# MAGIC %plan vc --include-unmodified

stroma/models/argentum/silver__care_site.sql

@@ -16,10 +16,10 @@ MODEL (
 );
 
 SELECT
-  cs.care_site_id::INT, /* A unique identifier for each Care Site. */
+  cs.care_site_id::BIGINT, /* A unique identifier for each Care Site. */
   cs.care_site_name::TEXT,
-  cs.place_of_service_concept_id::INT,
-  cs.location_id::INT,
+  cs.place_of_service_concept_id::BIGINT,
+  cs.location_id::BIGINT,
   cs.care_site_source_value::TEXT,
   cs.place_of_service_source_value::TEXT
 FROM bronze.care_site AS cs

stroma/models/argentum/silver__cost.sql

@@ -3,6 +3,15 @@ MODEL (
   kind FULL,
   cron '@monthly',
   grain cost_id,
+  references (
+    cost_event_id,
+    cost_domain_id,
+    cost_type_concept_id,
+    currency_concept_id,
+    payer_plan_period_id,
+    revenue_code_concept_id,
+    drg_concept_id
+  ),
   physical_properties ('delta.tuneFileSizesForRewrites' = FALSE, 'delta.targetFileSize' = '256mb'),
   description 'This table stores the cost details for medical events recorded in the OMOP clinical event tables',
   column_descriptions (

stroma/models/argentum/silver__death.sql

@@ -3,6 +3,12 @@ MODEL (
   kind FULL,
   cron '@monthly',
   grain person_id,
+  references (
+    person_id,
+    death_type_concept_id AS concept_id,
+    cause_concept_id AS concept_id,
+    cause_source_concept_id AS concept_id
+  ),
   physical_properties ('delta.tuneFileSizesForRewrites' = FALSE, 'delta.targetFileSize' = '256mb'),
   description 'This table stores records of death details for persons in the OMOP database.',
   column_descriptions (

stroma/models/argentum/silver__device_exposure.sql

@@ -3,6 +3,17 @@ MODEL (
   kind FULL,
   cron '@monthly',
   grain device_exposure_id,
+  references (
+    person_id,
+    device_concept_id AS concept_id,
+    device_type_concept_id AS concept_id,
+    provider_id AS provider_id,
+    visit_occurrence_id,
+    visit_detail_id,
+    device_source_concept_id AS concept_id,
+    unit_concept_id AS concept_id,
+    unit_source_concept_id AS concept_id
+  ),
   physical_properties ('delta.tuneFileSizesForRewrites' = FALSE, 'delta.targetFileSize' = '256mb'),
   description 'This table stores records of device exposures for persons in the OMOP database.',
   column_descriptions (
@@ -29,23 +40,23 @@ MODEL (
 );
 
 SELECT
-  de.device_exposure_id::INT,
-  de.person_id::INT,
-  de.device_concept_id::INT,
+  de.device_exposure_id::BIGINT,
+  de.person_id::BIGINT,
+  de.device_concept_id::BIGINT,
   de.device_exposure_start_date::DATE,
   de.device_exposure_start_datetime::TIMESTAMP,
   de.device_exposure_end_date::DATE,
   de.device_exposure_end_datetime::TIMESTAMP,
-  de.device_type_concept_id::INT,
+  de.device_type_concept_id::BIGINT,
   de.unique_device_id::TEXT,
   de.production_id::TEXT,
-  de.quantity::INT,
+  de.quantity::BIGINT,
   de.provider_id::TEXT,
-  de.visit_occurrence_id::INT,
-  de.visit_detail_id::INT,
+  de.visit_occurrence_id::BIGINT,
+  de.visit_detail_id::BIGINT,
   de.device_source_value::TEXT,
-  de.device_source_concept_id::INT,
-  de.unit_concept_id::INT,
+  de.device_source_concept_id::BIGINT,
+  de.unit_concept_id::BIGINT,
   de.unit_source_value::TEXT,
-  de.unit_source_concept_id::INT
+  de.unit_source_concept_id::BIGINT
 FROM bronze.device_exposure AS de

stroma/models/argentum/silver__drug_exposure.sql

@@ -3,6 +3,16 @@ MODEL (
   kind FULL,
   cron '@monthly',
   grain drug_exposure_id,
+  references (
+    person_id,
+    drug_concept_id AS concept_id,
+    drug_type_concept_id AS concept_id,
+    route_concept_id AS concept_id,
+    provider_id,
+    visit_occurrence_id,
+    visit_detail_id,
+    drug_source_concept_id AS concept_id
+    ),
   physical_properties ('delta.tuneFileSizesForRewrites' = FALSE, 'delta.targetFileSize' = '256mb'),
   description 'Records details of drugs a patient has been exposed to, including prescriptions and administrations.',
   column_descriptions (

stroma/models/argentum/silver__drug_strength.sql

@@ -2,6 +2,14 @@ MODEL (
   name silver.drug_strength,
   kind FULL,
   cron '@monthly',
+  references (
+    person_id,
+    drug_concept_id AS concept_id,
+    ingredient_concept_id AS concept_id,
+    amount_unit_concept_id AS concept_id,
+    numerator_unit_concept_id AS concept_id,
+    denominator_unit_concept_id AS concept_id
+    ),
   physical_properties ('delta.tuneFileSizesForRewrites' = FALSE, 'delta.targetFileSize' = '256mb'), 
   description 'The drug_strength table stores data on the strength and concentration of drug ingredients, including measurement units and validity periods.', 
   column_descriptions (

stroma/models/argentum/silver__episode.sql

@@ -2,6 +2,13 @@ MODEL (
   name silver.episode,
   kind FULL,
   cron '@monthly',
+  references (
+    person_id,
+    episode_concept_id AS concept_id,
+    episode_object_concept_id AS concept_id,
+    episode_type_concept_id AS concept_id,
+    episode_source_concept_id AS concept_id
+    ),
   physical_properties ('delta.tuneFileSizesForRewrites' = FALSE, 'delta.targetFileSize' = '256mb'),
   description 'Table containing information about episodes of care in the OMOP CDM',
   column_descriptions 

stroma/models/argentum/silver__episode_event.sql

@@ -2,6 +2,10 @@ MODEL (
   name silver.episode_event,
   kind FULL,
   cron '@monthly',
+  references (
+    episode_id,
+    episode_event_field_concept_id AS concept_id
+    ),
   physical_properties ('delta.tuneFileSizesForRewrites' = FALSE, 'delta.targetFileSize' = '256mb'),
   description 'Table capturing events associated with each episode in the OMOP CDM',
   column_descriptions (

stroma/models/argentum/silver__fact_relationship.sql

@@ -2,6 +2,11 @@ MODEL (
   name silver.fact_relationship,
   kind FULL,
   cron '@monthly',
+  references (
+    domain_concept_id_1 AS concept_id,
+    domain_concept_id_2 AS concept_id,
+    relationship_concept_id AS concept_id
+    ),
   physical_properties ('delta.tuneFileSizesForRewrites' = FALSE, 'delta.targetFileSize' = '256mb'),
   description 'Table capturing relationships between facts in the OMOP CDM',
   column_descriptions (

stroma/models/argentum/silver__location.sql

@@ -3,6 +3,9 @@ MODEL (
   kind FULL,
   cron '@monthly',
   grain location_id,
+  references (
+    country_concept_id AS concept_id
+    ),
   physical_properties ('delta.tuneFileSizesForRewrites' = FALSE, 'delta.targetFileSize' = '256mb'),
   description 'Table containing geographic and address information for persons and care sites in the OMOP CDM',
   column_descriptions (
@@ -22,7 +25,7 @@ MODEL (
 );
 
 SELECT
-  l.location_id::INT,
+  l.location_id::BIGINT,
   l.address_1::TEXT,
   l.address_2::TEXT,
   l.city::TEXT,

stroma/models/argentum/silver__measurement.sql

@@ -3,6 +3,20 @@ MODEL (
   kind FULL,
   cron '@monthly',
   grain unique_key,
+  references (
+    person_id,
+    measurement_concept_id AS concept_id,
+    measurement_type_concept_id AS concept_id,
+    operator_concept_id AS concept_id,
+    value_as_concept_id AS concept_id,
+    unit_concept_id AS concept_id,
+    provider_id,
+    visit_occurrence_id,
+    visit_detail_id,
+    measurement_source_concept_id AS concept_id,
+    unit_source_concept_id AS concept_id,
+    meas_event_field_concept_id AS concept_id
+    ),
   physical_properties ('delta.tuneFileSizesForRewrites' = FALSE, 'delta.targetFileSize' = '256mb'),
   description 'Table containing clinical measurements and observations for persons in the OMOP CDM',
   column_descriptions (

stroma/models/argentum/silver__note.sql

@@ -3,6 +3,17 @@ MODEL (
   kind FULL,
   cron '@monthly',
   grain note_id,
+  references (
+    person_id,
+    note_type_concept_id AS concept_id, 
+    note_class_concept_id AS concept_id,
+    encoding_concept_id AS concept_id,
+    language_concept_id AS concept_id,
+    provider_id,
+    visit_occurrence_id,
+    visit_detail_id,
+    note_event_field_concept_id AS concept_id
+    ),
   physical_properties ('delta.tuneFileSizesForRewrites' = FALSE, 'delta.targetFileSize' = '256mb'),
   description 'The note table contains unstructured clinical notes recorded during patient care, providing detailed narrative information.',
   column_descriptions (

stroma/models/argentum/silver__note_nlp.sql

@@ -3,6 +3,11 @@ MODEL (
   kind FULL,
   cron '@monthly',
   grain note_nlp_id,
+  references (
+    section_concept_id AS concept_id,
+    note_nlp_concept_id AS concept_id,
+    note_nlp_source_concept_id AS concept_id
+    ),
   physical_properties ('delta.tuneFileSizesForRewrites' = FALSE, 'delta.targetFileSize' = '256mb'),
   description 'The note_nlp table captures the results of natural language processing applied to clinical notes, detailing identified terms and their context.',
   column_descriptions (

stroma/models/argentum/silver__observation.sql

@@ -3,6 +3,19 @@ MODEL (
   kind FULL,
   cron '@monthly',
   grain observation_id,
+  references (
+    person_id,
+    observation_concept_id AS concept_id, 
+    observation_type_concept_id AS concept_id,
+    value_as_concept_id AS concept_id,
+    qualifier_concept_id AS concept_id,
+    unit_concept_id AS concept_id,
+    provider_id,
+    visit_occurrence_id,
+    visit_detail_id,
+    observation_source_concept_id AS concept_id,
+    obs_event_field_concept_id AS concept_id
+    ),
   physical_properties ('delta.tuneFileSizesForRewrites' = FALSE, 'delta.targetFileSize' = '256mb'),
   description 'The observation table captures clinical observations made about a person, including measurements, assessments, and other relevant clinical findings.',
   column_descriptions (
@@ -31,26 +44,27 @@ MODEL (
 );
 
 SELECT
-  o.observation_id::INT,
-  o.person_id::INT,
-  o.observation_concept_id::INT,
+  o.observation_id::BIGINT,
+  o.person_id::BIGINT,
+  o.observation_concept_id::BIGINT,
   o.observation_date::DATE,
   o.observation_datetime::TIMESTAMP,
-  o.observation_type_concept_id::INT,
+  o.observation_type_concept_id::BIGINT,
   o.value_as_number::REAL,
   o.value_as_string::TEXT,
-  o.value_as_concept_id::INT,
-  o.qualifier_concept_id::INT,
-  o.unit_concept_id::INT,
-  o.provider_id::INT,
-  o.visit_occurrence_id::INT,
-  o.visit_detail_id::INT,
+  o.value_as_concept_id::BIGINT,
+  o.qualifier_concept_id::BIGINT,
+  o.unit_concept_id::BIGINT,
+  o.provider_id::BIGINT,
+  o.visit_occurrence_id::BIGINT,
+  o.visit_detail_id::BIGINT,
   o.observation_source_value::TEXT,
-  o.observation_source_concept_id::INT,
+  o.observation_source_concept_id::BIGINT,
   o.unit_source_value::TEXT,
   o.qualifier_source_value::TEXT,
   o.value_source_value::TEXT,
-  o.observation_event_id::INT,
-  o.obs_event_field_concept_id::INT
+  o.observation_event_id::TEXT,
+  o.obs_event_field_concept_id::TEXT
 /* o.unique_key, */ /* o.datasource, */ /* o.updated_at */
-FROM bronze.observation AS o
+FROM bronze.observation AS o 
+

stroma/models/argentum/silver__payer_plan_period.sql

@@ -3,6 +3,17 @@ MODEL (
   kind FULL,
   cron '@monthly',
   grain payer_plan_period_id,
+  references (
+    person_id,
+    payer_concept_id AS concept_id, 
+    payer_source_concept_id AS concept_id,
+    plan_concept_id AS concept_id,
+    plan_source_concept_id AS concept_id,
+    sponsor_concept_id AS concept_id,
+    sponsor_source_concept_id AS concept_id,
+    stop_reason_concept_id AS concept_id,
+    stop_reason_source_concept_id AS concept_id
+    ),
   physical_properties ('delta.tuneFileSizesForRewrites' = FALSE, 'delta.targetFileSize' = '256mb'),
   description 'The payer_plan_period table captures details about the periods during which a person is covered by a specific payer and health plan.',
   column_descriptions (