Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Code used to build the CHoRUS year 1 cohort OMOP tables #1

Open
wants to merge 1 commit into
base: chorus_year1_cohort
Choose a base branch
from
Open

Code used to build the CHoRUS year 1 cohort OMOP tables #1

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
20 changes: 10 additions & 10 deletions conf/full.etlconf
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,20 +8,20 @@
"@hosp_dataset": "mimiciv_hosp",
"@icu_dataset": "mimiciv_icu",

"@voc_project": "odysseus-mimic-dev",
"@voc_dataset": "vocab_schema",
"@voc_project": "lcp-internal",
"@voc_dataset": "mimiciv_demo_omop",

"@wf_project": "odysseus-mimic-dev",
"@wf_dataset": "waveform_source_poc",
"@wf_project": "",
"@wf_dataset": "",

"@etl_project": "odysseus-mimic-dev",
"@etl_dataset": "mimiciv_full_cdm_*",
"@etl_project": "lcp-internal",
"@etl_dataset": "mimiciv_chorus_year1_cdm",

"@metrics_project": "odysseus-mimic-dev",
"@metrics_dataset": "mimiciv_full_metrics_*",
"@metrics_project": "lcp-internal",
"@metrics_dataset": "mimiciv_chorus_year1_metrics",

"@atlas_project": "odysseus-mimic-dev",
"@atlas_dataset": "mimiciv_full_current_cdm_*",
"@atlas_project": "lcp-internal",
"@atlas_dataset": "mimiciv_chorus_year1_atlas",

"@waveforms_csv_path": "gs://bucket..."

Expand Down
1 change: 0 additions & 1 deletion conf/workflow_etl.conf
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,6 @@
{"script": "etl/etl/lk_meas_chartevents.sql", "comment": ""},
{"script": "etl/etl/lk_meas_labevents.sql", "comment": "pick missing hadm_id by datetime"},
{"script": "etl/etl/lk_meas_specimen.sql", "comment": "pick missing hadm_id by datetime"},
{"script": "etl/etl/lk_meas_waveform.sql", "comment": "pick missing hadm_id by datetime"},

{"script": "etl/etl/lk_vis_part_2.sql", "comment": "add single day visits where there is still no hadm_id"},
{"script": "etl/etl/cdm_visit_occurrence.sql", "comment": ""},
Expand Down
2 changes: 0 additions & 2 deletions conf/workflow_staging.conf
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,8 +8,6 @@
{"script": "etl/staging/st_core.sql", "comment": ""},
{"script": "etl/staging/st_hosp.sql", "comment": ""},
{"script": "etl/staging/st_icu.sql", "comment": ""},
{"script": "etl/staging/st_waveform_poc_2.sql", "comment": ""},
{"script": "etl/staging/st_waveform_poc_3.sql", "comment": ""},

{"script": "etl/staging/voc_copy_to_target_dataset.sql", "comment": ""}
]
Expand Down
58 changes: 29 additions & 29 deletions etl/ddl/ddl_cdm_5_3_1.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@
/*OMOP CDM v5.3.1 14June2018*/


CREATE OR REPLACE TABLE `@etl_project`.@etl_dataset.cdm_cohort_definition (
CREATE OR REPLACE TABLE @etl_project.@etl_dataset.cdm_cohort_definition (
cohort_definition_id INT64 not null,
cohort_definition_name STRING not null,
cohort_definition_description STRING ,
Expand All @@ -18,7 +18,7 @@ CREATE OR REPLACE TABLE `@etl_project`.@etl_dataset.cdm_cohort_definition (
;


CREATE OR REPLACE TABLE `@etl_project`.@etl_dataset.cdm_attribute_definition (
CREATE OR REPLACE TABLE @etl_project.@etl_dataset.cdm_attribute_definition (
attribute_definition_id INT64 not null,
attribute_name STRING not null,
attribute_description STRING ,
Expand All @@ -28,7 +28,7 @@ CREATE OR REPLACE TABLE `@etl_project`.@etl_dataset.cdm_attribute_definition (
;


CREATE OR REPLACE TABLE `@etl_project`.@etl_dataset.cdm_cdm_source
CREATE OR REPLACE TABLE @etl_project.@etl_dataset.cdm_cdm_source
(
cdm_source_name STRING not null ,
cdm_source_abbreviation STRING ,
Expand All @@ -44,7 +44,7 @@ CREATE OR REPLACE TABLE `@etl_project`.@etl_dataset.cdm_cdm_source
;


CREATE OR REPLACE TABLE `@etl_project`.@etl_dataset.cdm_metadata
CREATE OR REPLACE TABLE @etl_project.@etl_dataset.cdm_metadata
(
metadata_concept_id INT64 not null ,
metadata_type_concept_id INT64 not null ,
Expand All @@ -59,7 +59,7 @@ CREATE OR REPLACE TABLE `@etl_project`.@etl_dataset.cdm_metadata


--HINT DISTRIBUTE_ON_KEY(person_id)
CREATE OR REPLACE TABLE `@etl_project`.@etl_dataset.cdm_person
CREATE OR REPLACE TABLE @etl_project.@etl_dataset.cdm_person
(
person_id INT64 not null ,
gender_concept_id INT64 not null ,
Expand All @@ -84,7 +84,7 @@ CREATE OR REPLACE TABLE `@etl_project`.@etl_dataset.cdm_person


--HINT DISTRIBUTE_ON_KEY(person_id)
CREATE OR REPLACE TABLE `@etl_project`.@etl_dataset.cdm_observation_period
CREATE OR REPLACE TABLE @etl_project.@etl_dataset.cdm_observation_period
(
observation_period_id INT64 not null ,
person_id INT64 not null ,
Expand All @@ -96,7 +96,7 @@ CREATE OR REPLACE TABLE `@etl_project`.@etl_dataset.cdm_observation_period


--HINT DISTRIBUTE_ON_KEY(person_id)
CREATE OR REPLACE TABLE `@etl_project`.@etl_dataset.cdm_specimen
CREATE OR REPLACE TABLE @etl_project.@etl_dataset.cdm_specimen
(
specimen_id INT64 not null ,
person_id INT64 not null ,
Expand All @@ -118,7 +118,7 @@ CREATE OR REPLACE TABLE `@etl_project`.@etl_dataset.cdm_specimen


--HINT DISTRIBUTE_ON_KEY(person_id)
CREATE OR REPLACE TABLE `@etl_project`.@etl_dataset.cdm_death
CREATE OR REPLACE TABLE @etl_project.@etl_dataset.cdm_death
(
person_id INT64 not null ,
death_date DATE not null ,
Expand All @@ -132,7 +132,7 @@ CREATE OR REPLACE TABLE `@etl_project`.@etl_dataset.cdm_death


--HINT DISTRIBUTE_ON_KEY(person_id)
CREATE OR REPLACE TABLE `@etl_project`.@etl_dataset.cdm_visit_occurrence
CREATE OR REPLACE TABLE @etl_project.@etl_dataset.cdm_visit_occurrence
(
visit_occurrence_id INT64 not null ,
person_id INT64 not null ,
Expand All @@ -156,7 +156,7 @@ CREATE OR REPLACE TABLE `@etl_project`.@etl_dataset.cdm_visit_occurrence


--HINT DISTRIBUTE_ON_KEY(person_id)
CREATE OR REPLACE TABLE `@etl_project`.@etl_dataset.cdm_visit_detail
CREATE OR REPLACE TABLE @etl_project.@etl_dataset.cdm_visit_detail
(
visit_detail_id INT64 not null ,
person_id INT64 not null ,
Expand All @@ -182,7 +182,7 @@ CREATE OR REPLACE TABLE `@etl_project`.@etl_dataset.cdm_visit_detail


--HINT DISTRIBUTE_ON_KEY(person_id)
CREATE OR REPLACE TABLE `@etl_project`.@etl_dataset.cdm_procedure_occurrence
CREATE OR REPLACE TABLE @etl_project.@etl_dataset.cdm_procedure_occurrence
(
procedure_occurrence_id INT64 not null ,
person_id INT64 not null ,
Expand All @@ -203,7 +203,7 @@ CREATE OR REPLACE TABLE `@etl_project`.@etl_dataset.cdm_procedure_occurrence


--HINT DISTRIBUTE_ON_KEY(person_id)
CREATE OR REPLACE TABLE `@etl_project`.@etl_dataset.cdm_drug_exposure
CREATE OR REPLACE TABLE @etl_project.@etl_dataset.cdm_drug_exposure
(
drug_exposure_id INT64 not null ,
person_id INT64 not null ,
Expand Down Expand Up @@ -233,7 +233,7 @@ CREATE OR REPLACE TABLE `@etl_project`.@etl_dataset.cdm_drug_exposure


--HINT DISTRIBUTE_ON_KEY(person_id)
CREATE OR REPLACE TABLE `@etl_project`.@etl_dataset.cdm_device_exposure
CREATE OR REPLACE TABLE @etl_project.@etl_dataset.cdm_device_exposure
(
device_exposure_id INT64 not null ,
person_id INT64 not null ,
Expand All @@ -255,7 +255,7 @@ CREATE OR REPLACE TABLE `@etl_project`.@etl_dataset.cdm_device_exposure


--HINT DISTRIBUTE_ON_KEY(person_id)
CREATE OR REPLACE TABLE `@etl_project`.@etl_dataset.cdm_condition_occurrence
CREATE OR REPLACE TABLE @etl_project.@etl_dataset.cdm_condition_occurrence
(
condition_occurrence_id INT64 not null ,
person_id INT64 not null ,
Expand All @@ -278,7 +278,7 @@ CREATE OR REPLACE TABLE `@etl_project`.@etl_dataset.cdm_condition_occurrence


--HINT DISTRIBUTE_ON_KEY(person_id)
CREATE OR REPLACE TABLE `@etl_project`.@etl_dataset.cdm_measurement
CREATE OR REPLACE TABLE @etl_project.@etl_dataset.cdm_measurement
(
measurement_id INT64 not null ,
person_id INT64 not null ,
Expand All @@ -305,7 +305,7 @@ CREATE OR REPLACE TABLE `@etl_project`.@etl_dataset.cdm_measurement


--HINT DISTRIBUTE_ON_KEY(person_id)
CREATE OR REPLACE TABLE `@etl_project`.@etl_dataset.cdm_note
CREATE OR REPLACE TABLE @etl_project.@etl_dataset.cdm_note
(
note_id INT64 not null ,
person_id INT64 not null ,
Expand All @@ -326,7 +326,7 @@ CREATE OR REPLACE TABLE `@etl_project`.@etl_dataset.cdm_note



CREATE OR REPLACE TABLE `@etl_project`.@etl_dataset.cdm_note_nlp
CREATE OR REPLACE TABLE @etl_project.@etl_dataset.cdm_note_nlp
(
note_nlp_id INT64 ,
note_id INT64 ,
Expand All @@ -347,7 +347,7 @@ CREATE OR REPLACE TABLE `@etl_project`.@etl_dataset.cdm_note_nlp


--HINT DISTRIBUTE_ON_KEY(person_id)
CREATE OR REPLACE TABLE `@etl_project`.@etl_dataset.cdm_observation
CREATE OR REPLACE TABLE @etl_project.@etl_dataset.cdm_observation
(
observation_id INT64 not null ,
person_id INT64 not null ,
Expand All @@ -371,7 +371,7 @@ CREATE OR REPLACE TABLE `@etl_project`.@etl_dataset.cdm_observation
;


CREATE OR REPLACE TABLE `@etl_project`.@etl_dataset.cdm_fact_relationship
CREATE OR REPLACE TABLE @etl_project.@etl_dataset.cdm_fact_relationship
(
domain_concept_id_1 INT64 not null ,
fact_id_1 INT64 not null ,
Expand All @@ -382,7 +382,7 @@ CREATE OR REPLACE TABLE `@etl_project`.@etl_dataset.cdm_fact_relationship
;


CREATE OR REPLACE TABLE `@etl_project`.@etl_dataset.cdm_location
CREATE OR REPLACE TABLE @etl_project.@etl_dataset.cdm_location
(
location_id INT64 not null ,
address_1 STRING ,
Expand All @@ -396,7 +396,7 @@ CREATE OR REPLACE TABLE `@etl_project`.@etl_dataset.cdm_location
;


CREATE OR REPLACE TABLE `@etl_project`.@etl_dataset.cdm_care_site
CREATE OR REPLACE TABLE @etl_project.@etl_dataset.cdm_care_site
(
care_site_id INT64 not null ,
care_site_name STRING ,
Expand All @@ -408,7 +408,7 @@ CREATE OR REPLACE TABLE `@etl_project`.@etl_dataset.cdm_care_site
;


CREATE OR REPLACE TABLE `@etl_project`.@etl_dataset.cdm_provider
CREATE OR REPLACE TABLE @etl_project.@etl_dataset.cdm_provider
(
provider_id INT64 not null ,
provider_name STRING ,
Expand All @@ -428,7 +428,7 @@ CREATE OR REPLACE TABLE `@etl_project`.@etl_dataset.cdm_provider


--HINT DISTRIBUTE_ON_KEY(person_id)
CREATE OR REPLACE TABLE `@etl_project`.@etl_dataset.cdm_payer_plan_period
CREATE OR REPLACE TABLE @etl_project.@etl_dataset.cdm_payer_plan_period
(
payer_plan_period_id INT64 not null ,
person_id INT64 not null ,
Expand All @@ -451,7 +451,7 @@ CREATE OR REPLACE TABLE `@etl_project`.@etl_dataset.cdm_payer_plan_period
;


CREATE OR REPLACE TABLE `@etl_project`.@etl_dataset.cdm_cost
CREATE OR REPLACE TABLE @etl_project.@etl_dataset.cdm_cost
(
cost_id INT64 not null ,
cost_event_id INT64 not null ,
Expand Down Expand Up @@ -480,7 +480,7 @@ CREATE OR REPLACE TABLE `@etl_project`.@etl_dataset.cdm_cost


--HINT DISTRIBUTE_ON_KEY(subject_id)
CREATE OR REPLACE TABLE `@etl_project`.@etl_dataset.cdm_cohort
CREATE OR REPLACE TABLE @etl_project.@etl_dataset.cdm_cohort
(
cohort_definition_id INT64 not null ,
subject_id INT64 not null ,
Expand All @@ -491,7 +491,7 @@ CREATE OR REPLACE TABLE `@etl_project`.@etl_dataset.cdm_cohort


--HINT DISTRIBUTE_ON_KEY(subject_id)
CREATE OR REPLACE TABLE `@etl_project`.@etl_dataset.cdm_cohort_attribute
CREATE OR REPLACE TABLE @etl_project.@etl_dataset.cdm_cohort_attribute
(
cohort_definition_id INT64 not null ,
subject_id INT64 not null ,
Expand All @@ -505,7 +505,7 @@ CREATE OR REPLACE TABLE `@etl_project`.@etl_dataset.cdm_cohort_attribute


--HINT DISTRIBUTE_ON_KEY(person_id)
CREATE OR REPLACE TABLE `@etl_project`.@etl_dataset.cdm_drug_era
CREATE OR REPLACE TABLE @etl_project.@etl_dataset.cdm_drug_era
(
drug_era_id INT64 not null ,
person_id INT64 not null ,
Expand All @@ -519,7 +519,7 @@ CREATE OR REPLACE TABLE `@etl_project`.@etl_dataset.cdm_drug_era


--HINT DISTRIBUTE_ON_KEY(person_id)
CREATE OR REPLACE TABLE `@etl_project`.@etl_dataset.cdm_dose_era
CREATE OR REPLACE TABLE @etl_project.@etl_dataset.cdm_dose_era
(
dose_era_id INT64 not null ,
person_id INT64 not null ,
Expand All @@ -533,7 +533,7 @@ CREATE OR REPLACE TABLE `@etl_project`.@etl_dataset.cdm_dose_era


--HINT DISTRIBUTE_ON_KEY(person_id)
CREATE OR REPLACE TABLE `@etl_project`.@etl_dataset.cdm_condition_era
CREATE OR REPLACE TABLE @etl_project.@etl_dataset.cdm_condition_era
(
condition_era_id INT64 not null ,
person_id INT64 not null ,
Expand Down
20 changes: 10 additions & 10 deletions etl/ddl/ddl_voc_5_3_1.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
/*OMOP CDM v5.3.1 14June2018*/

CREATE OR REPLACE TABLE `@etl_project`.@etl_dataset.voc_concept (
CREATE OR REPLACE TABLE @etl_project.@etl_dataset.voc_concept (
concept_id INT64 not null ,
concept_name STRING not null ,
domain_id STRING not null ,
Expand All @@ -15,7 +15,7 @@ CREATE OR REPLACE TABLE `@etl_project`.@etl_dataset.voc_concept (
;


CREATE OR REPLACE TABLE `@etl_project`.@etl_dataset.voc_vocabulary (
CREATE OR REPLACE TABLE @etl_project.@etl_dataset.voc_vocabulary (
vocabulary_id STRING not null,
vocabulary_name STRING not null,
vocabulary_reference STRING not null,
Expand All @@ -25,23 +25,23 @@ CREATE OR REPLACE TABLE `@etl_project`.@etl_dataset.voc_vocabulary (
;


CREATE OR REPLACE TABLE `@etl_project`.@etl_dataset.voc_domain (
CREATE OR REPLACE TABLE @etl_project.@etl_dataset.voc_domain (
domain_id STRING not null,
domain_name STRING not null,
domain_concept_id INT64 not null
)
;


CREATE OR REPLACE TABLE `@etl_project`.@etl_dataset.voc_concept_class (
CREATE OR REPLACE TABLE @etl_project.@etl_dataset.voc_concept_class (
concept_class_id STRING not null,
concept_class_name STRING not null,
concept_class_concept_id INT64 not null
)
;


CREATE OR REPLACE TABLE `@etl_project`.@etl_dataset.voc_concept_relationship (
CREATE OR REPLACE TABLE @etl_project.@etl_dataset.voc_concept_relationship (
concept_id_1 INT64 not null,
concept_id_2 INT64 not null,
relationship_id STRING not null,
Expand All @@ -52,7 +52,7 @@ CREATE OR REPLACE TABLE `@etl_project`.@etl_dataset.voc_concept_relationship (
;


CREATE OR REPLACE TABLE `@etl_project`.@etl_dataset.voc_relationship (
CREATE OR REPLACE TABLE @etl_project.@etl_dataset.voc_relationship (
relationship_id STRING not null,
relationship_name STRING not null,
is_hierarchical STRING not null,
Expand All @@ -63,15 +63,15 @@ CREATE OR REPLACE TABLE `@etl_project`.@etl_dataset.voc_relationship (
;


CREATE OR REPLACE TABLE `@etl_project`.@etl_dataset.voc_concept_synonym (
CREATE OR REPLACE TABLE @etl_project.@etl_dataset.voc_concept_synonym (
concept_id INT64 not null,
concept_synonym_name STRING not null,
language_concept_id INT64 not null
)
;


CREATE OR REPLACE TABLE `@etl_project`.@etl_dataset.voc_concept_ancestor (
CREATE OR REPLACE TABLE @etl_project.@etl_dataset.voc_concept_ancestor (
ancestor_concept_id INT64 not null,
descendant_concept_id INT64 not null,
min_levels_of_separation INT64 not null,
Expand All @@ -80,7 +80,7 @@ CREATE OR REPLACE TABLE `@etl_project`.@etl_dataset.voc_concept_ancestor (
;


CREATE OR REPLACE TABLE `@etl_project`.@etl_dataset.voc_source_to_concept_map (
CREATE OR REPLACE TABLE @etl_project.@etl_dataset.voc_source_to_concept_map (
source_code STRING not null,
source_concept_id INT64 not null,
source_vocabulary_id STRING not null,
Expand All @@ -94,7 +94,7 @@ CREATE OR REPLACE TABLE `@etl_project`.@etl_dataset.voc_source_to_concept_map (
;


CREATE OR REPLACE TABLE `@etl_project`.@etl_dataset.voc_drug_strength (
CREATE OR REPLACE TABLE @etl_project.@etl_dataset.voc_drug_strength (
drug_concept_id INT64 not null,
ingredient_concept_id INT64 not null,
amount_value FLOAT64 ,
Expand Down
Loading