Add cohort to partition_keys of Glue tables

Sage-Bionetworks · Sep 1, 2023 · f186062 · f186062
1 parent 5cd8b76
commit f186062
Show file tree

Hide file tree

Showing 2 changed files with 98 additions and 0 deletions.
diff --git a/src/glue/resources/table_columns.yaml b/src/glue/resources/table_columns.yaml
@@ -37,6 +37,9 @@ tables:
         Type: string
       - Name: export_end_date
         Type: string
+    partition_keys:
+      - Name: cohort
+        Type: string
   HealthKitV2Statistics:
     columns:
       - Name: HealthKitStatisticKey
@@ -61,6 +64,9 @@ tables:
         Type: string
       - Name: export_end_date
         Type: string
+    partition_keys:
+      - Name: cohort
+        Type: string
   HealthKitV2Samples:
     columns:
       - Name: HealthKitSampleKey
@@ -91,6 +97,9 @@ tables:
         Type: string
       - Name: export_end_date
         Type: string
+    partition_keys:
+      - Name: cohort
+        Type: string
   HealthKitV2Samples_Deleted:
     columns:
       - Name: HealthKitSampleKey
@@ -105,6 +114,9 @@ tables:
         Type: string
       - Name: export_end_date
         Type: string
+    partition_keys:
+      - Name: cohort
+        Type: string
   HealthKitV2ActivitySummaries:
     columns:
       - Name: HealthKitActivitySummaryKey
@@ -135,6 +147,9 @@ tables:
         Type: string
       - Name: export_end_date
         Type: string
+    partition_keys:
+      - Name: cohort
+        Type: string
   HealthKitV2Electrocardiogram:
     columns:
       - Name: HealthKitECGSampleKey
@@ -171,6 +186,9 @@ tables:
         Type: string
       - Name: export_end_date
         Type: string
+    partition_keys:
+      - Name: cohort
+        Type: string
   HealthKitV2Workouts:
     columns:
       - Name: HealthKitWorkoutKey
@@ -199,6 +217,9 @@ tables:
         Type: string
       - Name: export_end_date
         Type: string
+    partition_keys:
+      - Name: cohort
+        Type: string
   HealthKitV2Heartbeat:
     columns:
       - Name: HealthKitHeartbeatSampleKey
@@ -225,6 +246,9 @@ tables:
         Type: string
       - Name: export_end_date
         Type: string
+    partition_keys:
+      - Name: cohort
+        Type: string
   FitbitActivityLogs:
     columns:
       - Name: ParticipantIdentifier
@@ -273,6 +297,9 @@ tables:
         Type: string
       - Name: export_end_date
         Type: string
+    partition_keys:
+      - Name: cohort
+        Type: string
   FitbitDevices:
     columns:
       - Name: ParticipantIdentifier
@@ -291,6 +318,9 @@ tables:
         Type: string
       - Name: export_end_date
         Type: string
+    partition_keys:
+      - Name: cohort
+        Type: string
   FitbitDailyData:
     columns:
       - Name: ParticipantIdentifier
@@ -359,6 +389,9 @@ tables:
         Type: string
       - Name: export_end_date
         Type: string
+    partition_keys:
+      - Name: cohort
+        Type: string
   FitbitIntradayCombined:
     columns:
       - Name: ParticipantID
@@ -397,6 +430,9 @@ tables:
         Type: string
       - Name: export_end_date
         Type: string
+    partition_keys:
+      - Name: cohort
+        Type: string
   FitbitRestingHeartRates:
     columns:
       - Name: ParticipantIdentifier
@@ -411,6 +447,9 @@ tables:
         Type: string
       - Name: export_end_date
         Type: string
+    partition_keys:
+      - Name: cohort
+        Type: string
   FitbitSleepLogs:
     columns:
       - Name: ParticipantIdentifier
@@ -461,6 +500,9 @@ tables:
         Type: string
       - Name: export_end_date
         Type: string
+    partition_keys:
+      - Name: cohort
+        Type: string
   GoogleFitSamples:
     columns:
       - Name: GoogleFitSampleKey
@@ -505,6 +547,9 @@ tables:
         Type: string
       - Name: export_end_date
         Type: string
+    partition_keys:
+      - Name: cohort
+        Type: string
   SymptomLog:
     columns:
       - Name: DatapointKey
@@ -525,6 +570,9 @@ tables:
         Type: string
       - Name: export_end_date
         Type: string
+    partition_keys:
+      - Name: cohort
+        Type: string
   GarminActivityDetailsSummary:
     columns:
       - Name: ParticipantIdentifier
@@ -553,6 +601,9 @@ tables:
         Type: string
       - Name: export_end_date
         Type: string
+    partition_keys:
+      - Name: cohort
+        Type: string
   GarminActivitySummary:
     columns:
       - Name: ParticipantIdentifier
@@ -661,6 +712,9 @@ tables:
         Type: string
       - Name: export_end_date
         Type: string
+    partition_keys:
+      - Name: cohort
+        Type: string
   GarminBloodPressureSummary:
     columns:
       - Name: ParticipantIdentifier
@@ -687,6 +741,9 @@ tables:
         Type: string
       - Name: export_end_date
         Type: string
+    partition_keys:
+      - Name: cohort
+        Type: string
   GarminBodyCompositionSummary:
     columns:
       - Name: ParticipantIdentifier
@@ -717,6 +774,9 @@ tables:
         Type: string
       - Name: export_end_date
         Type: string
+    partition_keys:
+      - Name: cohort
+        Type: string
   GarminDailySummary:
     columns:
       - Name: ParticipantIdentifier
@@ -789,6 +849,9 @@ tables:
         Type: string
       - Name: export_end_date
         Type: string
+    partition_keys:
+      - Name: cohort
+        Type: string
   GarminEpochSummary:
     columns:
       - Name: ParticipantIdentifier
@@ -827,6 +890,9 @@ tables:
         Type: string
       - Name: export_end_date
         Type: string
+    partition_keys:
+      - Name: cohort
+        Type: string
   GarminHealthSnapshotSummary:
     columns:
       - Name: ParticipantIdentifier
@@ -851,6 +917,9 @@ tables:
         Type: string
       - Name: export_end_date
         Type: string
+    partition_keys:
+      - Name: cohort
+        Type: string
   GarminHrvSummary:
     columns:
       - Name: ParticipantIdentifier
@@ -879,6 +948,9 @@ tables:
         Type: string
       - Name: export_end_date
         Type: string
+    partition_keys:
+      - Name: cohort
+        Type: string
   GarminManuallyUpdatedActivitySummary:
     columns:
       - Name: ParticipantIdentifier
@@ -951,6 +1023,9 @@ tables:
         Type: string
       - Name: export_end_date
         Type: string
+    partition_keys:
+      - Name: cohort
+        Type: string
   GarminMoveIQActivitySummary:
     columns:
       - Name: ParticipantIdentifier
@@ -981,6 +1056,9 @@ tables:
         Type: string
       - Name: export_end_date
         Type: string
+    partition_keys:
+      - Name: cohort
+        Type: string
   GarminPulseOxSummary:
     columns:
       - Name: ParticipantIdentifier
@@ -1007,6 +1085,9 @@ tables:
         Type: string
       - Name: export_end_date
         Type: string
+    partition_keys:
+      - Name: cohort
+        Type: string
   GarminRespirationSummary:
     columns:
       - Name: ParticipantIdentifier
@@ -1029,6 +1110,9 @@ tables:
         Type: string
       - Name: export_end_date
         Type: string
+    partition_keys:
+      - Name: cohort
+        Type: string
   GarminSleepSummary:
     columns:
       - Name: ParticipantIdentifier
@@ -1075,6 +1159,9 @@ tables:
         Type: string
       - Name: export_end_date
         Type: string
+    partition_keys:
+      - Name: cohort
+        Type: string
   GarminStressDetailSummary:
     columns:
       - Name: ParticipantIdentifier
@@ -1101,6 +1188,9 @@ tables:
         Type: string
       - Name: export_end_date
         Type: string
+    partition_keys:
+      - Name: cohort
+        Type: string
   GarminThirdPartyDailySummary:
     columns:
       - Name: ParticipantIdentifier
@@ -1149,6 +1239,9 @@ tables:
         Type: string
       - Name: export_end_date
         Type: string
+    partition_keys:
+      - Name: cohort
+        Type: string
   GarminUserMetricsSummary:
     columns:
       - Name: ParticipantIdentifier
@@ -1171,3 +1264,6 @@ tables:
         Type: string
       - Name: export_end_date
         Type: string
+    partition_keys:
+      - Name: cohort
+        Type: string
diff --git a/templates/glue-tables.j2 b/templates/glue-tables.j2
@@ -27,6 +27,7 @@ Resources:
   {% do dataset.update({'table_name': 'dataset_' + v.lower()})%}
   {% set schema = sceptre_user_data.dataset_schemas.tables[v] %}
   {% do dataset.update({'columns': schema['columns']}) %}
+  {% do dataset.update({'partition_keys': schema['partition_keys']}) %}
   {% do dataset.update({'stackname_prefix': '{}'.format(v.replace('_',''))}) %}
   {% do datasets.append(dataset) %}
   {% endfor %}
@@ -53,6 +54,7 @@ Resources:
           compressionType: none
           typeOfData: file
         Retention: 0
+        PartitionKeys: {{ dataset.partition_keys }}
         StorageDescriptor:
           Columns: {{ dataset.columns }}
           Compressed: false