Add medium job types to JSON to Parquet job

Sage-Bionetworks · Nov 20, 2024 · 10ed80a · 10ed80a
1 parent b18c586
commit 10ed80a
Showing 1 changed file with 30 additions and 10 deletions.
diff --git a/templates/glue-job-JSONToParquet.j2 b/templates/glue-job-JSONToParquet.j2
@@ -30,27 +30,43 @@ Parameters:
   DefaultWorkerType:
     Type: String
     Description: >-
-        Which worker type to use for this job.
+        Which worker type to use for most data types
     Default: 'Standard'
 
-  LargeJobWorkerType:
+  MediumJobWorkerType:
     Type: String
     Description: >-
         Which worker type to use for this job.
+        Medium data types include: HealthKitV2Samples, HealthKitV2Electrocardiogram,
+        FitbitDailyData, FitbitSleepLogs
     Default: 'G.4X'
 
+  LargeJobWorkerType:
+    Type: String
+    Description: >-
+        Which worker type to use for this job.
+        Large data types include: FitbitIntradayCombined
+    Default: 'G.8X'
+
   DefaultNumberOfWorkers:
     Type: Number
     Description: >-
-        How many DPUs to allot to this job. This parameter is not used for types
-        FitbitIntradayCombined and HealthKitV2Samples.
+        How many DPUs to allot for most data types.
     Default: 1
 
+  MediumJobNumberOfWorkers:
+    Type: Number
+    Description: >-
+        How many DPUs to allot to this job.
+        Medium data types include: HealthKitV2Samples, HealthKitV2Electrocardiogram,
+        FitbitDailyData, FitbitSleepLogs
+    Default: 4
+
   LargeJobNumberOfWorkers:
     Type: Number
     Description: >-
-        How many DPUs to allot to this job. This parameter overrides `DefaultNumberOfWorkers`
-        for data types FitbitIntradayCombined and HealthKitV2Samples.
+        How many DPUs to allot to this job.
+        Large data types include: FitbitIntradayCombined
     Default: 8
 
   MaxRetries:
@@ -110,13 +126,17 @@ Resources:
       GlueVersion: !Ref GlueVersion
       MaxRetries: !Ref MaxRetries
       Name: !Sub "${Namespace}-{{ dataset["stackname_prefix"] }}-Job"
-      {% if (
+      {% if dataset["type"] == "FitbitIntradayCombined" -%}
+      WorkerType: !Ref LargeJobWorkerType
+      NumberOfWorkers: !Ref LargeJobNumberOfWorkers
+      {% elif (
         dataset["type"] == "HealthKitV2Samples"
-        or dataset["type"] == "FitbitIntradayCombined"
         or dataset["type"] == "HealthKitV2Electrocardiogram"
+        or dataset["type"] == "FitbitDailyData"
+        or dataset["type"] == "FitbitSleepLogs"
       ) -%}
-      WorkerType: !Ref LargeJobWorkerType
-      NumberOfWorkers: !Ref LargeJobNumberOfWorkers
+      WorkerType: !Ref MediumJobWorkerType
+      NumberOfWorkers: !Ref MediumJobNumberOfWorkers
       {% else -%}
       WorkerType: !Ref DefaultWorkerType
       NumberOfWorkers: !Ref DefaultNumberOfWorkers