set memory limit/request for blob storage connectors (#10861)

airbytehq · Mar 4, 2022 · fb7e259 · fb7e259
1 parent 21a8844
commit fb7e259
Showing 1 changed file with 38 additions and 12 deletions.
diff --git a/airbyte-config/init/src/main/resources/seed/destination_definitions.yaml b/airbyte-config/init/src/main/resources/seed/destination_definitions.yaml
@@ -4,6 +4,12 @@
   dockerImageTag: 0.1.3
   documentationUrl: https://docs.airbyte.io/integrations/destinations/azureblobstorage
   icon: azureblobstorage.svg
+  resourceRequirements:
+    jobSpecific:
+      - jobType: sync
+        resourceRequirements:
+          memory_limit: "1Gi"
+          memory_request: "1Gi"
 - name: Amazon SQS
   destinationDefinitionId: 0eeee7fb-518f-4045-bacc-9619e31c43ea
   dockerRepository: airbyte/destination-amazon-sqs
@@ -16,12 +22,24 @@
   dockerImageTag: 0.6.11
   documentationUrl: https://docs.airbyte.io/integrations/destinations/bigquery
   icon: bigquery.svg
+  resourceRequirements:
+    jobSpecific:
+      - jobType: sync
+        resourceRequirements:
+          memory_limit: "1Gi"
+          memory_request: "1Gi"
 - name: BigQuery (denormalized typed struct)
   destinationDefinitionId: 079d5540-f236-4294-ba7c-ade8fd918496
   dockerRepository: airbyte/destination-bigquery-denormalized
   dockerImageTag: 0.2.10
   documentationUrl: https://docs.airbyte.io/integrations/destinations/bigquery
   icon: bigquery.svg
+  resourceRequirements:
+    jobSpecific:
+      - jobType: sync
+        resourceRequirements:
+          memory_limit: "1Gi"
+          memory_request: "1Gi"
 - name: Cassandra
   destinationDefinitionId: 707456df-6f4f-4ced-b5c6-03f73bcad1c5
   dockerRepository: airbyte/destination-cassandra
@@ -63,6 +81,12 @@
   dockerImageTag: 0.1.24
   documentationUrl: https://docs.airbyte.io/integrations/destinations/gcs
   icon: googlecloudstorage.svg
+  resourceRequirements:
+    jobSpecific:
+      - jobType: sync
+        resourceRequirements:
+          memory_limit: "1Gi"
+          memory_request: "1Gi"
 - name: Google Firestore
   destinationDefinitionId: 27dc7500-6d1b-40b1-8b07-e2f2aea3c9f4
   dockerRepository: airbyte/destination-firestore
@@ -165,6 +189,12 @@
   dockerImageTag: 0.3.27
   documentationUrl: https://docs.airbyte.io/integrations/destinations/redshift
   icon: redshift.svg
+  resourceRequirements:
+    jobSpecific:
+      - jobType: sync
+        resourceRequirements:
+          memory_limit: "1Gi"
+          memory_request: "1Gi"
 - name: Rockset
   destinationDefinitionId: 2c9d93a7-9a17-4789-9de9-f46f0097eb70
   dockerRepository: airbyte/destination-rockset
@@ -176,6 +206,12 @@
   dockerImageTag: 0.2.9
   documentationUrl: https://docs.airbyte.io/integrations/destinations/s3
   icon: s3.svg
+  resourceRequirements:
+    jobSpecific:
+      - jobType: sync
+        resourceRequirements:
+          memory_limit: "1Gi"
+          memory_request: "1Gi"
 - name: SFTP-JSON
   destinationDefinitionId: e9810f61-4bab-46d2-bb22-edfc902e0644
   dockerRepository: airbyte/destination-sftp-json
@@ -189,21 +225,11 @@
   documentationUrl: https://docs.airbyte.io/integrations/destinations/snowflake
   icon: snowflake.svg
   resourceRequirements:
-    # this is our first example of setting these requirements. they are guesses, not data driven.
-    # setting default cpu higher, because we have found that check and discover can be cpu constrained for dbs.
-    default:
-      cpu_limit: "1.0"
-      cpu_request: "1.0"
-      memory_limit: "300Mi"
-      memory_request: "300Mi"
     jobSpecific:
-      # sync jobs are generally IO and memory bound and not cpu.
       - jobType: sync
         resourceRequirements:
-          cpu_limit: "0.5"
-          cpu_request: "0.5"
-          memory_limit: "600Mi"
-          memory_request: "600Mi"
+          memory_limit: "1Gi"
+          memory_request: "1Gi"
 - name: MariaDB ColumnStore
   destinationDefinitionId: 294a4790-429b-40ae-9516-49826b9702e1
   dockerRepository: airbyte/destination-mariadb-columnstore