Kube: Better Port Abstraction. (#4829)

Introduce a better port abstraction whose primary purpose is to confirm that ports are released when the Kube Pod Process is closed. This prevents issues like #4660 I'm also opening more ports so we can run at least 10 syncs in parallel.
airbytehq · Jul 21, 2021 · 350bb58 · 350bb58
1 parent 356ca18
commit 350bb58
Show file tree

Hide file tree

Showing 17 changed files with 327 additions and 680 deletions.
diff --git a/airbyte-config/models/src/main/java/io/airbyte/config/EnvConfigs.java b/airbyte-config/models/src/main/java/io/airbyte/config/EnvConfigs.java
@@ -28,6 +28,7 @@
 import io.airbyte.config.helpers.LogClientSingleton;
 import java.nio.file.Path;
 import java.util.Arrays;
+import java.util.HashSet;
 import java.util.Optional;
 import java.util.Set;
 import java.util.function.Function;
@@ -213,9 +214,11 @@ public String getTemporalHost() {
 
   @Override
   public Set<Integer> getTemporalWorkerPorts() {
-    return Arrays.stream(getEnvOrDefault(TEMPORAL_WORKER_PORTS, "").split(","))
-        .map(Integer::valueOf)
-        .collect(Collectors.toSet());
+    var ports = getEnvOrDefault(TEMPORAL_WORKER_PORTS, "");
+    if (ports.isEmpty()) {
+      return new HashSet<>();
+    }
+    return Arrays.stream(ports.split(",")).map(Integer::valueOf).collect(Collectors.toSet());
   }
 
   @Override

diff --git a/airbyte-integrations/connectors/source-jira/integration_tests/configured_catalog.json b/airbyte-integrations/connectors/source-jira/integration_tests/configured_catalog.json
@@ -7406,53 +7406,30 @@
         "name": "labels",
         "json_schema": {
           "$schema": "http://json-schema.org/draft-07/schema#",
-          "type": [
-            "object",
-            "null"
-          ],
+          "type": ["object", "null"],
           "properties": {
             "id": {
-              "type": [
-                "string",
-                "null"
-              ]
+              "type": ["string", "null"]
             },
             "key": {
-              "type": [
-                "string",
-                "null"
-              ]
+              "type": ["string", "null"]
             },
             "value": {
-              "type": [
-                "string",
-                "null"
-              ]
+              "type": ["string", "null"]
             },
             "name": {
-              "type": [
-                "string",
-                "null"
-              ]
+              "type": ["string", "null"]
             },
             "desc": {
-              "type": [
-                "string",
-                "null"
-              ]
+              "type": ["string", "null"]
             },
             "type": {
-              "type": [
-                "string",
-                "null"
-              ]
+              "type": ["string", "null"]
             }
           },
           "additionalProperties": true
         },
-        "supported_sync_modes": [
-          "full_refresh"
-        ],
+        "supported_sync_modes": ["full_refresh"],
         "source_defined_cursor": false
       },
       "sync_mode": "full_refresh",

diff --git a/airbyte-integrations/connectors/source-jira/integration_tests/full_configured_catalog.json b/airbyte-integrations/connectors/source-jira/integration_tests/full_configured_catalog.json
@@ -9593,53 +9593,30 @@
         "name": "labels",
         "json_schema": {
           "$schema": "http://json-schema.org/draft-07/schema#",
-          "type": [
-            "object",
-            "null"
-          ],
+          "type": ["object", "null"],
           "properties": {
             "id": {
-              "type": [
-                "string",
-                "null"
-              ]
+              "type": ["string", "null"]
             },
             "key": {
-              "type": [
-                "string",
-                "null"
-              ]
+              "type": ["string", "null"]
             },
             "value": {
-              "type": [
-                "string",
-                "null"
-              ]
+              "type": ["string", "null"]
             },
             "name": {
-              "type": [
-                "string",
-                "null"
-              ]
+              "type": ["string", "null"]
             },
             "desc": {
-              "type": [
-                "string",
-                "null"
-              ]
+              "type": ["string", "null"]
             },
             "type": {
-              "type": [
-                "string",
-                "null"
-              ]
+              "type": ["string", "null"]
             }
           },
           "additionalProperties": true
         },
-        "supported_sync_modes": [
-          "full_refresh"
-        ],
+        "supported_sync_modes": ["full_refresh"],
         "source_defined_cursor": false
       },
       "sync_mode": "full_refresh",

diff --git a/airbyte-integrations/connectors/source-jira/integration_tests/labels_catalog.json b/airbyte-integrations/connectors/source-jira/integration_tests/labels_catalog.json
@@ -1,39 +1,38 @@
 {
-    "streams": [
-      {
-        "stream": {
-          "name": "labels",
-          "json_schema": {
-            "$schema": "http://json-schema.org/draft-07/schema#",
-            "type": ["object", "null"],
-            "properties": {
-                "id": {
-                    "type": ["string", "null"]
-                },
-                "key": {
-                    "type": ["string", "null"]
-                },
-                "value": {
-                    "type": ["string", "null"]
-                },
-                "name": {
-                    "type": ["string", "null"]
-                },
-                "desc": {
-                    "type": ["string", "null"]
-                },
-                "type": {
-                    "type": ["string", "null"]
-                }
+  "streams": [
+    {
+      "stream": {
+        "name": "labels",
+        "json_schema": {
+          "$schema": "http://json-schema.org/draft-07/schema#",
+          "type": ["object", "null"],
+          "properties": {
+            "id": {
+              "type": ["string", "null"]
             },
-            "additionalProperties": true
+            "key": {
+              "type": ["string", "null"]
+            },
+            "value": {
+              "type": ["string", "null"]
+            },
+            "name": {
+              "type": ["string", "null"]
+            },
+            "desc": {
+              "type": ["string", "null"]
+            },
+            "type": {
+              "type": ["string", "null"]
+            }
           },
-          "supported_sync_modes": ["full_refresh"],
-          "source_defined_cursor": false
+          "additionalProperties": true
         },
-        "sync_mode": "full_refresh",
-        "destination_sync_mode": "overwrite"
-      }
-    ]
-  }
-
+        "supported_sync_modes": ["full_refresh"],
+        "source_defined_cursor": false
+      },
+      "sync_mode": "full_refresh",
+      "destination_sync_mode": "overwrite"
+    }
+  ]
+}