From 49c8d63ba5cba71e52a7836bc2b007bc55a562b4 Mon Sep 17 00:00:00 2001 From: alovew Date: Tue, 24 Jan 2023 10:35:08 -0800 Subject: [PATCH 1/2] Turn on auto-detect schema env variable --- .env | 2 +- .env.dev | 2 +- .../io/airbyte/commons/features/EnvVariableFeatureFlags.java | 2 +- .../src/main/java/io/airbyte/config/EnvConfigs.java | 2 +- .../process/AsyncOrchestratorPodProcessIntegrationTest.java | 2 +- charts/airbyte/templates/env-configmap.yaml | 2 +- kube/overlays/dev-integration-test/.env | 2 +- kube/overlays/dev/.env | 2 +- kube/overlays/stable-with-resource-limits/.env | 2 +- kube/overlays/stable/.env | 2 +- 10 files changed, 10 insertions(+), 10 deletions(-) diff --git a/.env b/.env index d9bf9f7d2048..f15310773da3 100644 --- a/.env +++ b/.env @@ -114,4 +114,4 @@ METRIC_CLIENT= OTEL_COLLECTOR_ENDPOINT="http://host.docker.internal:4317" USE_STREAM_CAPABLE_STATE=true -AUTO_DETECT_SCHEMA=false +AUTO_DETECT_SCHEMA=true diff --git a/.env.dev b/.env.dev index 4f25503451fc..19480e7072f9 100644 --- a/.env.dev +++ b/.env.dev @@ -28,7 +28,7 @@ SYNC_JOB_MAX_ATTEMPTS=3 SYNC_JOB_MAX_TIMEOUT_DAYS=3 WORKERS_MICRONAUT_ENVIRONMENTS=control-plane CRON_MICRONAUT_ENVIRONMENTS=control-plane -AUTO_DETECT_SCHEMA=false +AUTO_DETECT_SCHEMA=true # Sentry SENTRY_DSN="" diff --git a/airbyte-commons/src/main/java/io/airbyte/commons/features/EnvVariableFeatureFlags.java b/airbyte-commons/src/main/java/io/airbyte/commons/features/EnvVariableFeatureFlags.java index 248c5524ed56..b574d0a40cee 100644 --- a/airbyte-commons/src/main/java/io/airbyte/commons/features/EnvVariableFeatureFlags.java +++ b/airbyte-commons/src/main/java/io/airbyte/commons/features/EnvVariableFeatureFlags.java @@ -41,7 +41,7 @@ public boolean useStreamCapableState() { @Override public boolean autoDetectSchema() { - return getEnvOrDefault(AUTO_DETECT_SCHEMA, false, Boolean::parseBoolean); + return getEnvOrDefault(AUTO_DETECT_SCHEMA, true, Boolean::parseBoolean); } @Override diff --git a/airbyte-config/config-models/src/main/java/io/airbyte/config/EnvConfigs.java b/airbyte-config/config-models/src/main/java/io/airbyte/config/EnvConfigs.java index b68be54b055d..c939764eab65 100644 --- a/airbyte-config/config-models/src/main/java/io/airbyte/config/EnvConfigs.java +++ b/airbyte-config/config-models/src/main/java/io/airbyte/config/EnvConfigs.java @@ -1124,7 +1124,7 @@ public int getWorkflowFailureRestartDelaySeconds() { @Override public boolean getAutoDetectSchema() { - return getEnvOrDefault(AUTO_DETECT_SCHEMA, false); + return getEnvOrDefault(AUTO_DETECT_SCHEMA, true); } @Override diff --git a/airbyte-workers/src/test-integration/java/io/airbyte/workers/process/AsyncOrchestratorPodProcessIntegrationTest.java b/airbyte-workers/src/test-integration/java/io/airbyte/workers/process/AsyncOrchestratorPodProcessIntegrationTest.java index 9fa6ba491dd3..6bbbf800edf8 100644 --- a/airbyte-workers/src/test-integration/java/io/airbyte/workers/process/AsyncOrchestratorPodProcessIntegrationTest.java +++ b/airbyte-workers/src/test-integration/java/io/airbyte/workers/process/AsyncOrchestratorPodProcessIntegrationTest.java @@ -116,7 +116,7 @@ public void testAsyncOrchestratorPodProcess(final String pullPolicy) throws Inte null, null, null, - Map.of(EnvVariableFeatureFlags.USE_STREAM_CAPABLE_STATE, "true", EnvVariableFeatureFlags.AUTO_DETECT_SCHEMA, "false"), + Map.of(EnvVariableFeatureFlags.USE_STREAM_CAPABLE_STATE, "true", EnvVariableFeatureFlags.AUTO_DETECT_SCHEMA, "true"), serverPort); final Map portMap = Map.of( diff --git a/charts/airbyte/templates/env-configmap.yaml b/charts/airbyte/templates/env-configmap.yaml index a219395275e4..f55fd2e88e8b 100644 --- a/charts/airbyte/templates/env-configmap.yaml +++ b/charts/airbyte/templates/env-configmap.yaml @@ -63,7 +63,7 @@ data: ACTIVITY_MAX_DELAY_BETWEEN_ATTEMPTS_SECONDS: "" WORKFLOW_FAILURE_RESTART_DELAY_SECONDS: "" USE_STREAM_CAPABLE_STATE: "true" - AUTO_DETECT_SCHEMA: "false" + AUTO_DETECT_SCHEMA: "true" CONTAINER_ORCHESTRATOR_ENABLED: {{ .Values.worker.containerOrchestrator.enabled | quote }} CONTAINER_ORCHESTRATOR_IMAGE: {{ .Values.worker.containerOrchestrator.image | quote }} WORKERS_MICRONAUT_ENVIRONMENTS: "control-plane" diff --git a/kube/overlays/dev-integration-test/.env b/kube/overlays/dev-integration-test/.env index c0d8b1754bde..aad4911e7fda 100644 --- a/kube/overlays/dev-integration-test/.env +++ b/kube/overlays/dev-integration-test/.env @@ -95,4 +95,4 @@ USE_STREAM_CAPABLE_STATE=true SHOULD_RUN_NOTIFY_WORKFLOWS=false MAX_NOTIFY_WORKERS=5 -AUTO_DETECT_SCHEMA=false +AUTO_DETECT_SCHEMA=true diff --git a/kube/overlays/dev/.env b/kube/overlays/dev/.env index 8762e2ea5914..1bc8b49dfe33 100644 --- a/kube/overlays/dev/.env +++ b/kube/overlays/dev/.env @@ -96,4 +96,4 @@ WORKFLOW_FAILURE_RESTART_DELAY_SECONDS= USE_STREAM_CAPABLE_STATE=true SHOULD_RUN_NOTIFY_WORKFLOWS=false MAX_NOTIFY_WORKERS=5 -AUTO_DETECT_SCHEMA=false +AUTO_DETECT_SCHEMA=true diff --git a/kube/overlays/stable-with-resource-limits/.env b/kube/overlays/stable-with-resource-limits/.env index 63df173366ca..22203d6db585 100644 --- a/kube/overlays/stable-with-resource-limits/.env +++ b/kube/overlays/stable-with-resource-limits/.env @@ -98,4 +98,4 @@ WORKFLOW_FAILURE_RESTART_DELAY_SECONDS= USE_STREAM_CAPABLE_STATE=true SHOULD_RUN_NOTIFY_WORKFLOWS=false MAX_NOTIFY_WORKERS=5 -AUTO_DETECT_SCHEMA=false +AUTO_DETECT_SCHEMA=true diff --git a/kube/overlays/stable/.env b/kube/overlays/stable/.env index 46400b8ceeda..91ab84ac2d9b 100644 --- a/kube/overlays/stable/.env +++ b/kube/overlays/stable/.env @@ -96,4 +96,4 @@ WORKFLOW_FAILURE_RESTART_DELAY_SECONDS= USE_STREAM_CAPABLE_STATE=true SHOULD_RUN_NOTIFY_WORKFLOWS=false MAX_NOTIFY_WORKERS=5 -AUTO_DETECT_SCHEMA=false +AUTO_DETECT_SCHEMA=true From 0992bea622a556fa5a9fe0129b2bfd73ab7d9b15 Mon Sep 17 00:00:00 2001 From: Edmundo Ruiz Ghanem <168664+edmundito@users.noreply.github.com> Date: Wed, 25 Jan 2023 09:56:32 -0500 Subject: [PATCH 2/2] =?UTF-8?q?=F0=9F=AA=9F=20=F0=9F=94=A7=20Add=20auto-de?= =?UTF-8?q?tect=20schema=20as=20a=20default=20feature=20in=20webapp=20(#21?= =?UTF-8?q?820)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Add auto-detect schema as a default feature * Update snapshots --- .../CreateConnectionForm.test.tsx.snap | 130 +++++++++++++++++- .../src/hooks/services/Feature/constants.ts | 2 + .../ConnectionReplicationPage.test.tsx.snap | 130 +++++++++++++++++- 3 files changed, 254 insertions(+), 8 deletions(-) diff --git a/airbyte-webapp/src/components/connection/CreateConnectionForm/__snapshots__/CreateConnectionForm.test.tsx.snap b/airbyte-webapp/src/components/connection/CreateConnectionForm/__snapshots__/CreateConnectionForm.test.tsx.snap index a59078873892..5a7124860480 100644 --- a/airbyte-webapp/src/components/connection/CreateConnectionForm/__snapshots__/CreateConnectionForm.test.tsx.snap +++ b/airbyte-webapp/src/components/connection/CreateConnectionForm/__snapshots__/CreateConnectionForm.test.tsx.snap @@ -215,6 +215,128 @@ exports[`CreateConnectionForm should render 1`] = ` +
+
+
+ +
+
+
+
+ + +
+
+
+
+
+ Ignore +
+
+
+ +
+
+ +
+
+ +
+
+
+
+
+
+ +
+
+
+
+ + +
+
+
+
+
+ Ignore +
+
+
+ +
+
+ +
+
+ +
+
+