From 4ec8ad7dcf755ad766371ea1c9b764ad0e86c42c Mon Sep 17 00:00:00 2001 From: Jimmy Ma Date: Tue, 20 Jun 2023 08:51:19 -0700 Subject: [PATCH] Fix connector resource defaults location (#7342) --- .../src/main/resources/application.yml | 10 -------- .../ContainerOrchestratorFactoryTest.java | 23 ------------------- .../src/main/resources/application.yml | 10 ++++++++ .../WorkerConfigProviderMicronautTest.java | 23 +++++++++++++++++++ 4 files changed, 33 insertions(+), 33 deletions(-) diff --git a/airbyte-container-orchestrator/src/main/resources/application.yml b/airbyte-container-orchestrator/src/main/resources/application.yml index e84dacd1865..a30f680b066 100644 --- a/airbyte-container-orchestrator/src/main/resources/application.yml +++ b/airbyte-container-orchestrator/src/main/resources/application.yml @@ -77,16 +77,6 @@ airbyte: cpu-request: ${REPLICATION_ORCHESTRATOR_CPU_REQUEST:} memory-limit: ${REPLICATION_ORCHESTRATOR_MEMORY_LIMIT:} memory-request: ${REPLICATION_ORCHESTRATOR_MEMORY_REQUEST:} - source: - cpu-limit: ${SOURCE_CONTAINER_CPU_LIMIT:} - cpu-request: ${SOURCE_CONTAINER_CPU_REQUEST:0.5} - memory-limit: ${SOURCE_CONTAINER_MEMORY_LIMIT:} - memory-request: ${SOURCE_CONTAINER_MEMORY_REQUEST:} - source-database: - cpu-limit: ${SOURCE_DATABASE_CONTAINER_CPU_LIMIT:} - cpu-request: ${SOURCE_DATABASE_CONTAINER_CPU_REQUEST:1} - memory-limit: ${SOURCE_DATABASE_CONTAINER_MEMORY_LIMIT:} - memory-request: ${SOURCE_DATABASE_CONTAINER_MEMORY_REQUEST:} replication: persistence-flush-period-sec: ${REPLICATION_FLUSH_PERIOD_SECONDS:60} diff --git a/airbyte-container-orchestrator/src/test/java/io/airbyte/container_orchestrator/config/ContainerOrchestratorFactoryTest.java b/airbyte-container-orchestrator/src/test/java/io/airbyte/container_orchestrator/config/ContainerOrchestratorFactoryTest.java index a5487c9394a..5e0ae5acc61 100644 --- a/airbyte-container-orchestrator/src/test/java/io/airbyte/container_orchestrator/config/ContainerOrchestratorFactoryTest.java +++ b/airbyte-container-orchestrator/src/test/java/io/airbyte/container_orchestrator/config/ContainerOrchestratorFactoryTest.java @@ -11,8 +11,6 @@ import io.airbyte.commons.features.FeatureFlags; import io.airbyte.config.EnvConfigs; -import io.airbyte.config.ResourceRequirements; -import io.airbyte.config.ResourceRequirementsType; import io.airbyte.featureflag.FeatureFlagClient; import io.airbyte.featureflag.TestClient; import io.airbyte.persistence.job.models.JobRunConfig; @@ -32,7 +30,6 @@ import io.micronaut.test.extensions.junit5.annotation.MicronautTest; import jakarta.inject.Inject; import java.util.Map; -import java.util.Optional; import org.junit.jupiter.api.Test; // tests may be running on a real k8s environment, override the environment to something else for @@ -125,24 +122,4 @@ void jobOrchestrator() { assertTrue(caught, "invalid application name should have thrown an exception"); } - @Test - void checkDatabaseSourceResourceRequirements() { - final ResourceRequirements resourceRequirements = - workerConfigsProvider.getResourceRequirements(ResourceRequirementsType.SOURCE, Optional.of("database")); - - assertEquals("1", resourceRequirements.getCpuRequest()); - // This is verifying that we are inheriting the value from default. - assertEquals("1", resourceRequirements.getCpuLimit()); - } - - @Test - void checkSourceResourceRequirements() { - final ResourceRequirements resourceRequirements = - workerConfigsProvider.getResourceRequirements(ResourceRequirementsType.SOURCE, Optional.of("any")); - - assertEquals("0.5", resourceRequirements.getCpuRequest()); - // This is verifying that we are inheriting the value from default. - assertEquals("1", resourceRequirements.getCpuLimit()); - } - } diff --git a/airbyte-workers/src/main/resources/application.yml b/airbyte-workers/src/main/resources/application.yml index b98b8fa264d..2e3bc748962 100644 --- a/airbyte-workers/src/main/resources/application.yml +++ b/airbyte-workers/src/main/resources/application.yml @@ -142,6 +142,16 @@ airbyte: cpu-request: ${REPLICATION_ORCHESTRATOR_CPU_REQUEST:} memory-limit: ${REPLICATION_ORCHESTRATOR_MEMORY_LIMIT:} memory-request: ${REPLICATION_ORCHESTRATOR_MEMORY_REQUEST:} + source: + cpu-limit: ${SOURCE_CONTAINER_CPU_LIMIT:} + cpu-request: ${SOURCE_CONTAINER_CPU_REQUEST:0.5} + memory-limit: ${SOURCE_CONTAINER_MEMORY_LIMIT:} + memory-request: ${SOURCE_CONTAINER_MEMORY_REQUEST:} + source-database: + cpu-limit: ${SOURCE_DATABASE_CONTAINER_CPU_LIMIT:} + cpu-request: ${SOURCE_DATABASE_CONTAINER_CPU_REQUEST:1} + memory-limit: ${SOURCE_DATABASE_CONTAINER_MEMORY_LIMIT:} + memory-request: ${SOURCE_DATABASE_CONTAINER_MEMORY_REQUEST:} spec: annotations: ${SPEC_JOB_KUBE_ANNOTATIONS:} node-selectors: ${SPEC_JOB_KUBE_NODE_SELECTORS:} diff --git a/airbyte-workers/src/test/java/io/airbyte/workers/config/WorkerConfigProviderMicronautTest.java b/airbyte-workers/src/test/java/io/airbyte/workers/config/WorkerConfigProviderMicronautTest.java index 25566a5f77c..e66dcc4f3ef 100644 --- a/airbyte-workers/src/test/java/io/airbyte/workers/config/WorkerConfigProviderMicronautTest.java +++ b/airbyte-workers/src/test/java/io/airbyte/workers/config/WorkerConfigProviderMicronautTest.java @@ -7,12 +7,15 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNull; +import io.airbyte.config.ResourceRequirements; +import io.airbyte.config.ResourceRequirementsType; import io.airbyte.workers.WorkerConfigs; import io.airbyte.workers.config.WorkerConfigsProvider.ResourceType; import io.micronaut.context.annotation.Value; import io.micronaut.test.extensions.junit5.annotation.MicronautTest; import jakarta.inject.Inject; import jakarta.inject.Named; +import java.util.Optional; import org.junit.jupiter.api.Test; // We are overriding the default config with application-config-test.yaml for consistency of the @@ -73,4 +76,24 @@ void checkWorkerConfigProvider() { assertEquals("", specKubeConfig.getResourceRequirements().getMemoryRequest()); } + @Test + void checkDatabaseSourceResourceRequirements() { + final ResourceRequirements resourceRequirements = + workerConfigsProvider.getResourceRequirements(ResourceRequirementsType.SOURCE, Optional.of("database")); + + assertEquals("1", resourceRequirements.getCpuRequest()); + // This is verifying that we are inheriting the value from default. + assertEquals("default cpu limit", resourceRequirements.getCpuLimit()); + } + + @Test + void checkSourceResourceRequirements() { + final ResourceRequirements resourceRequirements = + workerConfigsProvider.getResourceRequirements(ResourceRequirementsType.SOURCE, Optional.of("any")); + + assertEquals("0.5", resourceRequirements.getCpuRequest()); + // This is verifying that we are inheriting the value from default. + assertEquals("default cpu limit", resourceRequirements.getCpuLimit()); + } + }