From e4707f5ed1c2ac538cdea060457645b50ea4d0c9 Mon Sep 17 00:00:00 2001 From: Augustin Date: Sat, 7 Jan 2023 02:12:50 +0100 Subject: [PATCH] normalization: delete supportsDbt and supportsNormalization from DestinationDefinitionSpecificationRead (#21005) * delete supportsDbt and supportsNormalization from DestinationDefinitionSpecificationRead * FE changes * add new fields to the DestinationDefinitionRead API model * update handlers * update api doc * update handlers * remove debug loggin * implement suggestions * update octavia-cli tests * fix supported check * update octavia-cli * fix integration tests * fix mocks * fix forgotten renaming Co-authored-by: Evan Tahler --- airbyte-api/src/main/openapi/config.yaml | 33 ++++- .../server/converters/ApiPojoConverters.java | 12 ++ .../DestinationDefinitionsHandler.java | 5 + .../server/handlers/SchedulerHandler.java | 2 - .../DestinationDefinitionsHandlerTest.java | 114 +++++++++++------ .../handlers/DestinationHandlerTest.java | 4 +- .../CreateConnectionForm.test.tsx | 11 +- .../connection/CatalogTree/BulkHeader.tsx | 2 +- .../connection/CatalogTree/CatalogSection.tsx | 2 +- .../CatalogTree/next/BulkEditPanel.test.tsx | 2 +- .../CatalogTree/next/BulkEditPanel.tsx | 2 +- .../ConnectionEditService.test.tsx | 11 +- .../ConnectionFormService.test.tsx | 11 +- .../ConnectionForm/ConnectionFormService.tsx | 13 +- .../ConnectionReplicationTab.test.tsx | 12 +- .../ConnectionTransformationTab.tsx | 6 +- .../StatusMainInfo.test.tsx | 4 +- .../test-utils/mock-data/mockDestination.ts | 23 +++- ...mockDestinationDefinitionSpecification.ts} | 4 +- .../components/OperationsSection.tsx | 3 +- .../ConnectionForm/formConfig.test.ts | 17 ++- .../Connection/ConnectionForm/formConfig.tsx | 12 +- .../api/generated-api-html/index.html | 115 +++++++++++++++++- .../destination_postgres/input_spec.yaml | 2 - .../destination_s3/input_spec.yaml | 2 - .../source_postgres/input_spec.yaml | 2 - .../test_generate/test_renderers.py | 2 +- octavia-cli/octavia_cli/apply/resources.py | 30 ++++- octavia-cli/octavia_cli/generate/renderers.py | 2 +- .../unit_tests/test_apply/test_resources.py | 26 +++- .../test_generate/test_renderers.py | 4 +- 31 files changed, 390 insertions(+), 100 deletions(-) rename airbyte-webapp/src/test-utils/mock-data/{mockDestinationDefinition.ts => mockDestinationDefinitionSpecification.ts} (98%) diff --git a/airbyte-api/src/main/openapi/config.yaml b/airbyte-api/src/main/openapi/config.yaml index 86ba95fc30d3..2d9722921bcd 100644 --- a/airbyte-api/src/main/openapi/config.yaml +++ b/airbyte-api/src/main/openapi/config.yaml @@ -2878,6 +2878,8 @@ components: - dockerRepository - dockerImageTag - documentationUrl + - supportsDbt + - normalizationConfig properties: destinationDefinitionId: $ref: "#/components/schemas/DestinationDefinitionId" @@ -2903,6 +2905,12 @@ components: format: date resourceRequirements: $ref: "#/components/schemas/ActorDefinitionResourceRequirements" + supportsDbt: + type: boolean + description: an optional flag indicating whether DBT is used in the normalization. If the flag value is NULL - DBT is not used. + normalizationConfig: + $ref: "#/components/schemas/NormalizationDestinationDefinitionConfig" + DestinationDefinitionReadList: type: object required: @@ -2977,10 +2985,6 @@ components: type: array items: $ref: "#/components/schemas/DestinationSyncMode" - supportsDbt: - type: boolean - supportsNormalization: - type: boolean # DESTINATION DestinationId: type: string @@ -4370,6 +4374,27 @@ components: type: array items: "$ref": "#/components/schemas/JobTypeResourceLimit" + NormalizationDestinationDefinitionConfig: + description: describes a normalization config for destination definition + type: object + required: + - supported + additionalProperties: false + properties: + supported: + type: boolean + description: whether the destination definition supports normalization. + default: false + normalizationRepository: + type: string + description: a field indicating the name of the repository to be used for normalization. If the value of the flag is NULL - normalization is not used. + normalizationTag: + type: string + description: a field indicating the tag of the docker repository to be used for normalization. + normalizationIntegrationType: + type: string + description: a field indicating the type of integration dialect to use for normalization. + JobTypeResourceLimit: description: sets resource requirements for a specific job type for an actor definition. these values override the default, if both are set. type: object diff --git a/airbyte-server/src/main/java/io/airbyte/server/converters/ApiPojoConverters.java b/airbyte-server/src/main/java/io/airbyte/server/converters/ApiPojoConverters.java index 701ef8445045..9b85de35c34b 100644 --- a/airbyte-server/src/main/java/io/airbyte/server/converters/ApiPojoConverters.java +++ b/airbyte-server/src/main/java/io/airbyte/server/converters/ApiPojoConverters.java @@ -15,6 +15,7 @@ import io.airbyte.api.model.generated.JobType; import io.airbyte.api.model.generated.JobTypeResourceLimit; import io.airbyte.api.model.generated.NonBreakingChangesPreference; +import io.airbyte.api.model.generated.NormalizationDestinationDefinitionConfig; import io.airbyte.api.model.generated.ResourceRequirements; import io.airbyte.commons.enums.Enums; import io.airbyte.config.BasicSchedule; @@ -81,6 +82,17 @@ public static ResourceRequirements resourceRequirementsToApi(final io.airbyte.co .memoryLimit(resourceReqs.getMemoryLimit()); } + public static NormalizationDestinationDefinitionConfig normalizationDestinationDefinitionConfigToApi(final io.airbyte.config.NormalizationDestinationDefinitionConfig normalizationDestinationDefinitionConfig) { + if (normalizationDestinationDefinitionConfig == null) { + return new NormalizationDestinationDefinitionConfig().supported(false); + } + return new NormalizationDestinationDefinitionConfig() + .supported(true) + .normalizationRepository(normalizationDestinationDefinitionConfig.getNormalizationRepository()) + .normalizationTag(normalizationDestinationDefinitionConfig.getNormalizationTag()) + .normalizationIntegrationType(normalizationDestinationDefinitionConfig.getNormalizationIntegrationType()); + } + public static ConnectionRead internalToConnectionRead(final StandardSync standardSync) { final ConnectionRead connectionRead = new ConnectionRead() .connectionId(standardSync.getConnectionId()) diff --git a/airbyte-server/src/main/java/io/airbyte/server/handlers/DestinationDefinitionsHandler.java b/airbyte-server/src/main/java/io/airbyte/server/handlers/DestinationDefinitionsHandler.java index 4b07f3bda272..2f21ad1dd38c 100644 --- a/airbyte-server/src/main/java/io/airbyte/server/handlers/DestinationDefinitionsHandler.java +++ b/airbyte-server/src/main/java/io/airbyte/server/handlers/DestinationDefinitionsHandler.java @@ -48,6 +48,7 @@ import java.time.LocalDate; import java.util.List; import java.util.Map.Entry; +import java.util.Objects; import java.util.UUID; import java.util.function.Supplier; import java.util.stream.Collectors; @@ -88,6 +89,7 @@ public DestinationDefinitionsHandler(final ConfigRepository configRepository, @VisibleForTesting static DestinationDefinitionRead buildDestinationDefinitionRead(final StandardDestinationDefinition standardDestinationDefinition) { try { + return new DestinationDefinitionRead() .destinationDefinitionId(standardDestinationDefinition.getDestinationDefinitionId()) .name(standardDestinationDefinition.getName()) @@ -98,6 +100,9 @@ static DestinationDefinitionRead buildDestinationDefinitionRead(final StandardDe .protocolVersion(standardDestinationDefinition.getProtocolVersion()) .releaseStage(getReleaseStage(standardDestinationDefinition)) .releaseDate(getReleaseDate(standardDestinationDefinition)) + .supportsDbt(Objects.requireNonNullElse(standardDestinationDefinition.getSupportsDbt(), false)) + .normalizationConfig( + ApiPojoConverters.normalizationDestinationDefinitionConfigToApi(standardDestinationDefinition.getNormalizationConfig())) .resourceRequirements(ApiPojoConverters.actorDefResourceReqsToApi(standardDestinationDefinition.getResourceRequirements())); } catch (final URISyntaxException | NullPointerException e) { throw new InternalServerKnownException("Unable to process retrieved latest destination definitions list", e); diff --git a/airbyte-server/src/main/java/io/airbyte/server/handlers/SchedulerHandler.java b/airbyte-server/src/main/java/io/airbyte/server/handlers/SchedulerHandler.java index 3fb92857aa9c..618100940edd 100644 --- a/airbyte-server/src/main/java/io/airbyte/server/handlers/SchedulerHandler.java +++ b/airbyte-server/src/main/java/io/airbyte/server/handlers/SchedulerHandler.java @@ -362,8 +362,6 @@ public DestinationDefinitionSpecificationRead getDestinationSpecification( .supportedDestinationSyncModes(Enums.convertListTo(spec.getSupportedDestinationSyncModes(), DestinationSyncMode.class)) .connectionSpecification(spec.getConnectionSpecification()) .documentationUrl(spec.getDocumentationUrl().toString()) - .supportsNormalization(spec.getSupportsNormalization()) - .supportsDbt(spec.getSupportsDBT()) .destinationDefinitionId(destinationDefinitionId); final Optional authSpec = OauthModelConverter.getAuthSpec(spec); diff --git a/airbyte-server/src/test/java/io/airbyte/server/handlers/DestinationDefinitionsHandlerTest.java b/airbyte-server/src/test/java/io/airbyte/server/handlers/DestinationDefinitionsHandlerTest.java index 32ba33b311bd..6c113a0e57af 100644 --- a/airbyte-server/src/test/java/io/airbyte/server/handlers/DestinationDefinitionsHandlerTest.java +++ b/airbyte-server/src/test/java/io/airbyte/server/handlers/DestinationDefinitionsHandlerTest.java @@ -37,6 +37,7 @@ import io.airbyte.config.Configs; import io.airbyte.config.EnvConfigs; import io.airbyte.config.JobConfig.ConfigType; +import io.airbyte.config.NormalizationDestinationDefinitionConfig; import io.airbyte.config.ResourceRequirements; import io.airbyte.config.StandardDestinationDefinition; import io.airbyte.config.persistence.ConfigNotFoundException; @@ -69,6 +70,8 @@ class DestinationDefinitionsHandlerTest { private ConfigRepository configRepository; private StandardDestinationDefinition destinationDefinition; + private StandardDestinationDefinition destinationDefinitionWithNormalization; + private DestinationDefinitionsHandler destinationDefinitionsHandler; private Supplier uuidSupplier; private SynchronousSchedulerClient schedulerSynchronousClient; @@ -82,6 +85,7 @@ void setUp() { configRepository = mock(ConfigRepository.class); uuidSupplier = mock(Supplier.class); destinationDefinition = generateDestinationDefinition(); + destinationDefinitionWithNormalization = generateDestinationDefinitionWithNormalization(); schedulerSynchronousClient = spy(SynchronousSchedulerClient.class); githubStore = mock(AirbyteGithubStore.class); destinationHandler = mock(DestinationHandler.class); @@ -114,12 +118,22 @@ private StandardDestinationDefinition generateDestinationDefinition() { .withResourceRequirements(new ActorDefinitionResourceRequirements().withDefault(new ResourceRequirements().withCpuRequest("2"))); } + private StandardDestinationDefinition generateDestinationDefinitionWithNormalization() { + final StandardDestinationDefinition definition = generateDestinationDefinition(); + return definition + .withSupportsDbt(true) + .withNormalizationConfig(new NormalizationDestinationDefinitionConfig() + .withNormalizationRepository("repository") + .withNormalizationTag("dev") + .withNormalizationIntegrationType("integration-type")); + } + @Test @DisplayName("listDestinationDefinition should return the right list") void testListDestinations() throws JsonValidationException, IOException, URISyntaxException { - final StandardDestinationDefinition destination2 = generateDestinationDefinition(); - when(configRepository.listStandardDestinationDefinitions(false)).thenReturn(Lists.newArrayList(destinationDefinition, destination2)); + when(configRepository.listStandardDestinationDefinitions(false)) + .thenReturn(Lists.newArrayList(destinationDefinition, destinationDefinitionWithNormalization)); final DestinationDefinitionRead expectedDestinationDefinitionRead1 = new DestinationDefinitionRead() .destinationDefinitionId(destinationDefinition.getDestinationDefinitionId()) @@ -131,24 +145,31 @@ void testListDestinations() throws JsonValidationException, IOException, URISynt .protocolVersion(destinationDefinition.getProtocolVersion()) .releaseStage(ReleaseStage.fromValue(destinationDefinition.getReleaseStage().value())) .releaseDate(LocalDate.parse(destinationDefinition.getReleaseDate())) + .supportsDbt(false) + .normalizationConfig(new io.airbyte.api.model.generated.NormalizationDestinationDefinitionConfig().supported(false)) .resourceRequirements(new io.airbyte.api.model.generated.ActorDefinitionResourceRequirements() ._default(new io.airbyte.api.model.generated.ResourceRequirements() .cpuRequest(destinationDefinition.getResourceRequirements().getDefault().getCpuRequest())) .jobSpecific(Collections.emptyList())); final DestinationDefinitionRead expectedDestinationDefinitionRead2 = new DestinationDefinitionRead() - .destinationDefinitionId(destination2.getDestinationDefinitionId()) - .name(destination2.getName()) - .dockerRepository(destination2.getDockerRepository()) - .dockerImageTag(destination2.getDockerImageTag()) - .documentationUrl(new URI(destination2.getDocumentationUrl())) - .icon(DestinationDefinitionsHandler.loadIcon(destination2.getIcon())) - .protocolVersion(destinationDefinition.getProtocolVersion()) - .releaseStage(ReleaseStage.fromValue(destinationDefinition.getReleaseStage().value())) - .releaseDate(LocalDate.parse(destinationDefinition.getReleaseDate())) + .destinationDefinitionId(destinationDefinitionWithNormalization.getDestinationDefinitionId()) + .name(destinationDefinitionWithNormalization.getName()) + .dockerRepository(destinationDefinitionWithNormalization.getDockerRepository()) + .dockerImageTag(destinationDefinitionWithNormalization.getDockerImageTag()) + .documentationUrl(new URI(destinationDefinitionWithNormalization.getDocumentationUrl())) + .icon(DestinationDefinitionsHandler.loadIcon(destinationDefinitionWithNormalization.getIcon())) + .protocolVersion(destinationDefinitionWithNormalization.getProtocolVersion()) + .releaseStage(ReleaseStage.fromValue(destinationDefinitionWithNormalization.getReleaseStage().value())) + .releaseDate(LocalDate.parse(destinationDefinitionWithNormalization.getReleaseDate())) + .supportsDbt(destinationDefinitionWithNormalization.getSupportsDbt()) + .normalizationConfig(new io.airbyte.api.model.generated.NormalizationDestinationDefinitionConfig().supported(true) + .normalizationRepository(destinationDefinitionWithNormalization.getNormalizationConfig().getNormalizationRepository()) + .normalizationTag(destinationDefinitionWithNormalization.getNormalizationConfig().getNormalizationTag()) + .normalizationIntegrationType(destinationDefinitionWithNormalization.getNormalizationConfig().getNormalizationIntegrationType())) .resourceRequirements(new io.airbyte.api.model.generated.ActorDefinitionResourceRequirements() ._default(new io.airbyte.api.model.generated.ResourceRequirements() - .cpuRequest(destination2.getResourceRequirements().getDefault().getCpuRequest())) + .cpuRequest(destinationDefinitionWithNormalization.getResourceRequirements().getDefault().getCpuRequest())) .jobSpecific(Collections.emptyList())); final DestinationDefinitionReadList actualDestinationDefinitionReadList = destinationDefinitionsHandler.listDestinationDefinitions(); @@ -161,10 +182,10 @@ void testListDestinations() throws JsonValidationException, IOException, URISynt @Test @DisplayName("listDestinationDefinitionsForWorkspace should return the right list") void testListDestinationDefinitionsForWorkspace() throws IOException, URISyntaxException { - final StandardDestinationDefinition destination2 = generateDestinationDefinition(); when(configRepository.listPublicDestinationDefinitions(false)).thenReturn(Lists.newArrayList(destinationDefinition)); - when(configRepository.listGrantedDestinationDefinitions(workspaceId, false)).thenReturn(Lists.newArrayList(destination2)); + when(configRepository.listGrantedDestinationDefinitions(workspaceId, false)) + .thenReturn(Lists.newArrayList(destinationDefinitionWithNormalization)); final DestinationDefinitionRead expectedDestinationDefinitionRead1 = new DestinationDefinitionRead() .destinationDefinitionId(destinationDefinition.getDestinationDefinitionId()) @@ -176,24 +197,31 @@ void testListDestinationDefinitionsForWorkspace() throws IOException, URISyntaxE .protocolVersion(destinationDefinition.getProtocolVersion()) .releaseStage(ReleaseStage.fromValue(destinationDefinition.getReleaseStage().value())) .releaseDate(LocalDate.parse(destinationDefinition.getReleaseDate())) + .supportsDbt(false) + .normalizationConfig(new io.airbyte.api.model.generated.NormalizationDestinationDefinitionConfig().supported(false)) .resourceRequirements(new io.airbyte.api.model.generated.ActorDefinitionResourceRequirements() ._default(new io.airbyte.api.model.generated.ResourceRequirements() .cpuRequest(destinationDefinition.getResourceRequirements().getDefault().getCpuRequest())) .jobSpecific(Collections.emptyList())); final DestinationDefinitionRead expectedDestinationDefinitionRead2 = new DestinationDefinitionRead() - .destinationDefinitionId(destination2.getDestinationDefinitionId()) - .name(destination2.getName()) - .dockerRepository(destination2.getDockerRepository()) - .dockerImageTag(destination2.getDockerImageTag()) - .documentationUrl(new URI(destination2.getDocumentationUrl())) - .icon(DestinationDefinitionsHandler.loadIcon(destination2.getIcon())) - .protocolVersion(destinationDefinition.getProtocolVersion()) - .releaseStage(ReleaseStage.fromValue(destinationDefinition.getReleaseStage().value())) - .releaseDate(LocalDate.parse(destinationDefinition.getReleaseDate())) + .destinationDefinitionId(destinationDefinitionWithNormalization.getDestinationDefinitionId()) + .name(destinationDefinitionWithNormalization.getName()) + .dockerRepository(destinationDefinitionWithNormalization.getDockerRepository()) + .dockerImageTag(destinationDefinitionWithNormalization.getDockerImageTag()) + .documentationUrl(new URI(destinationDefinitionWithNormalization.getDocumentationUrl())) + .icon(DestinationDefinitionsHandler.loadIcon(destinationDefinitionWithNormalization.getIcon())) + .protocolVersion(destinationDefinitionWithNormalization.getProtocolVersion()) + .releaseStage(ReleaseStage.fromValue(destinationDefinitionWithNormalization.getReleaseStage().value())) + .releaseDate(LocalDate.parse(destinationDefinitionWithNormalization.getReleaseDate())) + .supportsDbt(destinationDefinitionWithNormalization.getSupportsDbt()) + .normalizationConfig(new io.airbyte.api.model.generated.NormalizationDestinationDefinitionConfig().supported(true) + .normalizationRepository(destinationDefinitionWithNormalization.getNormalizationConfig().getNormalizationRepository()) + .normalizationTag(destinationDefinitionWithNormalization.getNormalizationConfig().getNormalizationTag()) + .normalizationIntegrationType(destinationDefinitionWithNormalization.getNormalizationConfig().getNormalizationIntegrationType())) .resourceRequirements(new io.airbyte.api.model.generated.ActorDefinitionResourceRequirements() ._default(new io.airbyte.api.model.generated.ResourceRequirements() - .cpuRequest(destination2.getResourceRequirements().getDefault().getCpuRequest())) + .cpuRequest(destinationDefinitionWithNormalization.getResourceRequirements().getDefault().getCpuRequest())) .jobSpecific(Collections.emptyList())); final DestinationDefinitionReadList actualDestinationDefinitionReadList = destinationDefinitionsHandler @@ -207,12 +235,11 @@ void testListDestinationDefinitionsForWorkspace() throws IOException, URISyntaxE @Test @DisplayName("listPrivateDestinationDefinitions should return the right list") void testListPrivateDestinationDefinitions() throws IOException, URISyntaxException { - final StandardDestinationDefinition destinationDefinition2 = generateDestinationDefinition(); when(configRepository.listGrantableDestinationDefinitions(workspaceId, false)).thenReturn( Lists.newArrayList( Map.entry(destinationDefinition, false), - Map.entry(destinationDefinition2, true))); + Map.entry(destinationDefinitionWithNormalization, true))); final DestinationDefinitionRead expectedDestinationDefinitionRead1 = new DestinationDefinitionRead() .destinationDefinitionId(destinationDefinition.getDestinationDefinitionId()) @@ -224,24 +251,31 @@ void testListPrivateDestinationDefinitions() throws IOException, URISyntaxExcept .protocolVersion(destinationDefinition.getProtocolVersion()) .releaseStage(ReleaseStage.fromValue(destinationDefinition.getReleaseStage().value())) .releaseDate(LocalDate.parse(destinationDefinition.getReleaseDate())) + .supportsDbt(false) + .normalizationConfig(new io.airbyte.api.model.generated.NormalizationDestinationDefinitionConfig().supported(false)) .resourceRequirements(new io.airbyte.api.model.generated.ActorDefinitionResourceRequirements() ._default(new io.airbyte.api.model.generated.ResourceRequirements() .cpuRequest(destinationDefinition.getResourceRequirements().getDefault().getCpuRequest())) .jobSpecific(Collections.emptyList())); final DestinationDefinitionRead expectedDestinationDefinitionRead2 = new DestinationDefinitionRead() - .destinationDefinitionId(destinationDefinition2.getDestinationDefinitionId()) - .name(destinationDefinition2.getName()) - .dockerRepository(destinationDefinition.getDockerRepository()) - .dockerImageTag(destinationDefinition.getDockerImageTag()) - .documentationUrl(new URI(destinationDefinition.getDocumentationUrl())) - .icon(DestinationDefinitionsHandler.loadIcon(destinationDefinition.getIcon())) - .protocolVersion(destinationDefinition.getProtocolVersion()) - .releaseStage(ReleaseStage.fromValue(destinationDefinition.getReleaseStage().value())) - .releaseDate(LocalDate.parse(destinationDefinition.getReleaseDate())) + .destinationDefinitionId(destinationDefinitionWithNormalization.getDestinationDefinitionId()) + .name(destinationDefinitionWithNormalization.getName()) + .dockerRepository(destinationDefinitionWithNormalization.getDockerRepository()) + .dockerImageTag(destinationDefinitionWithNormalization.getDockerImageTag()) + .documentationUrl(new URI(destinationDefinitionWithNormalization.getDocumentationUrl())) + .icon(DestinationDefinitionsHandler.loadIcon(destinationDefinitionWithNormalization.getIcon())) + .protocolVersion(destinationDefinitionWithNormalization.getProtocolVersion()) + .releaseStage(ReleaseStage.fromValue(destinationDefinitionWithNormalization.getReleaseStage().value())) + .releaseDate(LocalDate.parse(destinationDefinitionWithNormalization.getReleaseDate())) + .supportsDbt(destinationDefinitionWithNormalization.getSupportsDbt()) + .normalizationConfig(new io.airbyte.api.model.generated.NormalizationDestinationDefinitionConfig().supported(true) + .normalizationRepository(destinationDefinitionWithNormalization.getNormalizationConfig().getNormalizationRepository()) + .normalizationTag(destinationDefinitionWithNormalization.getNormalizationConfig().getNormalizationTag()) + .normalizationIntegrationType(destinationDefinitionWithNormalization.getNormalizationConfig().getNormalizationIntegrationType())) .resourceRequirements(new io.airbyte.api.model.generated.ActorDefinitionResourceRequirements() ._default(new io.airbyte.api.model.generated.ResourceRequirements() - .cpuRequest(destinationDefinition2.getResourceRequirements().getDefault().getCpuRequest())) + .cpuRequest(destinationDefinitionWithNormalization.getResourceRequirements().getDefault().getCpuRequest())) .jobSpecific(Collections.emptyList())); final PrivateDestinationDefinitionRead expectedDestinationDefinitionOptInRead1 = @@ -275,6 +309,8 @@ void testGetDestination() throws JsonValidationException, ConfigNotFoundExceptio .protocolVersion(destinationDefinition.getProtocolVersion()) .releaseStage(ReleaseStage.fromValue(destinationDefinition.getReleaseStage().value())) .releaseDate(LocalDate.parse(destinationDefinition.getReleaseDate())) + .supportsDbt(false) + .normalizationConfig(new io.airbyte.api.model.generated.NormalizationDestinationDefinitionConfig().supported(false)) .resourceRequirements(new io.airbyte.api.model.generated.ActorDefinitionResourceRequirements() ._default(new io.airbyte.api.model.generated.ResourceRequirements() .cpuRequest(destinationDefinition.getResourceRequirements().getDefault().getCpuRequest())) @@ -321,6 +357,8 @@ void testGetDefinitionWithGrantForWorkspace() throws JsonValidationException, Co .protocolVersion(destinationDefinition.getProtocolVersion()) .releaseStage(ReleaseStage.fromValue(destinationDefinition.getReleaseStage().value())) .releaseDate(LocalDate.parse(destinationDefinition.getReleaseDate())) + .supportsDbt(false) + .normalizationConfig(new io.airbyte.api.model.generated.NormalizationDestinationDefinitionConfig().supported(false)) .resourceRequirements(new io.airbyte.api.model.generated.ActorDefinitionResourceRequirements() ._default(new io.airbyte.api.model.generated.ResourceRequirements() .cpuRequest(destinationDefinition.getResourceRequirements().getDefault().getCpuRequest())) @@ -408,6 +446,8 @@ void testCreateCustomDestinationDefinition() throws URISyntaxException, IOExcept .icon(DestinationDefinitionsHandler.loadIcon(destination.getIcon())) .protocolVersion(DEFAULT_PROTOCOL_VERSION) .releaseStage(ReleaseStage.CUSTOM) + .supportsDbt(false) + .normalizationConfig(new io.airbyte.api.model.generated.NormalizationDestinationDefinitionConfig().supported(false)) .resourceRequirements(new io.airbyte.api.model.generated.ActorDefinitionResourceRequirements() ._default(new io.airbyte.api.model.generated.ResourceRequirements() .cpuRequest(destination.getResourceRequirements().getDefault().getCpuRequest())) @@ -574,6 +614,8 @@ void testGrantDestinationDefinitionToWorkspace() throws JsonValidationException, .protocolVersion(destinationDefinition.getProtocolVersion()) .releaseStage(ReleaseStage.fromValue(destinationDefinition.getReleaseStage().value())) .releaseDate(LocalDate.parse(destinationDefinition.getReleaseDate())) + .supportsDbt(false) + .normalizationConfig(new io.airbyte.api.model.generated.NormalizationDestinationDefinitionConfig().supported(false)) .resourceRequirements(new io.airbyte.api.model.generated.ActorDefinitionResourceRequirements() ._default(new io.airbyte.api.model.generated.ResourceRequirements() .cpuRequest(destinationDefinition.getResourceRequirements().getDefault().getCpuRequest())) diff --git a/airbyte-server/src/test/java/io/airbyte/server/handlers/DestinationHandlerTest.java b/airbyte-server/src/test/java/io/airbyte/server/handlers/DestinationHandlerTest.java index 19d263fdbc94..728d60bbee48 100644 --- a/airbyte-server/src/test/java/io/airbyte/server/handlers/DestinationHandlerTest.java +++ b/airbyte-server/src/test/java/io/airbyte/server/handlers/DestinationHandlerTest.java @@ -92,9 +92,7 @@ void setUp() throws IOException { destinationDefinitionSpecificationRead = new DestinationDefinitionSpecificationRead() .connectionSpecification(connectorSpecification.getConnectionSpecification()) .destinationDefinitionId(standardDestinationDefinition.getDestinationDefinitionId()) - .documentationUrl(connectorSpecification.getDocumentationUrl().toString()) - .supportsDbt(connectorSpecification.getSupportsDBT()) - .supportsNormalization(connectorSpecification.getSupportsNormalization()); + .documentationUrl(connectorSpecification.getDocumentationUrl().toString()); destinationConnection = DestinationHelpers.generateDestination(standardDestinationDefinition.getDestinationDefinitionId()); diff --git a/airbyte-webapp/src/components/CreateConnection/CreateConnectionForm.test.tsx b/airbyte-webapp/src/components/CreateConnection/CreateConnectionForm.test.tsx index af61ff6ba76d..e5ec8a8e258f 100644 --- a/airbyte-webapp/src/components/CreateConnection/CreateConnectionForm.test.tsx +++ b/airbyte-webapp/src/components/CreateConnection/CreateConnectionForm.test.tsx @@ -4,7 +4,10 @@ import userEvent from "@testing-library/user-event"; import React from "react"; import selectEvent from "react-select-event"; import { mockConnection } from "test-utils/mock-data/mockConnection"; -import { mockDestination } from "test-utils/mock-data/mockDestination"; +import { + mockDestinationDefinition, + mockDestinationDefinitionSpecification, +} from "test-utils/mock-data/mockDestination"; import { TestWrapper } from "test-utils/testutils"; import { defaultOssFeatures, FeatureItem } from "hooks/services/Feature"; @@ -13,7 +16,11 @@ import * as sourceHook from "hooks/services/useSourceHook"; import { CreateConnectionForm } from "./CreateConnectionForm"; jest.mock("services/connector/DestinationDefinitionSpecificationService", () => ({ - useGetDestinationDefinitionSpecification: () => mockDestination, + useGetDestinationDefinitionSpecification: () => mockDestinationDefinitionSpecification, +})); + +jest.mock("services/connector/DestinationDefinitionService", () => ({ + useDestinationDefinition: () => mockDestinationDefinition, })); jest.mock("services/workspaces/WorkspacesService", () => ({ diff --git a/airbyte-webapp/src/components/connection/CatalogTree/BulkHeader.tsx b/airbyte-webapp/src/components/connection/CatalogTree/BulkHeader.tsx index 4bbbe17f7774..07e5cf648c45 100644 --- a/airbyte-webapp/src/components/connection/CatalogTree/BulkHeader.tsx +++ b/airbyte-webapp/src/components/connection/CatalogTree/BulkHeader.tsx @@ -43,7 +43,7 @@ function calculateSharedFields(selectedBatchNodes: SyncSchemaStream[]) { export const BulkHeader: React.FC = () => { const { - destDefinition: { supportedDestinationSyncModes }, + destDefinitionSpecification: { supportedDestinationSyncModes }, } = useConnectionFormService(); const { selectedBatchNodes, options, onChangeOption, onApply, isActive, onCancel } = useBulkEditService(); diff --git a/airbyte-webapp/src/components/connection/CatalogTree/CatalogSection.tsx b/airbyte-webapp/src/components/connection/CatalogTree/CatalogSection.tsx index 7d75f818ea61..7b0b0b7bcaa2 100644 --- a/airbyte-webapp/src/components/connection/CatalogTree/CatalogSection.tsx +++ b/airbyte-webapp/src/components/connection/CatalogTree/CatalogSection.tsx @@ -48,7 +48,7 @@ const CatalogSectionInner: React.FC = ({ const isNewStreamsTableEnabled = process.env.REACT_APP_NEW_STREAMS_TABLE ?? false; const { - destDefinition: { supportedDestinationSyncModes }, + destDefinitionSpecification: { supportedDestinationSyncModes }, } = useConnectionFormService(); const { mode } = useConnectionFormService(); diff --git a/airbyte-webapp/src/components/connection/CatalogTree/next/BulkEditPanel.test.tsx b/airbyte-webapp/src/components/connection/CatalogTree/next/BulkEditPanel.test.tsx index 35ccc9f1af73..8fadfbbd9d90 100644 --- a/airbyte-webapp/src/components/connection/CatalogTree/next/BulkEditPanel.test.tsx +++ b/airbyte-webapp/src/components/connection/CatalogTree/next/BulkEditPanel.test.tsx @@ -174,7 +174,7 @@ jest.mock("hooks/services/BulkEdit/BulkEditService", () => ({ })); jest.mock("hooks/services/ConnectionForm/ConnectionFormService", () => ({ - useConnectionFormService: () => ({ destDefinition: ["overwrite", "append", "append_dedup"] }), + useConnectionFormService: () => ({ destDefinitionSpecification: ["overwrite", "append", "append_dedup"] }), })); const renderBulkEditPanel = () => diff --git a/airbyte-webapp/src/components/connection/CatalogTree/next/BulkEditPanel.tsx b/airbyte-webapp/src/components/connection/CatalogTree/next/BulkEditPanel.tsx index 800291990c4e..e101b786902a 100644 --- a/airbyte-webapp/src/components/connection/CatalogTree/next/BulkEditPanel.tsx +++ b/airbyte-webapp/src/components/connection/CatalogTree/next/BulkEditPanel.tsx @@ -75,7 +75,7 @@ export const getAvailableSyncModesOptions = ( export const BulkEditPanel: React.FC = () => { const { - destDefinition: { supportedDestinationSyncModes }, + destDefinitionSpecification: { supportedDestinationSyncModes }, } = useConnectionFormService(); const { selectedBatchNodes, options, onChangeOption, onApply, isActive, onCancel } = useBulkEditService(); const availableSyncModesOptions = useMemo( diff --git a/airbyte-webapp/src/hooks/services/ConnectionEdit/ConnectionEditService.test.tsx b/airbyte-webapp/src/hooks/services/ConnectionEdit/ConnectionEditService.test.tsx index 4e829cb10b1c..a7d5adc8de60 100644 --- a/airbyte-webapp/src/hooks/services/ConnectionEdit/ConnectionEditService.test.tsx +++ b/airbyte-webapp/src/hooks/services/ConnectionEdit/ConnectionEditService.test.tsx @@ -2,7 +2,10 @@ import { act, renderHook } from "@testing-library/react-hooks"; import React from "react"; import { mockCatalogDiff } from "test-utils/mock-data/mockCatalogDiff"; import { mockConnection } from "test-utils/mock-data/mockConnection"; -import { mockDestination } from "test-utils/mock-data/mockDestination"; +import { + mockDestinationDefinition, + mockDestinationDefinitionSpecification, +} from "test-utils/mock-data/mockDestination"; import { mockWorkspace } from "test-utils/mock-data/mockWorkspace"; import { TestWrapper } from "test-utils/testutils"; @@ -12,7 +15,11 @@ import { useConnectionFormService } from "../ConnectionForm/ConnectionFormServic import { ConnectionEditServiceProvider, useConnectionEditService } from "./ConnectionEditService"; jest.mock("services/connector/DestinationDefinitionSpecificationService", () => ({ - useGetDestinationDefinitionSpecification: () => mockDestination, + useGetDestinationDefinitionSpecification: () => mockDestinationDefinitionSpecification, +})); + +jest.mock("services/connector/DestinationDefinitionService", () => ({ + useDestinationDefinition: () => mockDestinationDefinition, })); jest.mock("services/workspaces/WorkspacesService", () => ({ diff --git a/airbyte-webapp/src/hooks/services/ConnectionForm/ConnectionFormService.test.tsx b/airbyte-webapp/src/hooks/services/ConnectionForm/ConnectionFormService.test.tsx index 35dc40bff32d..4405c5caa5dc 100644 --- a/airbyte-webapp/src/hooks/services/ConnectionForm/ConnectionFormService.test.tsx +++ b/airbyte-webapp/src/hooks/services/ConnectionForm/ConnectionFormService.test.tsx @@ -1,7 +1,10 @@ import { act, renderHook } from "@testing-library/react-hooks"; import React from "react"; import { mockConnection } from "test-utils/mock-data/mockConnection"; -import { mockDestination } from "test-utils/mock-data/mockDestination"; +import { + mockDestinationDefinition, + mockDestinationDefinitionSpecification, +} from "test-utils/mock-data/mockDestination"; import { mockWorkspace } from "test-utils/mock-data/mockWorkspace"; import { TestWrapper } from "test-utils/testutils"; @@ -14,7 +17,11 @@ import { } from "./ConnectionFormService"; jest.mock("services/connector/DestinationDefinitionSpecificationService", () => ({ - useGetDestinationDefinitionSpecification: () => mockDestination, + useGetDestinationDefinitionSpecification: () => mockDestinationDefinitionSpecification, +})); + +jest.mock("services/connector/DestinationDefinitionService", () => ({ + useDestinationDefinition: () => mockDestinationDefinition, })); jest.mock("services/workspaces/WorkspacesService", () => ({ diff --git a/airbyte-webapp/src/hooks/services/ConnectionForm/ConnectionFormService.tsx b/airbyte-webapp/src/hooks/services/ConnectionForm/ConnectionFormService.tsx index ff3baccd4d73..f5d030f46215 100644 --- a/airbyte-webapp/src/hooks/services/ConnectionForm/ConnectionFormService.tsx +++ b/airbyte-webapp/src/hooks/services/ConnectionForm/ConnectionFormService.tsx @@ -3,10 +3,12 @@ import { useIntl } from "react-intl"; import { ConnectionScheduleType, + DestinationDefinitionRead, DestinationDefinitionSpecificationRead, OperationRead, WebBackendConnectionRead, } from "core/request/AirbyteClient"; +import { useDestinationDefinition } from "services/connector/DestinationDefinitionService"; import { useGetDestinationDefinitionSpecification } from "services/connector/DestinationDefinitionSpecificationService"; import { FormError, generateMessageFromError } from "utils/errorStatusMessage"; import { @@ -63,7 +65,8 @@ export const tidyConnectionFormValues = ( interface ConnectionFormHook { connection: ConnectionOrPartialConnection; mode: ConnectionFormMode; - destDefinition: DestinationDefinitionSpecificationRead; + destDefinition: DestinationDefinitionRead; + destDefinitionSpecification: DestinationDefinitionSpecificationRead; initialValues: FormikConnectionFormValues; schemaError?: SchemaError; formId: string; @@ -78,8 +81,11 @@ const useConnectionForm = ({ schemaError, refreshSchema, }: ConnectionServiceProps): ConnectionFormHook => { - const destDefinition = useGetDestinationDefinitionSpecification(connection.destination.destinationDefinitionId); - const initialValues = useInitialValues(connection, destDefinition, mode !== "create"); + const destDefinition = useDestinationDefinition(connection.destination.destinationDefinitionId); + const destDefinitionSpecification = useGetDestinationDefinitionSpecification( + connection.destination.destinationDefinitionId + ); + const initialValues = useInitialValues(connection, destDefinition, destDefinitionSpecification, mode !== "create"); const { formatMessage } = useIntl(); const [submitError, setSubmitError] = useState(null); const formId = useUniqueFormId(); @@ -109,6 +115,7 @@ const useConnectionForm = ({ connection, mode, destDefinition, + destDefinitionSpecification, initialValues, schemaError, formId, diff --git a/airbyte-webapp/src/pages/ConnectionPage/pages/ConnectionItemPage/ConnectionReplicationTab.test.tsx b/airbyte-webapp/src/pages/ConnectionPage/pages/ConnectionItemPage/ConnectionReplicationTab.test.tsx index 737562bbc25a..7f6d2b114017 100644 --- a/airbyte-webapp/src/pages/ConnectionPage/pages/ConnectionItemPage/ConnectionReplicationTab.test.tsx +++ b/airbyte-webapp/src/pages/ConnectionPage/pages/ConnectionItemPage/ConnectionReplicationTab.test.tsx @@ -6,7 +6,10 @@ import userEvent from "@testing-library/user-event"; import React, { Suspense } from "react"; import selectEvent from "react-select-event"; import { mockConnection } from "test-utils/mock-data/mockConnection"; -import { mockDestination } from "test-utils/mock-data/mockDestination"; +import { + mockDestinationDefinition, + mockDestinationDefinitionSpecification, +} from "test-utils/mock-data/mockDestination"; import { mockWorkspace } from "test-utils/mock-data/mockWorkspace"; import { mockWorkspaceId } from "test-utils/mock-data/mockWorkspaceId"; import { TestWrapper } from "test-utils/testutils"; @@ -19,8 +22,13 @@ import * as connectionHook from "hooks/services/useConnectionHook"; import { ConnectionReplicationTab } from "./ConnectionReplicationTab"; jest.mock("services/connector/DestinationDefinitionSpecificationService", () => ({ - useGetDestinationDefinitionSpecification: () => mockDestination, + useGetDestinationDefinitionSpecification: () => mockDestinationDefinitionSpecification, })); + +jest.mock("services/connector/DestinationDefinitionService", () => ({ + useDestinationDefinition: () => mockDestinationDefinition, +})); + jest.setTimeout(10000); jest.mock("services/workspaces/WorkspacesService", () => ({ diff --git a/airbyte-webapp/src/pages/ConnectionPage/pages/ConnectionItemPage/ConnectionTransformationTab.tsx b/airbyte-webapp/src/pages/ConnectionPage/pages/ConnectionItemPage/ConnectionTransformationTab.tsx index 04a54ed5c5ee..6e2d3f6209d2 100644 --- a/airbyte-webapp/src/pages/ConnectionPage/pages/ConnectionItemPage/ConnectionTransformationTab.tsx +++ b/airbyte-webapp/src/pages/ConnectionPage/pages/ConnectionItemPage/ConnectionTransformationTab.tsx @@ -13,7 +13,7 @@ import { useConnectionEditService } from "hooks/services/ConnectionEdit/Connecti import { useConnectionFormService } from "hooks/services/ConnectionForm/ConnectionFormService"; import { FeatureItem, useFeature } from "hooks/services/Feature"; import { useCurrentWorkspace } from "hooks/services/useWorkspace"; -import { useGetDestinationDefinitionSpecification } from "services/connector/DestinationDefinitionSpecificationService"; +import { useDestinationDefinition } from "services/connector/DestinationDefinitionService"; import { FormikOnSubmit } from "types/formik"; import { NormalizationField } from "views/Connection/ConnectionForm/components/NormalizationField"; import { TransformationField } from "views/Connection/ConnectionForm/components/TransformationField"; @@ -95,11 +95,11 @@ const NormalizationCard: React.FC<{ export const ConnectionTransformationTab: React.FC = () => { const { connection, updateConnection } = useConnectionEditService(); const { mode } = useConnectionFormService(); - const definition = useGetDestinationDefinitionSpecification(connection.destination.destinationDefinitionId); + const definition = useDestinationDefinition(connection.destination.destinationDefinitionId); const workspace = useCurrentWorkspace(); useTrackPage(PageTrackingCodes.CONNECTIONS_ITEM_TRANSFORMATION); - const { supportsNormalization } = definition; + const supportsNormalization = Boolean(definition.normalizationConfig); const supportsDbt = useFeature(FeatureItem.AllowCustomDBT) && definition.supportsDbt; const supportsCloudDbtIntegration = useFeature(FeatureItem.AllowDBTCloudIntegration) && definition.supportsDbt; const noSupportedTransformations = !supportsNormalization && !supportsDbt && !supportsCloudDbtIntegration; diff --git a/airbyte-webapp/src/pages/ConnectionPage/pages/ConnectionItemPage/StatusMainInfo.test.tsx b/airbyte-webapp/src/pages/ConnectionPage/pages/ConnectionItemPage/StatusMainInfo.test.tsx index 80f22eb9635d..1665305d8523 100644 --- a/airbyte-webapp/src/pages/ConnectionPage/pages/ConnectionItemPage/StatusMainInfo.test.tsx +++ b/airbyte-webapp/src/pages/ConnectionPage/pages/ConnectionItemPage/StatusMainInfo.test.tsx @@ -1,5 +1,5 @@ import { render } from "@testing-library/react"; -import { mockDestinationDefinition } from "test-utils/mock-data/mockDestinationDefinition"; +import { mockDestinationDefinitionSpecification } from "test-utils/mock-data/mockDestinationDefinitionSpecification"; import { mockSourceDefinition } from "test-utils/mock-data/mockSourceDefinition"; import { mockConnection, TestWrapper } from "test-utils/testutils"; @@ -20,7 +20,7 @@ jest.doMock("services/connector/SourceDefinitionService", () => ({ })); jest.doMock("services/connector/DestinationDefinitionService", () => ({ - useDestinationDefinition: () => mockDestinationDefinition, + useDestinationDefinition: () => mockDestinationDefinitionSpecification, })); jest.doMock("views/Connection/ConnectionForm/components/refreshSourceSchemaWithConfirmationOnDirty", () => ({ diff --git a/airbyte-webapp/src/test-utils/mock-data/mockDestination.ts b/airbyte-webapp/src/test-utils/mock-data/mockDestination.ts index dc7474aeec6c..1be14d77bf21 100644 --- a/airbyte-webapp/src/test-utils/mock-data/mockDestination.ts +++ b/airbyte-webapp/src/test-utils/mock-data/mockDestination.ts @@ -1,7 +1,24 @@ -import { DestinationDefinitionSpecificationRead } from "core/request/AirbyteClient"; +import { DestinationDefinitionSpecificationRead, DestinationDefinitionRead } from "core/request/AirbyteClient"; import { ConnectorIds } from "utils/connectors"; -export const mockDestination: DestinationDefinitionSpecificationRead = { +export const mockDestinationDefinition: DestinationDefinitionRead = { + destinationDefinitionId: ConnectorIds.Destinations.Postgres, + name: "Postgres", + dockerRepository: "airbyte/destination-postgres", + dockerImageTag: "0.3.26", + documentationUrl: "https://docs.airbyte.com/integrations/destinations/postgres", + icon: '', + releaseStage: "alpha", + supportsDbt: true, + normalizationConfig: { + supported: true, + normalizationRepository: "airbyte/normalization", + normalizationTag: "0.2.25", + normalizationIntegrationType: "postgres", + }, +}; + +export const mockDestinationDefinitionSpecification: DestinationDefinitionSpecificationRead = { destinationDefinitionId: ConnectorIds.Destinations.Postgres, documentationUrl: "https://docs.airbyte.io/integrations/destinations/postgres", connectionSpecification: { @@ -334,6 +351,4 @@ export const mockDestination: DestinationDefinitionSpecificationRead = { }, }, supportedDestinationSyncModes: ["overwrite", "append", "append_dedup"], - supportsDbt: true, - supportsNormalization: true, }; diff --git a/airbyte-webapp/src/test-utils/mock-data/mockDestinationDefinition.ts b/airbyte-webapp/src/test-utils/mock-data/mockDestinationDefinitionSpecification.ts similarity index 98% rename from airbyte-webapp/src/test-utils/mock-data/mockDestinationDefinition.ts rename to airbyte-webapp/src/test-utils/mock-data/mockDestinationDefinitionSpecification.ts index 04ce0cf05416..6f201361fe66 100644 --- a/airbyte-webapp/src/test-utils/mock-data/mockDestinationDefinition.ts +++ b/airbyte-webapp/src/test-utils/mock-data/mockDestinationDefinitionSpecification.ts @@ -1,7 +1,7 @@ import { DestinationDefinitionSpecificationRead } from "core/request/AirbyteClient"; import { ConnectorIds } from "utils/connectors"; -export const mockDestinationDefinition: DestinationDefinitionSpecificationRead = { +export const mockDestinationDefinitionSpecification: DestinationDefinitionSpecificationRead = { destinationDefinitionId: ConnectorIds.Destinations.Postgres, documentationUrl: "https://docs.airbyte.io/integrations/destinations/postgres", connectionSpecification: { @@ -273,6 +273,4 @@ export const mockDestinationDefinition: DestinationDefinitionSpecificationRead = logs: { logLines: [] }, }, supportedDestinationSyncModes: ["overwrite", "append", "append_dedup"], - supportsDbt: true, - supportsNormalization: true, }; diff --git a/airbyte-webapp/src/views/Connection/ConnectionForm/components/OperationsSection.tsx b/airbyte-webapp/src/views/Connection/ConnectionForm/components/OperationsSection.tsx index 7ca075943f1f..af139c171966 100644 --- a/airbyte-webapp/src/views/Connection/ConnectionForm/components/OperationsSection.tsx +++ b/airbyte-webapp/src/views/Connection/ConnectionForm/components/OperationsSection.tsx @@ -24,9 +24,10 @@ export const OperationsSection: React.FC = ({ const { formatMessage } = useIntl(); const { - destDefinition: { supportsNormalization, supportsDbt }, + destDefinition: { normalizationConfig, supportsDbt }, } = useConnectionFormService(); + const supportsNormalization = normalizationConfig.supported; const supportsTransformations = useFeature(FeatureItem.AllowCustomDBT) && supportsDbt; if (!supportsNormalization && !supportsTransformations) { diff --git a/airbyte-webapp/src/views/Connection/ConnectionForm/formConfig.test.ts b/airbyte-webapp/src/views/Connection/ConnectionForm/formConfig.test.ts index d79693be6c19..5985e76f2fb8 100644 --- a/airbyte-webapp/src/views/Connection/ConnectionForm/formConfig.test.ts +++ b/airbyte-webapp/src/views/Connection/ConnectionForm/formConfig.test.ts @@ -1,6 +1,9 @@ import { renderHook } from "@testing-library/react-hooks"; import { mockConnection } from "test-utils/mock-data/mockConnection"; -import { mockDestination } from "test-utils/mock-data/mockDestination"; +import { + mockDestinationDefinition, + mockDestinationDefinitionSpecification, +} from "test-utils/mock-data/mockDestination"; import { mockWorkspace } from "test-utils/mock-data/mockWorkspace"; import { TestWrapper as wrapper } from "test-utils/testutils"; @@ -171,19 +174,25 @@ describe("#mapFormPropsToOperation", () => { describe("#useInitialValues", () => { it("should generate initial values w/ no 'not create' mode", () => { - const { result } = renderHook(() => useInitialValues(mockConnection, mockDestination)); + const { result } = renderHook(() => + useInitialValues(mockConnection, mockDestinationDefinition, mockDestinationDefinitionSpecification) + ); expect(result.current).toMatchSnapshot(); expect(result.current.name).toBeDefined(); }); it("should generate initial values w/ 'not create' mode: false", () => { - const { result } = renderHook(() => useInitialValues(mockConnection, mockDestination, false)); + const { result } = renderHook(() => + useInitialValues(mockConnection, mockDestinationDefinition, mockDestinationDefinitionSpecification, false) + ); expect(result.current).toMatchSnapshot(); expect(result.current.name).toBeDefined(); }); it("should generate initial values w/ 'not create' mode: true", () => { - const { result } = renderHook(() => useInitialValues(mockConnection, mockDestination, true)); + const { result } = renderHook(() => + useInitialValues(mockConnection, mockDestinationDefinition, mockDestinationDefinitionSpecification, true) + ); expect(result.current).toMatchSnapshot(); expect(result.current.name).toBeUndefined(); }); diff --git a/airbyte-webapp/src/views/Connection/ConnectionForm/formConfig.tsx b/airbyte-webapp/src/views/Connection/ConnectionForm/formConfig.tsx index 22168ab51394..33f8dc0d2efe 100644 --- a/airbyte-webapp/src/views/Connection/ConnectionForm/formConfig.tsx +++ b/airbyte-webapp/src/views/Connection/ConnectionForm/formConfig.tsx @@ -15,6 +15,7 @@ import { SOURCE_NAMESPACE_TAG } from "core/domain/connector/source"; import { ConnectionScheduleData, ConnectionScheduleType, + DestinationDefinitionRead, DestinationDefinitionSpecificationRead, DestinationSyncMode, Geography, @@ -316,7 +317,8 @@ export const getInitialNormalization = ( export const useInitialValues = ( connection: ConnectionOrPartialConnection, - destDefinition: DestinationDefinitionSpecificationRead, + destDefinition: DestinationDefinitionRead, + destDefinitionSpecification: DestinationDefinitionSpecificationRead, isNotCreateMode?: boolean ): FormikConnectionFormValues => { const workspace = useCurrentWorkspace(); @@ -344,7 +346,7 @@ export const useInitialValues = ( () => calculateInitialCatalog( connection.syncCatalog, - destDefinition?.supportedDestinationSyncModes || [], + destDefinitionSpecification?.supportedDestinationSyncModes || [], streamTransformsWithBreakingChange, isNotCreateMode, newStreamDescriptors @@ -352,7 +354,7 @@ export const useInitialValues = ( [ streamTransformsWithBreakingChange, connection.syncCatalog, - destDefinition?.supportedDestinationSyncModes, + destDefinitionSpecification?.supportedDestinationSyncModes, isNotCreateMode, newStreamDescriptors, ] @@ -381,7 +383,7 @@ export const useInitialValues = ( initialValues.transformations = getInitialTransformations(operations); } - if (destDefinition.supportsNormalization) { + if (destDefinition.normalizationConfig.supported) { initialValues.normalization = getInitialNormalization(operations, isNotCreateMode); } @@ -400,7 +402,7 @@ export const useInitialValues = ( connection.scheduleType, connection.source.name, destDefinition.supportsDbt, - destDefinition.supportsNormalization, + destDefinition.normalizationConfig, initialSchema, isNotCreateMode, workspace, diff --git a/docs/reference/api/generated-api-html/index.html b/docs/reference/api/generated-api-html/index.html index 47bf101ac228..013a03823f65 100644 --- a/docs/reference/api/generated-api-html/index.html +++ b/docs/reference/api/generated-api-html/index.html @@ -2596,8 +2596,15 @@

Example data

"dockerImageTag" : "dockerImageTag", "releaseDate" : "2000-01-23", "dockerRepository" : "dockerRepository", + "supportsDbt" : true, "name" : "name", "icon" : "icon", + "normalizationConfig" : { + "normalizationIntegrationType" : "normalizationIntegrationType", + "normalizationRepository" : "normalizationRepository", + "normalizationTag" : "normalizationTag", + "supported" : false + }, "protocolVersion" : "protocolVersion", "destinationDefinitionId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91" } @@ -2729,8 +2736,15 @@

Example data

"dockerImageTag" : "dockerImageTag", "releaseDate" : "2000-01-23", "dockerRepository" : "dockerRepository", + "supportsDbt" : true, "name" : "name", "icon" : "icon", + "normalizationConfig" : { + "normalizationIntegrationType" : "normalizationIntegrationType", + "normalizationRepository" : "normalizationRepository", + "normalizationTag" : "normalizationTag", + "supported" : false + }, "protocolVersion" : "protocolVersion", "destinationDefinitionId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91" } @@ -2817,8 +2831,15 @@

Example data

"dockerImageTag" : "dockerImageTag", "releaseDate" : "2000-01-23", "dockerRepository" : "dockerRepository", + "supportsDbt" : true, "name" : "name", "icon" : "icon", + "normalizationConfig" : { + "normalizationIntegrationType" : "normalizationIntegrationType", + "normalizationRepository" : "normalizationRepository", + "normalizationTag" : "normalizationTag", + "supported" : false + }, "protocolVersion" : "protocolVersion", "destinationDefinitionId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91" } @@ -2906,8 +2927,15 @@

Example data

"dockerImageTag" : "dockerImageTag", "releaseDate" : "2000-01-23", "dockerRepository" : "dockerRepository", + "supportsDbt" : true, "name" : "name", "icon" : "icon", + "normalizationConfig" : { + "normalizationIntegrationType" : "normalizationIntegrationType", + "normalizationRepository" : "normalizationRepository", + "normalizationTag" : "normalizationTag", + "supported" : false + }, "protocolVersion" : "protocolVersion", "destinationDefinitionId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91" }, @@ -2985,8 +3013,15 @@

Example data

"dockerImageTag" : "dockerImageTag", "releaseDate" : "2000-01-23", "dockerRepository" : "dockerRepository", + "supportsDbt" : true, "name" : "name", "icon" : "icon", + "normalizationConfig" : { + "normalizationIntegrationType" : "normalizationIntegrationType", + "normalizationRepository" : "normalizationRepository", + "normalizationTag" : "normalizationTag", + "supported" : false + }, "protocolVersion" : "protocolVersion", "destinationDefinitionId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91" }, { @@ -3017,8 +3052,15 @@

Example data

"dockerImageTag" : "dockerImageTag", "releaseDate" : "2000-01-23", "dockerRepository" : "dockerRepository", + "supportsDbt" : true, "name" : "name", "icon" : "icon", + "normalizationConfig" : { + "normalizationIntegrationType" : "normalizationIntegrationType", + "normalizationRepository" : "normalizationRepository", + "normalizationTag" : "normalizationTag", + "supported" : false + }, "protocolVersion" : "protocolVersion", "destinationDefinitionId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91" } ] @@ -3101,8 +3143,15 @@

Example data

"dockerImageTag" : "dockerImageTag", "releaseDate" : "2000-01-23", "dockerRepository" : "dockerRepository", + "supportsDbt" : true, "name" : "name", "icon" : "icon", + "normalizationConfig" : { + "normalizationIntegrationType" : "normalizationIntegrationType", + "normalizationRepository" : "normalizationRepository", + "normalizationTag" : "normalizationTag", + "supported" : false + }, "protocolVersion" : "protocolVersion", "destinationDefinitionId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91" }, { @@ -3133,8 +3182,15 @@

Example data

"dockerImageTag" : "dockerImageTag", "releaseDate" : "2000-01-23", "dockerRepository" : "dockerRepository", + "supportsDbt" : true, "name" : "name", "icon" : "icon", + "normalizationConfig" : { + "normalizationIntegrationType" : "normalizationIntegrationType", + "normalizationRepository" : "normalizationRepository", + "normalizationTag" : "normalizationTag", + "supported" : false + }, "protocolVersion" : "protocolVersion", "destinationDefinitionId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91" } ] @@ -3205,8 +3261,15 @@

Example data

"dockerImageTag" : "dockerImageTag", "releaseDate" : "2000-01-23", "dockerRepository" : "dockerRepository", + "supportsDbt" : true, "name" : "name", "icon" : "icon", + "normalizationConfig" : { + "normalizationIntegrationType" : "normalizationIntegrationType", + "normalizationRepository" : "normalizationRepository", + "normalizationTag" : "normalizationTag", + "supported" : false + }, "protocolVersion" : "protocolVersion", "destinationDefinitionId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91" }, { @@ -3237,8 +3300,15 @@

Example data

"dockerImageTag" : "dockerImageTag", "releaseDate" : "2000-01-23", "dockerRepository" : "dockerRepository", + "supportsDbt" : true, "name" : "name", "icon" : "icon", + "normalizationConfig" : { + "normalizationIntegrationType" : "normalizationIntegrationType", + "normalizationRepository" : "normalizationRepository", + "normalizationTag" : "normalizationTag", + "supported" : false + }, "protocolVersion" : "protocolVersion", "destinationDefinitionId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91" } ] @@ -3322,8 +3392,15 @@

Example data

"dockerImageTag" : "dockerImageTag", "releaseDate" : "2000-01-23", "dockerRepository" : "dockerRepository", + "supportsDbt" : true, "name" : "name", "icon" : "icon", + "normalizationConfig" : { + "normalizationIntegrationType" : "normalizationIntegrationType", + "normalizationRepository" : "normalizationRepository", + "normalizationTag" : "normalizationTag", + "supported" : false + }, "protocolVersion" : "protocolVersion", "destinationDefinitionId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91" }, @@ -3357,8 +3434,15 @@

Example data

"dockerImageTag" : "dockerImageTag", "releaseDate" : "2000-01-23", "dockerRepository" : "dockerRepository", + "supportsDbt" : true, "name" : "name", "icon" : "icon", + "normalizationConfig" : { + "normalizationIntegrationType" : "normalizationIntegrationType", + "normalizationRepository" : "normalizationRepository", + "normalizationTag" : "normalizationTag", + "supported" : false + }, "protocolVersion" : "protocolVersion", "destinationDefinitionId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91" }, @@ -3490,8 +3574,15 @@

Example data

"dockerImageTag" : "dockerImageTag", "releaseDate" : "2000-01-23", "dockerRepository" : "dockerRepository", + "supportsDbt" : true, "name" : "name", "icon" : "icon", + "normalizationConfig" : { + "normalizationIntegrationType" : "normalizationIntegrationType", + "normalizationRepository" : "normalizationRepository", + "normalizationTag" : "normalizationTag", + "supported" : false + }, "protocolVersion" : "protocolVersion", "destinationDefinitionId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91" } @@ -3553,14 +3644,12 @@

Example data

Content-Type: application/json
{
   "documentationUrl" : "documentationUrl",
-  "supportsNormalization" : true,
   "connectionSpecification" : {
     "user" : {
       "type" : "string"
     }
   },
   "supportedDestinationSyncModes" : [ null, null ],
-  "supportsDbt" : true,
   "destinationDefinitionId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91",
   "advancedAuth" : {
     "predicateValue" : "predicateValue",
@@ -4416,8 +4505,15 @@ 
Example data

       "dockerImageTag" : "dockerImageTag",
       "releaseDate" : "2000-01-23",
       "dockerRepository" : "dockerRepository",
+      "supportsDbt" : true,
       "name" : "name",
       "icon" : "icon",
+      "normalizationConfig" : {
+        "normalizationIntegrationType" : "normalizationIntegrationType",
+        "normalizationRepository" : "normalizationRepository",
+        "normalizationTag" : "normalizationTag",
+        "supported" : false
+      },
       "protocolVersion" : "protocolVersion",
       "destinationDefinitionId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91"
     }
@@ -10173,6 +10269,7 @@ 
Table of Contents

     
  • LogsRequestBody - 
    • 
     
  • NamespaceDefinitionType - 
    • 
     
  • NonBreakingChangesPreference - 
    • 
+    
  • NormalizationDestinationDefinitionConfig - 
    • 
     
  • NotFoundKnownExceptionInfo - 
    • 
     
  • Notification - 
    • 
     
  • NotificationRead - 
    • 
@@ -10808,6 +10905,8 @@ 
    DestinationDefinitionRead -
 
    releaseStage (optional)
    ReleaseStage  
    
 
    releaseDate (optional)
    date The date when this connector was first released, in yyyy-mm-dd format. format: date
    
 
    resourceRequirements (optional)
    ActorDefinitionResourceRequirements  
    
+
    supportsDbt 
    Boolean an optional flag indicating whether DBT is used in the normalization. If the flag value is NULL - DBT is not used. 
    
+
    normalizationConfig 
    NormalizationDestinationDefinitionConfig  
    
       
   
   
    
@@ -10828,8 +10927,6 @@ 
    DestinationDefinition
 
    advancedAuth (optional)
    AdvancedAuth  
    
 
    jobInfo 
    SynchronousJobRead  
    
 
    supportedDestinationSyncModes (optional)
    array[DestinationSyncMode]  
    
-
    supportsDbt (optional)
    Boolean  
    
-
    supportsNormalization (optional)
    Boolean  
    
     
      
   
   
    
@@ -11170,6 +11267,16 @@ 
    NonBreakingChangesPreference
           
      
   
+  
    
+    
    NormalizationDestinationDefinitionConfig -  Up
    
+    
    describes a normalization config for destination definition
    
+    
    
+      
    supported 
    Boolean whether the destination definition supports normalization. 
    
+
    normalizationRepository (optional)
    String a field indicating the name of the repository to be used for normalization. If the value of the flag is NULL - normalization is not used. 
    
+
    normalizationTag (optional)
    String a field indicating the tag of the docker repository to be used for normalization. 
    
+
    normalizationIntegrationType (optional)
    String a field indicating the type of integration dialect to use for normalization. 
    
+    
      
+  
    
   
    
     
    NotFoundKnownExceptionInfo -  Up
    
     
    
diff --git a/octavia-cli/integration_tests/test_generate/expected_rendered_yaml/destination_postgres/input_spec.yaml b/octavia-cli/integration_tests/test_generate/expected_rendered_yaml/destination_postgres/input_spec.yaml
index e96c1e340d33..dcab0bdcc097 100644
--- a/octavia-cli/integration_tests/test_generate/expected_rendered_yaml/destination_postgres/input_spec.yaml
+++ b/octavia-cli/integration_tests/test_generate/expected_rendered_yaml/destination_postgres/input_spec.yaml
@@ -171,8 +171,6 @@ spec:
                 airbyte_secret: true
                 order: 4
   supportsIncremental: true
-  supportsNormalization: true
-  supportsDBT: true
   supported_destination_sync_modes:
     - "overwrite"
     - "append"
diff --git a/octavia-cli/integration_tests/test_generate/expected_rendered_yaml/destination_s3/input_spec.yaml b/octavia-cli/integration_tests/test_generate/expected_rendered_yaml/destination_s3/input_spec.yaml
index 5dde0fd6567f..7e0f17ac5b30 100644
--- a/octavia-cli/integration_tests/test_generate/expected_rendered_yaml/destination_s3/input_spec.yaml
+++ b/octavia-cli/integration_tests/test_generate/expected_rendered_yaml/destination_s3/input_spec.yaml
@@ -325,8 +325,6 @@ spec:
                 type: "boolean"
                 default: true
   supportsIncremental: true
-  supportsNormalization: false
-  supportsDBT: false
   supported_destination_sync_modes:
     - "overwrite"
     - "append"
diff --git a/octavia-cli/integration_tests/test_generate/expected_rendered_yaml/source_postgres/input_spec.yaml b/octavia-cli/integration_tests/test_generate/expected_rendered_yaml/source_postgres/input_spec.yaml
index 3fecca207abe..bcc8a383ad76 100644
--- a/octavia-cli/integration_tests/test_generate/expected_rendered_yaml/source_postgres/input_spec.yaml
+++ b/octavia-cli/integration_tests/test_generate/expected_rendered_yaml/source_postgres/input_spec.yaml
@@ -235,6 +235,4 @@ spec:
                 type: "string"
                 airbyte_secret: true
                 order: 4
-  supportsNormalization: false
-  supportsDBT: false
   supported_destination_sync_modes: []
diff --git a/octavia-cli/integration_tests/test_generate/test_renderers.py b/octavia-cli/integration_tests/test_generate/test_renderers.py
index c7afbb4bc275..22cd2c0e234b 100644
--- a/octavia-cli/integration_tests/test_generate/test_renderers.py
+++ b/octavia-cli/integration_tests/test_generate/test_renderers.py
@@ -111,7 +111,7 @@ def test_expected_output_connection_renderer(octavia_tmp_project_directory, mock
     mock_destination = mocker.Mock(
         resource_id="my_destination_id",
         configuration_path="destination_configuration_path",
-        definition=mocker.Mock(supports_dbt=with_normalization, supports_normalization=with_normalization),
+        definition=mocker.Mock(supports_dbt=with_normalization, normalization_config=mocker.Mock(supported=with_normalization)),
     )
 
     renderer = ConnectionRenderer("my_new_connection", mock_source, mock_destination)
diff --git a/octavia-cli/octavia_cli/apply/resources.py b/octavia-cli/octavia_cli/apply/resources.py
index 7425ac61d0f9..ffb5867ed963 100644
--- a/octavia-cli/octavia_cli/apply/resources.py
+++ b/octavia-cli/octavia_cli/apply/resources.py
@@ -14,8 +14,10 @@
 import yaml
 from airbyte_api_client.api import (
     destination_api,
+    destination_definition_api,
     destination_definition_specification_api,
     source_api,
+    source_definition_api,
     source_definition_specification_api,
     web_backend_api,
 )
@@ -30,7 +32,9 @@
 from airbyte_api_client.model.connection_schedule_type import ConnectionScheduleType
 from airbyte_api_client.model.connection_status import ConnectionStatus
 from airbyte_api_client.model.destination_create import DestinationCreate
+from airbyte_api_client.model.destination_definition_id_request_body import DestinationDefinitionIdRequestBody
 from airbyte_api_client.model.destination_definition_id_with_workspace_id import DestinationDefinitionIdWithWorkspaceId
+from airbyte_api_client.model.destination_definition_read import DestinationDefinitionRead
 from airbyte_api_client.model.destination_definition_specification_read import DestinationDefinitionSpecificationRead
 from airbyte_api_client.model.destination_id_request_body import DestinationIdRequestBody
 from airbyte_api_client.model.destination_read import DestinationRead
@@ -44,7 +48,9 @@
 from airbyte_api_client.model.operator_type import OperatorType
 from airbyte_api_client.model.resource_requirements import ResourceRequirements
 from airbyte_api_client.model.source_create import SourceCreate
+from airbyte_api_client.model.source_definition_id_request_body import SourceDefinitionIdRequestBody
 from airbyte_api_client.model.source_definition_id_with_workspace_id import SourceDefinitionIdWithWorkspaceId
+from airbyte_api_client.model.source_definition_read import SourceDefinitionRead
 from airbyte_api_client.model.source_definition_specification_read import SourceDefinitionSpecificationRead
 from airbyte_api_client.model.source_discover_schema_request_body import SourceDiscoverSchemaRequestBody
 from airbyte_api_client.model.source_id_request_body import SourceIdRequestBody
@@ -431,6 +437,7 @@ def resource_id(self) -> Optional[str]:
 
 
 class SourceAndDestination(BaseResource):
+
     @property
     @abc.abstractmethod
     def definition(
@@ -438,6 +445,13 @@ def definition(
     ):  # pragma: no cover
         pass
 
+    @property
+    @abc.abstractmethod
+    def definition_specification(
+        self,
+    ):  # pragma: no cover
+        pass
+
     @property
     def definition_id(self):
         return self.raw_configuration["definition_id"]
@@ -508,7 +522,13 @@ def catalog(self) -> AirbyteCatalog:
         raise Exception("Could not discover schema for source", self.source_discover_schema_request_body, schema.job_info.logs)
 
     @property
-    def definition(self) -> SourceDefinitionSpecificationRead:
+    def definition(self) -> SourceDefinitionRead:
+        api_instance = source_definition_api.SourceDefinitionApi(self.api_client)
+        payload = SourceDefinitionIdRequestBody(source_definition_id=self.definition_id)
+        return api_instance.get_source_definition(payload)
+
+    @property
+    def definition_specification(self) -> SourceDefinitionSpecificationRead:
         api_instance = source_definition_specification_api.SourceDefinitionSpecificationApi(self.api_client)
         payload = SourceDefinitionIdWithWorkspaceId(source_definition_id=self.definition_id, workspace_id=self.workspace_id)
         return api_instance.get_source_definition_specification(payload)
@@ -552,7 +572,13 @@ def update_payload(self) -> DestinationUpdate:
         )
 
     @property
-    def definition(self) -> DestinationDefinitionSpecificationRead:
+    def definition(self) -> DestinationDefinitionRead:
+        api_instance = destination_definition_api.DestinationDefinitionApi(self.api_client)
+        payload = DestinationDefinitionIdRequestBody(destination_definition_id=self.definition_id)
+        return api_instance.get_destination_definition(payload)
+
+    @property
+    def definition_specification(self) -> DestinationDefinitionSpecificationRead:
         api_instance = destination_definition_specification_api.DestinationDefinitionSpecificationApi(self.api_client)
         payload = DestinationDefinitionIdWithWorkspaceId(destination_definition_id=self.definition_id, workspace_id=self.workspace_id)
         return api_instance.get_destination_definition_specification(payload)
diff --git a/octavia-cli/octavia_cli/generate/renderers.py b/octavia-cli/octavia_cli/generate/renderers.py
index 5a56264fdce4..28308f03627d 100644
--- a/octavia-cli/octavia_cli/generate/renderers.py
+++ b/octavia-cli/octavia_cli/generate/renderers.py
@@ -298,7 +298,7 @@ def _render(self) -> str:
                 "source_configuration_path": self.source.configuration_path,
                 "destination_configuration_path": self.destination.configuration_path,
                 "catalog": yaml_catalog,
-                "supports_normalization": self.destination.definition.supports_normalization,
+                "supports_normalization": self.destination.definition.normalization_config.supported,
                 "supports_dbt": self.destination.definition.supports_dbt,
             }
         )
diff --git a/octavia-cli/unit_tests/test_apply/test_resources.py b/octavia-cli/unit_tests/test_apply/test_resources.py
index 66014d435a42..9ee7043040a3 100644
--- a/octavia-cli/unit_tests/test_apply/test_resources.py
+++ b/octavia-cli/unit_tests/test_apply/test_resources.py
@@ -11,11 +11,13 @@
 from airbyte_api_client.model.connection_schedule_data_basic_schedule import ConnectionScheduleDataBasicSchedule
 from airbyte_api_client.model.connection_schedule_type import ConnectionScheduleType
 from airbyte_api_client.model.connection_status import ConnectionStatus
+from airbyte_api_client.model.destination_definition_id_request_body import DestinationDefinitionIdRequestBody
 from airbyte_api_client.model.destination_definition_id_with_workspace_id import DestinationDefinitionIdWithWorkspaceId
 from airbyte_api_client.model.namespace_definition_type import NamespaceDefinitionType
 from airbyte_api_client.model.operation_create import OperationCreate
 from airbyte_api_client.model.operator_type import OperatorType
 from airbyte_api_client.model.resource_requirements import ResourceRequirements
+from airbyte_api_client.model.source_definition_id_request_body import SourceDefinitionIdRequestBody
 from airbyte_api_client.model.source_definition_id_with_workspace_id import SourceDefinitionIdWithWorkspaceId
 from airbyte_api_client.model.web_backend_operation_create_or_update import WebBackendOperationCreateOrUpdate
 from octavia_cli.apply import resources, yaml_loaders
@@ -386,10 +388,19 @@ def test_catalog(self, mocker, mock_api_client, local_configuration):
         source.api_instance.discover_schema_for_source.assert_called_with(source.source_discover_schema_request_body)
 
     def test_definition(self, mocker, mock_api_client, local_configuration):
+        mocker.patch.object(resources.source_definition_api, "SourceDefinitionApi")
+        mock_api_instance = resources.source_definition_api.SourceDefinitionApi.return_value
+        source = resources.Source(mock_api_client, "workspace_id", local_configuration, "bar.yaml")
+        assert source.definition == mock_api_instance.get_source_definition.return_value
+        resources.source_definition_api.SourceDefinitionApi.assert_called_with(mock_api_client)
+        expected_payload = SourceDefinitionIdRequestBody(source_definition_id=source.definition_id)
+        mock_api_instance.get_source_definition.assert_called_with(expected_payload)
+
+    def test_definition_specification(self, mocker, mock_api_client, local_configuration):
         mocker.patch.object(resources.source_definition_specification_api, "SourceDefinitionSpecificationApi")
         mock_api_instance = resources.source_definition_specification_api.SourceDefinitionSpecificationApi.return_value
         source = resources.Source(mock_api_client, "workspace_id", local_configuration, "bar.yaml")
-        assert source.definition == mock_api_instance.get_source_definition_specification.return_value
+        assert source.definition_specification == mock_api_instance.get_source_definition_specification.return_value
         resources.source_definition_specification_api.SourceDefinitionSpecificationApi.assert_called_with(mock_api_client)
         expected_payload = SourceDefinitionIdWithWorkspaceId(source_definition_id=source.definition_id, workspace_id=source.workspace_id)
         mock_api_instance.get_source_definition_specification.assert_called_with(expected_payload)
@@ -423,10 +434,21 @@ def test_init(self, mocker, mock_api_client, local_configuration, state):
             assert destination.get_payload == resources.DestinationIdRequestBody(state.resource_id)
 
     def test_definition(self, mocker, mock_api_client, local_configuration):
+        mocker.patch.object(resources.destination_definition_api, "DestinationDefinitionApi")
+        mock_api_instance = resources.destination_definition_api.DestinationDefinitionApi.return_value
+        destination = resources.Destination(mock_api_client, "workspace_id", local_configuration, "bar.yaml")
+        assert destination.definition == mock_api_instance.get_destination_definition.return_value
+        resources.destination_definition_api.DestinationDefinitionApi.assert_called_with(mock_api_client)
+        expected_payload = DestinationDefinitionIdRequestBody(
+            destination_definition_id=destination.definition_id
+        )
+        mock_api_instance.get_destination_definition.assert_called_with(expected_payload)
+
+    def test_definition_specification(self, mocker, mock_api_client, local_configuration):
         mocker.patch.object(resources.destination_definition_specification_api, "DestinationDefinitionSpecificationApi")
         mock_api_instance = resources.destination_definition_specification_api.DestinationDefinitionSpecificationApi.return_value
         destination = resources.Destination(mock_api_client, "workspace_id", local_configuration, "bar.yaml")
-        assert destination.definition == mock_api_instance.get_destination_definition_specification.return_value
+        assert destination.definition_specification == mock_api_instance.get_destination_definition_specification.return_value
         resources.destination_definition_specification_api.DestinationDefinitionSpecificationApi.assert_called_with(mock_api_client)
         expected_payload = DestinationDefinitionIdWithWorkspaceId(
             destination_definition_id=destination.definition_id, workspace_id=destination.workspace_id
diff --git a/octavia-cli/unit_tests/test_generate/test_renderers.py b/octavia-cli/unit_tests/test_generate/test_renderers.py
index 478c71ffd249..cd76218fde00 100644
--- a/octavia-cli/unit_tests/test_generate/test_renderers.py
+++ b/octavia-cli/unit_tests/test_generate/test_renderers.py
@@ -355,7 +355,7 @@ def test_write_yaml(self, mocker, mock_source, mock_destination, overwrite):
                     "source_configuration_path": mock_source.configuration_path,
                     "destination_configuration_path": mock_destination.configuration_path,
                     "catalog": connection_renderer.catalog_to_yaml.return_value,
-                    "supports_normalization": connection_renderer.destination.definition.supports_normalization,
+                    "supports_normalization": connection_renderer.destination.definition.normalization_config.supported,
                     "supports_dbt": connection_renderer.destination.definition.supports_dbt,
                 }
             )
@@ -375,7 +375,7 @@ def test__render(self, mocker):
                 "source_configuration_path": connection_renderer.source.configuration_path,
                 "destination_configuration_path": connection_renderer.destination.configuration_path,
                 "catalog": connection_renderer.catalog_to_yaml.return_value,
-                "supports_normalization": connection_renderer.destination.definition.supports_normalization,
+                "supports_normalization": connection_renderer.destination.definition.normalization_config.supported,
                 "supports_dbt": connection_renderer.destination.definition.supports_dbt,
             }
         )