normalization: delete supportsDbt and supportsNormalization from Dest…

…inationDefinitionSpecificationRead (#21005) * delete supportsDbt and supportsNormalization from DestinationDefinitionSpecificationRead * FE changes * add new fields to the DestinationDefinitionRead API model * update handlers * update api doc * update handlers * remove debug loggin * implement suggestions * update octavia-cli tests * fix supported check * update octavia-cli * fix integration tests * fix mocks * fix forgotten renaming Co-authored-by: Evan Tahler <evan@airbyte.io>
airbytehq · Jan 7, 2023 · e4707f5 · e4707f5
1 parent 3c52168
commit e4707f5
Show file tree

Hide file tree

Showing 31 changed files with 390 additions and 100 deletions.
diff --git a/airbyte-api/src/main/openapi/config.yaml b/airbyte-api/src/main/openapi/config.yaml
@@ -2878,6 +2878,8 @@ components:
         - dockerRepository
         - dockerImageTag
         - documentationUrl
+        - supportsDbt
+        - normalizationConfig
       properties:
         destinationDefinitionId:
           $ref: "#/components/schemas/DestinationDefinitionId"
@@ -2903,6 +2905,12 @@ components:
           format: date
         resourceRequirements:
           $ref: "#/components/schemas/ActorDefinitionResourceRequirements"
+        supportsDbt:
+          type: boolean
+          description: an optional flag indicating whether DBT is used in the normalization. If the flag value is NULL - DBT is not used.
+        normalizationConfig:
+          $ref: "#/components/schemas/NormalizationDestinationDefinitionConfig"
+
     DestinationDefinitionReadList:
       type: object
       required:
@@ -2977,10 +2985,6 @@ components:
           type: array
           items:
             $ref: "#/components/schemas/DestinationSyncMode"
-        supportsDbt:
-          type: boolean
-        supportsNormalization:
-          type: boolean
     # DESTINATION
     DestinationId:
       type: string
@@ -4370,6 +4374,27 @@ components:
           type: array
           items:
             "$ref": "#/components/schemas/JobTypeResourceLimit"
+    NormalizationDestinationDefinitionConfig:
+      description: describes a normalization config for destination definition
+      type: object
+      required:
+        - supported
+      additionalProperties: false
+      properties:
+        supported:
+          type: boolean
+          description: whether the destination definition supports normalization.
+          default: false
+        normalizationRepository:
+          type: string
+          description: a field indicating the name of the repository to be used for normalization. If the value of the flag is NULL - normalization is not used.
+        normalizationTag:
+          type: string
+          description: a field indicating the tag of the docker repository to be used for normalization.
+        normalizationIntegrationType:
+          type: string
+          description: a field indicating the type of integration dialect to use for normalization.
+
     JobTypeResourceLimit:
       description: sets resource requirements for a specific job type for an actor definition. these values override the default, if both are set.
       type: object

diff --git a/airbyte-server/src/main/java/io/airbyte/server/converters/ApiPojoConverters.java b/airbyte-server/src/main/java/io/airbyte/server/converters/ApiPojoConverters.java
@@ -15,6 +15,7 @@
 import io.airbyte.api.model.generated.JobType;
 import io.airbyte.api.model.generated.JobTypeResourceLimit;
 import io.airbyte.api.model.generated.NonBreakingChangesPreference;
+import io.airbyte.api.model.generated.NormalizationDestinationDefinitionConfig;
 import io.airbyte.api.model.generated.ResourceRequirements;
 import io.airbyte.commons.enums.Enums;
 import io.airbyte.config.BasicSchedule;
@@ -81,6 +82,17 @@ public static ResourceRequirements resourceRequirementsToApi(final io.airbyte.co
         .memoryLimit(resourceReqs.getMemoryLimit());
   }
 
+  public static NormalizationDestinationDefinitionConfig normalizationDestinationDefinitionConfigToApi(final io.airbyte.config.NormalizationDestinationDefinitionConfig normalizationDestinationDefinitionConfig) {
+    if (normalizationDestinationDefinitionConfig == null) {
+      return new NormalizationDestinationDefinitionConfig().supported(false);
+    }
+    return new NormalizationDestinationDefinitionConfig()
+        .supported(true)
+        .normalizationRepository(normalizationDestinationDefinitionConfig.getNormalizationRepository())
+        .normalizationTag(normalizationDestinationDefinitionConfig.getNormalizationTag())
+        .normalizationIntegrationType(normalizationDestinationDefinitionConfig.getNormalizationIntegrationType());
+  }
+
   public static ConnectionRead internalToConnectionRead(final StandardSync standardSync) {
     final ConnectionRead connectionRead = new ConnectionRead()
         .connectionId(standardSync.getConnectionId())

diff --git a/airbyte-server/src/main/java/io/airbyte/server/handlers/DestinationDefinitionsHandler.java b/airbyte-server/src/main/java/io/airbyte/server/handlers/DestinationDefinitionsHandler.java
@@ -48,6 +48,7 @@
 import java.time.LocalDate;
 import java.util.List;
 import java.util.Map.Entry;
+import java.util.Objects;
 import java.util.UUID;
 import java.util.function.Supplier;
 import java.util.stream.Collectors;
@@ -88,6 +89,7 @@ public DestinationDefinitionsHandler(final ConfigRepository configRepository,
   @VisibleForTesting
   static DestinationDefinitionRead buildDestinationDefinitionRead(final StandardDestinationDefinition standardDestinationDefinition) {
     try {
+
       return new DestinationDefinitionRead()
           .destinationDefinitionId(standardDestinationDefinition.getDestinationDefinitionId())
           .name(standardDestinationDefinition.getName())
@@ -98,6 +100,9 @@ static DestinationDefinitionRead buildDestinationDefinitionRead(final StandardDe
           .protocolVersion(standardDestinationDefinition.getProtocolVersion())
           .releaseStage(getReleaseStage(standardDestinationDefinition))
           .releaseDate(getReleaseDate(standardDestinationDefinition))
+          .supportsDbt(Objects.requireNonNullElse(standardDestinationDefinition.getSupportsDbt(), false))
+          .normalizationConfig(
+              ApiPojoConverters.normalizationDestinationDefinitionConfigToApi(standardDestinationDefinition.getNormalizationConfig()))
           .resourceRequirements(ApiPojoConverters.actorDefResourceReqsToApi(standardDestinationDefinition.getResourceRequirements()));
     } catch (final URISyntaxException | NullPointerException e) {
       throw new InternalServerKnownException("Unable to process retrieved latest destination definitions list", e);

diff --git a/airbyte-server/src/main/java/io/airbyte/server/handlers/SchedulerHandler.java b/airbyte-server/src/main/java/io/airbyte/server/handlers/SchedulerHandler.java
@@ -362,8 +362,6 @@ public DestinationDefinitionSpecificationRead getDestinationSpecification(
         .supportedDestinationSyncModes(Enums.convertListTo(spec.getSupportedDestinationSyncModes(), DestinationSyncMode.class))
         .connectionSpecification(spec.getConnectionSpecification())
         .documentationUrl(spec.getDocumentationUrl().toString())
-        .supportsNormalization(spec.getSupportsNormalization())
-        .supportsDbt(spec.getSupportsDBT())
         .destinationDefinitionId(destinationDefinitionId);
 
     final Optional<AuthSpecification> authSpec = OauthModelConverter.getAuthSpec(spec);