Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

normalization: delete supportsDbt and supportsNormalization from DestinationDefinitionSpecificationRead #21005

Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
23 commits
Select commit Hold shift + click to select a range
6a26dc3
delete supportsDbt and supportsNormalization from DestinationDefiniti…
alafanechere Jan 4, 2023
bc5d1ec
FE changes
alafanechere Jan 4, 2023
362895e
add new fields to the DestinationDefinitionRead API model
alafanechere Jan 4, 2023
ad88c8a
update handlers
alafanechere Jan 4, 2023
314d423
update api doc
alafanechere Jan 4, 2023
200772e
update handlers
alafanechere Jan 4, 2023
764503c
Merge branch 'master' into augustin/normalization/clean-DestinationDe…
alafanechere Jan 4, 2023
83ac433
remove debug loggin
alafanechere Jan 4, 2023
08ee60d
Merge branch 'augustin/normalization/clean-DestinationDefinitionSpeci…
alafanechere Jan 4, 2023
dba6167
implement suggestions
alafanechere Jan 5, 2023
658d136
update octavia-cli tests
alafanechere Jan 5, 2023
71cc3f2
Merge branch 'master' into augustin/normalization/clean-DestinationDe…
alafanechere Jan 5, 2023
ec9b436
fix supported check
alafanechere Jan 6, 2023
938e571
update octavia-cli
alafanechere Jan 6, 2023
428f2be
fix integration tests
alafanechere Jan 6, 2023
528dfe3
fix mocks
alafanechere Jan 6, 2023
53ce4fd
fix forgotten renaming
alafanechere Jan 6, 2023
aa4fd23
Merge branch 'master' into augustin/normalization/clean-DestinationDe…
alafanechere Jan 6, 2023
bbb1f61
Merge branch 'master' into augustin/normalization/clean-DestinationDe…
alafanechere Jan 6, 2023
e3a3c24
Merge branch 'master' into augustin/normalization/clean-DestinationDe…
evantahler Jan 6, 2023
ee124be
Merge branch 'master' into augustin/normalization/clean-DestinationDe…
evantahler Jan 6, 2023
52b1ce7
Merge branch 'master' into augustin/normalization/clean-DestinationDe…
evantahler Jan 6, 2023
44f4cf8
Merge branch 'master' into augustin/normalization/clean-DestinationDe…
davinchia Jan 7, 2023
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
33 changes: 29 additions & 4 deletions airbyte-api/src/main/openapi/config.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -2878,6 +2878,8 @@ components:
- dockerRepository
- dockerImageTag
- documentationUrl
- supportsDbt
- normalizationConfig
properties:
destinationDefinitionId:
$ref: "#/components/schemas/DestinationDefinitionId"
Expand All @@ -2903,6 +2905,12 @@ components:
format: date
resourceRequirements:
$ref: "#/components/schemas/ActorDefinitionResourceRequirements"
supportsDbt:
type: boolean
description: an optional flag indicating whether DBT is used in the normalization. If the flag value is NULL - DBT is not used.
normalizationConfig:
$ref: "#/components/schemas/NormalizationDestinationDefinitionConfig"

DestinationDefinitionReadList:
type: object
required:
Expand Down Expand Up @@ -2977,10 +2985,6 @@ components:
type: array
items:
$ref: "#/components/schemas/DestinationSyncMode"
supportsDbt:
type: boolean
supportsNormalization:
type: boolean
# DESTINATION
DestinationId:
type: string
Expand Down Expand Up @@ -4370,6 +4374,27 @@ components:
type: array
items:
"$ref": "#/components/schemas/JobTypeResourceLimit"
NormalizationDestinationDefinitionConfig:
description: describes a normalization config for destination definition
type: object
required:
- supported
additionalProperties: false
properties:
supported:
type: boolean
description: whether the destination definition supports normalization.
default: false
normalizationRepository:
type: string
description: a field indicating the name of the repository to be used for normalization. If the value of the flag is NULL - normalization is not used.
normalizationTag:
type: string
description: a field indicating the tag of the docker repository to be used for normalization.
normalizationIntegrationType:
type: string
description: a field indicating the type of integration dialect to use for normalization.

JobTypeResourceLimit:
description: sets resource requirements for a specific job type for an actor definition. these values override the default, if both are set.
type: object
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@
import io.airbyte.api.model.generated.JobType;
import io.airbyte.api.model.generated.JobTypeResourceLimit;
import io.airbyte.api.model.generated.NonBreakingChangesPreference;
import io.airbyte.api.model.generated.NormalizationDestinationDefinitionConfig;
import io.airbyte.api.model.generated.ResourceRequirements;
import io.airbyte.commons.enums.Enums;
import io.airbyte.config.BasicSchedule;
Expand Down Expand Up @@ -81,6 +82,17 @@ public static ResourceRequirements resourceRequirementsToApi(final io.airbyte.co
.memoryLimit(resourceReqs.getMemoryLimit());
}

public static NormalizationDestinationDefinitionConfig normalizationDestinationDefinitionConfigToApi(final io.airbyte.config.NormalizationDestinationDefinitionConfig normalizationDestinationDefinitionConfig) {
if (normalizationDestinationDefinitionConfig == null) {
return new NormalizationDestinationDefinitionConfig().supported(false);
}
return new NormalizationDestinationDefinitionConfig()
.supported(true)
.normalizationRepository(normalizationDestinationDefinitionConfig.getNormalizationRepository())
.normalizationTag(normalizationDestinationDefinitionConfig.getNormalizationTag())
.normalizationIntegrationType(normalizationDestinationDefinitionConfig.getNormalizationIntegrationType());
}

public static ConnectionRead internalToConnectionRead(final StandardSync standardSync) {
final ConnectionRead connectionRead = new ConnectionRead()
.connectionId(standardSync.getConnectionId())
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -48,6 +48,7 @@
import java.time.LocalDate;
import java.util.List;
import java.util.Map.Entry;
import java.util.Objects;
import java.util.UUID;
import java.util.function.Supplier;
import java.util.stream.Collectors;
Expand Down Expand Up @@ -88,6 +89,7 @@ public DestinationDefinitionsHandler(final ConfigRepository configRepository,
@VisibleForTesting
static DestinationDefinitionRead buildDestinationDefinitionRead(final StandardDestinationDefinition standardDestinationDefinition) {
try {

return new DestinationDefinitionRead()
.destinationDefinitionId(standardDestinationDefinition.getDestinationDefinitionId())
.name(standardDestinationDefinition.getName())
Expand All @@ -98,6 +100,9 @@ static DestinationDefinitionRead buildDestinationDefinitionRead(final StandardDe
.protocolVersion(standardDestinationDefinition.getProtocolVersion())
.releaseStage(getReleaseStage(standardDestinationDefinition))
.releaseDate(getReleaseDate(standardDestinationDefinition))
.supportsDbt(Objects.requireNonNullElse(standardDestinationDefinition.getSupportsDbt(), false))
.normalizationConfig(
ApiPojoConverters.normalizationDestinationDefinitionConfigToApi(standardDestinationDefinition.getNormalizationConfig()))
.resourceRequirements(ApiPojoConverters.actorDefResourceReqsToApi(standardDestinationDefinition.getResourceRequirements()));
} catch (final URISyntaxException | NullPointerException e) {
throw new InternalServerKnownException("Unable to process retrieved latest destination definitions list", e);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -362,8 +362,6 @@ public DestinationDefinitionSpecificationRead getDestinationSpecification(
.supportedDestinationSyncModes(Enums.convertListTo(spec.getSupportedDestinationSyncModes(), DestinationSyncMode.class))
.connectionSpecification(spec.getConnectionSpecification())
.documentationUrl(spec.getDocumentationUrl().toString())
.supportsNormalization(spec.getSupportsNormalization())
.supportsDbt(spec.getSupportsDBT())
.destinationDefinitionId(destinationDefinitionId);

final Optional<AuthSpecification> authSpec = OauthModelConverter.getAuthSpec(spec);
Expand Down
Loading