-
Notifications
You must be signed in to change notification settings - Fork 4.2k
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Merge branch 'master' into leti/remove-oauth-position-ff
* master: Discover worker starts to use API to write schema result (#21875) 🪟 🎉 Connector Builder Landing Page (#22122) Fix pnpm cache path (#22418) Add additional shorter setup guides (#22318) Source Amazon Ads: fix reports stream records primary keys (#21677) Connector acceptance test: Fix discovered catalog caching for different configs (#22301) 🪟🐛 Make modal scrollable (#21973) only compute diff if the schema discovery actually succeeded (#22377) Source Klaviyo: fix schema (#22071) 🪟 🔧 Switch to `pnpm` for package managing (#22053) Source Sentry: turn on default availability strategy (#22303) Source freshdesk: deduplicate table names (#22164)
- Loading branch information
Showing
113 changed files
with
17,947 additions
and
58,452 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
66 changes: 66 additions & 0 deletions
66
airbyte-commons-worker/src/main/java/io/airbyte/workers/helper/CatalogClientConverters.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,66 @@ | ||
| /* | ||
| * Copyright (c) 2022 Airbyte, Inc., all rights reserved. | ||
| */ | ||
|
|
||
| package io.airbyte.workers.helper; | ||
|
|
||
| import io.airbyte.commons.enums.Enums; | ||
| import io.airbyte.commons.text.Names; | ||
| import io.airbyte.protocol.models.AirbyteStream; | ||
| import java.util.stream.Collectors; | ||
|
|
||
| /** | ||
| * Utilities to convert Catalog protocol to Catalog API client. This class was similar to existing | ||
| * logic in CatalogConverter.java; But code can't be shared because the protocol model is | ||
| * essentially converted to two different api models. Thus, if we need to change logic on either | ||
| * place we have to take care of the other one too. | ||
| */ | ||
| public class CatalogClientConverters { | ||
|
|
||
| /** | ||
| * Converts a protocol AirbyteCatalog to an OpenAPI client versioned AirbyteCatalog. | ||
| */ | ||
| public static io.airbyte.api.client.model.generated.AirbyteCatalog toAirbyteCatalogClientApi( | ||
| final io.airbyte.protocol.models.AirbyteCatalog catalog) { | ||
| return new io.airbyte.api.client.model.generated.AirbyteCatalog() | ||
| .streams(catalog.getStreams() | ||
| .stream() | ||
| .map(stream -> toAirbyteStreamClientApi(stream)) | ||
| .map(s -> new io.airbyte.api.client.model.generated.AirbyteStreamAndConfiguration() | ||
| .stream(s) | ||
| .config(generateDefaultConfiguration(s))) | ||
| .collect(Collectors.toList())); | ||
| } | ||
|
|
||
| private static io.airbyte.api.client.model.generated.AirbyteStreamConfiguration generateDefaultConfiguration( | ||
| final io.airbyte.api.client.model.generated.AirbyteStream stream) { | ||
| final io.airbyte.api.client.model.generated.AirbyteStreamConfiguration result = | ||
| new io.airbyte.api.client.model.generated.AirbyteStreamConfiguration() | ||
| .aliasName(Names.toAlphanumericAndUnderscore(stream.getName())) | ||
| .cursorField(stream.getDefaultCursorField()) | ||
| .destinationSyncMode(io.airbyte.api.client.model.generated.DestinationSyncMode.APPEND) | ||
| .primaryKey(stream.getSourceDefinedPrimaryKey()) | ||
| .selected(true); | ||
| if (stream.getSupportedSyncModes().size() > 0) { | ||
| result.setSyncMode(Enums.convertTo(stream.getSupportedSyncModes().get(0), | ||
| io.airbyte.api.client.model.generated.SyncMode.class)); | ||
| } else { | ||
| result.setSyncMode(io.airbyte.api.client.model.generated.SyncMode.INCREMENTAL); | ||
| } | ||
| return result; | ||
| } | ||
|
|
||
| private static io.airbyte.api.client.model.generated.AirbyteStream toAirbyteStreamClientApi( | ||
| final AirbyteStream stream) { | ||
| return new io.airbyte.api.client.model.generated.AirbyteStream() | ||
| .name(stream.getName()) | ||
| .jsonSchema(stream.getJsonSchema()) | ||
| .supportedSyncModes(Enums.convertListTo(stream.getSupportedSyncModes(), | ||
| io.airbyte.api.client.model.generated.SyncMode.class)) | ||
| .sourceDefinedCursor(stream.getSourceDefinedCursor()) | ||
| .defaultCursorField(stream.getDefaultCursorField()) | ||
| .sourceDefinedPrimaryKey(stream.getSourceDefinedPrimaryKey()) | ||
| .namespace(stream.getNamespace()); | ||
| } | ||
|
|
||
| } |
68 changes: 68 additions & 0 deletions
68
...e-commons-worker/src/test/java/io/airbyte/workers/helper/CatalogClientConvertersTest.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,68 @@ | ||
| /* | ||
| * Copyright (c) 2022 Airbyte, Inc., all rights reserved. | ||
| */ | ||
|
|
||
| package io.airbyte.workers.helper; | ||
|
|
||
| import static org.junit.jupiter.api.Assertions.assertEquals; | ||
|
|
||
| import com.google.common.collect.Lists; | ||
| import io.airbyte.commons.text.Names; | ||
| import io.airbyte.protocol.models.AirbyteCatalog; | ||
| import io.airbyte.protocol.models.AirbyteStream; | ||
| import io.airbyte.protocol.models.CatalogHelpers; | ||
| import io.airbyte.protocol.models.Field; | ||
| import io.airbyte.protocol.models.JsonSchemaType; | ||
| import io.airbyte.protocol.models.SyncMode; | ||
| import java.util.Collections; | ||
| import java.util.List; | ||
| import org.junit.jupiter.api.Test; | ||
|
|
||
| class CatalogClientConvertersTest { | ||
|
|
||
| public static final String ID_FIELD_NAME = "id"; | ||
| private static final String STREAM_NAME = "users-data"; | ||
| private static final AirbyteStream STREAM = new AirbyteStream() | ||
| .withName(STREAM_NAME) | ||
| .withJsonSchema( | ||
| CatalogHelpers.fieldsToJsonSchema(Field.of(ID_FIELD_NAME, JsonSchemaType.STRING))) | ||
| .withDefaultCursorField(Lists.newArrayList(ID_FIELD_NAME)) | ||
| .withSourceDefinedCursor(false) | ||
| .withSourceDefinedPrimaryKey(Collections.emptyList()) | ||
| .withSupportedSyncModes(List.of(SyncMode.FULL_REFRESH, SyncMode.INCREMENTAL)); | ||
|
|
||
| private static final io.airbyte.api.client.model.generated.AirbyteStream CLIENT_STREAM = | ||
| new io.airbyte.api.client.model.generated.AirbyteStream() | ||
| .name(STREAM_NAME) | ||
| .jsonSchema(CatalogHelpers.fieldsToJsonSchema(Field.of(ID_FIELD_NAME, JsonSchemaType.STRING))) | ||
| .defaultCursorField(Lists.newArrayList(ID_FIELD_NAME)) | ||
| .sourceDefinedCursor(false) | ||
| .sourceDefinedPrimaryKey(Collections.emptyList()) | ||
| .supportedSyncModes(List.of(io.airbyte.api.client.model.generated.SyncMode.FULL_REFRESH, | ||
| io.airbyte.api.client.model.generated.SyncMode.INCREMENTAL)); | ||
| private static final io.airbyte.api.client.model.generated.AirbyteStreamConfiguration CLIENT_DEFAULT_STREAM_CONFIGURATION = | ||
| new io.airbyte.api.client.model.generated.AirbyteStreamConfiguration() | ||
| .syncMode(io.airbyte.api.client.model.generated.SyncMode.FULL_REFRESH) | ||
| .cursorField(Lists.newArrayList(ID_FIELD_NAME)) | ||
| .destinationSyncMode(io.airbyte.api.client.model.generated.DestinationSyncMode.APPEND) | ||
| .primaryKey(Collections.emptyList()) | ||
| .aliasName(Names.toAlphanumericAndUnderscore(STREAM_NAME)) | ||
| .selected(true); | ||
|
|
||
| private static final AirbyteCatalog BASIC_MODEL_CATALOG = new AirbyteCatalog().withStreams( | ||
| Lists.newArrayList(STREAM)); | ||
|
|
||
| private static final io.airbyte.api.client.model.generated.AirbyteCatalog EXPECTED_CLIENT_CATALOG = | ||
| new io.airbyte.api.client.model.generated.AirbyteCatalog() | ||
| .streams(Lists.newArrayList( | ||
| new io.airbyte.api.client.model.generated.AirbyteStreamAndConfiguration() | ||
| .stream(CLIENT_STREAM) | ||
| .config(CLIENT_DEFAULT_STREAM_CONFIGURATION))); | ||
|
|
||
| @Test | ||
| void testConvertToClientAPI() { | ||
| assertEquals(EXPECTED_CLIENT_CATALOG, | ||
| CatalogClientConverters.toAirbyteCatalogClientApi(BASIC_MODEL_CATALOG)); | ||
| } | ||
|
|
||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.