Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Refactor Catalog API #1934

Merged
merged 5 commits into from
Feb 15, 2021
Merged

Refactor Catalog API #1934

Show file tree
Hide file tree
Changes from 4 commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
c04752d
Refactor API and drop Json Schema objects in BE/FE for objects closer…
ChristopheDuong Feb 4, 2021
26fe2da
Refactor API catalog classes
ChristopheDuong Feb 4, 2021
2430e36
Rename some fields
ChristopheDuong Feb 4, 2021
3c23c8c
Fix test
ChristopheDuong Feb 5, 2021
2726ef6
Merge remote-tracking branch 'origin/master' into api_catalog
ChristopheDuong Feb 8, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions airbyte-api/build.gradle
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@ task generateApiServer(type: GenerateTask) {
'SourceConfiguration' : 'com.fasterxml.jackson.databind.JsonNode',
'DestinationDefinitionSpecification': 'com.fasterxml.jackson.databind.JsonNode',
'DestinationConfiguration' : 'com.fasterxml.jackson.databind.JsonNode',
'StreamJsonSchema' : 'com.fasterxml.jackson.databind.JsonNode',
]

generateApiDocumentation = false
Expand Down Expand Up @@ -57,6 +58,7 @@ task generateApiClient(type: GenerateTask) {
'SourceConfiguration' : 'com.fasterxml.jackson.databind.JsonNode',
'DestinationDefinitionSpecification': 'com.fasterxml.jackson.databind.JsonNode',
'DestinationConfiguration' : 'com.fasterxml.jackson.databind.JsonNode',
'StreamJsonSchema' : 'com.fasterxml.jackson.databind.JsonNode',
]

library = "native"
Expand Down Expand Up @@ -89,6 +91,7 @@ task generateApiDocs(type: GenerateTask) {
'SourceConfiguration' : 'com.fasterxml.jackson.databind.JsonNode',
'DestinationDefinitionSpecification': 'com.fasterxml.jackson.databind.JsonNode',
'DestinationConfiguration' : 'com.fasterxml.jackson.databind.JsonNode',
'StreamJsonSchema' : 'com.fasterxml.jackson.databind.JsonNode',
]

generateApiDocumentation = false
Expand Down
95 changes: 51 additions & 44 deletions airbyte-api/src/main/openapi/config.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -386,7 +386,7 @@ paths:
post:
tags:
- source
summary: Discover the schema of the source
summary: Discover the schema catalog of the source
operationId: discoverSchemaForSource
requestBody:
content:
Expand Down Expand Up @@ -1366,13 +1366,13 @@ components:
items:
$ref: "#/components/schemas/SourceRead"
SourceDiscoverSchemaRead:
description: Returns the results of a discover schema job. If the job was not successful, the schema field will not be present. jobInfo will aways be present and its status be used to determine if the job was successful or not.
description: Returns the results of a discover catalog job. If the job was not successful, the catalog field will not be present. jobInfo will aways be present and its status be used to determine if the job was successful or not.
type: object
required:
- jobInfo
properties:
schema:
$ref: "#/components/schemas/SourceSchema"
catalog:
$ref: "#/components/schemas/AirbyteCatalog"
jobInfo:
$ref: "#/components/schemas/JobInfoRead"
# DESTINATION DEFINITION
Expand Down Expand Up @@ -1592,8 +1592,8 @@ components:
$ref: "#/components/schemas/SourceId"
destinationId:
$ref: "#/components/schemas/DestinationId"
syncSchema:
$ref: "#/components/schemas/SourceSchema"
syncCatalog:
$ref: "#/components/schemas/AirbyteCatalog"
schedule:
$ref: "#/components/schemas/ConnectionSchedule"
status:
Expand All @@ -1602,13 +1602,13 @@ components:
type: object
required:
- connectionId
- syncSchema
- syncCatalog
- status
properties:
connectionId:
$ref: "#/components/schemas/ConnectionId"
syncSchema:
$ref: "#/components/schemas/SourceSchema"
syncCatalog:
$ref: "#/components/schemas/AirbyteCatalog"
schedule:
$ref: "#/components/schemas/ConnectionSchedule"
status:
Expand All @@ -1627,7 +1627,7 @@ components:
- name
- sourceId
- destinationId
- syncSchema
- syncCatalog
- status
properties:
connectionId:
Expand All @@ -1638,8 +1638,8 @@ components:
$ref: "#/components/schemas/SourceId"
destinationId:
$ref: "#/components/schemas/DestinationId"
syncSchema:
$ref: "#/components/schemas/SourceSchema"
syncCatalog:
$ref: "#/components/schemas/AirbyteCatalog"
schedule:
$ref: "#/components/schemas/ConnectionSchedule"
status:
Expand Down Expand Up @@ -1692,65 +1692,72 @@ components:
properties:
logType:
$ref: "#/components/schemas/LogType"
# SCHEMA
SourceSchema:
description: describes the available schema.
# SCHEMA CATALOG
AirbyteCatalog:
description: describes the available schema (catalog).
type: object
required:
- streams
properties:
streams:
type: array
items:
$ref: "#/components/schemas/SourceSchemaStream"
SourceSchemaStream:
$ref: "#/components/schemas/AirbyteStreamAndConfiguration"
AirbyteStreamAndConfiguration:
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

What's the point of prepending entities with Airbyte? Most of the other fields don't have this prefix. What is more, I do not think that it actually brings any meaning to prepend all fields with Airbyte

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Also, @ChristopheDuong do you think its a good idea to combine different parts of this object with And? What if we add 1 more field here somewhere in future? It means that the current name will be obsolete.

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Stream is a reserved word in java and so that's why we tend to stick the word Airbyte in front of it. We will need to come up with a better name for this this thing. The and is not ideal, but we just don't have a better name yet.

description: each stream is split in two parts; the immutable schema from source and mutable configuration for destination
type: object
additionalProperties: false
properties:
stream:
$ref: "#/components/schemas/AirbyteStream"
config:
$ref: "#/components/schemas/AirbyteStreamConfiguration"
AirbyteStream:
description: the immutable schema defined by the source
type: object
additionalProperties: false
required:
- name
- fields
- supportedSyncModes
- json_schema
# todo (cgardens) - make required once sources are migrated
# - supported_sync_modes
properties:
# immutable
name:
type: string
cleanedName:
type: string
fields:
type: array
items:
$ref: "#/components/schemas/SourceSchemaField"
description: Stream's name.
jsonSchema:
description: Stream schema using Json Schema specs.
$ref: "#/components/schemas/StreamJsonSchema"
supportedSyncModes:
type: array
items:
$ref: "#/components/schemas/SyncMode"
sourceDefinedCursor:
description: If the source defines the cursor field, then it does any other cursor field inputs will be ignored. If it does not either the user_provided one is used or as a backup the default one is used.
type: boolean
defaultCursorField:
description: Path to the field that will be used to determine if a record is new or modified since the last sync. If not provided by the source, the end user will have to specify the comparable themselves.
type: array
items:
type: string
# configurable
selected:
type: boolean
StreamJsonSchema:
type: object
AirbyteStreamConfiguration:
description: the mutable part of the stream to configure the destination
type: object
additionalProperties: false
properties:
syncMode:
$ref: "#/components/schemas/SyncMode"
default: full_refresh
cursorField:
description: Path to the field that will be used to determine if a record is new or modified since the last sync. This field is REQUIRED if `sync_mode` is `incremental`. Otherwise it is ignored.
type: array
items:
type: string
SourceSchemaField:
type: object
required:
- name
- dataType
- selected
properties:
name:
type: string
cleanedName:
aliasName:
description: Alias name to the stream to be used in the destination
type: string
dataType:
$ref: "#/components/schemas/DataType"
selected:
type: boolean
DataType:
Expand Down Expand Up @@ -1942,7 +1949,7 @@ components:
- name
- sourceId
- destinationId
- syncSchema
- syncCatalog
- status
- source
- destination
Expand All @@ -1956,8 +1963,8 @@ components:
$ref: "#/components/schemas/SourceId"
destinationId:
$ref: "#/components/schemas/DestinationId"
syncSchema:
$ref: "#/components/schemas/SourceSchema"
syncCatalog:
$ref: "#/components/schemas/AirbyteCatalog"
schedule:
$ref: "#/components/schemas/ConnectionSchedule"
status:
Expand Down
8 changes: 8 additions & 0 deletions airbyte-commons/src/main/java/io/airbyte/commons/enums/Enums.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@
import com.google.common.collect.Maps;
import com.google.common.collect.Sets;
import java.util.Arrays;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.Set;
Expand All @@ -43,6 +44,13 @@ public static <T1 extends Enum<T1>, T2 extends Enum<T2>> T2 convertTo(T1 ie, Cla
return Enum.valueOf(oe, ie.name());
}

public static <T1 extends Enum<T1>, T2 extends Enum<T2>> List<T2> convertListTo(List<T1> ies, Class<T2> oe) {
return ies
.stream()
.map(ie -> convertTo(ie, oe))
.collect(Collectors.toList());
}

public static <T1 extends Enum<T1>, T2 extends Enum<T2>> boolean isCompatible(Class<T1> c1,
Class<T2> c2) {
Preconditions.checkArgument(c1.isEnum());
Expand Down
Loading